enum TokenizerState {
Data,
CharacterReferenceInData,
RcData,
CharacterReferenceInRcData,
RawText,
ScriptData,
PlainText,
TagOpen,
EndTagOpen,
TagName,
RcDataLessThan,
RcDataEndTagOpen,
RcDataEndTagName,
RawTextLessThanSign,
RawTextEndTagOpen,
RawTextEndTagName,
ScriptDataLessThanSign,
ScriptDataEndTagOpen,
ScriptDataEndTagName,
ScriptDataEscapeStart,
ScriptDataEscapeStartDash,
ScriptDataEscaped,
ScriptDataEscapedDash,
ScriptDataEscapedDashDash,
ScriptDataEscapedLessThanSign,
ScriptDataEscapedEndTagOpen,
ScriptDataEscapedEndTagName,
ScriptDataDoubleEscaped,
ScriptDataDoubleEscapedDash,
ScriptDataDoubleEscapedDashDash,
ScriptDataDoubleEscapedLessThanSign,
ScriptDataDoubleEscapeEnd,
BeforeAttributeName,
AttributeName,
AfterAttributeName,
BeforeAttributeValue,
AttributeValueDoubleQuoted,
AttributeValueSingleQuoted,
AttributeValueUnquoted,
CharacterReferenceInAttributeValue,
AfterAttributeValue,
SelfClosingStartTag,
BogusComment,
MarkupDeclarationOpen,
CommentStart,
CommentStartDash,
Comment,
CommentEndDash,
CommentEnd,
CommendEndBang,
DocType,
BeforeDocTypeName,
DocTypeName,
AfterDocTypeName,
AfterDocTypePublicKeyword,
DocTypePublicIdentifierDoubleQuoted,
DocTypePublicIdentifierSingleQuoted,
AfterDocTypePublicIdentifier,
BetweenDocTypePublicAndSystemIdentifiers,
AfterDocTypeSystemKeyword,
BeforeDocTypeSystemIdentifier,
DocTypeSystemIdentifierDoubleQuoted,
DocTypeSystemIdentifierSingleQuoted,
AfterDocTypeSystemIdentifier,
BogusDocType,
CdataSection,
}
struct TagBuilder {
name: String,
self_closing: bool,
end_tag: bool,
attributes: Vec<(Option<String>, Option<String>)>,
}
impl TagBuilder {
fn is_appropriate_end_tag(&self, last_start_tag: &Option<String>) -> bool {
last_start_tag
.as_ref()
.is_some_and(|name| name == &self.name)
}
}
impl From<TagBuilder> for Token {
fn from(value: TagBuilder) -> Self {
let attributes = value
.attributes
.into_iter()
.map(|(key, value)| (key.map(Into::into), value.map(Into::into)))
.collect();
if value.end_tag {
Token::EndTag {
tag_name: value.name.into(),
self_closing: value.self_closing,
attributes,
}
} else {
Token::StartTag {
tag_name: value.name.into(),
self_closing: value.self_closing,
attributes,
}
}
}
}
fn consume_character_reference(chars: &mut Peekable<Chars<'_>>) -> Option<char> {
match chars.next() {
Some('\t' | '\u{000a}' | '\u{000c}' | ' ' | '<' | '&') | None => None,
Some('#') => match chars.peek() {
Some('x' | 'X') => {
chars.next();
let mut digits = chars
.take_while(|c| c.is_ascii_hexdigit())
.collect::<Box<str>>();
if digits.is_empty() {
panic!("Parse error: no digits found after the hexadecimal specifier");
return None;
}
if chars.next_if(|c| *c == ';').is_none() {
panic!("Parse error: expected a semicolon after escaped digits");
}
let number = u32::from_str_radix(&digits, 16).expect("only hexadecimal digits");
if let Some(c) = match number {
0x00 => Some('\u{fffd}'),
0x0d => Some('\r'),
0x80 => Some('\u{20ac}'),
0x81 => Some('\u{0081}'),
0x82 => Some('\u{201a}'),
0x83 => Some('\u{0192}'),
0x84 => Some('\u{201e}'),
0x85 => Some('\u{2026}'),
0x86 => Some('\u{2020}'),
0x87 => Some('\u{2021}'),
0x88 => Some('\u{02c6}'),
0x89 => Some('\u{2030}'),
0x8a => Some('\u{0160}'),
0x8b => Some('\u{2039}'),
0x8c => Some('\u{0152}'),
0x8d => Some('\u{008d}'),
0x8e => Some('\u{017d}'),
0x8f => Some('\u{008f}'),
0x90 => Some('\u{0090}'),
0x91 => Some('\u{2018}'),
0x92 => Some('\u{2019}'),
0x93 => Some('\u{201c}'),
0x94 => Some('\u{201d}'),
0x95 => Some('\u{2022}'),
0x96 => Some('\u{2013}'),
0x97 => Some('\u{2014}'),
0x98 => Some('\u{02dc}'),
0x99 => Some('\u{2122}'),
0x9a => Some('\u{0161}'),
0x9b => Some('\u{203a}'),
0x9c => Some('\u{0152}'),
0x9d => Some('\u{009d}'),
0x9e => Some('\u{017e}'),
0x9f => Some('\u{0178}'),
_ => None,
} {
panic!("Parse error: invalid character");
return Some(c);
}
if (0xd800 < number && number < 0xdfff) || number > 0x10ffff {
panic!("Parse error: invalid character reference");
return Some(char::REPLACEMENT_CHARACTER);
}
match char::from_u32(number) {
Some(char) => Some(char),
None => panic!("Parse error: invalid character reference"),
}
}
_ => {
let mut digits = chars
.take_while(|c| c.is_ascii_digit())
.collect::<Box<str>>();
if digits.is_empty() {
panic!("Parse error: no digits found after the ampersand");
return None;
}
if chars.next_if(|c| *c == ';').is_none() {
panic!("Parse error: expected a semicolon after escaped digits");
}
let number = u32::from_str_radix(&digits, 10).expect("only decimal digits");
if let Some(c) = match number {
0x00 => Some('\u{fffd}'),
0x0d => Some('\r'),
0x80 => Some('\u{20ac}'),
0x81 => Some('\u{0081}'),
0x82 => Some('\u{201a}'),
0x83 => Some('\u{0192}'),
0x84 => Some('\u{201e}'),
0x85 => Some('\u{2026}'),
0x86 => Some('\u{2020}'),
0x87 => Some('\u{2021}'),
0x88 => Some('\u{02c6}'),
0x89 => Some('\u{2030}'),
0x8a => Some('\u{0160}'),
0x8b => Some('\u{2039}'),
0x8c => Some('\u{0152}'),
0x8d => Some('\u{008d}'),
0x8e => Some('\u{017d}'),
0x8f => Some('\u{008f}'),
0x90 => Some('\u{0090}'),
0x91 => Some('\u{2018}'),
0x92 => Some('\u{2019}'),
0x93 => Some('\u{201c}'),
0x94 => Some('\u{201d}'),
0x95 => Some('\u{2022}'),
0x96 => Some('\u{2013}'),
0x97 => Some('\u{2014}'),
0x98 => Some('\u{02dc}'),
0x99 => Some('\u{2122}'),
0x9a => Some('\u{0161}'),
0x9b => Some('\u{203a}'),
0x9c => Some('\u{0152}'),
0x9d => Some('\u{009d}'),
0x9e => Some('\u{017e}'),
0x9f => Some('\u{0178}'),
_ => None,
} {
panic!("Parse error: invalid character");
return Some(c);
}
if (0xd800 < number && number < 0xdfff) || number > 0x10ffff {
panic!("Parse error: invalid character reference");
return Some(char::REPLACEMENT_CHARACTER);
}
match char::from_u32(number) {
Some(char) => Some(char),
None => panic!("Parse error: invalid character reference"),
}
}
},
_ => todo!("Parse named character reference"),
}
}
fn tokenize(source: &str) -> Box<[Token]> {
let mut chars = source.chars().peekable();
let mut tokens = Vec::new();
let mut state = TokenizerState::Data;
let mut tag_builder = TagBuilder {
name: String::new(),
self_closing: false,
end_tag: false,
attributes: Vec::new(),
};
let mut last_start_tag = None;
let mut temp_buffer = String::new();
loop {
match state {
TokenizerState::Data => match chars.next() {
Some('&') => state = TokenizerState::CharacterReferenceInData,
Some('<') => state = TokenizerState::TagOpen,
Some('\0') => {
panic!("Parse error: NUL character");
tokens.push(Token::Character('\0'));
}
None => break,
Some(c) => tokens.push(Token::Character(c)),
},
TokenizerState::CharacterReferenceInData => {
tokens.push(Token::Character(
consume_character_reference(&mut chars).unwrap_or('&'),
));
state = TokenizerState::Data;
}
TokenizerState::RcData => match chars.next() {
Some('&') => state = TokenizerState::CharacterReferenceInRcData,
Some('<') => state = TokenizerState::RcDataLessThan,
Some('\0') => {
panic!("Parse error: NUL character");
tokens.push(Token::Character(char::REPLACEMENT_CHARACTER));
}
None => break,
Some(c) => tokens.push(Token::Character(c)),
},
TokenizerState::CharacterReferenceInRcData => {
tokens.push(Token::Character(
consume_character_reference(&mut chars).unwrap_or('&'),
));
state = TokenizerState::RcData;
}
TokenizerState::RawText => match chars.next() {
Some('<') => state = TokenizerState::RawTextLessThanSign,
Some('\0') => {
panic!("Parse error: NUL character");
tokens.push(Token::Character(char::REPLACEMENT_CHARACTER));
}
None => break,
Some(c) => tokens.push(Token::Character(c)),
},
TokenizerState::ScriptData => match chars.next() {
Some('<') => state = TokenizerState::ScriptDataLessThanSign,
Some('\0') => {
panic!("Parse error: NUL character");
tokens.push(Token::Character(char::REPLACEMENT_CHARACTER));
}
None => break,
Some(c) => tokens.push(Token::Character(c)),
},
TokenizerState::PlainText => match chars.next() {
Some('\0') => {
panic!("Parse error: NUL character");
tokens.push(Token::Character(char::REPLACEMENT_CHARACTER));
}
None => break,
Some(c) => tokens.push(Token::Character(c)),
},
TokenizerState::TagOpen => match chars.next() {
Some('!') => state = TokenizerState::MarkupDeclarationOpen,
Some('/') => state = TokenizerState::EndTagOpen,
Some(c) if c.is_ascii_uppercase() => {
state = TokenizerState::TagName;
tag_builder = TagBuilder {
name: String::from(c.to_ascii_lowercase()),
end_tag: false,
self_closing: false,
attributes: Vec::new(),
};
}
Some(c) if c.is_ascii_lowercase() => {
state = TokenizerState::TagName;
tag_builder = TagBuilder {
name: c.to_string(),
end_tag: false,
self_closing: false,
attributes: Vec::new(),
}
}
Some('?') => {
panic!("Parse error: bogus comment");
state = TokenizerState::BogusComment;
}
None => {
panic!("Parse error: unfinished tag");
tokens.push(Token::Character('<'));
state = TokenizerState::Data;
}
_ => {
chars.next_back();
panic!("Parse error: less than sign without a tag");
tokens.push(Token::Character('<'));
state = TokenizerState::Data;
}
},
TokenizerState::EndTagOpen => match chars.next() {
Some(c) if c.is_ascii_uppercase() => {
state = TokenizerState::TagName;
tag_builder = TagBuilder {
name: String::from(c.to_ascii_lowercase()),
end_tag: true,
self_closing: false,
attributes: Vec::new(),
}
}
Some(c) if c.is_ascii_lowercase() => {
state = TokenizerState::TagName;
tag_builder = TagBuilder {
name: c.to_string(),
end_tag: true,
self_closing: false,
attributes: Vec::new(),
}
}
Some('>') => {
panic!("Parse error: end tag with no name");
state = TokenizerState::Data;
}
None => {
panic!("Parse error: unfinished end tag");
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
state = TokenizerState::Data;
}
_ => {
chars.next_back();
panic!("Parse error: less than sign without a tag");
tokens.push(Token::Character('<'));
state = TokenizerState::Data;
}
},
TokenizerState::TagName => match chars.next() {
Some('\t' | '\u{000a}' | '\u{000c}' | ' ') => {
state = TokenizerState::BeforeAttributeName
}
Some('/') => state = TokenizerState::SelfClosingStartTag,
Some('>') => {
state = TokenizerState::Data;
if !tag_builder.end_tag {
last_start_tag = Some(tag_builder.name.clone());
}
tokens.push(tag_builder.into());
}
Some(c) if c.is_ascii_uppercase() => tag_builder.name.push(c.to_ascii_lowercase()),
Some('\0') => {
panic!("Parse error: NUL character in tag name");
tag_builder.name.push(char::REPLACEMENT_CHARACTER);
}
None => {
panic!("Parse error: unfinished tag");
state = TokenizerState::Data;
}
Some(c) => tag_builder.name.push(c),
},
TokenizerState::RcDataLessThan => match chars.next() {
Some('/') => {
temp_buffer.clear();
state = TokenizerState::RcDataEndTagOpen;
}
_ => {
tokens.push(Token::Character('<'));
chars.next_back();
state = TokenizerState::RcData;
}
},
TokenizerState::RcDataEndTagOpen => match chars.next() {
Some(c) if c.is_ascii_uppercase() => {
tag_builder = TagBuilder {
end_tag: true,
name: c.to_ascii_lowercase().to_string(),
self_closing: false,
attributes: Vec::new(),
};
temp_buffer = c.to_string();
state = TokenizerState::RcDataEndTagName;
}
Some(c) if c.is_ascii_lowercase() => {
tag_builder = TagBuilder {
end_tag: true,
name: c.to_string(),
self_closing: false,
attributes: Vec::new(),
};
temp_buffer = c.to_string();
state = TokenizerState::RcDataEndTagName;
}
Some(c) => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
chars.next_back();
state = TokenizerState::RcData;
}
None => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
state = TokenizerState::RcData;
}
},
TokenizerState::RcDataEndTagName => match chars.next() {
Some('\t' | '\u{000a}' | '\u{000c}' | ' ')
if tag_builder.is_appropriate_end_tag(&last_start_tag) =>
{
state = TokenizerState::BeforeAttributeName
}
Some('/') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
state = TokenizerState::SelfClosingStartTag
}
Some('>') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
tokens.push(tag_builder.into());
state = TokenizerState::Data;
}
Some(c) if c.is_ascii_uppercase() => {
tag_builder.name.push(c.to_ascii_lowercase());
temp_buffer.push(c);
}
Some(c) if c.is_ascii_lowercase() => {
tag_builder.name.push(c);
temp_buffer.push(c);
}
None => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
for char in temp_buffer.chars() {
tokens.push(Token::Character(char));
}
state = TokenizerState::RcData;
}
_ => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
for char in temp_buffer.chars() {
tokens.push(Token::Character(char));
}
chars.next_back();
state = TokenizerState::RcData;
}
},
TokenizerState::RawTextLessThanSign => match chars.next() {
Some('/') => {
temp_buffer = String::new();
state = TokenizerState::RawTextEndTagOpen;
}
None => {
tokens.push(Token::Character('<'));
state = TokenizerState::RawText;
}
_ => {
tokens.push(Token::Character('<'));
chars.next_back();
state = TokenizerState::RawText;
}
},
TokenizerState::RawTextEndTagOpen => match chars.next() {
Some(c) if c.is_ascii_uppercase() => {
tag_builder = TagBuilder {
name: c.to_ascii_lowercase().to_string(),
self_closing: false,
end_tag: true,
attributes: Vec::new(),
};
temp_buffer.push(c);
state = TokenizerState::RawTextEndTagName;
}
Some(c) if c.is_ascii_lowercase() => {
tag_builder = TagBuilder {
name: c.to_string(),
self_closing: false,
end_tag: true,
attributes: Vec::new(),
};
temp_buffer.push(c);
state = TokenizerState::RawTextEndTagName;
}
None => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
state = TokenizerState::RawText;
}
Some(_) => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
state = TokenizerState::RawText;
chars.next_back();
}
},
TokenizerState::RawTextEndTagName => match chars.next() {
Some('\t' | '\u{000a}' | '\u{000c}' | ' ')
if tag_builder.is_appropriate_end_tag(&last_start_tag) =>
{
state = TokenizerState::BeforeAttributeName
}
Some('/') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
state = TokenizerState::SelfClosingStartTag
}
Some('>') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
tokens.push(tag_builder.into());
state = TokenizerState::Data;
}
Some(c) if c.is_ascii_uppercase() => {
tag_builder.name.push(c.to_ascii_lowercase());
temp_buffer.push(c);
}
Some(c) if c.is_ascii_lowercase() => {
tag_builder.name.push(c);
temp_buffer.push(c);
}
None => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
for char in temp_buffer.chars() {
tokens.push(Token::Character(char));
}
state = TokenizerState::RawText;
}
_ => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
for char in temp_buffer.chars() {
tokens.push(Token::Character(char));
}
chars.next_back();
state = TokenizerState::RawText;
}
},
TokenizerState::ScriptDataLessThanSign => match chars.next() {
Some('/') => {
temp_buffer = String::new();
state = TokenizerState::ScriptDataEndTagOpen;
}
Some('!') => {
state = TokenizerState::ScriptDataEscapeStart;
tokens.push(Token::Character('<'));
tokens.push(Token::Character('!'));
}
None => {
tokens.push(Token::Character('<'));
state = TokenizerState::ScriptData;
}
_ => {
tokens.push(Token::Character('<'));
state = TokenizerState::ScriptData;
chars.next_back();
}
},
TokenizerState::ScriptDataEndTagOpen => match chars.next() {
Some(c) if c.is_ascii_uppercase() => {
tag_builder = TagBuilder {
name: c.to_ascii_lowercase().to_string(),
self_closing: false,
end_tag: true,
attributes: Vec::new(),
};
temp_buffer.push(c);
state = TokenizerState::ScriptDataEndTagName;
}
Some(c) if c.is_ascii_lowercase() => {
tag_builder = TagBuilder {
name: c.to_string(),
self_closing: false,
end_tag: true,
attributes: Vec::new(),
};
temp_buffer.push(c);
state = TokenizerState::ScriptDataEndTagName;
}
None => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
state = TokenizerState::ScriptData;
}
_ => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
chars.next_back();
state = TokenizerState::ScriptData;
}
},
TokenizerState::ScriptDataEndTagName => match chars.next() {
Some('\t' | '\u{000a}' | '\u{000c}' | ' ')
if tag_builder.is_appropriate_end_tag(&last_start_tag) =>
{
state = TokenizerState::BeforeAttributeName
}
Some('/') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
state = TokenizerState::SelfClosingStartTag
}
Some('>') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
tokens.push(tag_builder.into());
state = TokenizerState::Data;
}
Some(c) if c.is_ascii_uppercase() => {
tag_builder.name.push(c.to_ascii_lowercase());
temp_buffer.push(c);
}
Some(c) if c.is_ascii_lowercase() => {
tag_builder.name.push(c);
temp_buffer.push(c);
}
None => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
for char in temp_buffer.chars() {
tokens.push(Token::Character(char));
}
state = TokenizerState::ScriptData;
}
_ => {
tokens.push(Token::Character('<'));
tokens.push(Token::Character('/'));
for char in temp_buffer.chars() {
tokens.push(Token::Character(char));
}
chars.next_back();
state = TokenizerState::ScriptData;
}
},
TokenizerState::ScriptDataEscapeStart => match chars.next() {
Some('-') => {
state = TokenizerState::ScriptDataEscapeStartDash;
tokens.push(Token::Character('-'));
}
None => state = TokenizerState::ScriptData,
_ => {
chars.next_back();
state = TokenizerState::ScriptData;
}
},
TokenizerState::ScriptDataEscapeStartDash => match chars.next() {
Some('-') => {
state = TokenizerState::ScriptDataEscapedDashDash;
tokens.push(Token::Character('-'));
}
None => state = TokenizerState::ScriptData,
_ => {
chars.next_back();
state = TokenizerState::ScriptData;
}
},
TokenizerState::ScriptDataEscaped => match chars.next() {
Some('-') => {
state = TokenizerState::ScriptDataEscapedDash;
tokens.push(Token::Character('-'));
}
Some('<') => state = TokenizerState::ScriptDataEscapedLessThanSign,
Some('\0') => {
panic!("Parse error: unexpected NUL character");
tokens.push(Token::Character(char::REPLACEMENT_CHARACTER));
}
None => {
panic!("Parse error: unexpected end of file")
}
Some(c) => tokens.push(Token::Character(c)),
},
}
}
tokens.into_boxed_slice()
}
|
