From 8399983fd45bc0f11eac9def280c0833ffb93c04 Mon Sep 17 00:00:00 2001 From: Mica White Date: Sun, 12 Jul 2026 19:49:14 -0400 Subject: initial commit --- src/old_tokenizer.rs | 720 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 720 insertions(+) create mode 100644 src/old_tokenizer.rs (limited to 'src/old_tokenizer.rs') diff --git a/src/old_tokenizer.rs b/src/old_tokenizer.rs new file mode 100644 index 0000000..9f9b79d --- /dev/null +++ b/src/old_tokenizer.rs @@ -0,0 +1,720 @@ +enum TokenizerState { + Data, + CharacterReferenceInData, + RcData, + CharacterReferenceInRcData, + RawText, + ScriptData, + PlainText, + TagOpen, + EndTagOpen, + TagName, + RcDataLessThan, + RcDataEndTagOpen, + RcDataEndTagName, + RawTextLessThanSign, + RawTextEndTagOpen, + RawTextEndTagName, + ScriptDataLessThanSign, + ScriptDataEndTagOpen, + ScriptDataEndTagName, + ScriptDataEscapeStart, + ScriptDataEscapeStartDash, + ScriptDataEscaped, + ScriptDataEscapedDash, + ScriptDataEscapedDashDash, + ScriptDataEscapedLessThanSign, + ScriptDataEscapedEndTagOpen, + ScriptDataEscapedEndTagName, + ScriptDataDoubleEscaped, + ScriptDataDoubleEscapedDash, + ScriptDataDoubleEscapedDashDash, + ScriptDataDoubleEscapedLessThanSign, + ScriptDataDoubleEscapeEnd, + BeforeAttributeName, + AttributeName, + AfterAttributeName, + BeforeAttributeValue, + AttributeValueDoubleQuoted, + AttributeValueSingleQuoted, + AttributeValueUnquoted, + CharacterReferenceInAttributeValue, + AfterAttributeValue, + SelfClosingStartTag, + BogusComment, + MarkupDeclarationOpen, + CommentStart, + CommentStartDash, + Comment, + CommentEndDash, + CommentEnd, + CommendEndBang, + DocType, + BeforeDocTypeName, + DocTypeName, + AfterDocTypeName, + AfterDocTypePublicKeyword, + DocTypePublicIdentifierDoubleQuoted, + DocTypePublicIdentifierSingleQuoted, + AfterDocTypePublicIdentifier, + BetweenDocTypePublicAndSystemIdentifiers, + AfterDocTypeSystemKeyword, + BeforeDocTypeSystemIdentifier, + DocTypeSystemIdentifierDoubleQuoted, + DocTypeSystemIdentifierSingleQuoted, + AfterDocTypeSystemIdentifier, + BogusDocType, + CdataSection, +} + +struct TagBuilder { + name: String, + self_closing: bool, + end_tag: bool, + attributes: Vec<(Option, Option)>, +} + +impl TagBuilder { + fn is_appropriate_end_tag(&self, last_start_tag: &Option) -> bool { + last_start_tag + .as_ref() + .is_some_and(|name| name == &self.name) + } +} + +impl From for Token { + fn from(value: TagBuilder) -> Self { + let attributes = value + .attributes + .into_iter() + .map(|(key, value)| (key.map(Into::into), value.map(Into::into))) + .collect(); + if value.end_tag { + Token::EndTag { + tag_name: value.name.into(), + self_closing: value.self_closing, + attributes, + } + } else { + Token::StartTag { + tag_name: value.name.into(), + self_closing: value.self_closing, + attributes, + } + } + } +} + +fn consume_character_reference(chars: &mut Peekable>) -> Option { + match chars.next() { + Some('\t' | '\u{000a}' | '\u{000c}' | ' ' | '<' | '&') | None => None, + Some('#') => match chars.peek() { + Some('x' | 'X') => { + chars.next(); + let mut digits = chars + .take_while(|c| c.is_ascii_hexdigit()) + .collect::>(); + if digits.is_empty() { + panic!("Parse error: no digits found after the hexadecimal specifier"); + return None; + } + if chars.next_if(|c| *c == ';').is_none() { + panic!("Parse error: expected a semicolon after escaped digits"); + } + + let number = u32::from_str_radix(&digits, 16).expect("only hexadecimal digits"); + if let Some(c) = match number { + 0x00 => Some('\u{fffd}'), + 0x0d => Some('\r'), + 0x80 => Some('\u{20ac}'), + 0x81 => Some('\u{0081}'), + 0x82 => Some('\u{201a}'), + 0x83 => Some('\u{0192}'), + 0x84 => Some('\u{201e}'), + 0x85 => Some('\u{2026}'), + 0x86 => Some('\u{2020}'), + 0x87 => Some('\u{2021}'), + 0x88 => Some('\u{02c6}'), + 0x89 => Some('\u{2030}'), + 0x8a => Some('\u{0160}'), + 0x8b => Some('\u{2039}'), + 0x8c => Some('\u{0152}'), + 0x8d => Some('\u{008d}'), + 0x8e => Some('\u{017d}'), + 0x8f => Some('\u{008f}'), + 0x90 => Some('\u{0090}'), + 0x91 => Some('\u{2018}'), + 0x92 => Some('\u{2019}'), + 0x93 => Some('\u{201c}'), + 0x94 => Some('\u{201d}'), + 0x95 => Some('\u{2022}'), + 0x96 => Some('\u{2013}'), + 0x97 => Some('\u{2014}'), + 0x98 => Some('\u{02dc}'), + 0x99 => Some('\u{2122}'), + 0x9a => Some('\u{0161}'), + 0x9b => Some('\u{203a}'), + 0x9c => Some('\u{0152}'), + 0x9d => Some('\u{009d}'), + 0x9e => Some('\u{017e}'), + 0x9f => Some('\u{0178}'), + _ => None, + } { + panic!("Parse error: invalid character"); + return Some(c); + } + + if (0xd800 < number && number < 0xdfff) || number > 0x10ffff { + panic!("Parse error: invalid character reference"); + return Some(char::REPLACEMENT_CHARACTER); + } + + match char::from_u32(number) { + Some(char) => Some(char), + None => panic!("Parse error: invalid character reference"), + } + } + _ => { + let mut digits = chars + .take_while(|c| c.is_ascii_digit()) + .collect::>(); + if digits.is_empty() { + panic!("Parse error: no digits found after the ampersand"); + return None; + } + if chars.next_if(|c| *c == ';').is_none() { + panic!("Parse error: expected a semicolon after escaped digits"); + } + + let number = u32::from_str_radix(&digits, 10).expect("only decimal digits"); + if let Some(c) = match number { + 0x00 => Some('\u{fffd}'), + 0x0d => Some('\r'), + 0x80 => Some('\u{20ac}'), + 0x81 => Some('\u{0081}'), + 0x82 => Some('\u{201a}'), + 0x83 => Some('\u{0192}'), + 0x84 => Some('\u{201e}'), + 0x85 => Some('\u{2026}'), + 0x86 => Some('\u{2020}'), + 0x87 => Some('\u{2021}'), + 0x88 => Some('\u{02c6}'), + 0x89 => Some('\u{2030}'), + 0x8a => Some('\u{0160}'), + 0x8b => Some('\u{2039}'), + 0x8c => Some('\u{0152}'), + 0x8d => Some('\u{008d}'), + 0x8e => Some('\u{017d}'), + 0x8f => Some('\u{008f}'), + 0x90 => Some('\u{0090}'), + 0x91 => Some('\u{2018}'), + 0x92 => Some('\u{2019}'), + 0x93 => Some('\u{201c}'), + 0x94 => Some('\u{201d}'), + 0x95 => Some('\u{2022}'), + 0x96 => Some('\u{2013}'), + 0x97 => Some('\u{2014}'), + 0x98 => Some('\u{02dc}'), + 0x99 => Some('\u{2122}'), + 0x9a => Some('\u{0161}'), + 0x9b => Some('\u{203a}'), + 0x9c => Some('\u{0152}'), + 0x9d => Some('\u{009d}'), + 0x9e => Some('\u{017e}'), + 0x9f => Some('\u{0178}'), + _ => None, + } { + panic!("Parse error: invalid character"); + return Some(c); + } + + if (0xd800 < number && number < 0xdfff) || number > 0x10ffff { + panic!("Parse error: invalid character reference"); + return Some(char::REPLACEMENT_CHARACTER); + } + + match char::from_u32(number) { + Some(char) => Some(char), + None => panic!("Parse error: invalid character reference"), + } + } + }, + _ => todo!("Parse named character reference"), + } +} + +fn tokenize(source: &str) -> Box<[Token]> { + let mut chars = source.chars().peekable(); + let mut tokens = Vec::new(); + let mut state = TokenizerState::Data; + let mut tag_builder = TagBuilder { + name: String::new(), + self_closing: false, + end_tag: false, + attributes: Vec::new(), + }; + let mut last_start_tag = None; + let mut temp_buffer = String::new(); + loop { + match state { + TokenizerState::Data => match chars.next() { + Some('&') => state = TokenizerState::CharacterReferenceInData, + Some('<') => state = TokenizerState::TagOpen, + Some('\0') => { + panic!("Parse error: NUL character"); + tokens.push(Token::Character('\0')); + } + None => break, + Some(c) => tokens.push(Token::Character(c)), + }, + TokenizerState::CharacterReferenceInData => { + tokens.push(Token::Character( + consume_character_reference(&mut chars).unwrap_or('&'), + )); + state = TokenizerState::Data; + } + TokenizerState::RcData => match chars.next() { + Some('&') => state = TokenizerState::CharacterReferenceInRcData, + Some('<') => state = TokenizerState::RcDataLessThan, + Some('\0') => { + panic!("Parse error: NUL character"); + tokens.push(Token::Character(char::REPLACEMENT_CHARACTER)); + } + None => break, + Some(c) => tokens.push(Token::Character(c)), + }, + TokenizerState::CharacterReferenceInRcData => { + tokens.push(Token::Character( + consume_character_reference(&mut chars).unwrap_or('&'), + )); + state = TokenizerState::RcData; + } + TokenizerState::RawText => match chars.next() { + Some('<') => state = TokenizerState::RawTextLessThanSign, + Some('\0') => { + panic!("Parse error: NUL character"); + tokens.push(Token::Character(char::REPLACEMENT_CHARACTER)); + } + None => break, + Some(c) => tokens.push(Token::Character(c)), + }, + TokenizerState::ScriptData => match chars.next() { + Some('<') => state = TokenizerState::ScriptDataLessThanSign, + Some('\0') => { + panic!("Parse error: NUL character"); + tokens.push(Token::Character(char::REPLACEMENT_CHARACTER)); + } + None => break, + Some(c) => tokens.push(Token::Character(c)), + }, + TokenizerState::PlainText => match chars.next() { + Some('\0') => { + panic!("Parse error: NUL character"); + tokens.push(Token::Character(char::REPLACEMENT_CHARACTER)); + } + None => break, + Some(c) => tokens.push(Token::Character(c)), + }, + TokenizerState::TagOpen => match chars.next() { + Some('!') => state = TokenizerState::MarkupDeclarationOpen, + Some('/') => state = TokenizerState::EndTagOpen, + Some(c) if c.is_ascii_uppercase() => { + state = TokenizerState::TagName; + tag_builder = TagBuilder { + name: String::from(c.to_ascii_lowercase()), + end_tag: false, + self_closing: false, + attributes: Vec::new(), + }; + } + Some(c) if c.is_ascii_lowercase() => { + state = TokenizerState::TagName; + tag_builder = TagBuilder { + name: c.to_string(), + end_tag: false, + self_closing: false, + attributes: Vec::new(), + } + } + Some('?') => { + panic!("Parse error: bogus comment"); + state = TokenizerState::BogusComment; + } + None => { + panic!("Parse error: unfinished tag"); + tokens.push(Token::Character('<')); + state = TokenizerState::Data; + } + _ => { + chars.next_back(); + panic!("Parse error: less than sign without a tag"); + tokens.push(Token::Character('<')); + state = TokenizerState::Data; + } + }, + TokenizerState::EndTagOpen => match chars.next() { + Some(c) if c.is_ascii_uppercase() => { + state = TokenizerState::TagName; + tag_builder = TagBuilder { + name: String::from(c.to_ascii_lowercase()), + end_tag: true, + self_closing: false, + attributes: Vec::new(), + } + } + Some(c) if c.is_ascii_lowercase() => { + state = TokenizerState::TagName; + tag_builder = TagBuilder { + name: c.to_string(), + end_tag: true, + self_closing: false, + attributes: Vec::new(), + } + } + Some('>') => { + panic!("Parse error: end tag with no name"); + state = TokenizerState::Data; + } + None => { + panic!("Parse error: unfinished end tag"); + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + state = TokenizerState::Data; + } + _ => { + chars.next_back(); + panic!("Parse error: less than sign without a tag"); + tokens.push(Token::Character('<')); + state = TokenizerState::Data; + } + }, + TokenizerState::TagName => match chars.next() { + Some('\t' | '\u{000a}' | '\u{000c}' | ' ') => { + state = TokenizerState::BeforeAttributeName + } + Some('/') => state = TokenizerState::SelfClosingStartTag, + Some('>') => { + state = TokenizerState::Data; + if !tag_builder.end_tag { + last_start_tag = Some(tag_builder.name.clone()); + } + tokens.push(tag_builder.into()); + } + Some(c) if c.is_ascii_uppercase() => tag_builder.name.push(c.to_ascii_lowercase()), + Some('\0') => { + panic!("Parse error: NUL character in tag name"); + tag_builder.name.push(char::REPLACEMENT_CHARACTER); + } + None => { + panic!("Parse error: unfinished tag"); + state = TokenizerState::Data; + } + Some(c) => tag_builder.name.push(c), + }, + TokenizerState::RcDataLessThan => match chars.next() { + Some('/') => { + temp_buffer.clear(); + state = TokenizerState::RcDataEndTagOpen; + } + _ => { + tokens.push(Token::Character('<')); + chars.next_back(); + state = TokenizerState::RcData; + } + }, + TokenizerState::RcDataEndTagOpen => match chars.next() { + Some(c) if c.is_ascii_uppercase() => { + tag_builder = TagBuilder { + end_tag: true, + name: c.to_ascii_lowercase().to_string(), + self_closing: false, + attributes: Vec::new(), + }; + temp_buffer = c.to_string(); + state = TokenizerState::RcDataEndTagName; + } + Some(c) if c.is_ascii_lowercase() => { + tag_builder = TagBuilder { + end_tag: true, + name: c.to_string(), + self_closing: false, + attributes: Vec::new(), + }; + temp_buffer = c.to_string(); + state = TokenizerState::RcDataEndTagName; + } + Some(c) => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + chars.next_back(); + state = TokenizerState::RcData; + } + None => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + state = TokenizerState::RcData; + } + }, + TokenizerState::RcDataEndTagName => match chars.next() { + Some('\t' | '\u{000a}' | '\u{000c}' | ' ') + if tag_builder.is_appropriate_end_tag(&last_start_tag) => + { + state = TokenizerState::BeforeAttributeName + } + + Some('/') if tag_builder.is_appropriate_end_tag(&last_start_tag) => { + state = TokenizerState::SelfClosingStartTag + } + Some('>') if tag_builder.is_appropriate_end_tag(&last_start_tag) => { + tokens.push(tag_builder.into()); + state = TokenizerState::Data; + } + Some(c) if c.is_ascii_uppercase() => { + tag_builder.name.push(c.to_ascii_lowercase()); + temp_buffer.push(c); + } + Some(c) if c.is_ascii_lowercase() => { + tag_builder.name.push(c); + temp_buffer.push(c); + } + None => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + for char in temp_buffer.chars() { + tokens.push(Token::Character(char)); + } + state = TokenizerState::RcData; + } + _ => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + for char in temp_buffer.chars() { + tokens.push(Token::Character(char)); + } + chars.next_back(); + state = TokenizerState::RcData; + } + }, + TokenizerState::RawTextLessThanSign => match chars.next() { + Some('/') => { + temp_buffer = String::new(); + state = TokenizerState::RawTextEndTagOpen; + } + None => { + tokens.push(Token::Character('<')); + state = TokenizerState::RawText; + } + _ => { + tokens.push(Token::Character('<')); + chars.next_back(); + state = TokenizerState::RawText; + } + }, + TokenizerState::RawTextEndTagOpen => match chars.next() { + Some(c) if c.is_ascii_uppercase() => { + tag_builder = TagBuilder { + name: c.to_ascii_lowercase().to_string(), + self_closing: false, + end_tag: true, + attributes: Vec::new(), + }; + temp_buffer.push(c); + state = TokenizerState::RawTextEndTagName; + } + Some(c) if c.is_ascii_lowercase() => { + tag_builder = TagBuilder { + name: c.to_string(), + self_closing: false, + end_tag: true, + attributes: Vec::new(), + }; + temp_buffer.push(c); + state = TokenizerState::RawTextEndTagName; + } + None => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + state = TokenizerState::RawText; + } + Some(_) => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + state = TokenizerState::RawText; + chars.next_back(); + } + }, + TokenizerState::RawTextEndTagName => match chars.next() { + Some('\t' | '\u{000a}' | '\u{000c}' | ' ') + if tag_builder.is_appropriate_end_tag(&last_start_tag) => + { + state = TokenizerState::BeforeAttributeName + } + + Some('/') if tag_builder.is_appropriate_end_tag(&last_start_tag) => { + state = TokenizerState::SelfClosingStartTag + } + Some('>') if tag_builder.is_appropriate_end_tag(&last_start_tag) => { + tokens.push(tag_builder.into()); + state = TokenizerState::Data; + } + Some(c) if c.is_ascii_uppercase() => { + tag_builder.name.push(c.to_ascii_lowercase()); + temp_buffer.push(c); + } + Some(c) if c.is_ascii_lowercase() => { + tag_builder.name.push(c); + temp_buffer.push(c); + } + None => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + for char in temp_buffer.chars() { + tokens.push(Token::Character(char)); + } + state = TokenizerState::RawText; + } + _ => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + for char in temp_buffer.chars() { + tokens.push(Token::Character(char)); + } + chars.next_back(); + state = TokenizerState::RawText; + } + }, + TokenizerState::ScriptDataLessThanSign => match chars.next() { + Some('/') => { + temp_buffer = String::new(); + state = TokenizerState::ScriptDataEndTagOpen; + } + Some('!') => { + state = TokenizerState::ScriptDataEscapeStart; + tokens.push(Token::Character('<')); + tokens.push(Token::Character('!')); + } + None => { + tokens.push(Token::Character('<')); + state = TokenizerState::ScriptData; + } + _ => { + tokens.push(Token::Character('<')); + state = TokenizerState::ScriptData; + chars.next_back(); + } + }, + TokenizerState::ScriptDataEndTagOpen => match chars.next() { + Some(c) if c.is_ascii_uppercase() => { + tag_builder = TagBuilder { + name: c.to_ascii_lowercase().to_string(), + self_closing: false, + end_tag: true, + attributes: Vec::new(), + }; + temp_buffer.push(c); + state = TokenizerState::ScriptDataEndTagName; + } + Some(c) if c.is_ascii_lowercase() => { + tag_builder = TagBuilder { + name: c.to_string(), + self_closing: false, + end_tag: true, + attributes: Vec::new(), + }; + temp_buffer.push(c); + state = TokenizerState::ScriptDataEndTagName; + } + None => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + state = TokenizerState::ScriptData; + } + _ => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + chars.next_back(); + state = TokenizerState::ScriptData; + } + }, + TokenizerState::ScriptDataEndTagName => match chars.next() { + Some('\t' | '\u{000a}' | '\u{000c}' | ' ') + if tag_builder.is_appropriate_end_tag(&last_start_tag) => + { + state = TokenizerState::BeforeAttributeName + } + + Some('/') if tag_builder.is_appropriate_end_tag(&last_start_tag) => { + state = TokenizerState::SelfClosingStartTag + } + Some('>') if tag_builder.is_appropriate_end_tag(&last_start_tag) => { + tokens.push(tag_builder.into()); + state = TokenizerState::Data; + } + Some(c) if c.is_ascii_uppercase() => { + tag_builder.name.push(c.to_ascii_lowercase()); + temp_buffer.push(c); + } + Some(c) if c.is_ascii_lowercase() => { + tag_builder.name.push(c); + temp_buffer.push(c); + } + None => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + for char in temp_buffer.chars() { + tokens.push(Token::Character(char)); + } + state = TokenizerState::ScriptData; + } + _ => { + tokens.push(Token::Character('<')); + tokens.push(Token::Character('/')); + for char in temp_buffer.chars() { + tokens.push(Token::Character(char)); + } + chars.next_back(); + state = TokenizerState::ScriptData; + } + }, + TokenizerState::ScriptDataEscapeStart => match chars.next() { + Some('-') => { + state = TokenizerState::ScriptDataEscapeStartDash; + tokens.push(Token::Character('-')); + } + None => state = TokenizerState::ScriptData, + _ => { + chars.next_back(); + state = TokenizerState::ScriptData; + } + }, + TokenizerState::ScriptDataEscapeStartDash => match chars.next() { + Some('-') => { + state = TokenizerState::ScriptDataEscapedDashDash; + tokens.push(Token::Character('-')); + } + None => state = TokenizerState::ScriptData, + _ => { + chars.next_back(); + state = TokenizerState::ScriptData; + } + }, + TokenizerState::ScriptDataEscaped => match chars.next() { + Some('-') => { + state = TokenizerState::ScriptDataEscapedDash; + tokens.push(Token::Character('-')); + } + Some('<') => state = TokenizerState::ScriptDataEscapedLessThanSign, + Some('\0') => { + panic!("Parse error: unexpected NUL character"); + tokens.push(Token::Character(char::REPLACEMENT_CHARACTER)); + } + None => { + panic!("Parse error: unexpected end of file") + } + Some(c) => tokens.push(Token::Character(c)), + }, + } + } + + tokens.into_boxed_slice() +} -- cgit v1.2.3