initial commitHEADmain

1 files changed, 720 insertions, 0 deletions

diff --git a/src/old_tokenizer.rs b/src/old_tokenizer.rs
new file mode 100644
index 0000000..9f9b79d
--- /dev/null
+++ b/src/old_tokenizer.rs
@@ -0,0 +1,720 @@
+enum TokenizerState {
+	Data,
+	CharacterReferenceInData,
+	RcData,
+	CharacterReferenceInRcData,
+	RawText,
+	ScriptData,
+	PlainText,
+	TagOpen,
+	EndTagOpen,
+	TagName,
+	RcDataLessThan,
+	RcDataEndTagOpen,
+	RcDataEndTagName,
+	RawTextLessThanSign,
+	RawTextEndTagOpen,
+	RawTextEndTagName,
+	ScriptDataLessThanSign,
+	ScriptDataEndTagOpen,
+	ScriptDataEndTagName,
+	ScriptDataEscapeStart,
+	ScriptDataEscapeStartDash,
+	ScriptDataEscaped,
+	ScriptDataEscapedDash,
+	ScriptDataEscapedDashDash,
+	ScriptDataEscapedLessThanSign,
+	ScriptDataEscapedEndTagOpen,
+	ScriptDataEscapedEndTagName,
+	ScriptDataDoubleEscaped,
+	ScriptDataDoubleEscapedDash,
+	ScriptDataDoubleEscapedDashDash,
+	ScriptDataDoubleEscapedLessThanSign,
+	ScriptDataDoubleEscapeEnd,
+	BeforeAttributeName,
+	AttributeName,
+	AfterAttributeName,
+	BeforeAttributeValue,
+	AttributeValueDoubleQuoted,
+	AttributeValueSingleQuoted,
+	AttributeValueUnquoted,
+	CharacterReferenceInAttributeValue,
+	AfterAttributeValue,
+	SelfClosingStartTag,
+	BogusComment,
+	MarkupDeclarationOpen,
+	CommentStart,
+	CommentStartDash,
+	Comment,
+	CommentEndDash,
+	CommentEnd,
+	CommendEndBang,
+	DocType,
+	BeforeDocTypeName,
+	DocTypeName,
+	AfterDocTypeName,
+	AfterDocTypePublicKeyword,
+	DocTypePublicIdentifierDoubleQuoted,
+	DocTypePublicIdentifierSingleQuoted,
+	AfterDocTypePublicIdentifier,
+	BetweenDocTypePublicAndSystemIdentifiers,
+	AfterDocTypeSystemKeyword,
+	BeforeDocTypeSystemIdentifier,
+	DocTypeSystemIdentifierDoubleQuoted,
+	DocTypeSystemIdentifierSingleQuoted,
+	AfterDocTypeSystemIdentifier,
+	BogusDocType,
+	CdataSection,
+}
+
+struct TagBuilder {
+	name: String,
+	self_closing: bool,
+	end_tag: bool,
+	attributes: Vec<(Option<String>, Option<String>)>,
+}
+
+impl TagBuilder {
+	fn is_appropriate_end_tag(&self, last_start_tag: &Option<String>) -> bool {
+		last_start_tag
+			.as_ref()
+			.is_some_and(|name| name == &self.name)
+	}
+}
+
+impl From<TagBuilder> for Token {
+	fn from(value: TagBuilder) -> Self {
+		let attributes = value
+			.attributes
+			.into_iter()
+			.map(|(key, value)| (key.map(Into::into), value.map(Into::into)))
+			.collect();
+		if value.end_tag {
+			Token::EndTag {
+				tag_name: value.name.into(),
+				self_closing: value.self_closing,
+				attributes,
+			}
+		} else {
+			Token::StartTag {
+				tag_name: value.name.into(),
+				self_closing: value.self_closing,
+				attributes,
+			}
+		}
+	}
+}
+
+fn consume_character_reference(chars: &mut Peekable<Chars<'_>>) -> Option<char> {
+	match chars.next() {
+		Some('\t' | '\u{000a}' | '\u{000c}' | ' ' | '<' | '&') | None => None,
+		Some('#') => match chars.peek() {
+			Some('x' | 'X') => {
+				chars.next();
+				let mut digits = chars
+					.take_while(|c| c.is_ascii_hexdigit())
+					.collect::<Box<str>>();
+				if digits.is_empty() {
+					panic!("Parse error: no digits found after the hexadecimal specifier");
+					return None;
+				}
+				if chars.next_if(|c| *c == ';').is_none() {
+					panic!("Parse error: expected a semicolon after escaped digits");
+				}
+
+				let number = u32::from_str_radix(&digits, 16).expect("only hexadecimal digits");
+				if let Some(c) = match number {
+					0x00 => Some('\u{fffd}'),
+					0x0d => Some('\r'),
+					0x80 => Some('\u{20ac}'),
+					0x81 => Some('\u{0081}'),
+					0x82 => Some('\u{201a}'),
+					0x83 => Some('\u{0192}'),
+					0x84 => Some('\u{201e}'),
+					0x85 => Some('\u{2026}'),
+					0x86 => Some('\u{2020}'),
+					0x87 => Some('\u{2021}'),
+					0x88 => Some('\u{02c6}'),
+					0x89 => Some('\u{2030}'),
+					0x8a => Some('\u{0160}'),
+					0x8b => Some('\u{2039}'),
+					0x8c => Some('\u{0152}'),
+					0x8d => Some('\u{008d}'),
+					0x8e => Some('\u{017d}'),
+					0x8f => Some('\u{008f}'),
+					0x90 => Some('\u{0090}'),
+					0x91 => Some('\u{2018}'),
+					0x92 => Some('\u{2019}'),
+					0x93 => Some('\u{201c}'),
+					0x94 => Some('\u{201d}'),
+					0x95 => Some('\u{2022}'),
+					0x96 => Some('\u{2013}'),
+					0x97 => Some('\u{2014}'),
+					0x98 => Some('\u{02dc}'),
+					0x99 => Some('\u{2122}'),
+					0x9a => Some('\u{0161}'),
+					0x9b => Some('\u{203a}'),
+					0x9c => Some('\u{0152}'),
+					0x9d => Some('\u{009d}'),
+					0x9e => Some('\u{017e}'),
+					0x9f => Some('\u{0178}'),
+					_ => None,
+				} {
+					panic!("Parse error: invalid character");
+					return Some(c);
+				}
+
+				if (0xd800 < number && number < 0xdfff) || number > 0x10ffff {
+					panic!("Parse error: invalid character reference");
+					return Some(char::REPLACEMENT_CHARACTER);
+				}
+
+				match char::from_u32(number) {
+					Some(char) => Some(char),
+					None => panic!("Parse error: invalid character reference"),
+				}
+			}
+			_ => {
+				let mut digits = chars
+					.take_while(|c| c.is_ascii_digit())
+					.collect::<Box<str>>();
+				if digits.is_empty() {
+					panic!("Parse error: no digits found after the ampersand");
+					return None;
+				}
+				if chars.next_if(|c| *c == ';').is_none() {
+					panic!("Parse error: expected a semicolon after escaped digits");
+				}
+
+				let number = u32::from_str_radix(&digits, 10).expect("only decimal digits");
+				if let Some(c) = match number {
+					0x00 => Some('\u{fffd}'),
+					0x0d => Some('\r'),
+					0x80 => Some('\u{20ac}'),
+					0x81 => Some('\u{0081}'),
+					0x82 => Some('\u{201a}'),
+					0x83 => Some('\u{0192}'),
+					0x84 => Some('\u{201e}'),
+					0x85 => Some('\u{2026}'),
+					0x86 => Some('\u{2020}'),
+					0x87 => Some('\u{2021}'),
+					0x88 => Some('\u{02c6}'),
+					0x89 => Some('\u{2030}'),
+					0x8a => Some('\u{0160}'),
+					0x8b => Some('\u{2039}'),
+					0x8c => Some('\u{0152}'),
+					0x8d => Some('\u{008d}'),
+					0x8e => Some('\u{017d}'),
+					0x8f => Some('\u{008f}'),
+					0x90 => Some('\u{0090}'),
+					0x91 => Some('\u{2018}'),
+					0x92 => Some('\u{2019}'),
+					0x93 => Some('\u{201c}'),
+					0x94 => Some('\u{201d}'),
+					0x95 => Some('\u{2022}'),
+					0x96 => Some('\u{2013}'),
+					0x97 => Some('\u{2014}'),
+					0x98 => Some('\u{02dc}'),
+					0x99 => Some('\u{2122}'),
+					0x9a => Some('\u{0161}'),
+					0x9b => Some('\u{203a}'),
+					0x9c => Some('\u{0152}'),
+					0x9d => Some('\u{009d}'),
+					0x9e => Some('\u{017e}'),
+					0x9f => Some('\u{0178}'),
+					_ => None,
+				} {
+					panic!("Parse error: invalid character");
+					return Some(c);
+				}
+
+				if (0xd800 < number && number < 0xdfff) || number > 0x10ffff {
+					panic!("Parse error: invalid character reference");
+					return Some(char::REPLACEMENT_CHARACTER);
+				}
+
+				match char::from_u32(number) {
+					Some(char) => Some(char),
+					None => panic!("Parse error: invalid character reference"),
+				}
+			}
+		},
+		_ => todo!("Parse named character reference"),
+	}
+}
+
+fn tokenize(source: &str) -> Box<[Token]> {
+	let mut chars = source.chars().peekable();
+	let mut tokens = Vec::new();
+	let mut state = TokenizerState::Data;
+	let mut tag_builder = TagBuilder {
+		name: String::new(),
+		self_closing: false,
+		end_tag: false,
+		attributes: Vec::new(),
+	};
+	let mut last_start_tag = None;
+	let mut temp_buffer = String::new();
+	loop {
+		match state {
+			TokenizerState::Data => match chars.next() {
+				Some('&') => state = TokenizerState::CharacterReferenceInData,
+				Some('<') => state = TokenizerState::TagOpen,
+				Some('\0') => {
+					panic!("Parse error: NUL character");
+					tokens.push(Token::Character('\0'));
+				}
+				None => break,
+				Some(c) => tokens.push(Token::Character(c)),
+			},
+			TokenizerState::CharacterReferenceInData => {
+				tokens.push(Token::Character(
+					consume_character_reference(&mut chars).unwrap_or('&'),
+				));
+				state = TokenizerState::Data;
+			}
+			TokenizerState::RcData => match chars.next() {
+				Some('&') => state = TokenizerState::CharacterReferenceInRcData,
+				Some('<') => state = TokenizerState::RcDataLessThan,
+				Some('\0') => {
+					panic!("Parse error: NUL character");
+					tokens.push(Token::Character(char::REPLACEMENT_CHARACTER));
+				}
+				None => break,
+				Some(c) => tokens.push(Token::Character(c)),
+			},
+			TokenizerState::CharacterReferenceInRcData => {
+				tokens.push(Token::Character(
+					consume_character_reference(&mut chars).unwrap_or('&'),
+				));
+				state = TokenizerState::RcData;
+			}
+			TokenizerState::RawText => match chars.next() {
+				Some('<') => state = TokenizerState::RawTextLessThanSign,
+				Some('\0') => {
+					panic!("Parse error: NUL character");
+					tokens.push(Token::Character(char::REPLACEMENT_CHARACTER));
+				}
+				None => break,
+				Some(c) => tokens.push(Token::Character(c)),
+			},
+			TokenizerState::ScriptData => match chars.next() {
+				Some('<') => state = TokenizerState::ScriptDataLessThanSign,
+				Some('\0') => {
+					panic!("Parse error: NUL character");
+					tokens.push(Token::Character(char::REPLACEMENT_CHARACTER));
+				}
+				None => break,
+				Some(c) => tokens.push(Token::Character(c)),
+			},
+			TokenizerState::PlainText => match chars.next() {
+				Some('\0') => {
+					panic!("Parse error: NUL character");
+					tokens.push(Token::Character(char::REPLACEMENT_CHARACTER));
+				}
+				None => break,
+				Some(c) => tokens.push(Token::Character(c)),
+			},
+			TokenizerState::TagOpen => match chars.next() {
+				Some('!') => state = TokenizerState::MarkupDeclarationOpen,
+				Some('/') => state = TokenizerState::EndTagOpen,
+				Some(c) if c.is_ascii_uppercase() => {
+					state = TokenizerState::TagName;
+					tag_builder = TagBuilder {
+						name: String::from(c.to_ascii_lowercase()),
+						end_tag: false,
+						self_closing: false,
+						attributes: Vec::new(),
+					};
+				}
+				Some(c) if c.is_ascii_lowercase() => {
+					state = TokenizerState::TagName;
+					tag_builder = TagBuilder {
+						name: c.to_string(),
+						end_tag: false,
+						self_closing: false,
+						attributes: Vec::new(),
+					}
+				}
+				Some('?') => {
+					panic!("Parse error: bogus comment");
+					state = TokenizerState::BogusComment;
+				}
+				None => {
+					panic!("Parse error: unfinished tag");
+					tokens.push(Token::Character('<'));
+					state = TokenizerState::Data;
+				}
+				_ => {
+					chars.next_back();
+					panic!("Parse error: less than sign without a tag");
+					tokens.push(Token::Character('<'));
+					state = TokenizerState::Data;
+				}
+			},
+			TokenizerState::EndTagOpen => match chars.next() {
+				Some(c) if c.is_ascii_uppercase() => {
+					state = TokenizerState::TagName;
+					tag_builder = TagBuilder {
+						name: String::from(c.to_ascii_lowercase()),
+						end_tag: true,
+						self_closing: false,
+						attributes: Vec::new(),
+					}
+				}
+				Some(c) if c.is_ascii_lowercase() => {
+					state = TokenizerState::TagName;
+					tag_builder = TagBuilder {
+						name: c.to_string(),
+						end_tag: true,
+						self_closing: false,
+						attributes: Vec::new(),
+					}
+				}
+				Some('>') => {
+					panic!("Parse error: end tag with no name");
+					state = TokenizerState::Data;
+				}
+				None => {
+					panic!("Parse error: unfinished end tag");
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					state = TokenizerState::Data;
+				}
+				_ => {
+					chars.next_back();
+					panic!("Parse error: less than sign without a tag");
+					tokens.push(Token::Character('<'));
+					state = TokenizerState::Data;
+				}
+			},
+			TokenizerState::TagName => match chars.next() {
+				Some('\t' | '\u{000a}' | '\u{000c}' | ' ') => {
+					state = TokenizerState::BeforeAttributeName
+				}
+				Some('/') => state = TokenizerState::SelfClosingStartTag,
+				Some('>') => {
+					state = TokenizerState::Data;
+					if !tag_builder.end_tag {
+						last_start_tag = Some(tag_builder.name.clone());
+					}
+					tokens.push(tag_builder.into());
+				}
+				Some(c) if c.is_ascii_uppercase() => tag_builder.name.push(c.to_ascii_lowercase()),
+				Some('\0') => {
+					panic!("Parse error: NUL character in tag name");
+					tag_builder.name.push(char::REPLACEMENT_CHARACTER);
+				}
+				None => {
+					panic!("Parse error: unfinished tag");
+					state = TokenizerState::Data;
+				}
+				Some(c) => tag_builder.name.push(c),
+			},
+			TokenizerState::RcDataLessThan => match chars.next() {
+				Some('/') => {
+					temp_buffer.clear();
+					state = TokenizerState::RcDataEndTagOpen;
+				}
+				_ => {
+					tokens.push(Token::Character('<'));
+					chars.next_back();
+					state = TokenizerState::RcData;
+				}
+			},
+			TokenizerState::RcDataEndTagOpen => match chars.next() {
+				Some(c) if c.is_ascii_uppercase() => {
+					tag_builder = TagBuilder {
+						end_tag: true,
+						name: c.to_ascii_lowercase().to_string(),
+						self_closing: false,
+						attributes: Vec::new(),
+					};
+					temp_buffer = c.to_string();
+					state = TokenizerState::RcDataEndTagName;
+				}
+				Some(c) if c.is_ascii_lowercase() => {
+					tag_builder = TagBuilder {
+						end_tag: true,
+						name: c.to_string(),
+						self_closing: false,
+						attributes: Vec::new(),
+					};
+					temp_buffer = c.to_string();
+					state = TokenizerState::RcDataEndTagName;
+				}
+				Some(c) => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					chars.next_back();
+					state = TokenizerState::RcData;
+				}
+				None => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					state = TokenizerState::RcData;
+				}
+			},
+			TokenizerState::RcDataEndTagName => match chars.next() {
+				Some('\t' | '\u{000a}' | '\u{000c}' | ' ')
+					if tag_builder.is_appropriate_end_tag(&last_start_tag) =>
+				{
+					state = TokenizerState::BeforeAttributeName
+				}
+
+				Some('/') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
+					state = TokenizerState::SelfClosingStartTag
+				}
+				Some('>') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
+					tokens.push(tag_builder.into());
+					state = TokenizerState::Data;
+				}
+				Some(c) if c.is_ascii_uppercase() => {
+					tag_builder.name.push(c.to_ascii_lowercase());
+					temp_buffer.push(c);
+				}
+				Some(c) if c.is_ascii_lowercase() => {
+					tag_builder.name.push(c);
+					temp_buffer.push(c);
+				}
+				None => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					for char in temp_buffer.chars() {
+						tokens.push(Token::Character(char));
+					}
+					state = TokenizerState::RcData;
+				}
+				_ => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					for char in temp_buffer.chars() {
+						tokens.push(Token::Character(char));
+					}
+					chars.next_back();
+					state = TokenizerState::RcData;
+				}
+			},
+			TokenizerState::RawTextLessThanSign => match chars.next() {
+				Some('/') => {
+					temp_buffer = String::new();
+					state = TokenizerState::RawTextEndTagOpen;
+				}
+				None => {
+					tokens.push(Token::Character('<'));
+					state = TokenizerState::RawText;
+				}
+				_ => {
+					tokens.push(Token::Character('<'));
+					chars.next_back();
+					state = TokenizerState::RawText;
+				}
+			},
+			TokenizerState::RawTextEndTagOpen => match chars.next() {
+				Some(c) if c.is_ascii_uppercase() => {
+					tag_builder = TagBuilder {
+						name: c.to_ascii_lowercase().to_string(),
+						self_closing: false,
+						end_tag: true,
+						attributes: Vec::new(),
+					};
+					temp_buffer.push(c);
+					state = TokenizerState::RawTextEndTagName;
+				}
+				Some(c) if c.is_ascii_lowercase() => {
+					tag_builder = TagBuilder {
+						name: c.to_string(),
+						self_closing: false,
+						end_tag: true,
+						attributes: Vec::new(),
+					};
+					temp_buffer.push(c);
+					state = TokenizerState::RawTextEndTagName;
+				}
+				None => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					state = TokenizerState::RawText;
+				}
+				Some(_) => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					state = TokenizerState::RawText;
+					chars.next_back();
+				}
+			},
+			TokenizerState::RawTextEndTagName => match chars.next() {
+				Some('\t' | '\u{000a}' | '\u{000c}' | ' ')
+					if tag_builder.is_appropriate_end_tag(&last_start_tag) =>
+				{
+					state = TokenizerState::BeforeAttributeName
+				}
+
+				Some('/') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
+					state = TokenizerState::SelfClosingStartTag
+				}
+				Some('>') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
+					tokens.push(tag_builder.into());
+					state = TokenizerState::Data;
+				}
+				Some(c) if c.is_ascii_uppercase() => {
+					tag_builder.name.push(c.to_ascii_lowercase());
+					temp_buffer.push(c);
+				}
+				Some(c) if c.is_ascii_lowercase() => {
+					tag_builder.name.push(c);
+					temp_buffer.push(c);
+				}
+				None => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					for char in temp_buffer.chars() {
+						tokens.push(Token::Character(char));
+					}
+					state = TokenizerState::RawText;
+				}
+				_ => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					for char in temp_buffer.chars() {
+						tokens.push(Token::Character(char));
+					}
+					chars.next_back();
+					state = TokenizerState::RawText;
+				}
+			},
+			TokenizerState::ScriptDataLessThanSign => match chars.next() {
+				Some('/') => {
+					temp_buffer = String::new();
+					state = TokenizerState::ScriptDataEndTagOpen;
+				}
+				Some('!') => {
+					state = TokenizerState::ScriptDataEscapeStart;
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('!'));
+				}
+				None => {
+					tokens.push(Token::Character('<'));
+					state = TokenizerState::ScriptData;
+				}
+				_ => {
+					tokens.push(Token::Character('<'));
+					state = TokenizerState::ScriptData;
+					chars.next_back();
+				}
+			},
+			TokenizerState::ScriptDataEndTagOpen => match chars.next() {
+				Some(c) if c.is_ascii_uppercase() => {
+					tag_builder = TagBuilder {
+						name: c.to_ascii_lowercase().to_string(),
+						self_closing: false,
+						end_tag: true,
+						attributes: Vec::new(),
+					};
+					temp_buffer.push(c);
+					state = TokenizerState::ScriptDataEndTagName;
+				}
+				Some(c) if c.is_ascii_lowercase() => {
+					tag_builder = TagBuilder {
+						name: c.to_string(),
+						self_closing: false,
+						end_tag: true,
+						attributes: Vec::new(),
+					};
+					temp_buffer.push(c);
+					state = TokenizerState::ScriptDataEndTagName;
+				}
+				None => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					state = TokenizerState::ScriptData;
+				}
+				_ => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					chars.next_back();
+					state = TokenizerState::ScriptData;
+				}
+			},
+			TokenizerState::ScriptDataEndTagName => match chars.next() {
+				Some('\t' | '\u{000a}' | '\u{000c}' | ' ')
+					if tag_builder.is_appropriate_end_tag(&last_start_tag) =>
+				{
+					state = TokenizerState::BeforeAttributeName
+				}
+
+				Some('/') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
+					state = TokenizerState::SelfClosingStartTag
+				}
+				Some('>') if tag_builder.is_appropriate_end_tag(&last_start_tag) => {
+					tokens.push(tag_builder.into());
+					state = TokenizerState::Data;
+				}
+				Some(c) if c.is_ascii_uppercase() => {
+					tag_builder.name.push(c.to_ascii_lowercase());
+					temp_buffer.push(c);
+				}
+				Some(c) if c.is_ascii_lowercase() => {
+					tag_builder.name.push(c);
+					temp_buffer.push(c);
+				}
+				None => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					for char in temp_buffer.chars() {
+						tokens.push(Token::Character(char));
+					}
+					state = TokenizerState::ScriptData;
+				}
+				_ => {
+					tokens.push(Token::Character('<'));
+					tokens.push(Token::Character('/'));
+					for char in temp_buffer.chars() {
+						tokens.push(Token::Character(char));
+					}
+					chars.next_back();
+					state = TokenizerState::ScriptData;
+				}
+			},
+			TokenizerState::ScriptDataEscapeStart => match chars.next() {
+				Some('-') => {
+					state = TokenizerState::ScriptDataEscapeStartDash;
+					tokens.push(Token::Character('-'));
+				}
+				None => state = TokenizerState::ScriptData,
+				_ => {
+					chars.next_back();
+					state = TokenizerState::ScriptData;
+				}
+			},
+			TokenizerState::ScriptDataEscapeStartDash => match chars.next() {
+				Some('-') => {
+					state = TokenizerState::ScriptDataEscapedDashDash;
+					tokens.push(Token::Character('-'));
+				}
+				None => state = TokenizerState::ScriptData,
+				_ => {
+					chars.next_back();
+					state = TokenizerState::ScriptData;
+				}
+			},
+			TokenizerState::ScriptDataEscaped => match chars.next() {
+				Some('-') => {
+					state = TokenizerState::ScriptDataEscapedDash;
+					tokens.push(Token::Character('-'));
+				}
+				Some('<') => state = TokenizerState::ScriptDataEscapedLessThanSign,
+				Some('\0') => {
+					panic!("Parse error: unexpected NUL character");
+					tokens.push(Token::Character(char::REPLACEMENT_CHARACTER));
+				}
+				None => {
+					panic!("Parse error: unexpected end of file")
+				}
+				Some(c) => tokens.push(Token::Character(c)),
+			},
+		}
+	}
+
+	tokens.into_boxed_slice()
+}