use bumpalo::Bump; use snob::csets::CharacterSet; #[derive(Debug, Clone, Copy)] pub struct Token<'a> { pub span: Span, pub ty: TokenType<'a>, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Span { pub start: usize, pub end: usize, } #[derive(Debug, Clone, Copy)] pub enum TokenType<'a> { Tag { is_end: bool, original_tag_name: &'a str, lowercase_tag_name: &'a str, self_closing: bool, attribute_tokens: &'a [AttributeToken<'a>], greater_than_exists: bool, }, Comment { terminated: bool, data: &'a str, }, Data(&'a str), } #[derive(Debug, Clone, Copy)] pub struct AttributeToken<'a> { pub span: Span, pub ty: AttributeTokenType<'a>, } #[derive(Debug, Clone, Copy)] pub enum AttributeTokenType<'a> { Identifier { lowercase_name: &'a str, original_name: &'a str, }, String { double_quoted: bool, terminated: bool, value: &'a str, }, EqualSign, } impl From for miette::SourceSpan { fn from(value: Span) -> Self { miette::SourceSpan::from((value.start, value.start - value.end)) } } fn advance_over_whitespace(scanner: &mut snob::Scanner) { if let Some(position) = scanner.many(snob::csets::AsciiWhitespace) { scanner.goto(position).expect("a valid position"); } } fn upto_str(scanner: &mut snob::Scanner, str: &str) -> Option { let first_char = str .chars() .next() .expect("Cannot use upto_str on an empty string"); while let Some(position) = scanner.upto(first_char) { if scanner.starts_with(str).is_some() { return Some(position); } } None } fn scan_attribute_token<'a>( scanner: &mut snob::Scanner, arena: &'a Bump, ) -> Option> { advance_over_whitespace(scanner); let start = scanner.position(); if scanner.advance_if_starts_with("=").is_some() { Some(AttributeToken { span: Span { start, end: scanner.position(), }, ty: AttributeTokenType::EqualSign, }) } else if let Some(position) = scanner.any("'\"") { let delimiter = scanner .goto(position) .expect("a valid position") .chars() .next() .expect("one character, either double quote or single quote"); let position = scanner.upto(delimiter).unwrap_or(scanner.len()); let value = scanner.goto(position).expect("a valid position"); let value = arena.alloc_str(&value); let terminated = scanner .advance_if_starts_with(delimiter.to_string()) .is_some(); Some(AttributeToken { span: Span { start, end: scanner.position(), }, ty: AttributeTokenType::String { double_quoted: delimiter == '"', terminated, value, }, }) } else if let Some(position) = scanner.many(snob::csets::AsciiLetters) { let value = scanner.goto(position).expect("a valid position"); let value = arena.alloc_str(&value); let lowercase_value = arena.alloc_str(value); lowercase_value.make_ascii_lowercase(); Some(AttributeToken { span: Span { start, end: scanner.position(), }, ty: AttributeTokenType::Identifier { original_name: value, lowercase_name: value, }, }) } else { None } } fn scan_data<'a>(scanner: &mut snob::Scanner, arena: &'a Bump) -> &'a str { let position = scanner.upto('<'); let data = scanner .goto(position.unwrap_or(scanner.len())) .expect("valid position"); arena.alloc_str(&data) } fn scan_comment<'a>(scanner: &mut snob::Scanner, arena: &'a Bump) -> Option> { scanner.advance_if_starts_with("") else { let data = scanner.goto(scanner.len()).expect("a valid position"); let data = arena.alloc_str(&data); return Some(TokenType::Comment { terminated: false, data, }); }; let data = scanner.goto(position).expect("a valid position"); let data = arena.alloc_str(&data); scanner .advance_if_starts_with("-->") .expect("an end to the comment"); Some(TokenType::Comment { terminated: true, data, }) } fn scan_tag<'a>(scanner: &mut snob::Scanner, arena: &'a Bump) -> Option> { scanner.advance_if_starts_with("<")?; advance_over_whitespace(scanner); let is_end = scanner.advance_if_starts_with("/").is_some(); advance_over_whitespace(scanner); let position = scanner .upto(snob::csets::AsciiWhitespace.union("/>")) .unwrap_or(scanner.len()); let original_tag_name = scanner.goto(position).expect("a valid position"); let original_tag_name = arena.alloc_str(&original_tag_name); let lowercase_tag_name = arena.alloc_str(original_tag_name); lowercase_tag_name.make_ascii_lowercase(); advance_over_whitespace(scanner); let mut attribute_tokens = Vec::new(); while let Some(attribute) = scan_attribute_token(scanner, arena) { attribute_tokens.push(attribute); advance_over_whitespace(scanner); } let attribute_tokens = arena.alloc_slice_copy(&attribute_tokens); let self_closing = scanner.advance_if_starts_with("/").is_some(); advance_over_whitespace(scanner); let greater_than_exists = scanner.advance_if_starts_with(">").is_some(); Some(TokenType::Tag { is_end, original_tag_name, lowercase_tag_name, self_closing, attribute_tokens, greater_than_exists, }) } pub fn tokenize<'a>(source: &str, arena: &'a Bump) -> &'a [Token<'a>] { let mut scanner = snob::Scanner::new(source); let mut tokens = Vec::new(); loop { if scanner.is_at_end() { break; } let start = scanner.position(); let token = if scanner.starts_with("