1 files changed, 231 insertions, 0 deletions
diff --git a/src/tokenize.rs b/src/tokenize.rs
new file mode 100644
index 0000000..43ad305
--- /dev/null
+++ b/src/tokenize.rs
@@ -0,0 +1,231 @@
+use bumpalo::Bump;
+use snob::csets::CharacterSet;
+
+#[derive(Debug, Clone, Copy)]
+pub struct Token<'a> {
+	pub span: Span,
+	pub ty: TokenType<'a>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Span {
+	pub start: usize,
+	pub end: usize,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum TokenType<'a> {
+	Tag {
+		is_end: bool,
+		original_tag_name: &'a str,
+		lowercase_tag_name: &'a str,
+		self_closing: bool,
+		attribute_tokens: &'a [AttributeToken<'a>],
+		greater_than_exists: bool,
+	},
+	Comment {
+		terminated: bool,
+		data: &'a str,
+	},
+	Data(&'a str),
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct AttributeToken<'a> {
+	pub span: Span,
+	pub ty: AttributeTokenType<'a>,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum AttributeTokenType<'a> {
+	Identifier {
+		lowercase_name: &'a str,
+		original_name: &'a str,
+	},
+	String {
+		double_quoted: bool,
+		terminated: bool,
+		value: &'a str,
+	},
+	EqualSign,
+}
+
+impl From<Span> for miette::SourceSpan {
+	fn from(value: Span) -> Self {
+		miette::SourceSpan::from((value.start, value.start - value.end))
+	}
+}
+
+fn advance_over_whitespace(scanner: &mut snob::Scanner) {
+	if let Some(position) = scanner.many(snob::csets::AsciiWhitespace) {
+		scanner.goto(position).expect("a valid position");
+	}
+}
+
+fn upto_str(scanner: &mut snob::Scanner, str: &str) -> Option<usize> {
+	let first_char = str
+		.chars()
+		.next()
+		.expect("Cannot use upto_str on an empty string");
+	while let Some(position) = scanner.upto(first_char) {
+		if scanner.starts_with(str).is_some() {
+			return Some(position);
+		}
+	}
+
+	None
+}
+
+fn scan_attribute_token<'a>(
+	scanner: &mut snob::Scanner,
+	arena: &'a Bump,
+) -> Option<AttributeToken<'a>> {
+	advance_over_whitespace(scanner);
+	let start = scanner.position();
+
+	if scanner.advance_if_starts_with("=").is_some() {
+		Some(AttributeToken {
+			span: Span {
+				start,
+				end: scanner.position(),
+			},
+			ty: AttributeTokenType::EqualSign,
+		})
+	} else if let Some(position) = scanner.any("'\"") {
+		let delimiter = scanner
+			.goto(position)
+			.expect("a valid position")
+			.chars()
+			.next()
+			.expect("one character, either double quote or single quote");
+		let position = scanner.upto(delimiter).unwrap_or(scanner.len());
+		let value = scanner.goto(position).expect("a valid position");
+		let value = arena.alloc_str(&value);
+		let terminated = scanner
+			.advance_if_starts_with(delimiter.to_string())
+			.is_some();
+		Some(AttributeToken {
+			span: Span {
+				start,
+				end: scanner.position(),
+			},
+			ty: AttributeTokenType::String {
+				double_quoted: delimiter == '"',
+				terminated,
+				value,
+			},
+		})
+	} else if let Some(position) = scanner.many(snob::csets::AsciiLetters) {
+		let value = scanner.goto(position).expect("a valid position");
+		let value = arena.alloc_str(&value);
+		let lowercase_value = arena.alloc_str(value);
+		lowercase_value.make_ascii_lowercase();
+		Some(AttributeToken {
+			span: Span {
+				start,
+				end: scanner.position(),
+			},
+			ty: AttributeTokenType::Identifier {
+				original_name: value,
+				lowercase_name: value,
+			},
+		})
+	} else {
+		None
+	}
+}
+
+fn scan_data<'a>(scanner: &mut snob::Scanner, arena: &'a Bump) -> &'a str {
+	let position = scanner.upto('<');
+	let data = scanner
+		.goto(position.unwrap_or(scanner.len()))
+		.expect("valid position");
+	arena.alloc_str(&data)
+}
+
+fn scan_comment<'a>(scanner: &mut snob::Scanner, arena: &'a Bump) -> Option<TokenType<'a>> {
+	scanner.advance_if_starts_with("<!--")?;
+
+	let Some(position) = upto_str(scanner, "-->") else {
+		let data = scanner.goto(scanner.len()).expect("a valid position");
+		let data = arena.alloc_str(&data);
+		return Some(TokenType::Comment {
+			terminated: false,
+			data,
+		});
+	};
+	let data = scanner.goto(position).expect("a valid position");
+	let data = arena.alloc_str(&data);
+	scanner
+		.advance_if_starts_with("-->")
+		.expect("an end to the comment");
+
+	Some(TokenType::Comment {
+		terminated: true,
+		data,
+	})
+}
+
+fn scan_tag<'a>(scanner: &mut snob::Scanner, arena: &'a Bump) -> Option<TokenType<'a>> {
+	scanner.advance_if_starts_with("<")?;
+	advance_over_whitespace(scanner);
+
+	let is_end = scanner.advance_if_starts_with("/").is_some();
+	advance_over_whitespace(scanner);
+
+	let position = scanner
+		.upto(snob::csets::AsciiWhitespace.union("/>"))
+		.unwrap_or(scanner.len());
+	let original_tag_name = scanner.goto(position).expect("a valid position");
+	let original_tag_name = arena.alloc_str(&original_tag_name);
+	let lowercase_tag_name = arena.alloc_str(original_tag_name);
+	lowercase_tag_name.make_ascii_lowercase();
+	advance_over_whitespace(scanner);
+
+	let mut attribute_tokens = Vec::new();
+	while let Some(attribute) = scan_attribute_token(scanner, arena) {
+		attribute_tokens.push(attribute);
+		advance_over_whitespace(scanner);
+	}
+	let attribute_tokens = arena.alloc_slice_copy(&attribute_tokens);
+
+	let self_closing = scanner.advance_if_starts_with("/").is_some();
+	advance_over_whitespace(scanner);
+	let greater_than_exists = scanner.advance_if_starts_with(">").is_some();
+
+	Some(TokenType::Tag {
+		is_end,
+		original_tag_name,
+		lowercase_tag_name,
+		self_closing,
+		attribute_tokens,
+		greater_than_exists,
+	})
+}
+
+pub fn tokenize<'a>(source: &str, arena: &'a Bump) -> &'a [Token<'a>] {
+	let mut scanner = snob::Scanner::new(source);
+	let mut tokens = Vec::new();
+	loop {
+		if scanner.is_at_end() {
+			break;
+		}
+
+		let start = scanner.position();
+		let token = if scanner.starts_with("<!--").is_some() {
+			scan_comment(&mut scanner, arena).expect("a comment")
+		} else if scanner.starts_with("<").is_some() {
+			scan_tag(&mut scanner, arena).expect("a tag")
+		} else {
+			TokenType::Data(scan_data(&mut scanner, arena))
+		};
+		let end = scanner.position();
+
+		tokens.push(Token {
+			span: Span { start, end },
+			ty: token,
+		});
+	}
+
+	arena.alloc_slice_copy(&tokens)
+}