path: root/src/tokenize.rs

use bumpalo::Bump;
use snob::csets::CharacterSet;

#[derive(Debug, Clone, Copy)]
pub struct Token<'a> {
	pub span: Span,
	pub ty: TokenType<'a>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Span {
	pub start: usize,
	pub end: usize,
}

#[derive(Debug, Clone, Copy)]
pub enum TokenType<'a> {
	Tag {
		is_end: bool,
		original_tag_name: &'a str,
		lowercase_tag_name: &'a str,
		self_closing: bool,
		attribute_tokens: &'a [AttributeToken<'a>],
		greater_than_exists: bool,
	},
	Comment {
		terminated: bool,
		data: &'a str,
	},
	Data(&'a str),
}

#[derive(Debug, Clone, Copy)]
pub struct AttributeToken<'a> {
	pub span: Span,
	pub ty: AttributeTokenType<'a>,
}

#[derive(Debug, Clone, Copy)]
pub enum AttributeTokenType<'a> {
	Identifier {
		lowercase_name: &'a str,
		original_name: &'a str,
	},
	String {
		double_quoted: bool,
		terminated: bool,
		value: &'a str,
	},
	EqualSign,
}

impl From<Span> for miette::SourceSpan {
	fn from(value: Span) -> Self {
		miette::SourceSpan::from((value.start, value.start - value.end))
	}
}

fn advance_over_whitespace(scanner: &mut snob::Scanner) {
	if let Some(position) = scanner.many(snob::csets::AsciiWhitespace) {
		scanner.goto(position).expect("a valid position");
	}
}

fn upto_str(scanner: &mut snob::Scanner, str: &str) -> Option<usize> {
	let first_char = str
		.chars()
		.next()
		.expect("Cannot use upto_str on an empty string");
	while let Some(position) = scanner.upto(first_char) {
		if scanner.starts_with(str).is_some() {
			return Some(position);
		}
	}

	None
}

fn scan_attribute_token<'a>(
	scanner: &mut snob::Scanner,
	arena: &'a Bump,
) -> Option<AttributeToken<'a>> {
	advance_over_whitespace(scanner);
	let start = scanner.position();

	if scanner.advance_if_starts_with("=").is_some() {
		Some(AttributeToken {
			span: Span {
				start,
				end: scanner.position(),
			},
			ty: AttributeTokenType::EqualSign,
		})
	} else if let Some(position) = scanner.any("'\"") {
		let delimiter = scanner
			.goto(position)
			.expect("a valid position")
			.chars()
			.next()
			.expect("one character, either double quote or single quote");
		let position = scanner.upto(delimiter).unwrap_or(scanner.len());
		let value = scanner.goto(position).expect("a valid position");
		let value = arena.alloc_str(&value);
		let terminated = scanner
			.advance_if_starts_with(delimiter.to_string())
			.is_some();
		Some(AttributeToken {
			span: Span {
				start,
				end: scanner.position(),
			},
			ty: AttributeTokenType::String {
				double_quoted: delimiter == '"',
				terminated,
				value,
			},
		})
	} else if let Some(position) = scanner.many(snob::csets::AsciiLetters) {
		let value = scanner.goto(position).expect("a valid position");
		let value = arena.alloc_str(&value);
		let lowercase_value = arena.alloc_str(value);
		lowercase_value.make_ascii_lowercase();
		Some(AttributeToken {
			span: Span {
				start,
				end: scanner.position(),
			},
			ty: AttributeTokenType::Identifier {
				original_name: value,
				lowercase_name: value,
			},
		})
	} else {
		None
	}
}

fn scan_data<'a>(scanner: &mut snob::Scanner, arena: &'a Bump) -> &'a str {
	let position = scanner.upto('<');
	let data = scanner
		.goto(position.unwrap_or(scanner.len()))
		.expect("valid position");
	arena.alloc_str(&data)
}

fn scan_comment<'a>(scanner: &mut snob::Scanner, arena: &'a Bump) -> Option<TokenType<'a>> {
	scanner.advance_if_starts_with("<!--")?;

	let Some(position) = upto_str(scanner, "-->") else {
		let data = scanner.goto(scanner.len()).expect("a valid position");
		let data = arena.alloc_str(&data);
		return Some(TokenType::Comment {
			terminated: false,
			data,
		});
	};
	let data = scanner.goto(position).expect("a valid position");
	let data = arena.alloc_str(&data);
	scanner
		.advance_if_starts_with("-->")
		.expect("an end to the comment");

	Some(TokenType::Comment {
		terminated: true,
		data,
	})
}

fn scan_tag<'a>(scanner: &mut snob::Scanner, arena: &'a Bump) -> Option<TokenType<'a>> {
	scanner.advance_if_starts_with("<")?;
	advance_over_whitespace(scanner);

	let is_end = scanner.advance_if_starts_with("/").is_some();
	advance_over_whitespace(scanner);

	let position = scanner
		.upto(snob::csets::AsciiWhitespace.union("/>"))
		.unwrap_or(scanner.len());
	let original_tag_name = scanner.goto(position).expect("a valid position");
	let original_tag_name = arena.alloc_str(&original_tag_name);
	let lowercase_tag_name = arena.alloc_str(original_tag_name);
	lowercase_tag_name.make_ascii_lowercase();
	advance_over_whitespace(scanner);

	let mut attribute_tokens = Vec::new();
	while let Some(attribute) = scan_attribute_token(scanner, arena) {
		attribute_tokens.push(attribute);
		advance_over_whitespace(scanner);
	}
	let attribute_tokens = arena.alloc_slice_copy(&attribute_tokens);

	let self_closing = scanner.advance_if_starts_with("/").is_some();
	advance_over_whitespace(scanner);
	let greater_than_exists = scanner.advance_if_starts_with(">").is_some();

	Some(TokenType::Tag {
		is_end,
		original_tag_name,
		lowercase_tag_name,
		self_closing,
		attribute_tokens,
		greater_than_exists,
	})
}

pub fn tokenize<'a>(source: &str, arena: &'a Bump) -> &'a [Token<'a>] {
	let mut scanner = snob::Scanner::new(source);
	let mut tokens = Vec::new();
	loop {
		if scanner.is_at_end() {
			break;
		}

		let start = scanner.position();
		let token = if scanner.starts_with("<!--").is_some() {
			scan_comment(&mut scanner, arena).expect("a comment")
		} else if scanner.starts_with("<").is_some() {
			scan_tag(&mut scanner, arena).expect("a tag")
		} else {
			TokenType::Data(scan_data(&mut scanner, arena))
		};
		let end = scanner.position();

		tokens.push(Token {
			span: Span { start, end },
			ty: token,
		});
	}

	arena.alloc_slice_copy(&tokens)
}