commit cca85f7e89bab2663604f403819ef7583b38d749
parent 47004c278480b7c2285e0a7941494fd499129e27
Author: Jack Mordaunt <jackmordaunt.dev@gmail.com>
Date: Thu, 5 Sep 2024 14:22:37 +0800
lex: tokenize the doctype
Signed-off-by: Jack Mordaunt <jackmordaunt.dev@gmail.com>
Diffstat:
1 file changed, 15 insertions(+), 6 deletions(-)
diff --git a/lex.odin b/lex.odin
@@ -10,7 +10,7 @@ import "core:testing"
import "core:time"
import "core:unicode"
-DOCTYPE_LITERAL :: "<!DOCTYPE html>"
+DOCTYPE_LITERAL :: "<!DOCTYPE"
Token_Span :: struct {
kind: Token_Kind,
@@ -39,6 +39,7 @@ Token_Kind :: enum u8 {
Double_Quote_String_Literal,
Single_Quote_String_Literal,
Raw_Text,
+ Doctype,
}
Lexer :: struct {
@@ -47,11 +48,12 @@ Lexer :: struct {
}
lexer_next :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
- // Avoid emitting the doctype.
- // NOTE(jfm): we could emit this and ignore it in the parser.
if peek := lexer_peek(l, len(DOCTYPE_LITERAL)); peek == DOCTYPE_LITERAL {
- lexer_advance(l, len(DOCTYPE_LITERAL))
- return lexer_next(l)
+ start := l.cursor
+ if !lexer_advance_until(l, `>`) do return t, false
+ end := l.cursor
+ defer lexer_advance(l)
+ return Token_Span{kind = .Doctype, start = start, end = end}, true
}
peek := lexer_peek(l)
@@ -342,7 +344,14 @@ test_lex_comment_3 :: proc(t: ^testing.T) {
@(test)
test_lex_doctype :: proc(t: ^testing.T) {
source := "<!DOCTYPE html>"
- want := []Token{}
+ want := []Token{Token_Span{kind = .Doctype, start = 0, end = 14}}
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_exotic_doctype :: proc(t: ^testing.T) {
+ source := "<!DOCTYPE foobar>"
+ want := []Token{Token_Span{kind = .Doctype, start = 0, end = 16}}
_lexer_test(t, source, want)
}
