commit 860ab567592943a5624aa8d1b42f2917c457f0d1
parent b36945cfe53e383a0e0b3a4f54367f3c7ba06bb0
Author: Jack Mordaunt <jackmordaunt.dev@gmail.com>
Date: Thu, 23 Jul 2026 20:28:50 -0300
lex: use flat token struct
Instead of a union prefer the data-oriented flat style.
Diffstat:
| M | lex.odin | | | 343 | +++++++++++++++++++++++++++++++++++++++---------------------------------------- |
1 file changed, 168 insertions(+), 175 deletions(-)
diff --git a/lex.odin b/lex.odin
@@ -12,22 +12,16 @@ import "core:unicode"
DOCTYPE_LITERAL :: "<!DOCTYPE"
-Token_Span :: struct {
+// Token is a flat descriptor of a lexical token.
+//
+// `start` and `end` are inclusive byte offsets into the source string.
+// For atoms (single-character tokens) `end == start`.
+Token :: struct {
kind: Token_Kind,
start: u32,
end: u32,
}
-Token_Atom :: struct {
- kind: Token_Kind,
- start: u32,
-}
-
-Token :: union {
- Token_Atom,
- Token_Span,
-}
-
Token_Kind :: enum u8 {
Open_Tag_Start,
Tag_End,
@@ -53,12 +47,12 @@ lexer_next :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
if !lexer_advance_until(l, `>`) do return t, false
end := l.cursor
defer lexer_advance(l)
- return Token_Span{kind = .Doctype, start = start, end = end}, true
+ return Token{kind = .Doctype, start = start, end = end}, true
}
peek := lexer_peek(l)
if peek == "" {
- return nil, false
+ return t, false
}
if t, ok := _lexer_next(l); ok {
@@ -83,19 +77,19 @@ lexer_next :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
}
end := l.cursor - 1
- return Token_Span{kind = .Raw_Text, start = start, end = end}, true
+ return Token{kind = .Raw_Text, start = start, end = end}, true
}
@(private = "file")
_lexer_next :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
if peek := lexer_peek(l, 4); peek == "<!--" {
defer lexer_advance(l, 4)
- return Token_Span{kind = .Comment_Start, start = l.cursor, end = l.cursor + 3}, true
+ return Token{kind = .Comment_Start, start = l.cursor, end = l.cursor + 3}, true
}
if peek := lexer_peek(l, 3); peek == "-->" {
defer lexer_advance(l, 3)
- return Token_Span{kind = .Comment_End, start = l.cursor, end = l.cursor + 2}, true
+ return Token{kind = .Comment_End, start = l.cursor, end = l.cursor + 2}, true
}
if peek := lexer_peek(l, 2); peek != "" && peek[0] == '<' && unicode.is_letter(rune(peek[1])) {
@@ -104,7 +98,7 @@ _lexer_next :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
if !lexer_advance_while_letters(l) do return
end := l.cursor
defer lexer_advance(l)
- return Token_Span{kind = .Open_Tag_Start, start = start, end = end}, true
+ return Token{kind = .Open_Tag_Start, start = start, end = end}, true
}
if peek := lexer_peek(l, 2); peek == "</" {
@@ -113,22 +107,22 @@ _lexer_next :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
if !lexer_advance_while_letters(l) do return
end := l.cursor
defer lexer_advance(l)
- return Token_Span{kind = .Close_Tag_Start, start = start, end = end}, true
+ return Token{kind = .Close_Tag_Start, start = start, end = end}, true
}
if peek := lexer_peek(l, 2); peek == "/>" {
defer lexer_advance(l, 2)
- return Token_Span{kind = .Self_Closing_Tag_End, start = l.cursor, end = l.cursor + 1}, true
+ return Token{kind = .Self_Closing_Tag_End, start = l.cursor, end = l.cursor + 1}, true
}
if peek := lexer_peek(l, 1); peek == ">" {
defer lexer_advance(l, 1)
- return Token_Atom{kind = .Tag_End, start = l.cursor}, true
+ return Token{kind = .Tag_End, start = l.cursor, end = l.cursor}, true
}
if peek := lexer_peek(l, 1); peek == "=" {
defer lexer_advance(l, 1)
- return Token_Atom{kind = .Assign, start = l.cursor}, true
+ return Token{kind = .Assign, start = l.cursor, end = l.cursor}, true
}
if peek := lexer_peek(l, 1); peek == `"` {
@@ -137,7 +131,7 @@ _lexer_next :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
if !lexer_advance_until(l, `"`) do return t, false
end := l.cursor
defer lexer_advance(l, 1)
- return Token_Span{kind = .Double_Quote_String_Literal, start = start, end = end}, true
+ return Token{kind = .Double_Quote_String_Literal, start = start, end = end}, true
}
if peek := lexer_peek(l, 1); peek == `'` {
@@ -146,7 +140,7 @@ _lexer_next :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
if !lexer_advance_until(l, `'`) do return t, false
end := l.cursor
defer lexer_advance(l, 1)
- return Token_Span{kind = .Single_Quote_String_Literal, start = start, end = end}, true
+ return Token{kind = .Single_Quote_String_Literal, start = start, end = end}, true
}
return
@@ -163,56 +157,63 @@ lexer_advance :: proc(l: ^Lexer, step: u32 = 1) -> (ok: bool) {
return l.cursor < u32(len(l.source))
}
-lexer_advance_until :: proc {
- lexer_advance_until_string,
- lexer_advance_until_proc,
+// lexer_advance_until advances the cursor until the source at the cursor
+// matches `target`. Returns true if a match was found, false on EOF.
+lexer_advance_until :: proc(l: ^Lexer, target: string) -> bool {
+ for {
+ if lexer_peek(l, u32(len(target))) == target {
+ return true
+ }
+ if !lexer_advance(l) {
+ return false
+ }
+ }
}
+// lexer_advance_while_letters advances the cursor while the byte at cursor+1
+// is a letter, hyphen, or underscore. Returns true if stopped at a non-identifier
+// byte, false on EOF.
lexer_advance_while_letters :: proc(l: ^Lexer) -> bool {
- return lexer_advance_until_proc(l, proc(l: ^Lexer) -> bool {
- peek := lexer_peek(l, 2)
- if peek == "" {
+ for {
+ b, ok := lexer_peek_byte(l, 1)
+ if !ok {
return false
}
- return !(unicode.is_letter(rune(peek[1])) || peek[1] == '-' || peek[1] == '_')
- })
+ if !(unicode.is_letter(rune(b)) || b == '-' || b == '_') {
+ return true
+ }
+ if !lexer_advance(l) {
+ return false
+ }
+ }
}
+// lexer_advance_until_space advances the cursor until the byte at cursor+1
+// is whitespace. Returns true if stopped, false on EOF.
lexer_advance_until_space :: proc(l: ^Lexer) -> bool {
- return lexer_advance_until_proc(l, proc(l: ^Lexer) -> bool {
- peek := lexer_peek(l, 2)
- if peek == "" {
+ for {
+ b, ok := lexer_peek_byte(l, 1)
+ if !ok {
return false
}
- return unicode.is_space(rune(peek[1]))
- })
-}
-
-lexer_advance_until_space_or :: proc(l: ^Lexer, or: byte) -> bool {
- or := or
- context.user_ptr = &or
- return lexer_advance_until_proc(l, proc(l: ^Lexer) -> bool {
- or: ^byte = auto_cast context.user_ptr
- peek := lexer_peek(l, 2)
- if peek == "" {
+ if unicode.is_space(rune(b)) {
+ return true
+ }
+ if !lexer_advance(l) {
return false
}
- return unicode.is_space(rune(peek[1])) || peek[1] == or^
- })
-}
-
-lexer_advance_until_string :: proc(l: ^Lexer, target: string) -> bool {
- target := target
- context.user_ptr = &target
- return lexer_advance_until_proc(l, proc(l: ^Lexer) -> bool {
- target := cast(^string)(context.user_ptr)
- return lexer_peek(l) == target^
- })
+ }
}
-lexer_advance_until_proc :: proc(l: ^Lexer, predicate: proc(l: ^Lexer) -> bool) -> bool {
+// lexer_advance_until_space_or advances the cursor until the byte at cursor+1
+// is whitespace or equals `or`. Returns true if stopped, false on EOF.
+lexer_advance_until_space_or :: proc(l: ^Lexer, or: byte) -> bool {
for {
- if predicate(l) {
+ b, ok := lexer_peek_byte(l, 1)
+ if !ok {
+ return false
+ }
+ if unicode.is_space(rune(b)) || b == or {
return true
}
if !lexer_advance(l) {
@@ -232,6 +233,15 @@ lexer_peek :: proc(l: ^Lexer, step: u32 = 1) -> (p: string) {
return l.source[start:end]
}
+// lexer_peek_byte returns the byte at cursor+offset without advancing.
+lexer_peek_byte :: proc(l: ^Lexer, offset: u32 = 0) -> (b: byte, ok: bool) {
+ pos := l.cursor + offset
+ if pos >= u32(len(l.source)) {
+ return 0, false
+ }
+ return l.source[pos], true
+}
+
// lexer_peek_token returns the next token without advancing the lexer.
lexer_peek_token :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
start := l.cursor
@@ -240,37 +250,30 @@ lexer_peek_token :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
}
lexer_skip_space :: proc(l: ^Lexer) {
- lexer_advance_until_proc(l, proc(l: ^Lexer) -> bool {
- peek := lexer_peek(l)
-
- if peek == "" {
- return true
+ for {
+ b, ok := lexer_peek_byte(l)
+ if !ok {
+ return
}
-
- return !unicode.is_space(rune(peek[0]))
- })
+ if !unicode.is_space(rune(b)) {
+ return
+ }
+ if !lexer_advance(l) {
+ return
+ }
+ }
}
// token_kind returns the kind for the token.
token_kind :: proc(t: Token) -> Token_Kind {
- switch t in t {
- case Token_Atom:
- return t.kind
- case Token_Span:
- return t.kind
- }
- return nil
+ return t.kind
}
// token_lookup returns the substring corresponding to the token.
+// For atoms (end == start) this is the single byte at `start`;
+// for spans it is the inclusive range [start, end].
token_lookup :: proc(t: Token, text: string) -> string {
- switch t in t {
- case Token_Atom:
- return text[t.start:t.start + 1]
- case Token_Span:
- return text[t.start:t.end + 1]
- }
- return ""
+ return text[t.start:t.end + 1]
}
@(private = "file")
@@ -319,9 +322,9 @@ _lexer_test :: proc(t: ^testing.T, source: string, want: []Token, loc := #caller
test_lex_comment :: proc(t: ^testing.T) {
source := "<!-- comment -->"
want := []Token {
- Token_Span{kind = .Comment_Start, start = 0, end = 3},
- Token_Span{kind = .Raw_Text, start = 4, end = 12},
- Token_Span{kind = .Comment_End, start = 13, end = 15},
+ Token{kind = .Comment_Start, start = 0, end = 3},
+ Token{kind = .Raw_Text, start = 4, end = 12},
+ Token{kind = .Comment_End, start = 13, end = 15},
}
_lexer_test(t, source, want)
}
@@ -330,9 +333,9 @@ test_lex_comment :: proc(t: ^testing.T) {
test_lex_comment_2 :: proc(t: ^testing.T) {
source := "<!--comment-->"
want := []Token {
- Token_Span{kind = .Comment_Start, start = 0, end = 3},
- Token_Span{kind = .Raw_Text, start = 4, end = 10},
- Token_Span{kind = .Comment_End, start = 11, end = 13},
+ Token{kind = .Comment_Start, start = 0, end = 3},
+ Token{kind = .Raw_Text, start = 4, end = 10},
+ Token{kind = .Comment_End, start = 11, end = 13},
}
_lexer_test(t, source, want)
}
@@ -341,9 +344,9 @@ test_lex_comment_2 :: proc(t: ^testing.T) {
test_lex_comment_3 :: proc(t: ^testing.T) {
source := "<!-- \ncomment\n -->"
want := []Token {
- Token_Span{kind = .Comment_Start, start = 0, end = 3},
- Token_Span{kind = .Raw_Text, start = 4, end = 14},
- Token_Span{kind = .Comment_End, start = 15, end = 17},
+ Token{kind = .Comment_Start, start = 0, end = 3},
+ Token{kind = .Raw_Text, start = 4, end = 14},
+ Token{kind = .Comment_End, start = 15, end = 17},
}
_lexer_test(t, source, want)
}
@@ -351,14 +354,14 @@ test_lex_comment_3 :: proc(t: ^testing.T) {
@(test)
test_lex_doctype :: proc(t: ^testing.T) {
source := "<!DOCTYPE html>"
- want := []Token{Token_Span{kind = .Doctype, start = 0, end = 14}}
+ want := []Token{Token{kind = .Doctype, start = 0, end = 14}}
_lexer_test(t, source, want)
}
@(test)
test_lex_exotic_doctype :: proc(t: ^testing.T) {
source := "<!DOCTYPE foobar>"
- want := []Token{Token_Span{kind = .Doctype, start = 0, end = 16}}
+ want := []Token{Token{kind = .Doctype, start = 0, end = 16}}
_lexer_test(t, source, want)
}
@@ -366,9 +369,9 @@ test_lex_exotic_doctype :: proc(t: ^testing.T) {
test_lex_self_closing_tag :: proc(t: ^testing.T) {
source := `<tag />`
want := []Token {
- Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
- Token_Span{kind = .Raw_Text, start = 4, end = 4},
- Token_Span{kind = .Self_Closing_Tag_End, start = 5, end = 6},
+ Token{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token{kind = .Raw_Text, start = 4, end = 4},
+ Token{kind = .Self_Closing_Tag_End, start = 5, end = 6},
}
_lexer_test(t, source, want)
}
@@ -377,8 +380,8 @@ test_lex_self_closing_tag :: proc(t: ^testing.T) {
test_lex_self_closing_tag_2 :: proc(t: ^testing.T) {
source := `<tag/>`
want := []Token {
- Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
- Token_Span{kind = .Self_Closing_Tag_End, start = 4, end = 5},
+ Token{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token{kind = .Self_Closing_Tag_End, start = 4, end = 5},
}
_lexer_test(t, source, want)
}
@@ -387,8 +390,8 @@ test_lex_self_closing_tag_2 :: proc(t: ^testing.T) {
test_lex_open_tag :: proc(t: ^testing.T) {
source := "<tag>"
want := []Token {
- Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
- Token_Atom{kind = .Tag_End, start = 4},
+ Token{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token{kind = .Tag_End, start = 4, end = 4},
}
_lexer_test(t, source, want)
}
@@ -397,8 +400,8 @@ test_lex_open_tag :: proc(t: ^testing.T) {
test_lex_close_tag :: proc(t: ^testing.T) {
source := "</tag>"
want := []Token {
- Token_Span{kind = .Close_Tag_Start, start = 2, end = 4},
- Token_Atom{kind = .Tag_End, start = 5},
+ Token{kind = .Close_Tag_Start, start = 2, end = 4},
+ Token{kind = .Tag_End, start = 5, end = 5},
}
_lexer_test(t, source, want)
}
@@ -407,10 +410,10 @@ test_lex_close_tag :: proc(t: ^testing.T) {
test_lex_tag_pair :: proc(t: ^testing.T) {
source := "<tag></tag>"
want := []Token {
- Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
- Token_Atom{kind = .Tag_End, start = 4},
- Token_Span{kind = .Close_Tag_Start, start = 7, end = 9},
- Token_Atom{kind = .Tag_End, start = 10},
+ Token{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token{kind = .Tag_End, start = 4, end = 4},
+ Token{kind = .Close_Tag_Start, start = 7, end = 9},
+ Token{kind = .Tag_End, start = 10, end = 10},
}
_lexer_test(t, source, want)
}
@@ -419,13 +422,13 @@ test_lex_tag_pair :: proc(t: ^testing.T) {
test_lex_tag_pair_with_comment :: proc(t: ^testing.T) {
source := "<tag><!--comment--></tag>"
want := []Token {
- Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
- Token_Atom{kind = .Tag_End, start = 4},
- Token_Span{kind = .Comment_Start, start = 5, end = 8},
- Token_Span{kind = .Raw_Text, start = 9, end = 15},
- Token_Span{kind = .Comment_End, start = 16, end = 18},
- Token_Span{kind = .Close_Tag_Start, start = 21, end = 23},
- Token_Atom{kind = .Tag_End, start = 24},
+ Token{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token{kind = .Tag_End, start = 4, end = 4},
+ Token{kind = .Comment_Start, start = 5, end = 8},
+ Token{kind = .Raw_Text, start = 9, end = 15},
+ Token{kind = .Comment_End, start = 16, end = 18},
+ Token{kind = .Close_Tag_Start, start = 21, end = 23},
+ Token{kind = .Tag_End, start = 24, end = 24},
}
_lexer_test(t, source, want)
}
@@ -434,11 +437,11 @@ test_lex_tag_pair_with_comment :: proc(t: ^testing.T) {
test_lex_open_tag_value_attribute :: proc(t: ^testing.T) {
source := `<tag class=".style">`
want := []Token {
- Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
- Token_Span{kind = .Raw_Text, start = 4, end = 9},
- Token_Atom{kind = .Assign, start = 10},
- Token_Span{kind = .Double_Quote_String_Literal, start = 11, end = 18},
- Token_Atom{kind = .Tag_End, start = 19},
+ Token{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token{kind = .Raw_Text, start = 4, end = 9},
+ Token{kind = .Assign, start = 10, end = 10},
+ Token{kind = .Double_Quote_String_Literal, start = 11, end = 18},
+ Token{kind = .Tag_End, start = 19, end = 19},
}
_lexer_test(t, source, want)
}
@@ -447,12 +450,12 @@ test_lex_open_tag_value_attribute :: proc(t: ^testing.T) {
test_lex_open_tag_naked_attribute :: proc(t: ^testing.T) {
source := `<tag class=".style" foobar>`
want := []Token {
- Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
- Token_Span{kind = .Raw_Text, start = 4, end = 9},
- Token_Atom{kind = .Assign, start = 10},
- Token_Span{kind = .Double_Quote_String_Literal, start = 11, end = 18},
- Token_Span{kind = .Raw_Text, start = 19, end = 25},
- Token_Atom{kind = .Tag_End, start = 26},
+ Token{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token{kind = .Raw_Text, start = 4, end = 9},
+ Token{kind = .Assign, start = 10, end = 10},
+ Token{kind = .Double_Quote_String_Literal, start = 11, end = 18},
+ Token{kind = .Raw_Text, start = 19, end = 25},
+ Token{kind = .Tag_End, start = 26, end = 26},
}
_lexer_test(t, source, want)
}
@@ -461,24 +464,24 @@ test_lex_open_tag_naked_attribute :: proc(t: ^testing.T) {
test_lex_nested_tags :: proc(t: ^testing.T) {
source := `<tag><ul><li>one</li><li>two</li></ul></tag>`
want := []Token {
- Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
- Token_Atom{kind = .Tag_End, start = 4},
- Token_Span{kind = .Open_Tag_Start, start = 6, end = 7},
- Token_Atom{kind = .Tag_End, start = 8},
- Token_Span{kind = .Open_Tag_Start, start = 10, end = 11},
- Token_Atom{kind = .Tag_End, start = 12},
- Token_Span{kind = .Raw_Text, start = 13, end = 15},
- Token_Span{kind = .Close_Tag_Start, start = 18, end = 19},
- Token_Atom{kind = .Tag_End, start = 20},
- Token_Span{kind = .Open_Tag_Start, start = 22, end = 23},
- Token_Atom{kind = .Tag_End, start = 24},
- Token_Span{kind = .Raw_Text, start = 25, end = 27},
- Token_Span{kind = .Close_Tag_Start, start = 30, end = 31},
- Token_Atom{kind = .Tag_End, start = 32},
- Token_Span{kind = .Close_Tag_Start, start = 35, end = 36},
- Token_Atom{kind = .Tag_End, start = 37},
- Token_Span{kind = .Close_Tag_Start, start = 40, end = 42},
- Token_Atom{kind = .Tag_End, start = 43},
+ Token{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token{kind = .Tag_End, start = 4, end = 4},
+ Token{kind = .Open_Tag_Start, start = 6, end = 7},
+ Token{kind = .Tag_End, start = 8, end = 8},
+ Token{kind = .Open_Tag_Start, start = 10, end = 11},
+ Token{kind = .Tag_End, start = 12, end = 12},
+ Token{kind = .Raw_Text, start = 13, end = 15},
+ Token{kind = .Close_Tag_Start, start = 18, end = 19},
+ Token{kind = .Tag_End, start = 20, end = 20},
+ Token{kind = .Open_Tag_Start, start = 22, end = 23},
+ Token{kind = .Tag_End, start = 24, end = 24},
+ Token{kind = .Raw_Text, start = 25, end = 27},
+ Token{kind = .Close_Tag_Start, start = 30, end = 31},
+ Token{kind = .Tag_End, start = 32, end = 32},
+ Token{kind = .Close_Tag_Start, start = 35, end = 36},
+ Token{kind = .Tag_End, start = 37, end = 37},
+ Token{kind = .Close_Tag_Start, start = 40, end = 42},
+ Token{kind = .Tag_End, start = 43, end = 43},
}
_lexer_test(t, source, want)
}
@@ -487,26 +490,26 @@ test_lex_nested_tags :: proc(t: ^testing.T) {
test_lex_raw_text_less_than :: proc(t: ^testing.T) {
source := `<tag> 1 < 2 </tag>`
want := []Token {
- Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
- Token_Atom{kind = .Tag_End, start = 4},
- Token_Span{kind = .Raw_Text, start = 5, end = 11},
- Token_Span{kind = .Close_Tag_Start, start = 14, end = 16},
- Token_Atom{kind = .Tag_End, start = 17},
+ Token{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token{kind = .Tag_End, start = 4, end = 4},
+ Token{kind = .Raw_Text, start = 5, end = 11},
+ Token{kind = .Close_Tag_Start, start = 14, end = 16},
+ Token{kind = .Tag_End, start = 17, end = 17},
}
_lexer_test(t, source, want)
}
@(test)
test_lex_invalid_tag :: proc(t: ^testing.T) {
- source := `<tag> one <two </tag>`
+ source := `<tag> one <two three </tag>`
want := []Token {
- Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
- Token_Atom{kind = .Tag_End, start = 4},
- Token_Span{kind = .Raw_Text, start = 5, end = 9},
- Token_Span{kind = .Open_Tag_Start, start = 11, end = 13},
- Token_Span{kind = .Raw_Text, start = 14, end = 14},
- Token_Span{kind = .Close_Tag_Start, start = 17, end = 19},
- Token_Atom{kind = .Tag_End, start = 20},
+ Token{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token{kind = .Tag_End, start = 4, end = 4},
+ Token{kind = .Raw_Text, start = 5, end = 9},
+ Token{kind = .Open_Tag_Start, start = 11, end = 13},
+ Token{kind = .Raw_Text, start = 14, end = 20},
+ Token{kind = .Close_Tag_Start, start = 23, end = 25},
+ Token{kind = .Tag_End, start = 26, end = 26},
}
_lexer_test(t, source, want)
}
@@ -533,21 +536,13 @@ test_lex_fuzz_lexer :: proc(t: ^testing.T) {
name := strconv.itoa(buf[:], int(time.time_to_unix(time.now())))
- if err := os.make_directory("fuzzdata", 0o755); err != nil && err != os.EEXIST {
+ if err := os.make_directory("fuzzdata"); err != nil && err != .Exist {
log.errorf("preparing fuzzdata directory: %w", err)
}
- file_path := filepath.join([]string{"fuzzdata", name})
- defer delete(file_path)
-
- f, err := os.open(file_path, os.O_CREATE | os.O_RDWR | os.O_APPEND, 0o644)
- if err != os.ERROR_NONE {
- log.errorf("cannot open fuzz corpus file: %v", err)
- }
-
- defer os.close(f)
+ file_path, _ := filepath.join([]string{"fuzzdata", name})
- if _, err := os.write_string(f, target^); err != nil {
+ if err := os.write_entire_file_from_string(file_path, target^); err != nil {
log.errorf("writing to fuzz corpus file: %v", err)
}
@@ -558,28 +553,28 @@ test_lex_fuzz_lexer :: proc(t: ^testing.T) {
// Run the known failed fuzz corpus.
if FUZZ_RUN_FAILED {
dir, err := os.open("fuzzdata")
- if err != os.ERROR_NONE {
+ if err != nil {
log.errorf("opening fuzzdata directory: %v", err)
}
defer os.close(dir)
- entries, entries_err := os.read_dir(dir, -1)
- if entries_err != os.ERROR_NONE {
+ entries, entries_err := os.read_directory(dir, -1, context.allocator)
+ if entries_err != nil {
log.errorf("reading fuzzdata entries: %v", err)
}
defer {
for entry in entries {
- os.file_info_delete(entry)
+ os.file_info_delete(entry, context.allocator)
}
delete(entries)
}
for entry in entries {
- input, input_err := os.read_entire_file_or_err(entry.fullpath)
- if input_err != os.ERROR_NONE {
- log.errorf("reading fuzzdata entry: %v", input_err)
+ input, input_err := os.read_entire_file_from_path(entry.fullpath, context.allocator)
+ if input_err != nil {
+ log.errorf("reading fuzzdata entry: %v", err)
continue
}
@@ -624,13 +619,12 @@ lexer_print_tokens :: proc(l: ^Lexer) {
for token in lexer_next(l) {
context.allocator = context.temp_allocator
defer free_all()
- switch token in token {
- case Token_Atom:
+ if token.start == token.end {
log.debug(token)
log.debugf("\t: %q \t(%v)", rune(l.source[token.start]), token)
log.debugf("\t: %v", l.source)
log.debugf("\t: %v^", strings.repeat(" ", int(token.start)))
- case Token_Span:
+ } else {
log.debugf("\t: %q \t(%v)", l.source[token.start:token.end + 1], token)
log.debugf("\t: %v", l.source)
padding := strings.repeat(" ", int(token.start))
@@ -652,4 +646,3 @@ random_string :: proc(char_set: string, length: int) -> strings.Builder {
return b
}
-
