commit 47004c278480b7c2285e0a7941494fd499129e27
Author: Jack Mordaunt <jackmordaunt.dev@gmail.com>
Date: Thu, 5 Sep 2024 10:03:39 +0800
initial sketch
Signed-off-by: Jack Mordaunt <jackmordaunt.dev@gmail.com>
Diffstat:
| A | .gitignore | | | 1 | + |
| A | LICENSE | | | 21 | +++++++++++++++++++++ |
| A | README.md | | | 43 | +++++++++++++++++++++++++++++++++++++++++++ |
| A | example/main.odin | | | 16 | ++++++++++++++++ |
| A | html.odin | | | 59 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | iterate.odin | | | 47 | +++++++++++++++++++++++++++++++++++++++++++++++ |
| A | lex.odin | | | 639 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | parse.odin | | | 246 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
8 files changed, 1072 insertions(+), 0 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+*.bin
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Jack Mordaunt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -0,0 +1,43 @@
+# html
+
+This package offers a simple HTML parser, motivated by a desire to query the DOM and
+extract information from it.
+
+The current parser is NOT spec compliant, and is not guaranteed to work on _all_ HTML input.
+This may change.
+
+## usage
+
+```
+package main
+
+import html "../"
+import "core:fmt"
+
+main :: proc() {
+ doc := html.parse("<html><ul><li>one</li><li>two</li><li>three</li></ul></html>")
+ defer html.document_delete(doc)
+
+ iter := html.node_iterator_from_document(doc)
+
+ for node in html.node_iterator_depth_first(&iter) {
+ fmt.println(html.node_to_string(node))
+ }
+}
+```
+
+All strings on the Node are a slice into the original input string.
+The dynamic arrays for the attributes and children can be deleted with [html.document_delete].
+
+## roadmap
+
+- record parse errors
+- spec compliance
+ - respect content model: eg special hadling for `<script>`, `<pre>`, etc
+- stream in source data with a reader
+- support unicode input instead of just ascii
+
+## potholes
+
+Special tags like `<script>` are not handled specially. Such a tag is expected to have it's inner HTML
+be raw text. This version of the parser will parse script content as HTML.
diff --git a/example/main.odin b/example/main.odin
@@ -0,0 +1,16 @@
+package main
+
+import html "../"
+import "core:fmt"
+
+main :: proc() {
+ doc := html.parse("<html><ul><li>one</li><li>two</li><li>three</li></ul></html>")
+ defer html.document_delete(doc)
+
+ iter := html.node_iterator_from_document(doc)
+
+ for node in html.node_iterator_depth_first(&iter) {
+ fmt.println(html.node_to_string(node))
+ }
+}
+
diff --git a/html.odin b/html.odin
@@ -0,0 +1,59 @@
+package html
+
+// Document is the root level data structure containing the entire document.
+Document :: struct {
+ root: Node_Tag,
+}
+
+// Node_Tag is a tag element that might contain other tags.
+Node_Tag :: struct {
+ name: string,
+ attributes: [dynamic]Attribute,
+ children: [dynamic]Node,
+}
+
+// Node_Text is a raw text element.
+Node_Text :: struct {
+ text: string,
+}
+
+// Node represents datum within an HTML document.
+Node :: union {
+ Node_Tag,
+ Node_Text,
+}
+
+Attribute :: struct {
+ name: string,
+ value: string,
+}
+
+node_to_string :: proc(n: Node) -> string {
+ switch v in n {
+ case Node_Text:
+ return v.text
+ case Node_Tag:
+ return v.name
+ }
+ return "<unknown>"
+}
+
+document_delete :: proc(doc: Document) {
+ iter := node_iterator_from_document(doc)
+ defer node_iterator_delete(iter)
+
+ for node in node_iterator_depth_first(&iter) {
+ node_delete(node)
+ }
+}
+
+node_delete :: proc(n: Node) {
+ switch n in n {
+ case Node_Text:
+ return
+ case Node_Tag:
+ delete(n.attributes)
+ delete(n.children)
+ }
+}
+
diff --git a/iterate.odin b/iterate.odin
@@ -0,0 +1,47 @@
+package html
+
+Node_Iterator :: struct {
+ stack: [dynamic]Node,
+}
+
+node_iterator_delete :: proc(it: Node_Iterator) {
+ delete(it.stack)
+}
+
+node_iterator_from_document :: proc(doc: Document) -> (it: Node_Iterator) {
+ append(&it.stack, doc.root)
+ return
+}
+
+node_iterator_depth_first :: proc(it: ^Node_Iterator) -> (n: Node, ok: bool) {
+ entry := pop_safe(&it.stack) or_return
+
+ switch node in entry {
+ case Node_Text:
+ return entry, true
+ case Node_Tag:
+ #reverse for child in node.children {
+ append(&it.stack, child)
+ }
+ return entry, true
+ }
+
+ return
+}
+
+node_iterator_breadth_first :: proc(it: ^Node_Iterator) -> (n: Node, ok: bool) {
+ entry := pop_front_safe(&it.stack) or_return
+
+ switch node in entry {
+ case Node_Text:
+ return entry, true
+ case Node_Tag:
+ for child in node.children {
+ append(&it.stack, child)
+ }
+ return entry, true
+ }
+
+ return
+}
+
diff --git a/lex.odin b/lex.odin
@@ -0,0 +1,639 @@
+package html
+
+import "core:log"
+import "core:math/rand"
+import "core:os"
+import "core:path/filepath"
+import "core:strconv"
+import "core:strings"
+import "core:testing"
+import "core:time"
+import "core:unicode"
+
+DOCTYPE_LITERAL :: "<!DOCTYPE html>"
+
+Token_Span :: struct {
+ kind: Token_Kind,
+ start: u32,
+ end: u32,
+}
+
+Token_Atom :: struct {
+ kind: Token_Kind,
+ start: u32,
+}
+
+Token :: union {
+ Token_Atom,
+ Token_Span,
+}
+
+Token_Kind :: enum u8 {
+ Open_Tag_Start,
+ Tag_End,
+ Close_Tag_Start,
+ Self_Closing_Tag_End,
+ Assign,
+ Comment_Start,
+ Comment_End,
+ Double_Quote_String_Literal,
+ Single_Quote_String_Literal,
+ Raw_Text,
+}
+
+Lexer :: struct {
+ cursor: u32,
+ source: string,
+}
+
+lexer_next :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
+ // Avoid emitting the doctype.
+ // NOTE(jfm): we could emit this and ignore it in the parser.
+ if peek := lexer_peek(l, len(DOCTYPE_LITERAL)); peek == DOCTYPE_LITERAL {
+ lexer_advance(l, len(DOCTYPE_LITERAL))
+ return lexer_next(l)
+ }
+
+ peek := lexer_peek(l)
+ if peek == "" {
+ return nil, false
+ }
+
+ if t, ok := _lexer_next(l); ok {
+ return t, true
+ }
+
+ // Advance until we find an interesting token.
+ // The cursor is reset to undo the advancing that occurs within _lexer_next.
+ start := l.cursor
+ for {
+ if !lexer_advance(l) {
+ break
+ }
+
+ start := l.cursor
+ _, ok := _lexer_next(l)
+ l.cursor = start
+
+ if ok {
+ break
+ }
+ }
+ end := l.cursor - 1
+
+ return Token_Span{kind = .Raw_Text, start = start, end = end}, true
+}
+
+@(private = "file")
+_lexer_next :: proc(l: ^Lexer) -> (t: Token, ok: bool) {
+ if peek := lexer_peek(l, 4); peek == "<!--" {
+ defer lexer_advance(l, 4)
+ return Token_Span{kind = .Comment_Start, start = l.cursor, end = l.cursor + 3}, true
+ }
+
+ if peek := lexer_peek(l, 3); peek == "-->" {
+ defer lexer_advance(l, 3)
+ return Token_Span{kind = .Comment_End, start = l.cursor, end = l.cursor + 2}, true
+ }
+
+ if peek := lexer_peek(l, 2); peek != "" && peek[0] == '<' && unicode.is_letter(rune(peek[1])) {
+ lexer_advance(l)
+ start := l.cursor
+ if !lexer_advance_while_letters(l) do return
+ end := l.cursor
+ defer lexer_advance(l)
+ return Token_Span{kind = .Open_Tag_Start, start = start, end = end}, true
+ }
+
+ if peek := lexer_peek(l, 2); peek == "</" {
+ lexer_advance(l, 2)
+ start := l.cursor
+ if !lexer_advance_while_letters(l) do return
+ end := l.cursor
+ defer lexer_advance(l)
+ return Token_Span{kind = .Close_Tag_Start, start = start, end = end}, true
+ }
+
+ if peek := lexer_peek(l, 2); peek == "/>" {
+ defer lexer_advance(l, 2)
+ return Token_Span{kind = .Self_Closing_Tag_End, start = l.cursor, end = l.cursor + 1}, true
+ }
+
+ if peek := lexer_peek(l, 1); peek == ">" {
+ defer lexer_advance(l, 1)
+ return Token_Atom{kind = .Tag_End, start = l.cursor}, true
+ }
+
+ if peek := lexer_peek(l, 1); peek == "=" {
+ defer lexer_advance(l, 1)
+ return Token_Atom{kind = .Assign, start = l.cursor}, true
+ }
+
+ if peek := lexer_peek(l, 1); peek == `"` {
+ start := l.cursor
+ lexer_advance(l, 1)
+ if !lexer_advance_until(l, `"`) do return t, false
+ end := l.cursor
+ defer lexer_advance(l, 1)
+ return Token_Span{kind = .Double_Quote_String_Literal, start = start, end = end}, true
+ }
+
+ if peek := lexer_peek(l, 1); peek == `'` {
+ start := l.cursor
+ lexer_advance(l, 1)
+ if !lexer_advance_until(l, `'`) do return t, false
+ end := l.cursor
+ defer lexer_advance(l, 1)
+ return Token_Span{kind = .Single_Quote_String_Literal, start = start, end = end}, true
+ }
+
+ return
+
+}
+
+lexer_expect :: proc(l: ^Lexer, kind: Token_Kind) -> (t: Token, ok: bool) {
+ t = lexer_next(l) or_return
+ return t, token_kind(t) == kind
+}
+
+lexer_advance :: proc(l: ^Lexer, step: u32 = 1) -> (ok: bool) {
+ l.cursor += step
+ return l.cursor < u32(len(l.source))
+}
+
+lexer_advance_until :: proc {
+ lexer_advance_until_string,
+ lexer_advance_until_proc,
+}
+
+lexer_advance_while_letters :: proc(l: ^Lexer) -> bool {
+ return lexer_advance_until_proc(l, proc(l: ^Lexer) -> bool {
+ peek := lexer_peek(l, 2)
+ if peek == "" {
+ return false
+ }
+ return !(unicode.is_letter(rune(peek[1])) || peek[1] == '-' || peek[1] == '_')
+ })
+}
+
+lexer_advance_until_space :: proc(l: ^Lexer) -> bool {
+ return lexer_advance_until_proc(l, proc(l: ^Lexer) -> bool {
+ peek := lexer_peek(l, 2)
+ if peek == "" {
+ return false
+ }
+ return unicode.is_space(rune(peek[1]))
+ })
+}
+
+lexer_advance_until_space_or :: proc(l: ^Lexer, or: byte) -> bool {
+ or := or
+ context.user_ptr = &or
+ return lexer_advance_until_proc(l, proc(l: ^Lexer) -> bool {
+ or: ^byte = auto_cast context.user_ptr
+ peek := lexer_peek(l, 2)
+ if peek == "" {
+ return false
+ }
+ return unicode.is_space(rune(peek[1])) || peek[1] == or^
+ })
+}
+
+lexer_advance_until_string :: proc(l: ^Lexer, target: string) -> bool {
+ target := target
+ context.user_ptr = &target
+ return lexer_advance_until_proc(l, proc(l: ^Lexer) -> bool {
+ target := cast(^string)(context.user_ptr)
+ return lexer_peek(l) == target^
+ })
+}
+
+lexer_advance_until_proc :: proc(l: ^Lexer, predicate: proc(l: ^Lexer) -> bool) -> bool {
+ for {
+ if predicate(l) {
+ return true
+ }
+ if !lexer_advance(l) {
+ return false
+ }
+ }
+}
+
+lexer_peek :: proc(l: ^Lexer, step: u32 = 1) -> (p: string) {
+ start := l.cursor
+ end := l.cursor + step
+
+ if end > u32(len(l.source)) {
+ return ""
+ }
+
+ return l.source[start:end]
+}
+
+lexer_skip_space :: proc(l: ^Lexer) {
+ lexer_advance_until_proc(l, proc(l: ^Lexer) -> bool {
+ peek := lexer_peek(l)
+
+ if peek == "" {
+ return true
+ }
+
+ return !unicode.is_space(rune(peek[0]))
+ })
+}
+
+// token_kind returns the kind for the token.
+token_kind :: proc(t: Token) -> Token_Kind {
+ switch t in t {
+ case Token_Atom:
+ return t.kind
+ case Token_Span:
+ return t.kind
+ }
+ return nil
+}
+
+// token_lookup returns the substring corresponding to the token.
+token_lookup :: proc(t: Token, text: string) -> string {
+ switch t in t {
+ case Token_Atom:
+ return text[t.start:t.start + 1]
+ case Token_Span:
+ return text[t.start:t.end + 1]
+ }
+ return ""
+}
+
+@(private = "file")
+_lexer_test :: proc(t: ^testing.T, source: string, want: []Token, loc := #caller_location) {
+ defer free_all()
+
+ lexer: Lexer
+ lexer.source = source
+
+ got: [dynamic]Token
+
+ defer if testing.failed(t) {
+ log.debugf("got: %v", got)
+ }
+
+ for ii := 0; true; ii += 1 {
+ token, ok := lexer_next(&lexer)
+ append(&got, token)
+
+ log.debugf("%d/%d: token %v %v", ii + 1, len(want), ok, token, location = loc)
+ log.debugf("lexer: %v", lexer, location = loc)
+
+ if !ok {
+ if ii <= len(want) - 1 {
+ log.errorf(
+ "short output: got %d tokens; (%v), want %d %v",
+ ii,
+ token,
+ len(want),
+ want,
+ location = loc,
+ )
+ }
+ break
+ }
+
+ if ii >= len(want) {
+ log.errorf("got more tokens than expected: %v", token, location = loc)
+ } else {
+ testing.expect_value(t, token, want[ii], loc = loc)
+ }
+ }
+}
+
+@(test)
+test_lex_comment :: proc(t: ^testing.T) {
+ source := "<!-- comment -->"
+ want := []Token {
+ Token_Span{kind = .Comment_Start, start = 0, end = 3},
+ Token_Span{kind = .Raw_Text, start = 4, end = 12},
+ Token_Span{kind = .Comment_End, start = 13, end = 15},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_comment_2 :: proc(t: ^testing.T) {
+ source := "<!--comment-->"
+ want := []Token {
+ Token_Span{kind = .Comment_Start, start = 0, end = 3},
+ Token_Span{kind = .Raw_Text, start = 4, end = 10},
+ Token_Span{kind = .Comment_End, start = 11, end = 13},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_comment_3 :: proc(t: ^testing.T) {
+ source := "<!-- \ncomment\n -->"
+ want := []Token {
+ Token_Span{kind = .Comment_Start, start = 0, end = 3},
+ Token_Span{kind = .Raw_Text, start = 4, end = 14},
+ Token_Span{kind = .Comment_End, start = 15, end = 17},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_doctype :: proc(t: ^testing.T) {
+ source := "<!DOCTYPE html>"
+ want := []Token{}
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_self_closing_tag :: proc(t: ^testing.T) {
+ source := `<tag />`
+ want := []Token {
+ Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token_Span{kind = .Raw_Text, start = 4, end = 4},
+ Token_Span{kind = .Self_Closing_Tag_End, start = 5, end = 6},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_self_closing_tag_2 :: proc(t: ^testing.T) {
+ source := `<tag/>`
+ want := []Token {
+ Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token_Span{kind = .Self_Closing_Tag_End, start = 4, end = 5},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_open_tag :: proc(t: ^testing.T) {
+ source := "<tag>"
+ want := []Token {
+ Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token_Atom{kind = .Tag_End, start = 4},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_close_tag :: proc(t: ^testing.T) {
+ source := "</tag>"
+ want := []Token {
+ Token_Span{kind = .Close_Tag_Start, start = 2, end = 4},
+ Token_Atom{kind = .Tag_End, start = 5},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_tag_pair :: proc(t: ^testing.T) {
+ source := "<tag></tag>"
+ want := []Token {
+ Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token_Atom{kind = .Tag_End, start = 4},
+ Token_Span{kind = .Close_Tag_Start, start = 7, end = 9},
+ Token_Atom{kind = .Tag_End, start = 10},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_tag_pair_with_comment :: proc(t: ^testing.T) {
+ source := "<tag><!--comment--></tag>"
+ want := []Token {
+ Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token_Atom{kind = .Tag_End, start = 4},
+ Token_Span{kind = .Comment_Start, start = 5, end = 8},
+ Token_Span{kind = .Raw_Text, start = 9, end = 15},
+ Token_Span{kind = .Comment_End, start = 16, end = 18},
+ Token_Span{kind = .Close_Tag_Start, start = 21, end = 23},
+ Token_Atom{kind = .Tag_End, start = 24},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_open_tag_value_attribute :: proc(t: ^testing.T) {
+ source := `<tag class=".style">`
+ want := []Token {
+ Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token_Span{kind = .Raw_Text, start = 4, end = 9},
+ Token_Atom{kind = .Assign, start = 10},
+ Token_Span{kind = .Double_Quote_String_Literal, start = 11, end = 18},
+ Token_Atom{kind = .Tag_End, start = 19},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_open_tag_naked_attribute :: proc(t: ^testing.T) {
+ source := `<tag class=".style" foobar>`
+ want := []Token {
+ Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token_Span{kind = .Raw_Text, start = 4, end = 9},
+ Token_Atom{kind = .Assign, start = 10},
+ Token_Span{kind = .Double_Quote_String_Literal, start = 11, end = 18},
+ Token_Span{kind = .Raw_Text, start = 19, end = 25},
+ Token_Atom{kind = .Tag_End, start = 26},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_nested_tags :: proc(t: ^testing.T) {
+ source := `<tag><ul><li>one</li><li>two</li></ul></tag>`
+ want := []Token {
+ Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token_Atom{kind = .Tag_End, start = 4},
+ Token_Span{kind = .Open_Tag_Start, start = 6, end = 7},
+ Token_Atom{kind = .Tag_End, start = 8},
+ Token_Span{kind = .Open_Tag_Start, start = 10, end = 11},
+ Token_Atom{kind = .Tag_End, start = 12},
+ Token_Span{kind = .Raw_Text, start = 13, end = 15},
+ Token_Span{kind = .Close_Tag_Start, start = 18, end = 19},
+ Token_Atom{kind = .Tag_End, start = 20},
+ Token_Span{kind = .Open_Tag_Start, start = 22, end = 23},
+ Token_Atom{kind = .Tag_End, start = 24},
+ Token_Span{kind = .Raw_Text, start = 25, end = 27},
+ Token_Span{kind = .Close_Tag_Start, start = 30, end = 31},
+ Token_Atom{kind = .Tag_End, start = 32},
+ Token_Span{kind = .Close_Tag_Start, start = 35, end = 36},
+ Token_Atom{kind = .Tag_End, start = 37},
+ Token_Span{kind = .Close_Tag_Start, start = 40, end = 42},
+ Token_Atom{kind = .Tag_End, start = 43},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_raw_text_less_than :: proc(t: ^testing.T) {
+ source := `<tag> 1 < 2 </tag>`
+ want := []Token {
+ Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token_Atom{kind = .Tag_End, start = 4},
+ Token_Span{kind = .Raw_Text, start = 5, end = 11},
+ Token_Span{kind = .Close_Tag_Start, start = 14, end = 16},
+ Token_Atom{kind = .Tag_End, start = 17},
+ }
+ _lexer_test(t, source, want)
+}
+
+@(test)
+test_lex_invalid_tag :: proc(t: ^testing.T) {
+ source := `<tag> one <two </tag>`
+ want := []Token {
+ Token_Span{kind = .Open_Tag_Start, start = 1, end = 3},
+ Token_Atom{kind = .Tag_End, start = 4},
+ Token_Span{kind = .Raw_Text, start = 5, end = 9},
+ Token_Span{kind = .Open_Tag_Start, start = 11, end = 13},
+ Token_Span{kind = .Raw_Text, start = 14, end = 14},
+ Token_Span{kind = .Close_Tag_Start, start = 17, end = 19},
+ Token_Atom{kind = .Tag_End, start = 20},
+ }
+ _lexer_test(t, source, want)
+}
+
+FUZZ_ENABLED :: #config(FUZZ, false)
+FUZZ_RUN_FAILED :: #config(FUZZ_RUN_FAILED, false)
+
+@(test)
+@(disabled = !FUZZ_ENABLED)
+test_lex_fuzz_lexer :: proc(t: ^testing.T) {
+ if !FUZZ_ENABLED do return
+
+ target: string
+
+ // cleanup saves the failed fuzz target to a file.
+ testing.cleanup(t, proc(userdata: rawptr) {
+ target: ^string = auto_cast userdata
+
+ if target == nil || target^ == "" {
+ return
+ }
+
+ buf: [64]byte
+
+ name := strconv.itoa(buf[:], int(time.time_to_unix(time.now())))
+
+ if err := os.make_directory("fuzzdata", 0o755); err != nil && err != os.EEXIST {
+ log.errorf("preparing fuzzdata directory: %w", err)
+ }
+
+ file_path := filepath.join([]string{"fuzzdata", name})
+ defer delete(file_path)
+
+ f, err := os.open(file_path, os.O_CREATE | os.O_RDWR | os.O_APPEND, 0o644)
+ if err != os.ERROR_NONE {
+ log.errorf("cannot open fuzz corpus file: %v", err)
+ }
+
+ defer os.close(f)
+
+ if _, err := os.write_string(f, target^); err != nil {
+ log.errorf("writing to fuzz corpus file: %v", err)
+ }
+
+ }, &target)
+
+ log.info("fuzzing ...")
+
+ // Run the known failed fuzz corpus.
+ if FUZZ_RUN_FAILED {
+ dir, err := os.open("fuzzdata")
+ if err != os.ERROR_NONE {
+ log.errorf("opening fuzzdata directory: %v", err)
+ }
+
+ defer os.close(dir)
+
+ entries, entries_err := os.read_dir(dir, -1)
+ if entries_err != os.ERROR_NONE {
+ log.errorf("reading fuzzdata entries: %v", err)
+ }
+
+ defer {
+ for entry in entries {
+ os.file_info_delete(entry)
+ }
+ delete(entries)
+ }
+
+ for entry in entries {
+ input, input_err := os.read_entire_file_or_err(entry.fullpath)
+ if input_err != os.ERROR_NONE {
+ log.errorf("reading fuzzdata entry: %v", input_err)
+ continue
+ }
+
+ l: Lexer
+ l.source = string(input)
+
+ log.debugf("%q", l.source)
+ lexer_print_tokens(&l)
+ }
+
+ return
+ }
+
+ // Explore new randomized inputs.
+
+ rand.reset(t.seed)
+
+ // Define the pool characters.
+ // Whitespace and angle brackets are weighted higher.
+ character_set :: "abcdefghijklmnopqrstuvwxyz <<<<<>>>>>/=\"!"
+
+ for ii := 1; true; ii += 1 {
+ source := random_string(character_set, rand.int_max(100))
+ defer strings.builder_destroy(&source)
+
+ src := strings.to_string(source)
+ target = src
+
+ l: Lexer
+ l.source = src
+
+ for {
+ _, ok := lexer_next(&l)
+ if !ok {
+ break
+ }
+ }
+ }
+}
+
+lexer_print_tokens :: proc(l: ^Lexer) {
+ for token in lexer_next(l) {
+ context.allocator = context.temp_allocator
+ defer free_all()
+ switch token in token {
+ case Token_Atom:
+ log.debug(token)
+ log.debugf("\t: %q \t(%v)", rune(l.source[token.start]), token)
+ log.debugf("\t: %v", l.source)
+ log.debugf("\t: %v^", strings.repeat(" ", int(token.start)))
+ case Token_Span:
+ log.debugf("\t: %q \t(%v)", l.source[token.start:token.end + 1], token)
+ log.debugf("\t: %v", l.source)
+ padding := strings.repeat(" ", int(token.start))
+ underline := ""
+ if token.end - token.start - 1 >= 0 {
+ underline = strings.repeat("_", int(token.end - token.start - 1))
+ }
+ log.debugf("\t: %v^%v^", padding, underline)
+ }
+ }
+}
+
+random_string :: proc(char_set: string, length: int) -> strings.Builder {
+ b: strings.Builder
+
+ for ii in 0 ..< length {
+ strings.write_byte(&b, char_set[rand.int_max(len(char_set))])
+ }
+
+ return b
+}
+
diff --git a/parse.odin b/parse.odin
@@ -0,0 +1,246 @@
+package html
+
+/*
+ This parsing logic DOES NOT attempt to validate the DOM object model;
+ <head> and <body> tags are not generated, nor are tables validated.
+
+ Instead this logic gives a tree that literally describes the tags provided,
+ with the exception that the root tag is always <html>.
+
+ The parsing algorithm is a simple recursive descent.
+*/
+
+import "core:log"
+import "core:strings"
+import "core:testing"
+
+parse :: proc(text: string) -> Document {
+ l: Lexer
+ l.source = text
+ return _parse_document(&l)
+}
+
+@(private = "file")
+_parse_document :: proc(l: ^Lexer) -> (doc: Document) {
+ doc.root.name = "html"
+ doc.root.children = _parse_children(l)
+ return
+}
+
+@(private = "file")
+_parse_attributes :: proc(l: ^Lexer) -> (attrs: [dynamic]Attribute) {
+ for token in lexer_next(l) {
+ #partial switch token_kind(token) {
+ case .Tag_End:
+ return
+ case .Raw_Text:
+ // Raw_Text might contain leading whitespace, and might contain several
+ // space-delimited boolean attributes.
+ source := token_lookup(token, l.source)
+ for field in strings.fields_iterator(&source) {
+ append(&attrs, Attribute{name = field})
+ }
+ case .Assign:
+ token, ok := lexer_next(l)
+ if !ok {
+ return
+ }
+ #partial switch token_kind(token) {
+ case .Double_Quote_String_Literal, .Single_Quote_String_Literal:
+ if len(attrs) == 0 {
+ log.errorf(
+ "skipping unexpected string literal (missing applicable attribute): %q",
+ token_lookup(token, l.source),
+ )
+ continue
+ }
+ attrs[len(attrs) - 1].value = token_lookup(token, l.source)
+ case:
+ log.errorf("expecting string literal after assign")
+ continue
+ }
+ }
+ }
+ return
+}
+
+@(private = "file")
+_parse_children :: proc(l: ^Lexer) -> (ch: [dynamic]Node) {
+ for token in lexer_next(l) {
+ #partial switch token_kind(token) {
+ case .Raw_Text:
+ append(&ch, Node_Text{text = token_lookup(token, l.source)})
+ case .Open_Tag_Start:
+ node: Node_Tag
+ node.name = token_lookup(token, l.source)
+ node.attributes = _parse_attributes(l)
+ if node.name == "html" {
+ continue
+ }
+ node.children = _parse_children(l)
+ append(&ch, node)
+ case .Close_Tag_Start:
+ if _, ok := lexer_expect(l, .Tag_End); !ok {
+ log.errorf("expected tag end after tag identifier")
+ return
+ }
+ return
+ case:
+ log.errorf("unexpected token while parsing text and nodes: %v", token)
+ }
+ }
+ return
+}
+
+@(test)
+test_parse_doctype :: proc(t: ^testing.T) {
+ text := "<!DOCTYPE html>"
+
+ want := []Node{Node_Tag{name = "html"}}
+
+ _test_parse(t, text, want)
+}
+
+@(test)
+test_parse_single_tag :: proc(t: ^testing.T) {
+ text := "<button />"
+
+ want := []Node{Node_Tag{name = "html"}, Node_Tag{name = "button"}}
+
+ _test_parse(t, text, want)
+}
+
+@(test)
+test_parse_single_tag_with_attribute :: proc(t: ^testing.T) {
+ text := `<button class=".style"/>`
+
+ want := []Node {
+ Node_Tag{name = "html"},
+ Node_Tag {
+ name = "button",
+ attributes = [dynamic]Attribute{{name = "class", value = ".style"}},
+ },
+ }
+
+ _test_parse(t, text, want)
+}
+
+@(test)
+test_parse_single_tag_with_attributes :: proc(t: ^testing.T) {
+ text := `<button class=".style" attribute foo="bar" data-form/>`
+
+ want := []Node {
+ Node_Tag{name = "html"},
+ Node_Tag {
+ name = "button",
+ attributes = [dynamic]Attribute {
+ {name = "class", value = ".style"},
+ {name = "attribute", value = ""},
+ {name = "foo", value = "bar"},
+ {name = "data-form", value = ""},
+ },
+ },
+ }
+
+ _test_parse(t, text, want)
+}
+
+@(test)
+test_parse_nested_tags :: proc(t: ^testing.T) {
+ text := "<ul><li>one</li><li>two</li><li>three</li></ul>"
+
+ want := []Node {
+ Node_Tag{name = "html"},
+ Node_Tag{name = "ul"},
+ Node_Tag{name = "li"},
+ Node_Text{text = "one"},
+ Node_Tag{name = "li"},
+ Node_Text{text = "two"},
+ Node_Tag{name = "li"},
+ Node_Text{text = "three"},
+ }
+
+ _test_parse(t, text, want)
+}
+
+@(test)
+test_parse_raw_text :: proc(t: ^testing.T) {
+ text := "<ul><li> one < two < three </li></ul>"
+
+ want := []Node {
+ Node_Tag{name = "html"},
+ Node_Tag{name = "ul"},
+ Node_Tag{name = "li"},
+ Node_Text{text = " one < two < three "},
+ }
+
+ _test_parse(t, text, want)
+}
+
+@(test)
+test_parse_invalid_tag :: proc(t: ^testing.T) {
+ text := "<ul><li> one <two three </li></ul>"
+
+ want := []Node {
+ Node_Tag{name = "html"},
+ Node_Tag{name = "ul"},
+ Node_Tag{name = "li"},
+ Node_Text{text = " one "},
+ Node_Tag{name = "two"},
+ Node_Text{text = " three "},
+ }
+
+ _test_parse(t, text, want)
+}
+
+@(test)
+test_parse_explicit_html_tag :: proc(t: ^testing.T) {
+ text := "<html><ul><li>one</li><li>two</li><li>three</li></ul></html>"
+
+ want := []Node {
+ Node_Tag{name = "html"},
+ Node_Tag{name = "ul"},
+ Node_Tag{name = "li"},
+ Node_Text{text = "one"},
+ Node_Tag{name = "li"},
+ Node_Text{text = "two"},
+ Node_Tag{name = "li"},
+ Node_Text{text = "three"},
+ }
+
+ _test_parse(t, text, want)
+}
+
+_test_parse :: proc(t: ^testing.T, text: string, want: []Node) {
+ defer free_all()
+
+ doc := parse(text)
+
+ it := node_iterator_from_document(doc)
+
+ ii: int
+ for node in node_iterator_depth_first(&it) {
+ defer ii += 1
+ if ii >= len(want) {
+ continue
+ }
+ want := want[ii]
+ switch node in node {
+ case Node_Tag:
+ #partial switch want in want {
+ case Node_Tag:
+ testing.expect_value(t, node.name, want.name)
+ case:
+ log.errorf("got %q, want %q", node_to_string(node), node_to_string(want))
+ }
+ case Node_Text:
+ #partial switch want in want {
+ case Node_Text:
+ testing.expect_value(t, node.text, want.text)
+ case:
+ log.errorf("got %q, want %q", node_to_string(node), node_to_string(want))
+ }
+ }
+ }
+}
+
