commit 27ccb4ac697098313c6620adf4a04cb5bf619736
parent 55c8a0957794545579c72f1b791a5abee71eb041
Author: Jack Mordaunt <jackmordaunt.dev@gmail.com>
Date: Wed, 23 Sep 2026 20:23:28 -0300
odin: read TypeScript, JavaScript, Python and Rust through ast-grep
The ast-grep reading is a third kind of sidecar: the Go tool's patterns
for the TypeScript grammars and its node kinds for Python and Rust,
written as a temporary rule set per scan, with the answer in the shape
the parser sidecars give, so the change and the index read every
language one way. Comments of every language but Go are read by shape.
The Go tool ordered ast-grep matches by map iteration, which made the
unreferenced-symbol finding's anchor and id change between runs; both
tools now read matches in file and line order. On a repository holding
all three languages the two tools report the same 13 findings, id for
id, and on this repository's largest commit the same 25.
Diffstat:
9 files changed, 734 insertions(+), 21 deletions(-)
diff --git a/kinds.go b/kinds.go
@@ -11,6 +11,7 @@ import (
"os"
"path/filepath"
"regexp"
+ "slices"
"strings"
)
@@ -126,6 +127,14 @@ func (f *KindFrontend) read(matches []grepMatch) []kindMatch {
for _, m := range byLine {
out = append(out, m)
}
+ // In file and line order, so that what is read is the same on every
+ // run.
+ slices.SortFunc(out, func(a, b kindMatch) int {
+ if c := strings.Compare(a.File, b.File); c != 0 {
+ return c
+ }
+ return a.Line - b.Line
+ })
return out
}
diff --git a/odin/analyser/analyser_test.odin b/odin/analyser/analyser_test.odin
@@ -202,9 +202,15 @@ nearest_walks_up :: proc(t: ^testing.T) {
testing.expect(t, os.make_directory_all(join(root, path)) == nil)
}
for path in ([]string{"go.mod", "sidecar/govet/go.mod"}) {
- testing.expect(t, os.write_entire_file(join(root, path), transmute([]byte)string("module x")) == nil)
+ testing.expect(
+ t,
+ os.write_entire_file(join(root, path), transmute([]byte)string("module x")) == nil,
+ )
}
- pkgs := go_packages(root, {"a/a.go", "sidecar/govet/main.go", "sidecar/gofront/main.go", "vendor/x/x.go"})
+ pkgs := go_packages(
+ root,
+ {"a/a.go", "sidecar/govet/main.go", "sidecar/gofront/main.go", "vendor/x/x.go"},
+ )
testing.expect_value(t, fmt.tprint(pkgs), `["./a", "./sidecar/gofront"]`)
}
diff --git a/odin/change/change.odin b/odin/change/change.odin
@@ -661,7 +661,7 @@ read_file :: proc(c: ^Change, t: tree.Tree, name: string, file: frontend.File) {
name = strings.clone(decl.name),
file = name,
line = decl.line,
- body = text(lines, decl.line, decl.end_line),
+ body = body_of(decl, lines),
},
)
}
@@ -681,17 +681,24 @@ read_file :: proc(c: ^Change, t: tree.Tree, name: string, file: frontend.File) {
pkg = strings.clone(file.pkg),
}
if decl.kind == "func" {
- symbol.body = text(lines, decl.line, decl.end_line)
+ symbol.body = body_of(decl, lines)
}
append(&c.symbols, symbol)
}
- for comment in file.comments {
- if touched[comment.line] {
- append(
- &c.comments,
- Located{text = strings.clone(comment.text), file = name, line = comment.line},
- )
+ // Go's sidecar reads comments with its parser; every other language's
+ // are read by shape, the same way for a parsed file and an unparsed
+ // one.
+ if s, _ := frontend.sidecar_for(name); s == .Go {
+ for comment in file.comments {
+ if touched[comment.line] {
+ append(
+ &c.comments,
+ Located{text = strings.clone(comment.text), file = name, line = comment.line},
+ )
+ }
}
+ } else {
+ append(&c.comments, ..comment_prose(source, name, added)[:])
}
}
@@ -740,7 +747,7 @@ index :: proc(t: tree.Tree, allocator := context.allocator) -> (out: []Declared,
text = strings.clone(decl.text),
}
if decl.kind == "func" {
- entry.body = text(lines, decl.line, decl.end_line)
+ entry.body = body_of(decl, lines)
}
append(&declared, entry)
}
@@ -749,6 +756,15 @@ index :: proc(t: tree.Tree, allocator := context.allocator) -> (out: []Declared,
return declared[:], ok
}
+// body_of is a declaration's whole text: the reader's own match where it
+// has one, else the file's lines from its first to its last.
+body_of :: proc(decl: frontend.Decl, lines: []string, allocator := context.allocator) -> string {
+ if decl.body != "" {
+ return strings.clone(decl.body, allocator)
+ }
+ return text(lines, decl.line, decl.end_line, allocator)
+}
+
// scan asks a sidecar about tracked files, handing it their paths in the
// tree, and returns each answer under the tracked name it was asked for.
scan :: proc(
diff --git a/odin/check/check.odin b/odin/check/check.odin
@@ -11,6 +11,7 @@ import "base:runtime"
import "core:fmt"
import "core:slice"
import "core:strings"
+import "core:sync"
import "core:text/regex"
import "core:unicode"
@@ -264,22 +265,24 @@ grammar_of :: proc(path: string) -> string {
// The patterns the checks match are compiled once and kept for the
// program; a check runs over every line of a change, and a pattern is not
-// worth compiling per line.
+// worth compiling per line. The cache is shared between threads, as the
+// test runner's are, so it is locked.
@(private)
patterns: map[string]regex.Regular_Expression
@(private)
-scratch: regex.Capture
+patterns_lock: sync.Mutex
@(init)
init_patterns :: proc "contextless" () {
context = runtime.default_context()
patterns = make(map[string]regex.Regular_Expression, runtime.heap_allocator())
- scratch = regex.preallocate_capture(runtime.heap_allocator())
}
// rx is the compiled form of a pattern, in Go's syntax as far as the two
// engines share it.
rx :: proc(pattern: string) -> regex.Regular_Expression {
+ sync.mutex_lock(&patterns_lock)
+ defer sync.mutex_unlock(&patterns_lock)
if re, ok := patterns[pattern]; ok {
return re
}
@@ -293,7 +296,7 @@ rx :: proc(pattern: string) -> regex.Regular_Expression {
// matches reports whether a pattern matches anywhere in the text.
matches :: proc(pattern, text: string) -> bool {
- _, ok := regex.match(rx(pattern), text, &scratch)
+ _, ok := regex.match(rx(pattern), text, context.temp_allocator)
return ok
}
@@ -317,11 +320,12 @@ capture :: proc(
// capture_end is where the first match ends, for a reader that continues
// from there.
capture_end :: proc(pattern, text: string) -> (end: int, ok: bool) {
- _, ok = regex.match(rx(pattern), text, &scratch)
+ cap: regex.Capture
+ cap, ok = regex.match(rx(pattern), text, context.temp_allocator)
if !ok {
return 0, false
}
- return scratch.pos[0][1], true
+ return cap.pos[0][1], true
}
// find_all is every match of a pattern in the text, whole.
diff --git a/odin/check/check_test.odin b/odin/check/check_test.odin
@@ -518,6 +518,7 @@ covered_spares_a_rename :: proc(t: ^testing.T) {
@(test)
names_are_measured :: proc(t: ^testing.T) {
+ context.allocator = context.temp_allocator
Stutter :: struct {
s: change.Symbol,
fires: bool,
diff --git a/odin/frontend/frontend.odin b/odin/frontend/frontend.odin
@@ -24,6 +24,9 @@ Decl :: struct {
local: bool,
text: string,
doc: string,
+ // body is the declaration's whole text where the reader has it as
+ // one match; empty where the caller reads it from the file by lines.
+ body: string,
}
// Comment is one comment, located by its first line, marker stripped.
@@ -47,10 +50,15 @@ Output :: struct {
files: []File,
}
-// Sidecar names the parser a language is read through.
+// Sidecar names the parser a language is read through: a program built
+// on the language's own parser for Go and Odin, and ast-grep's grammars
+// for the rest.
Sidecar :: enum {
Go,
Odin,
+ Script,
+ Python,
+ Rust,
}
// Scan_Error says why a scan returned nothing: the sidecar is not on the
@@ -69,17 +77,27 @@ binary :: proc(sidecar: Sidecar) -> string {
return "review-go"
case .Odin:
return "odin-review-extract"
+ case .Script, .Python, .Rust:
+ return "ast-grep"
}
return ""
}
-// sidecar_for is the sidecar that reads a path, by its extension.
+// sidecar_for is the sidecar that reads a path, by its extension. The
+// ast-grep languages are covered only while ast-grep is on the path; their
+// files fall to the comment reader otherwise.
sidecar_for :: proc(path: string) -> (sidecar: Sidecar, covered: bool) {
switch {
case strings.has_suffix(path, ".go"):
return .Go, true
case strings.has_suffix(path, ".odin"):
return .Odin, true
+ case grammar_of(path) != "":
+ return .Script, installed(.Script)
+ case strings.has_suffix(path, ".py"):
+ return .Python, installed(.Python)
+ case strings.has_suffix(path, ".rs"):
+ return .Rust, installed(.Rust)
}
return .Go, false
}
@@ -107,6 +125,9 @@ scan :: proc(
if !installed(sidecar) {
return out, .Not_Installed
}
+ if sidecar == .Script || sidecar == .Python || sidecar == .Rust {
+ return grep_scan(sidecar, files, allocator)
+ }
argv := make([]string, len(files) + 1, context.temp_allocator)
argv[0] = binary(sidecar)
copy(argv[1:], files)
diff --git a/odin/frontend/frontend_test.odin b/odin/frontend/frontend_test.odin
@@ -92,3 +92,152 @@ scan_of_nothing_asks_nothing :: proc(t: ^testing.T) {
testing.expect_value(t, err, Scan_Error.None)
testing.expect_value(t, len(out.files), 0)
}
+
+write_fixture :: proc(t: ^testing.T, name, src: string) -> string {
+ return write_temp(t, name, src)
+}
+
+@(test)
+ast_grep_reads_typescript :: proc(t: ^testing.T) {
+ if !installed(.Script) {
+ testing.fail_now(t, "ast-grep is not on the path")
+ }
+ path := write_fixture(
+ t,
+ "review_grep_fixture.ts",
+ `import { x } from "y";
+
+// Limit bounds the work.
+export const limit: number = 3;
+const hidden = 4;
+export type Pair = [number, number];
+interface Shape { size: number }
+export function read(a: string): number {
+ const inner = 1;
+ return inner;
+}
+export default function main() {}
+describe('icons', () => {
+ it('decodes the largest', () => { expect(size).toBe(1); });
+ test.skip("reads a binary", () => {});
+});
+`,
+ )
+ defer os.remove(path)
+ out, err := scan(.Script, {path}, context.temp_allocator)
+ testing.expect_value(t, err, Scan_Error.None)
+ testing.expect_value(t, len(out.files), 1)
+ by := make(map[string]Decl, context.temp_allocator)
+ for d in out.files[0].decls {
+ by[d.name] = d
+ }
+ testing.expect_value(t, len(by), 8)
+ testing.expect_value(t, by["limit"].kind, "value")
+ testing.expect(t, by["limit"].exported)
+ testing.expect_value(t, by["limit"].doc, "Limit bounds the work.")
+ testing.expect_value(t, by["limit"].line, 4)
+ testing.expect(t, !by["hidden"].exported)
+ testing.expect_value(t, by["Pair"].kind, "type")
+ testing.expect_value(t, by["Shape"].kind, "type")
+ testing.expect_value(t, by["read"].kind, "func")
+ testing.expect_value(t, by["read"].end_line, 11)
+ testing.expect_value(t, by["main"].kind, "func")
+ testing.expect(t, by["decodes the largest"].test)
+ testing.expect(t, by["reads a binary"].test)
+ _, nested := by["inner"]
+ testing.expect(t, !nested, "a declaration inside a body is not top level")
+}
+
+@(test)
+ast_grep_reads_python_and_rust :: proc(t: ^testing.T) {
+ if !installed(.Python) {
+ testing.fail_now(t, "ast-grep is not on the path")
+ }
+ py := write_fixture(
+ t,
+ "review_grep_fixture.py",
+ `"""Icons."""
+MAX_ICONS = 12
+_hidden: int = 4
+
+# Reads the icons a file holds.
+def read_icons(src):
+ inner = 1
+ def nested():
+ return inner
+ return []
+
+class Reader:
+ size = 0
+ def method(self):
+ return 1
+
+def test_smoke():
+ read_icons("y")
+`,
+ )
+ defer os.remove(py)
+ out, err := scan(.Python, {py}, context.temp_allocator)
+ testing.expect_value(t, err, Scan_Error.None)
+ by := make(map[string]Decl, context.temp_allocator)
+ for d in out.files[0].decls {
+ by[d.name] = d
+ }
+ testing.expect_value(t, len(by), 5)
+ testing.expect_value(t, by["MAX_ICONS"].kind, "value")
+ testing.expect(t, by["MAX_ICONS"].exported)
+ testing.expect(t, !by["_hidden"].exported)
+ testing.expect_value(t, by["read_icons"].kind, "func")
+ testing.expect_value(t, by["read_icons"].doc, "Reads the icons a file holds.")
+ testing.expect_value(t, by["Reader"].kind, "type")
+ testing.expect(t, by["test_smoke"].test)
+ testing.expect_value(t, by["test_smoke"].kind, "func")
+
+ rs := write_fixture(
+ t,
+ "review_grep_fixture.rs",
+ `pub const LIMIT: usize = 3;
+static mut COUNT: i32 = 0;
+
+/// A shape.
+pub struct Shape { size: usize }
+
+impl Shape {
+ pub fn area(&self) -> usize { self.size }
+ fn hidden(&self) -> usize { 0 }
+}
+
+#[test]
+fn reads_shapes() {
+ assert_eq!(count(), 1);
+}
+`,
+ )
+ defer os.remove(rs)
+ out, err = scan(.Rust, {rs}, context.temp_allocator)
+ testing.expect_value(t, err, Scan_Error.None)
+ clear(&by)
+ for d in out.files[0].decls {
+ by[d.name] = d
+ }
+ testing.expect_value(t, len(by), 6)
+ testing.expect(t, by["LIMIT"].exported)
+ testing.expect(t, !by["COUNT"].exported)
+ testing.expect_value(t, by["Shape"].kind, "type")
+ testing.expect_value(t, by["Shape"].doc, "A shape.")
+ testing.expect(t, by["area"].exported)
+ testing.expect(t, !by["hidden"].exported)
+ testing.expect(t, by["reads_shapes"].test)
+ testing.expect_value(t, by["reads_shapes"].end_line, 15)
+}
+
+@(test)
+doc_above_reads_the_block :: proc(t: ^testing.T) {
+ lines := []string{"x", "/** Two", " * lines */", "// and one", "const a = 1"}
+ testing.expect_value(t, doc_above(lines, 5, context.temp_allocator), "Two lines and one")
+ testing.expect_value(t, doc_above(lines, 1, context.temp_allocator), "")
+ testing.expect_value(t, scratch_ext("a/b.mjs"), ".js")
+ testing.expect_value(t, grammar_of("a.tsx"), "tsx")
+ testing.expect(t, typed("export type $N = $$$B"))
+ testing.expect(t, !typed("export const $N = $$$V"))
+}
diff --git a/odin/frontend/grep.odin b/odin/frontend/grep.odin
@@ -0,0 +1,489 @@
+package frontend
+
+// TypeScript, JavaScript, Python and Rust are read through ast-grep, which
+// matches patterns and node kinds against the tree-sitter syntax tree
+// rather than against lines. A pattern over lines misses an indented
+// constant inside a block; a query over syntax does not. The answer is
+// shaped like a sidecar's, so the change reads every language one way.
+
+import "core:encoding/json"
+import "core:fmt"
+import "core:os"
+import "core:path/filepath"
+import "core:slice"
+import "core:strconv"
+import "core:strings"
+import "core:text/regex"
+import "jfm:sh"
+
+// Pattern is one top-level declaration shape a TypeScript grammar binds.
+// Export status is part of the pattern so a symbol's audience is known;
+// the plain forms also match the inner node of an export, which the
+// reader resolves by preferring the exported match.
+Pattern :: struct {
+ kind: string,
+ exported: bool,
+ pattern: string,
+}
+
+@(private = "file")
+ts_patterns := []Pattern {
+ {"value", false, "const $N = $$$V"},
+ {"value", true, "export const $N = $$$V"},
+ {"value", false, "const $N: $$$T = $$$V"},
+ {"value", true, "export const $N: $$$T = $$$V"},
+ {"value", false, "let $N = $$$V"},
+ {"value", true, "export let $N = $$$V"},
+ {"value", true, "export let $N: $$$T = $$$V"},
+ {"value", false, "var $N = $$$V"},
+ {"value", true, "export var $N = $$$V"},
+ {"type", false, "type $N = $$$B"},
+ {"type", true, "export type $N = $$$B"},
+ {"type", false, "interface $N { $$$B }"},
+ {"type", true, "export interface $N { $$$B }"},
+ {"type", false, "enum $N { $$$V }"},
+ {"type", true, "export enum $N { $$$V }"},
+ {"type", false, "class $N { $$$B }"},
+ {"type", true, "export class $N { $$$B }"},
+ {"func", false, "function $N($$$P) { $$$B }"},
+ {"func", true, "export function $N($$$P) { $$$B }"},
+ {"func", true, "export function $N($$$P): $$$R { $$$B }"},
+ {"func", false, "async function $N($$$P) { $$$B }"},
+ {"func", true, "export async function $N($$$P) { $$$B }"},
+ {"func", true, "export async function $N($$$P): $$$R { $$$B }"},
+ {"func", true, "export default function $N($$$P) { $$$B }"},
+ {"func", true, "export default function $N($$$P): $$$R { $$$B }"},
+}
+
+// ts_test_patterns bind the runner's calls, in the shapes Node's
+// node:test, Bun's bun:test, Deno's Deno.test and the Jest family write
+// them. A test inside a describe block is still a test, so these carry no
+// top-level constraint. The name is $S when the test is named by a string
+// and $N when by a function.
+@(private = "file")
+ts_test_patterns := []string {
+ "test($S, $$$B)",
+ "it($S, $$$B)",
+ "test.$M($S, $$$B)",
+ "it.$M($S, $$$B)",
+ "test.$M($$$T)($S, $$$B)",
+ "it.$M($$$T)($S, $$$B)",
+ "Deno.test($S, $$$B)",
+ "Deno.test.$M($S, $$$B)",
+ "Deno.test({ name: $S, $$$R })",
+ "Deno.test(function $N($$$P) { $$$B })",
+}
+
+// Kind_Rule is one declaration shape read by node kind: the ast-grep rule
+// that matches it, the kind the tool reports it as, and the expression
+// that reads its name out of the match text.
+Kind_Rule :: struct {
+ id: string,
+ kind: string,
+ rule: string,
+ name: string,
+ test: bool,
+}
+
+@(private = "file")
+py_top :: " not:\n inside:\n any:\n - kind: function_definition\n - kind: class_definition\n stopBy: end\n"
+
+// Python: module-level functions, classes and assignments, and any
+// function named test_. A leading underscore is the language's whole
+// notion of private.
+@(private = "file")
+py_rules := []Kind_Rule {
+ {
+ "py-func",
+ "func",
+ " kind: function_definition\n" + py_top,
+ `^\s*(?:async\s+)?def\s+(\w+)`,
+ false,
+ },
+ {"py-class", "type", " kind: class_definition\n" + py_top, `^\s*class\s+(\w+)`, false},
+ {
+ "py-value",
+ "value",
+ " kind: assignment\n inside:\n kind: expression_statement\n inside:\n kind: module\n",
+ `^\s*(\w+)\s*(?::[^=]*)?=`,
+ false,
+ },
+ {
+ "py-test",
+ "func",
+ " kind: function_definition\n has:\n field: name\n regex: ^test_\n",
+ `^\s*(?:async\s+)?def\s+(\w+)`,
+ true,
+ },
+}
+
+// Rust: functions wherever they are declared, impl methods included, the
+// type items, constants and statics, and any function a test attribute
+// precedes. pub is the whole notion of exported.
+@(private = "file")
+rs_rules := []Kind_Rule {
+ {"rs-func", "func", " kind: function_item\n", `\bfn\s+(\w+)`, false},
+ {
+ "rs-type",
+ "type",
+ " any:\n - kind: struct_item\n - kind: enum_item\n - kind: type_item\n - kind: trait_item\n",
+ `\b(?:struct|enum|type|trait)\s+(\w+)`,
+ false,
+ },
+ {
+ "rs-value",
+ "value",
+ " any:\n - kind: const_item\n - kind: static_item\n",
+ `\b(?:const|static)\s+(?:mut\s+)?(\w+)`,
+ false,
+ },
+ {
+ "rs-test",
+ "func",
+ " kind: function_item\n follows:\n kind: attribute_item\n regex: '^#\\[[\\w:]*test(\\(|\\])'\n",
+ `\bfn\s+(\w+)`,
+ true,
+ },
+}
+
+// grammar_of is the ast-grep grammar a path is parsed with, or empty
+// where none of the pattern grammars reads it.
+grammar_of :: proc(path: string) -> string {
+ switch {
+ case strings.has_suffix(path, ".ts"):
+ return "ts"
+ case strings.has_suffix(path, ".tsx"):
+ return "tsx"
+ case strings.has_suffix(path, ".js") ||
+ strings.has_suffix(path, ".jsx") ||
+ strings.has_suffix(path, ".mjs") ||
+ strings.has_suffix(path, ".cjs"):
+ return "js"
+ }
+ return ""
+}
+
+// scratch_ext is the extension a file is materialised under, which is
+// what ast-grep infers the grammar from. The module variants of
+// JavaScript are the same grammar under another name.
+scratch_ext :: proc(path: string) -> string {
+ ext := filepath.ext(path)
+ if ext == ".mjs" || ext == ".cjs" {
+ return ".js"
+ }
+ return ext
+}
+
+// typed is whether a pattern needs TypeScript's grammar: an annotation,
+// an interface, a type alias or an enum is not JavaScript.
+typed :: proc(pattern: string) -> bool {
+ bare := strings.trim_prefix(pattern, "export ")
+ return(
+ strings.contains(pattern, ": $$$") ||
+ strings.has_prefix(bare, "type ") ||
+ strings.contains(pattern, "interface ") ||
+ strings.contains(pattern, "enum ") \
+ )
+}
+
+// write_rules emits the query set for a language into the rules
+// directory. The top-level rule keeps TypeScript declarations out of
+// function bodies; the test rules go anywhere.
+write_rules :: proc(sidecar: Sidecar, dir: string) -> bool {
+ switch sidecar {
+ case .Script:
+ Table :: struct {
+ prefix, language: string,
+ }
+ for table in ([]Table{{"ts", "TypeScript"}, {"tsx", "TSX"}, {"js", "JavaScript"}}) {
+ for p, i in ts_patterns {
+ if table.prefix == "js" && typed(p.pattern) {
+ continue
+ }
+ status := "export" if p.exported else "plain"
+ id := fmt.tprintf("%s-%s-%s-%d", table.prefix, p.kind, status, i)
+ body := fmt.tprintf(
+ "id: %s\nlanguage: %s\nseverity: info\nrule:\n pattern: %q\n not:\n inside:\n kind: statement_block\n stopBy: end\n",
+ id,
+ table.language,
+ p.pattern,
+ )
+ write_rule(dir, id, body) or_return
+ }
+ for pattern, i in ts_test_patterns {
+ id := fmt.tprintf("%s-test-%d", table.prefix, i)
+ write_rule(
+ dir,
+ id,
+ fmt.tprintf(
+ "id: %s\nlanguage: %s\nseverity: info\nrule:\n pattern: %q\n",
+ id,
+ table.language,
+ pattern,
+ ),
+ ) or_return
+ }
+ }
+ case .Python, .Rust:
+ language := "Python" if sidecar == .Python else "Rust"
+ for r in (py_rules if sidecar == .Python else rs_rules) {
+ write_rule(
+ dir,
+ r.id,
+ fmt.tprintf(
+ "id: %s\nlanguage: %s\nseverity: info\nrule:\n%s",
+ r.id,
+ language,
+ r.rule,
+ ),
+ ) or_return
+ }
+ case .Go, .Odin:
+ return false
+ }
+ return true
+}
+
+write_rule :: proc(dir, id, body: string) -> bool {
+ path :=
+ filepath.join(
+ {dir, strings.concatenate({id, ".yml"}, context.temp_allocator)},
+ context.temp_allocator,
+ ) or_else id
+ return os.write_entire_file(path, transmute([]byte)body) == nil
+}
+
+// Grep_Match is one ast-grep finding, as its --json prints it.
+Grep_Match :: struct {
+ rule: string `json:"ruleId"`,
+ text: string `json:"text"`,
+ file: string `json:"file"`,
+ range: struct {
+ start: struct {
+ line: int `json:"line"`,
+ } `json:"start"`,
+ } `json:"range"`,
+ meta: struct {
+ single: map[string]struct {
+ text: string `json:"text"`,
+ } `json:"single"`,
+ } `json:"metaVariables"`,
+}
+
+// grep_scan materialises the files into a scratch directory under
+// numbered names, writes the rules, asks ast-grep for the matches, and
+// reads them back into the sidecar shape, one File per path given.
+grep_scan :: proc(
+ sidecar: Sidecar,
+ files: []string,
+ allocator := context.allocator,
+) -> (
+ out: Output,
+ err: Scan_Error,
+) {
+ temp := os.temp_directory(context.temp_allocator) or_else ""
+ dir, made := os.make_directory_temp(temp, "review-grep-*", context.temp_allocator)
+ if made != nil {
+ return out, .Failed
+ }
+ defer os.remove_all(dir)
+ rules := filepath.join({dir, "rules"}, context.temp_allocator) or_else ""
+ if os.make_directory_all(rules) != nil {
+ return out, .Failed
+ }
+ config := filepath.join({dir, "sgconfig.yml"}, context.temp_allocator) or_else ""
+ if os.write_entire_file(config, transmute([]byte)string("ruleDirs:\n - rules\n")) != nil {
+ return out, .Failed
+ }
+ if !write_rules(sidecar, rules) {
+ return out, .Failed
+ }
+ argv := make([dynamic]string, context.temp_allocator)
+ append(&argv, "ast-grep", "scan", "-c", config, "--json")
+ sources := make([][]byte, len(files), context.temp_allocator)
+ for name, i in files {
+ source, read_err := os.read_entire_file_from_path(name, allocator)
+ if read_err != nil {
+ continue
+ }
+ sources[i] = source
+ scratch :=
+ filepath.join(
+ {dir, fmt.tprintf("%04d%s", i, scratch_ext(name))},
+ context.temp_allocator,
+ ) or_else ""
+ if os.write_entire_file(scratch, source) != nil {
+ return out, .Failed
+ }
+ append(&argv, scratch)
+ }
+ r := sh.exec(argv[:], allocator = context.temp_allocator)
+ if !r.ok && len(r.stdout) == 0 {
+ return out, .Failed
+ }
+ matches: []Grep_Match
+ if json.unmarshal_string(r.stdout, &matches, allocator = context.temp_allocator) != nil {
+ return out, .Unreadable
+ }
+ out.files = make([]File, len(files), allocator)
+ for name, i in files {
+ out.files[i] = File {
+ name = strings.clone(name, allocator),
+ imports = {},
+ decls = {},
+ comments = {},
+ }
+ }
+ // One declaration, several matches: the plain forms see the inner
+ // node of an export, and a test function is also a function. Keep the
+ // exported reading, and the test reading, of each line.
+ best := make(map[string]Grep_Match, context.temp_allocator)
+ for m in matches {
+ base := strings.trim_suffix(filepath.base(m.file), filepath.ext(m.file))
+ index, numbered := strconv.parse_int(base)
+ if !numbered || index < 0 || index >= len(files) {
+ continue
+ }
+ key := fmt.tprintf("%d:%d", index, m.range.start.line)
+ if old, seen := best[key]; seen && !prefer(m.rule, old.rule) {
+ continue
+ }
+ best[strings.clone(key, context.temp_allocator)] = m
+ }
+ decls := make([][dynamic]Decl, len(files), context.temp_allocator)
+ for i in 0 ..< len(files) {
+ decls[i] = make([dynamic]Decl, allocator)
+ }
+ for key, m in best {
+ index, _ := strconv.parse_int(key[:strings.index_byte(key, ':')])
+ lines := strings.split_lines(string(sources[index]), context.temp_allocator)
+ if decl, ok := read_match(sidecar, m, lines, allocator); ok {
+ append(&decls[index], decl)
+ }
+ }
+ for i in 0 ..< len(files) {
+ slice.sort_by_cmp(decls[i][:], proc(a, b: Decl) -> slice.Ordering {
+ return .Less if a.line < b.line else (.Greater if a.line > b.line else .Equal)
+ })
+ out.files[i].decls = decls[i][:]
+ }
+ return out, .None
+}
+
+// prefer is whether a new match outranks the one already kept for a
+// line: an export over a plain form, a test over the function it also is.
+prefer :: proc(rule, old: string) -> bool {
+ if strings.contains(rule, "export") && !strings.contains(old, "export") {
+ return true
+ }
+ return strings.contains(rule, "test") && !strings.contains(old, "test")
+}
+
+// read_match reads one declaration out of a match: its name from the
+// bound metavariable or the match text, its kind from the rule, and its
+// audience from the language's notion of it.
+read_match :: proc(
+ sidecar: Sidecar,
+ m: Grep_Match,
+ lines: []string,
+ allocator := context.allocator,
+) -> (
+ decl: Decl,
+ ok: bool,
+) {
+ line := m.range.start.line + 1
+ decl.line = line
+ decl.end_line = line + strings.count(m.text, "\n")
+ decl.test = strings.contains(m.rule, "test")
+ switch sidecar {
+ case .Script:
+ name := ""
+ for key in ([]string{"N", "S"}) {
+ if bound, found := m.meta.single[key]; found {
+ name = bound.text
+ break
+ }
+ }
+ if decl.test {
+ name = strings.trim(name, "\"'`")
+ }
+ if name == "" {
+ return decl, false
+ }
+ decl.name = strings.clone(name, allocator)
+ decl.kind = rule_kind(m.rule)
+ decl.exported = strings.contains(m.rule, "export")
+ case .Python, .Rust:
+ rule, known := kind_rule(sidecar, m.rule)
+ if !known {
+ return decl, false
+ }
+ re, err := regex.create(rule.name, {}, context.temp_allocator, context.temp_allocator)
+ if err != nil {
+ return decl, false
+ }
+ cap, matched := regex.match(re, m.text, context.temp_allocator)
+ if !matched || len(cap.groups) < 2 {
+ return decl, false
+ }
+ decl.name = strings.clone(cap.groups[1], allocator)
+ decl.kind = rule.kind
+ decl.exported =
+ !strings.has_prefix(decl.name, "_") if sidecar == .Python else strings.has_prefix(strings.trim_space(m.text), "pub")
+ case .Go, .Odin:
+ return decl, false
+ }
+ if line - 1 < len(lines) {
+ decl.text = strings.clone(strings.trim_space(lines[line - 1]), allocator)
+ }
+ decl.doc = doc_above(lines, line, allocator)
+ decl.body = strings.clone(m.text, allocator)
+ return decl, true
+}
+
+kind_rule :: proc(sidecar: Sidecar, id: string) -> (Kind_Rule, bool) {
+ for r in (py_rules if sidecar == .Python else rs_rules) {
+ if r.id == id {
+ return r, true
+ }
+ }
+ return {}, false
+}
+
+// rule_kind turns a rule id back into the shape the findings report.
+rule_kind :: proc(rule: string) -> string {
+ switch {
+ case strings.contains(rule, "-func-"), strings.contains(rule, "-test-"):
+ return "func"
+ case strings.contains(rule, "-type-"):
+ return "type"
+ }
+ return "value"
+}
+
+// doc_above collects the comment block ending on the line before the
+// declaration, as the doc a reader would attach to it, in the C-family
+// shapes and Python's.
+doc_above :: proc(lines: []string, line: int, allocator := context.allocator) -> string {
+ parts := make([dynamic]string, context.temp_allocator)
+ for i := line - 2; i >= 0; i -= 1 {
+ trimmed := strings.trim_space(lines[i])
+ if !strings.has_prefix(trimmed, "//") &&
+ !strings.has_prefix(trimmed, "*") &&
+ !strings.has_prefix(trimmed, "/*") &&
+ (!strings.has_prefix(trimmed, "#") || strings.has_prefix(trimmed, "#!")) {
+ break
+ }
+ trimmed = strings.trim_prefix(trimmed, "#")
+ trimmed = strings.trim_prefix(trimmed, "///")
+ trimmed = strings.trim_prefix(trimmed, "//")
+ trimmed = strings.trim_prefix(trimmed, "/**")
+ trimmed = strings.trim_prefix(trimmed, "/*")
+ trimmed = strings.trim_suffix(trimmed, "*/")
+ part := strings.trim_space(strings.trim_prefix(trimmed, "*"))
+ if part != "" {
+ inject_at(&parts, 0, part)
+ }
+ }
+ return strings.join(parts[:], " ", allocator)
+}
diff --git a/tsfrontend.go b/tsfrontend.go
@@ -6,6 +6,7 @@ import (
"os"
"os/exec"
"path/filepath"
+ "slices"
"strconv"
"strings"
)
@@ -285,7 +286,7 @@ func (g TSFrontend) Change(root, rev string, c *Change, added map[string][]int)
best[key] = m
}
}
- for _, m := range best {
+ for _, m := range ordered(best) {
if strings.Contains(m.Rule, "-test-") {
name := unquote(m.name())
if name == "" || !touched[m.File][m.Range.Start.Line+1] {
@@ -352,7 +353,7 @@ func (g TSFrontend) Whole(root, rev string) ([]Declared, error) {
}
}
var index []Declared
- for _, m := range best {
+ for _, m := range ordered(best) {
name := m.name()
if name == "" || strings.Contains(m.Rule, "-test-") {
continue
@@ -374,6 +375,23 @@ func (g TSFrontend) Whole(root, rev string) ([]Declared, error) {
return index, nil
}
+// ordered is the best matches in file and line order, so that what is
+// read from them — and the finding anchored on the first of them — is the
+// same on every run.
+func ordered(best map[string]grepMatch) []grepMatch {
+ out := make([]grepMatch, 0, len(best))
+ for _, m := range best {
+ out = append(out, m)
+ }
+ slices.SortFunc(out, func(a, b grepMatch) int {
+ if c := strings.Compare(a.File, b.File); c != 0 {
+ return c
+ }
+ return a.Range.Start.Line - b.Range.Start.Line
+ })
+ return out
+}
+
// ruleKind turns a rule id back into the shape the findings report.
func ruleKind(rule string) string {
switch {
