From b0ad90ff243fa57223a5901f79fbcba342279766 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:17:30 -0700 Subject: [PATCH 1/4] Stage IR rows with perl when it is there, awk otherwise Staging copies every IR CSV into a Souffle fact file and drops rows whose tab-separated field count differs from the header's. BSD awk spends most of that time splitting fields it never reads: on a ~1,300-file Python project the step took 16-19 s of wall time per build, 13.6 CPU-s of it on the largest IR file (expressions). perl counts tabs instead and stages the same bytes (identical md5 over every staged file of three projects, two Python and one Java) at about a tenth of the CPU. The filter moves to graph/pipeline/stage-rows.sh. It picks perl when it runs (macOS, mainstream Linux, and Git for Windows' bash, which is the shell the pipeline runs under on win32) and falls back to the same awk program where perl is missing, so no machine loses the step. AXIOM_STAGE_WITH=awk forces the fallback. graph/test/tools/stage-rows-test.sh holds both to the same stdout and stderr on the edge cases (trailing tabs, empty lines, CRLF, no final newline, torn rows) and runs as a Java suite preflight. Refs #1835 --- graph/pipeline/run-souffle.sh | 6 ++- graph/pipeline/stage-rows.sh | 50 ++++++++++++++++++++++ graph/test/java/run-tests.sh | 7 ++++ graph/test/tools/stage-rows-test.sh | 65 +++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 graph/pipeline/stage-rows.sh create mode 100644 graph/test/tools/stage-rows-test.sh diff --git a/graph/pipeline/run-souffle.sh b/graph/pipeline/run-souffle.sh index f903e654b..b0ce60a1f 100755 --- a/graph/pipeline/run-souffle.sh +++ b/graph/pipeline/run-souffle.sh @@ -88,6 +88,8 @@ ENG="$SRC/$LANG_ARG/engine"; ENG2="$SRC/$LANG_ARG/engine-ii"; DL="$SRC/$LANG_ARG [ -d "$ENG" ] || { echo "no rule set for --language=$LANG_ARG (looked in $ENG)" >&2; exit 1; } # shellcheck source=souffle-include.sh . "$SRC/pipeline/souffle-include.sh" +# shellcheck source=stage-rows.sh +. "$SRC/pipeline/stage-rows.sh" # shellcheck source=engine.conf . "$SRC/pipeline/engine.conf" # Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies). @@ -817,7 +819,7 @@ while IFS=$'\t' read -r rel csv; do # deterministically on some inputs — takes down a run of tens of thousands of sites. The # header's field count is the contract; a row that does not meet it is dropped and # COUNTED, so the loss is visible rather than fatal and never silent. - if [ -f "$CLIENT/$csv.csv" ]; then awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$CLIENT/$csv.csv" > "$FACTS/$rel.facts" + if [ -f "$CLIENT/$csv.csv" ]; then stage_rows "$CLIENT/$csv.csv" > "$FACTS/$rel.facts" else : > "$FACTS/$rel.facts"; fi CLIENT_INPUTS="$CLIENT_INPUTS$rel"$'\n' done < <(read_map "$TPL/client-ir.map"; if [ -f "$TPL/client-extra.map" ]; then read_map "$TPL/client-extra.map"; fi) @@ -879,7 +881,7 @@ if [ ! -d "$LIBDIR" ]; then : > "$TMPDIR_L/$rel.facts" for root in "${LIB_ROOTS[@]}"; do while IFS= read -r mod; do - [ -f "$mod/$csv.csv" ] && awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$mod/$csv.csv" >> "$TMPDIR_L/$rel.facts" + [ -f "$mod/$csv.csv" ] && stage_rows "$mod/$csv.csv" >> "$TMPDIR_L/$rel.facts" done < <(lib_modules "$root") done done < <(printf '%s' "$LIB_SIG_RELS") diff --git a/graph/pipeline/stage-rows.sh b/graph/pipeline/stage-rows.sh new file mode 100644 index 000000000..200ff7670 --- /dev/null +++ b/graph/pipeline/stage-rows.sh @@ -0,0 +1,50 @@ +# stage-rows.sh: copy one IR CSV into a Soufflé fact file, dropping torn rows. Sourced by run-souffle.sh. +# +# stage_rows FILE print FILE's rows without its header, keeping only the rows whose tab-separated +# field count equals the header's; report how many were dropped on stderr. +# +# The filter is line based: a row is one newline-terminated line, a field count is the number of tabs +# plus one (an empty line has none), and a last line without a newline is printed with one. Those are +# exactly awk -F'\t' semantics, which is what this step used to run. +# +# PERL FIRST, AWK AS THE FALLBACK. The two print byte-identical output, but BSD awk (macOS) spends most +# of a build's staging time splitting fields it never reads: 13.6 CPU-s on a 352 MB expressions file, +# against 0.65 s for perl, which only counts tabs. perl ships with macOS, every mainstream Linux +# distribution and Git for Windows' bash (the shell this runs under on win32); a minimal image without +# it (Alpine, distroless) gets the same rows from awk, just slower. graph/test/tools/stage-rows-test.sh +# holds both to the same bytes on the edge cases (trailing tabs, empty lines, CRLF, no final newline). +# +# AXIOM_STAGE_WITH=awk forces the fallback (the test uses it; so can anyone bisecting a staging change). + +if [ "${AXIOM_STAGE_WITH:-}" != awk ] && command -v perl >/dev/null 2>&1 && perl -e 1 >/dev/null 2>&1; then + STAGE_ROWS_TOOL=perl +else + STAGE_ROWS_TOOL=awk +fi + +stage_rows(){ + if [ "$STAGE_ROWS_TOOL" = perl ]; then + # Bytes in, bytes out (no PerlIO layers, whatever PERL_UNICODE says), and $/ pinned to "\n". + perl -e ' + my $f = $ARGV[0]; + open(my $h, "<:raw", $f) or die "cannot read $f: $!\n"; + binmode STDOUT, ":raw"; + local $/ = "\n"; + my $hdr = <$h>; + exit 0 unless defined $hdr; + my $hl = length($hdr) - (substr($hdr, -1) eq "\n" ? 1 : 0); + my $n = $hl ? ($hdr =~ tr/\t//) + 1 : 0; + my $bad = 0; + while (my $l = <$h>) { + my $nl = substr($l, -1) eq "\n"; + my $nf = (length($l) - ($nl ? 1 : 0)) ? ($l =~ tr/\t//) + 1 : 0; + if ($nf == $n) { print($nl ? $l : "$l\n") } else { $bad++ } + } + close($h); + close(STDOUT) or die "cannot write the staged rows of $f: $!\n"; + printf STDERR " ! dropped %d malformed row(s) from %s\n", $bad, $f if $bad > 0; + ' "$1" + else + awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$1" + fi +} diff --git a/graph/test/java/run-tests.sh b/graph/test/java/run-tests.sh index 18db81e1d..e2a48d386 100755 --- a/graph/test/java/run-tests.sh +++ b/graph/test/java/run-tests.sh @@ -99,6 +99,13 @@ fi if ! bash "$ROOT/graph/test/tools/lib-cache-key-test.sh"; then echo "FAIL: the library-facts cache key is not a function of the staged modules' content (#588)"; exit 1 fi +# ── Staging must not depend on which row filter the machine has ───────────── +# graph/pipeline/stage-rows.sh runs perl where it exists and awk where it does not; both must stage +# the same bytes, or the facts every language solves from would differ by machine. +if ! bash "$ROOT/graph/test/tools/stage-rows-test.sh"; then + echo "aborting: perl and awk stage different rows from the same IR" + exit 1 +fi # ── The -I for soufflé's headers must be the one that actually compiles ────── # Also a preflight, and for the same reason: the old resolution returned a path that only built on # the machine it was written on, and every run here compiles the engine through it. diff --git a/graph/test/tools/stage-rows-test.sh b/graph/test/tools/stage-rows-test.sh new file mode 100644 index 000000000..67abcca69 --- /dev/null +++ b/graph/test/tools/stage-rows-test.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# Staging copies every IR CSV into a Soufflé fact file and drops the rows whose field count differs +# from the header's (graph/pipeline/stage-rows.sh). It runs perl where perl exists and awk where it +# does not, so the two must print the same bytes: a difference would make the facts, and so the +# graph, a function of which tools the machine happens to have. +# +# Each fixture below is staged by both, and the stdout bytes and the stderr report must match. The +# fixtures are the shapes where a line-based tab counter and awk's field splitting could part: a +# trailing tab (an empty last field), an empty line, CRLF, a last line with no newline, a header with +# no newline, an empty file, a quoted field holding a tab (both see two fields: neither parses quotes), +# bytes that are not UTF-8, and torn rows of both kinds (too few fields, too many). +# A NUL byte is left out on purpose: BSD awk ends the line at it (and drops the row as torn) while gawk +# and perl keep the whole line, so awk does not agree with itself there and is no reference. +# ───────────────────────────────────────────────────────────────────────────── +set -uo pipefail +unset AXIOM_STAGE_WITH +ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" + +command -v perl >/dev/null 2>&1 || { echo "stage-rows-test: FAILED (no perl on PATH; this test compares perl against awk)"; exit 1; } + +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +fail=0; checks=0 +bad(){ checks=$((checks+1)); printf ' FAIL %s\n' "$1"; fail=1; } +ok(){ checks=$((checks+1)); } + +# The tool the sourced file picks: perl here, awk when forced. +( . "$ROOT/graph/pipeline/stage-rows.sh"; [ "$STAGE_ROWS_TOOL" = perl ] ) && ok || bad "stage-rows.sh did not pick perl although perl is on PATH" +( AXIOM_STAGE_WITH=awk; . "$ROOT/graph/pipeline/stage-rows.sh"; [ "$STAGE_ROWS_TOOL" = awk ] ) && ok || bad "AXIOM_STAGE_WITH=awk did not force awk" + +printf 'a\tb\tc\n1\t2\t3\n4\t5\t6\n' > "$W/clean.csv" +printf 'a\tb\tc\n1\t2\t3\n1\t2\n1\t2\t3\t4\n7\t8\t9\n' > "$W/torn.csv" +printf 'a\tb\tc\n1\t2\t\n\t\t\n1\t2\t3\t\n' > "$W/trailing-tab.csv" +printf 'a\tb\tc\n1\t2\t3\n\n4\t5\t6\n\n' > "$W/empty-line.csv" +printf 'a\tb\tc\r\n1\t2\t3\r\n4\t5\r\n' > "$W/crlf.csv" +printf 'a\tb\tc\n1\t2\t3\n4\t5\t6' > "$W/no-final-newline.csv" +printf 'a\tb\tc\n1\t2' > "$W/torn-last-line.csv" +printf 'a\tb\tc' > "$W/header-only-no-newline.csv" +printf 'a\tb\tc\n' > "$W/header-only.csv" +: > "$W/empty.csv" +printf 'a\tb\n"x\ty"\tz\n"p\tq"\n' > "$W/quoted-tab.csv" +printf 'a\tb\n\xff\xfe\t\xc3\x28\n' > "$W/bytes.csv" +printf 'a\n1\n\n2\n' > "$W/one-column.csv" +printf '\n\n1\n\n' > "$W/empty-header.csv" + +n=0 +for f in "$W"/*.csv; do + n=$((n+1)); name="$(basename "$f")" + ( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$f" ) > "$W/p.out" 2> "$W/p.err"; prc=$? + ( AXIOM_STAGE_WITH=awk; . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$f" ) > "$W/a.out" 2> "$W/a.err"; arc=$? + [ "$prc" = 0 ] && [ "$arc" = 0 ] && ok || bad "$name: exit status perl=$prc awk=$arc" + cmp -s "$W/p.out" "$W/a.out" && ok || { bad "$name: staged rows differ"; od -c "$W/a.out" | sed 's/^/ awk /'; od -c "$W/p.out" | sed 's/^/ perl /'; } + cmp -s "$W/p.err" "$W/a.err" && ok || bad "$name: report differs: perl '$(cat "$W/p.err")' awk '$(cat "$W/a.err")'" +done + +# Not vacuous: the fixtures must hold rows that are kept and rows that are dropped. +( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$W/torn.csv" ) > "$W/t.out" 2> "$W/t.err" +[ "$(printf '1\t2\t3\n7\t8\t9\n')" = "$(cat "$W/t.out")" ] && ok || bad "torn.csv: expected the two whole rows, got '$(cat "$W/t.out")'" +grep -q "dropped 2 malformed row(s) from $W/torn.csv" "$W/t.err" && ok || bad "torn.csv: expected a report of 2 dropped rows, got '$(cat "$W/t.err")'" +# The control: a clean file comes through as its rows, header removed, nothing reported. +( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$W/clean.csv" ) > "$W/c.out" 2> "$W/c.err" +tail -n +2 "$W/clean.csv" | cmp -s - "$W/c.out" && [ ! -s "$W/c.err" ] && ok || bad "clean.csv: not passed through unchanged" + +[ "$n" -ge 14 ] || bad "only $n fixtures were staged" +[ $fail = 0 ] && echo "stage-rows-test: ok ($checks checks over $n fixtures)" || { echo "stage-rows-test: FAILED"; exit 1; } From 957a55e1eec54b63006a3ec65db5a8f044242063 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:38:05 -0700 Subject: [PATCH 2/4] axiomcode-index: read the IR with a line splitter, skip a VACUUM with nothing to reclaim, do not rebuild bundle indexes On a ~1,300-file Python project axiomcode-index took 12-14 s. Three parts of that bought nothing: - csv.DictReader over the IR. The reader is QUOTE_NONE, so a row is one line and a field is the text between tabs; ir_rows() does exactly that with str.split and dict(zip()) at about half the cost. It keeps the reader's semantics where they are not obvious: \r, \n and \r\n each end a row, blank lines are skipped, a short row gets None for the missing columns and a long one keeps the rest under the key None. tests/index_reader.py compares the two on fixtures (quoted tabs and newlines, CRLF, lone CR, repeated header names, invalid UTF-8) and on any IR directory given to it. - VACUUM of a database the bundle wrote seconds earlier. It now runs only when this run dropped a table it recreates (a re-index) or the file has free pages; a freshly bundled graph has neither. - ce_caller, ce_callee and fa_caller duplicate the bundle's own idx_call_edges_caller_id, idx_call_edges_callee_method_id and idx_field_access_caller_id. They are created only when the graph has no index on that column, so an older graph keeps getting them. graph.sqlite holds the same rows in every table (rowids included) on two Python projects and one Java project; only index_meta.built_at and the three duplicate indexes differ. Refs #1835 --- .../skills/axiomcode/scripts/axiomcode-index | 58 ++++++++-- tests/index_reader.py | 106 ++++++++++++++++++ 2 files changed, 154 insertions(+), 10 deletions(-) create mode 100644 tests/index_reader.py diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index index 2751c4430..b5ddd518d 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-index @@ -20,8 +20,7 @@ to --src). The language's IR is read through the ADAPTER table below — header positions — mirroring graph/bundle/languages.ts. When the IR is absent the index degrades: symbols come from methods/types alone, refs/literals/comments are empty, and index_meta says so. """ -import csv, os, re, sqlite3, sys, time, glob, collections, functools -csv.field_size_limit(10**9) +import os, re, sqlite3, sys, time, glob, collections, functools INDEX_VERSION = '9' # bump when the tables' CONTENT changes shape (v2: JavaScript arrows named after their variable; v3: paths table, sites view, no variable row for a bound function; # v5: JavaScript fields owned by their class, computed-key members named by their key, anonymous class expressions named by their binding; # v6: TypeScript class-property arrows named after their field; @@ -273,13 +272,40 @@ def is_test_file(fp): def rows(name): p = os.path.join(IR, name) if not HAVE_IR or not os.path.exists(p): return iter(()) - f = open(p, newline='', encoding='utf-8', errors='replace') - return csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE) + return ir_rows(p) + +def ir_rows(p): + """The rows of one IR file as dicts keyed by its header, EXACTLY as csv.DictReader(f, delimiter='\\t', + quoting=csv.QUOTE_NONE) over open(p, newline='') reads them, at about half the cost (the expressions file is + most of this script's time on a large repository). That reader treats a quote as an ordinary character, so a row + is one line and a field is the text between tabs: a line ends at \\n, \\r or \\r\\n (newline='' splits on all + three and keeps them, and each ends a record), a blank line is skipped, a short row has None for its missing + columns and a long one keeps the rest as a list under the key None. The writer's RFC 4180 quoting is NOT undone + here, as it was not before (see unquoted() below). tests/index_reader.py holds the two to the same dicts.""" + with open(p, newline='', encoding='utf-8', errors='replace') as f: + head = f.readline() + if not head: return + if head[-1:] == '\n': head = head[:-1] + if head[-1:] == '\r': head = head[:-1] + keys = head.split('\t') if head else [] + n = len(keys) + for line in f: + if line[-1:] == '\n': line = line[:-1] + if line[-1:] == '\r': line = line[:-1] + if not line: continue + v = line.split('\t') + d = dict(zip(keys, v)) + if len(v) != n: + if len(v) > n: d[None] = v[n:] + else: + for k in keys[len(v):]: d[k] = None + yield d # an older augment step left `symbols` as a VIEW; a name is dropped as whichever it is +dropped = False # whether this run frees pages: it decides the VACUUM at the end for name in ('symbols', 'refs', 'literals', 'comments', 'nesting', 'type_refs', 'decorations', 'skipped', 'index_meta', 'paths', 'callers', 'callees', 'source', 'sites'): kind = c.execute("SELECT type FROM sqlite_master WHERE name=?", (name,)).fetchone() - if kind: c.execute(f"DROP {kind[0].upper()} IF EXISTS {name}") + if kind: c.execute(f"DROP {kind[0].upper()} IF EXISTS {name}"); dropped = True c.executescript(''' DROP TABLE IF EXISTS symbols; DROP TABLE IF EXISTS refs; DROP TABLE IF EXISTS literals; DROP TABLE IF EXISTS comments; DROP TABLE IF EXISTS nesting; DROP TABLE IF EXISTS type_refs; DROP TABLE IF EXISTS decorations; DROP TABLE IF EXISTS skipped; DROP TABLE IF EXISTS index_meta; DROP TABLE IF EXISTS paths; @@ -765,11 +791,21 @@ if dec: c.executemany("INSERT INTO skipped VALUES (?,?)", ((rel(r.get('filePath') or r.get('file') or ''), r.get('reason') or r.get('skipReason') or '') for r in rows(A['skipped']))) # ── views: the joins written once ──────────────────────────────────────────────────────────────── +# Who calls a method and what it calls are looked up by call_edges(callee_method_id) and call_edges(caller_id); the +# fields a caller reads by field_access(caller_id): via_base_rows asks per single-target call site, and without that +# index each ask scanned the whole table (65,797 rows × 5,162 sites on an 8,619-file repository = 24 s of a 171 s warm +# → 1.4 s). The bundle indexes all three columns itself (idx__, schema.ts `indexed`), so they are made +# here only for a graph that lacks one: building the same index twice cost a second pass over the largest tables. +def has_index_on(table, cols): + for row in c.execute(f"PRAGMA index_list({table})").fetchall(): + if row[4] == 0 and [r[2] for r in c.execute(f"PRAGMA index_info({row[1]})")] == cols: return True # not partial + return False +for _name, _table, _col in (('ce_callee', 'call_edges', 'callee_method_id'), ('ce_caller', 'call_edges', 'caller_id'), ('fa_caller', 'field_access', 'caller_id')): + if not has_index_on(_table, [_col]): c.execute(f"CREATE INDEX IF NOT EXISTS {_name} ON {_table}({_col})") c.executescript(''' CREATE INDEX IF NOT EXISTS symbols_name ON symbols(name); CREATE INDEX IF NOT EXISTS symbols_display ON symbols(display); CREATE INDEX IF NOT EXISTS symbols_file ON symbols(file, line); CREATE INDEX IF NOT EXISTS symbols_mid ON symbols(method_id); CREATE INDEX IF NOT EXISTS symbols_tid ON symbols(type_id); CREATE INDEX IF NOT EXISTS refs_name ON refs(name); CREATE INDEX IF NOT EXISTS type_refs_name ON type_refs(name); CREATE INDEX IF NOT EXISTS decorations_owner ON decorations(owner_id); CREATE INDEX IF NOT EXISTS literals_value ON literals(value); CREATE INDEX IF NOT EXISTS comments_file ON comments(file); -CREATE INDEX IF NOT EXISTS ce_callee ON call_edges(callee_method_id); CREATE INDEX IF NOT EXISTS ce_caller ON call_edges(caller_id); -- lookups by id and by file + line: without them each per-site or per-test query scans the whole table CREATE INDEX IF NOT EXISTS symbols_id ON symbols(id); CREATE INDEX IF NOT EXISTS refs_file_line ON refs(file, line); CREATE INDEX IF NOT EXISTS literals_file_line ON literals(file, line); @@ -777,9 +813,6 @@ CREATE INDEX IF NOT EXISTS literals_file_line ON literals(file, line); -- the parts of a partial type (type_parts): without these each ask scanned type_refs / symbols, 1.2 s of a 1.8 s impact -- on a 1,300-file Java repository → 0.5-0.75 s, answers unchanged CREATE INDEX IF NOT EXISTS type_refs_file_line ON type_refs(file, line); CREATE INDEX IF NOT EXISTS symbols_qn ON symbols(qualified_name); --- the fields a caller reads: via_base_rows asks per single-target call site, and without the index each ask --- scanned the whole table (65,797 rows × 5,162 sites on an 8,619-file repository = 24 s of a 171 s warm → 1.4 s) -CREATE INDEX IF NOT EXISTS fa_caller ON field_access(caller_id); -- who calls a method: one row per (site, target), with the engine's tier. caller_id is symbols.id: a -- Java call written in a field initializer has the TYPE as its caller, and that row is kept, not dropped CREATE VIEW callers AS @@ -804,7 +837,12 @@ meta = dict(language=LANG, repo=REPO, source_dir=SRC, ir=IR if HAVE_IR else '', symbols=n('symbols'), refs=n('refs'), type_refs=n('type_refs'), decorations=n('decorations'), literals=n('literals'), comments=n('comments'), nested_types=n('nesting'), skipped_files=n('skipped')) c.executemany("INSERT INTO index_meta VALUES (?,?)", [(k, str(v)) for k, v in meta.items()]) con.commit() -con.execute("VACUUM") # the tables above are dropped and recreated on every run; without this the file only grows +# The tables above are dropped and recreated on every run, so a re-index leaves their old pages free and the file only +# grows without a VACUUM. A graph the bundle has just written has nothing to reclaim: no table of this script existed +# in it, and nothing freed a page (freelist_count). Rewriting it anyway cost 4.2 s of a 12 s index on a ~1,300-file +# Python repository, for the same rows. +if dropped or c.execute("PRAGMA freelist_count").fetchone()[0] > 0: + con.execute("VACUUM") con.close() # THE ASSERTED LINKS (ax_links.py, axiomcode-links.tsv) are applied to every graph as it is indexed, after the engine's # export and before the graph is published, so a rebuild keeps them and no query ever sees a graph without them: each is diff --git a/tests/index_reader.py b/tests/index_reader.py new file mode 100644 index 000000000..bc43878aa --- /dev/null +++ b/tests/index_reader.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""tests/index_reader.py: axiomcode-index reads the IR exactly as csv.DictReader did, row for row and key for key. + +axiomcode-index reads every IR row through ir_rows(), a line splitter that replaced +csv.DictReader(f, delimiter='\\t', quoting=csv.QUOTE_NONE) because the DictReader was most of the index's time on a +large repository. The index's tables are made of these dicts, so any difference is a different graph. Each fixture is +read by both and the lists of dicts must be equal: the shapes where a line splitter and the csv module could part +are a quoted field holding a tab or a newline (QUOTE_NONE does not unquote, so both split it), CRLF, a lone CR, a +blank line, a short row (None for the missing columns), a long row (the rest under the key None), a repeated header +name, a blank header, an empty file, a header with no rows, no final newline, bytes that are not UTF-8, and the +Unicode line separators that str.splitlines() would split on and the csv module does not. + + python3 tests/index_reader.py [IR_DIR ...] each IR_DIR: every *.csv in it is compared as well +""" +import ast, csv, glob, itertools, os, sys, tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SCRIPT = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode-index') +csv.field_size_limit(10**9) +MISSING = object() + + +def load_ir_rows(): + """ir_rows() out of the script without running it (the script indexes a graph at import).""" + tree = ast.parse(open(SCRIPT, encoding='utf-8').read()) + fn = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'ir_rows'] + if not fn: + sys.exit('FAIL: axiomcode-index defines no ir_rows()') + ns = {} + exec(compile(ast.Module(body=fn, type_ignores=[]), SCRIPT, 'exec'), ns) + return ns['ir_rows'] + + +def dictreader(p): + with open(p, newline='', encoding='utf-8', errors='replace') as f: + return list(csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)) + + +FIXTURES = { + 'plain': b'a\tb\tc\n1\t2\t3\n4\t5\t6\n', + 'quoted-tab': b'a\tb\tc\n"x\ty"\t2\t3\n1\t"p\tq"\t3\n', + 'quoted-newline': b'a\tb\tc\n"line one\nline two"\t2\t3\n4\t5\t6\n', + 'quoted-crlf-inside': b'a\tb\n"x\r\ny"\tz\n', + 'doubled-quotes': b'a\tb\n"@r.post(""/orders"")"\t1\n"""Fee"""\t2\n', + 'crlf': b'a\tb\tc\r\n1\t2\t3\r\n4\t5\t6\r\n', + 'lone-cr': b'a\tb\r1\t2\r3\t4\r', + 'mixed-endings': b'a\tb\n1\t2\r\n3\t4\r5\t6\n', + 'blank-lines': b'a\tb\n\n1\t2\n\r\n\n3\t4\n\n', + 'short-row': b'a\tb\tc\n1\t2\n1\n', + 'long-row': b'a\tb\n1\t2\t3\t4\n', + 'trailing-tab': b'a\tb\n1\t\n\t\n', + 'repeated-header': b'id\tname\tid\n1\tx\t2\n3\n', + 'blank-header': b'\n1\t2\n3\n', + 'empty': b'', + 'header-only': b'a\tb\n', + 'header-only-no-newline': b'a\tb', + 'no-final-newline': b'a\tb\n1\t2\n3\t4', + 'not-utf8': b'a\tb\n\xff\xfe\t\xc3\x28\n', + 'unicode-separators': 'a\tb\nx
y\tz\u0085w\n1\u000b2\t3\u000c4\n'.encode('utf-8'), + 'spaces': b'a\tb\n 1 \t 2 \n', +} + + +def main(argv): + ir_rows = load_ir_rows() + fails = checks = 0 + rows_compared = 0 + with tempfile.TemporaryDirectory() as w: + for name, data in FIXTURES.items(): + p = os.path.join(w, name + '.csv') + open(p, 'wb').write(data) + want, got = dictreader(p), list(ir_rows(p)) + checks += 1; rows_compared += len(want) + if want != got: + fails += 1 + print(f'FAIL {name}: csv.DictReader {want!r}\n ir_rows {got!r}') + # not vacuous: the fixtures hold rows, and the shapes the reader special-cases are among them + p = os.path.join(w, 'short-row.csv') + checks += 1 + if list(ir_rows(p)) != [{'a': '1', 'b': '2', 'c': None}, {'a': '1', 'b': None, 'c': None}]: + fails += 1; print(f'FAIL short-row: {list(ir_rows(p))!r}') + p = os.path.join(w, 'long-row.csv') + checks += 1 + if list(ir_rows(p)) != [{'a': '1', 'b': '2', None: ['3', '4']}]: + fails += 1; print(f'FAIL long-row: {list(ir_rows(p))!r}') + # a real IR: every file of every directory named on the command line + files = 0 + for d in argv: + for p in sorted(glob.glob(os.path.join(d, '*.csv'))): + files += 1; checks += 1 + n = 0 + with open(p, newline='', encoding='utf-8', errors='replace') as f: # streamed: an IR file can be 10^6 rows + for a, b in itertools.zip_longest(csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE), ir_rows(p), fillvalue=MISSING): + n += 1 + if a != b: + fails += 1; print(f'FAIL {p} row {n}: csv.DictReader {a!r}\n ir_rows {b!r}'); break + rows_compared += n + if rows_compared < 20: + fails += 1; print(f'FAIL only {rows_compared} rows compared') + print(f"index_reader: {'ok' if not fails else 'FAILED'} ({checks - fails}/{checks} checks, {rows_compared} rows," + f" {len(FIXTURES)} fixtures{f', {files} IR files' if files else ''})") + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) From 4bf09a96294518c6d238ab2f5ab5cc0f4635c796 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:38:05 -0700 Subject: [PATCH 3/4] AXIOMCODE_REINDEX=1 rebuilds a graph whose files are unchanged The build's messages, SKILL.md and ax_fresh all describe AXIOMCODE_REINDEX=1 as a rebuild ("rebuilds it with this one", "to force"). It skipped the newer-build shortcut and the remembered languages, but both up-to-date checks still answered "graph up to date" whenever the files matched the file table, so it rebuilt nothing. Both checks now step aside when it is set. tests/refresh.py checks that `index` with AXIOMCODE_REINDEX=1 rebuilds over an unchanged tree; the existing check beside it, the same `index` without it, is the control. Refs #1835 --- .../axiomcode/skills/axiomcode/scripts/axiomcode-build | 6 ++++-- tests/refresh.py | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build index 7a52d595c..28ec84d31 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build @@ -316,14 +316,16 @@ if [ -f "$OUT/corrupt" ]; then echo "rebuilding: $(head -1 "$OUT/corrupt")"; fi if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && NEWER="$(AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" newer "$REPO")"; then echo "$NEWER"; exit 0 fi -if [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then +# AXIOMCODE_REINDEX=1 IS A REBUILD, as every message that names it promises ("rebuilds it with this one", "to force"): +# it skips both up-to-date answers below, which otherwise said "graph up to date" whenever the files matched the table. +if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then if [ "$(cat "$OUT/stamp")" != "$STAMP" ]; then echo "$STAMP" > "$OUT/stamp"; INDEXED_TREE="$OLD_INDEXED"; set_base "$OLD_INDEXED"; PREV=""; keep_base_graph; commit_base; fi set_base_files has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO" [ -f "$REPO/.axiomcode/engine" ] || printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine" || true echo "graph up to date (${LANGS//,/, }) at $OUT/graph.sqlite"; exit 0 fi -if [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then +if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO" python3 "$H/axiomcode-impact" --warm "$REPO" || true # export the graph's facts now (~9 s on 1,373 files): the first # query pays it otherwise, and for the plugin that first query is inside hooks/changes.py's timeout=14, several at once. diff --git a/tests/refresh.py b/tests/refresh.py index 2a02b7c6a..405029879 100644 --- a/tests/refresh.py +++ b/tests/refresh.py @@ -3,7 +3,7 @@ Each language gets a throwaway git repository with a real graph. Then, per language: - up to date `index` with nothing changed does not rebuild + up to date `index` with nothing changed does not rebuild; with AXIOMCODE_REINDEX=1 it does query an edit adds a function; a query verb, which starts the refresher and waits for it, finds it baseline `changed` answers the same before and after the refresh absorbed the edit: the refresh moves the graph, not the baseline edits are measured against. The answer must name the edit (a pass on two @@ -110,6 +110,13 @@ def check(ok, why, detail=''): check(m.get('refresh_reason') == 'axiomcode index' and m.get('refreshed_at', '').endswith('Z'), f'{lang}: the graph records when and why it was built', json.dumps(m)) t0 = time.time(); again = sh(repo, AX, 'index', '.', '--lang', lang, env=env); took = time.time() - t0 check('graph up to date' in again.stdout, f'{lang}: `index` with nothing changed does not rebuild ({took:.1f}s)', again.stdout + again.stderr) + # AXIOMCODE_REINDEX=1 rebuilds with nothing changed, as SKILL.md and the build's own messages promise; it used to + # answer "graph up to date" whenever the files matched the table. The control is the check above: the same + # `index` without it does not rebuild + built = meta().get('refreshed_at') + rx = sh(repo, AX, 'index', '.', '--lang', lang, env=dict(env, AXIOMCODE_REINDEX='1')) + check(rx.returncode == 0 and 'graph up to date' not in rx.stdout and 'building' in rx.stdout and meta().get('refreshed_at') != built, + f'{lang}: AXIOMCODE_REINDEX=1 `index` rebuilds although nothing changed', rx.stdout + rx.stderr) # ── built by an older axiomcode ──────────────────────────────────────────────────────────────────── # no file changed, but the table says another engine built the graph (what an update leaves behind): not up From cd0bf00c5277727cfb78d82ec4c68a9ebfbc678b Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:15:30 -0700 Subject: [PATCH 4/4] tests/README: list index_reader.py Refs #1835 --- tests/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/README.md b/tests/README.md index 06150d2fd..bd5377d2d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -26,6 +26,9 @@ One check needs no graph and is its own script: signature edit is reported before it lands with nothing upstream did; `--range` from a branch left behind its remote reads from the remote's fork (indexes a small project) + python3 tests/index_reader.py axiomcode-index's IR reader returns the same rows csv.DictReader did, on quoted tabs + and newlines, CRLF, a lone CR, a short or long row and invalid UTF-8; pass IR + directories to compare them too (no graph needed) python3 tests/refresh.py the graph refreshes itself after an edit in every language: a query sees the edit, `changed` answers the same before and after, a burst costs one rebuild and queries during it answer (#1305; builds real graphs, needs the engine)