swapnilpaliwal-sd · Oct 15, 2026 · Oct 15, 2026 · Oct 15, 2026 · Oct 15, 2026
@@ -88,6 +88,8 @@ ENG="$SRC/$LANG_ARG/engine"; ENG2="$SRC/$LANG_ARG/engine-ii"; DL="$SRC/$LANG_ARG
 [ -d "$ENG" ] || { echo "no rule set for --language=$LANG_ARG (looked in $ENG)" >&2; exit 1; }
 # shellcheck source=souffle-include.sh
 . "$SRC/pipeline/souffle-include.sh"
+# shellcheck source=stage-rows.sh
+. "$SRC/pipeline/stage-rows.sh"
 # shellcheck source=engine.conf
 . "$SRC/pipeline/engine.conf"
 # Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies).
@@ -817,7 +819,7 @@ while IFS=$'\t' read -r rel csv; do
   # deterministically on some inputs — takes down a run of tens of thousands of sites. The
   # header's field count is the contract; a row that does not meet it is dropped and
   # COUNTED, so the loss is visible rather than fatal and never silent.
-  if [ -f "$CLIENT/$csv.csv" ]; then awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf "  ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}'  "$CLIENT/$csv.csv" > "$FACTS/$rel.facts"
+  if [ -f "$CLIENT/$csv.csv" ]; then stage_rows "$CLIENT/$csv.csv" > "$FACTS/$rel.facts"
   else : > "$FACTS/$rel.facts"; fi
   CLIENT_INPUTS="$CLIENT_INPUTS$rel"$'\n'
 done < <(read_map "$TPL/client-ir.map"; if [ -f "$TPL/client-extra.map" ]; then read_map "$TPL/client-extra.map"; fi)
@@ -879,7 +881,7 @@ if [ ! -d "$LIBDIR" ]; then
     : > "$TMPDIR_L/$rel.facts"
     for root in "${LIB_ROOTS[@]}"; do
       while IFS= read -r mod; do
-        [ -f "$mod/$csv.csv" ] && awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf "  ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}'  "$mod/$csv.csv" >> "$TMPDIR_L/$rel.facts"
+        [ -f "$mod/$csv.csv" ] && stage_rows "$mod/$csv.csv" >> "$TMPDIR_L/$rel.facts"
       done < <(lib_modules "$root")
     done
   done < <(printf '%s' "$LIB_SIG_RELS")

@@ -0,0 +1,50 @@
+# stage-rows.sh: copy one IR CSV into a Soufflé fact file, dropping torn rows. Sourced by run-souffle.sh.
+#
+#   stage_rows FILE    print FILE's rows without its header, keeping only the rows whose tab-separated
+#                      field count equals the header's; report how many were dropped on stderr.
+#
+# The filter is line based: a row is one newline-terminated line, a field count is the number of tabs
+# plus one (an empty line has none), and a last line without a newline is printed with one. Those are
+# exactly awk -F'\t' semantics, which is what this step used to run.
+#
+# PERL FIRST, AWK AS THE FALLBACK. The two print byte-identical output, but BSD awk (macOS) spends most
+# of a build's staging time splitting fields it never reads: 13.6 CPU-s on a 352 MB expressions file,
+# against 0.65 s for perl, which only counts tabs. perl ships with macOS, every mainstream Linux
+# distribution and Git for Windows' bash (the shell this runs under on win32); a minimal image without
+# it (Alpine, distroless) gets the same rows from awk, just slower. graph/test/tools/stage-rows-test.sh
+# holds both to the same bytes on the edge cases (trailing tabs, empty lines, CRLF, no final newline).
+#
+# AXIOM_STAGE_WITH=awk forces the fallback (the test uses it; so can anyone bisecting a staging change).
+
+if [ "${AXIOM_STAGE_WITH:-}" != awk ] && command -v perl >/dev/null 2>&1 && perl -e 1 >/dev/null 2>&1; then
+  STAGE_ROWS_TOOL=perl
+else
+  STAGE_ROWS_TOOL=awk
+fi
+
+stage_rows(){
+  if [ "$STAGE_ROWS_TOOL" = perl ]; then
+    # Bytes in, bytes out (no PerlIO layers, whatever PERL_UNICODE says), and $/ pinned to "\n".
+    perl -e '
+      my $f = $ARGV[0];
+      open(my $h, "<:raw", $f) or die "cannot read $f: $!\n";
+      binmode STDOUT, ":raw";
+      local $/ = "\n";
+      my $hdr = <$h>;
+      exit 0 unless defined $hdr;
+      my $hl = length($hdr) - (substr($hdr, -1) eq "\n" ? 1 : 0);
+      my $n = $hl ? ($hdr =~ tr/\t//) + 1 : 0;
+      my $bad = 0;
+      while (my $l = <$h>) {
+        my $nl = substr($l, -1) eq "\n";
+        my $nf = (length($l) - ($nl ? 1 : 0)) ? ($l =~ tr/\t//) + 1 : 0;
+        if ($nf == $n) { print($nl ? $l : "$l\n") } else { $bad++ }
+      }
+      close($h);
+      close(STDOUT) or die "cannot write the staged rows of $f: $!\n";
+      printf STDERR "  ! dropped %d malformed row(s) from %s\n", $bad, $f if $bad > 0;
+    ' "$1"
+  else
+    awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf "  ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$1"
+  fi
+}
@@ -99,6 +99,13 @@ fi
 if ! bash "$ROOT/graph/test/tools/lib-cache-key-test.sh"; then
   echo "FAIL: the library-facts cache key is not a function of the staged modules' content (#588)"; exit 1
 fi
+# ── Staging must not depend on which row filter the machine has ─────────────
+# graph/pipeline/stage-rows.sh runs perl where it exists and awk where it does not; both must stage
+# the same bytes, or the facts every language solves from would differ by machine.
+if ! bash "$ROOT/graph/test/tools/stage-rows-test.sh"; then
+  echo "aborting: perl and awk stage different rows from the same IR"
+  exit 1
+fi
 # ── The -I for soufflé's headers must be the one that actually compiles ──────
 # Also a preflight, and for the same reason: the old resolution returned a path that only built on
 # the machine it was written on, and every run here compiles the engine through it.

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────────────────────
+# Staging copies every IR CSV into a Soufflé fact file and drops the rows whose field count differs
+# from the header's (graph/pipeline/stage-rows.sh). It runs perl where perl exists and awk where it
+# does not, so the two must print the same bytes: a difference would make the facts, and so the
+# graph, a function of which tools the machine happens to have.
+#
+# Each fixture below is staged by both, and the stdout bytes and the stderr report must match. The
+# fixtures are the shapes where a line-based tab counter and awk's field splitting could part: a
+# trailing tab (an empty last field), an empty line, CRLF, a last line with no newline, a header with
+# no newline, an empty file, a quoted field holding a tab (both see two fields: neither parses quotes),
+# bytes that are not UTF-8, and torn rows of both kinds (too few fields, too many).
+# A NUL byte is left out on purpose: BSD awk ends the line at it (and drops the row as torn) while gawk
+# and perl keep the whole line, so awk does not agree with itself there and is no reference.
+# ─────────────────────────────────────────────────────────────────────────────
+set -uo pipefail
+unset AXIOM_STAGE_WITH
+ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")"
+
+command -v perl >/dev/null 2>&1 || { echo "stage-rows-test: FAILED (no perl on PATH; this test compares perl against awk)"; exit 1; }
+
+W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT
+fail=0; checks=0
+bad(){ checks=$((checks+1)); printf '  FAIL  %s\n' "$1"; fail=1; }
+ok(){  checks=$((checks+1)); }
+
+# The tool the sourced file picks: perl here, awk when forced.
+( . "$ROOT/graph/pipeline/stage-rows.sh"; [ "$STAGE_ROWS_TOOL" = perl ] ) && ok || bad "stage-rows.sh did not pick perl although perl is on PATH"
+( AXIOM_STAGE_WITH=awk; . "$ROOT/graph/pipeline/stage-rows.sh"; [ "$STAGE_ROWS_TOOL" = awk ] ) && ok || bad "AXIOM_STAGE_WITH=awk did not force awk"
+
+printf 'a\tb\tc\n1\t2\t3\n4\t5\t6\n'                         > "$W/clean.csv"
+printf 'a\tb\tc\n1\t2\t3\n1\t2\n1\t2\t3\t4\n7\t8\t9\n'       > "$W/torn.csv"
+printf 'a\tb\tc\n1\t2\t\n\t\t\n1\t2\t3\t\n'                  > "$W/trailing-tab.csv"
+printf 'a\tb\tc\n1\t2\t3\n\n4\t5\t6\n\n'                      > "$W/empty-line.csv"
+printf 'a\tb\tc\r\n1\t2\t3\r\n4\t5\r\n'                       > "$W/crlf.csv"
+printf 'a\tb\tc\n1\t2\t3\n4\t5\t6'                            > "$W/no-final-newline.csv"
+printf 'a\tb\tc\n1\t2'                                        > "$W/torn-last-line.csv"
+printf 'a\tb\tc'                                              > "$W/header-only-no-newline.csv"
+printf 'a\tb\tc\n'                                            > "$W/header-only.csv"
+: > "$W/empty.csv"
+printf 'a\tb\n"x\ty"\tz\n"p\tq"\n'                            > "$W/quoted-tab.csv"
+printf 'a\tb\n\xff\xfe\t\xc3\x28\n'                          > "$W/bytes.csv"
+printf 'a\n1\n\n2\n'                                          > "$W/one-column.csv"
+printf '\n\n1\n\n'                                            > "$W/empty-header.csv"
+
+n=0
+for f in "$W"/*.csv; do
+  n=$((n+1)); name="$(basename "$f")"
+  ( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$f" ) > "$W/p.out" 2> "$W/p.err"; prc=$?
+  ( AXIOM_STAGE_WITH=awk; . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$f" ) > "$W/a.out" 2> "$W/a.err"; arc=$?
+  [ "$prc" = 0 ] && [ "$arc" = 0 ] && ok || bad "$name: exit status perl=$prc awk=$arc"
+  cmp -s "$W/p.out" "$W/a.out" && ok || { bad "$name: staged rows differ"; od -c "$W/a.out" | sed 's/^/        awk  /'; od -c "$W/p.out" | sed 's/^/        perl /'; }
+  cmp -s "$W/p.err" "$W/a.err" && ok || bad "$name: report differs: perl '$(cat "$W/p.err")' awk '$(cat "$W/a.err")'"
+done
+
+# Not vacuous: the fixtures must hold rows that are kept and rows that are dropped.
+( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$W/torn.csv" ) > "$W/t.out" 2> "$W/t.err"
+[ "$(printf '1\t2\t3\n7\t8\t9\n')" = "$(cat "$W/t.out")" ] && ok || bad "torn.csv: expected the two whole rows, got '$(cat "$W/t.out")'"
+grep -q "dropped 2 malformed row(s) from $W/torn.csv" "$W/t.err" && ok || bad "torn.csv: expected a report of 2 dropped rows, got '$(cat "$W/t.err")'"
+# The control: a clean file comes through as its rows, header removed, nothing reported.
+( . "$ROOT/graph/pipeline/stage-rows.sh"; stage_rows "$W/clean.csv" ) > "$W/c.out" 2> "$W/c.err"
+tail -n +2 "$W/clean.csv" | cmp -s - "$W/c.out" && [ ! -s "$W/c.err" ] && ok || bad "clean.csv: not passed through unchanged"
+
+[ "$n" -ge 14 ] || bad "only $n fixtures were staged"
+[ $fail = 0 ] && echo "stage-rows-test: ok ($checks checks over $n fixtures)" || { echo "stage-rows-test: FAILED"; exit 1; }
@@ -316,14 +316,16 @@ if [ -f "$OUT/corrupt" ]; then echo "rebuilding: $(head -1 "$OUT/corrupt")"; fi
 if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && NEWER="$(AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" newer "$REPO")"; then
   echo "$NEWER"; exit 0
 fi
-if [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then
+# AXIOMCODE_REINDEX=1 IS A REBUILD, as every message that names it promises ("rebuilds it with this one", "to force"):
+# it skips both up-to-date answers below, which otherwise said "graph up to date" whenever the files matched the table.
+if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then
   if [ "$(cat "$OUT/stamp")" != "$STAMP" ]; then echo "$STAMP" > "$OUT/stamp"; INDEXED_TREE="$OLD_INDEXED"; set_base "$OLD_INDEXED"; PREV=""; keep_base_graph; commit_base; fi
   set_base_files
   has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO"
   [ -f "$REPO/.axiomcode/engine" ] || printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine" || true
   echo "graph up to date (${LANGS//,/, }) at $OUT/graph.sqlite"; exit 0
 fi
-if [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then
+if [ -z "${AXIOMCODE_REINDEX:-}" ] && [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then
   has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO"
   python3 "$H/axiomcode-impact" --warm "$REPO" || true   # export the graph's facts now (~9 s on 1,373 files): the first
 # query pays it otherwise, and for the plugin that first query is inside hooks/changes.py's timeout=14, several at once.