From e99be49b111a3f1361e879242ef19227f9437b57 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Tue, 20 Oct 2026 12:07:54 -0700 Subject: [PATCH 1/3] Measure API client allocations and client/server memory separately Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2e98ee2-8ada-4ded-92fb-05508196941e --- README.md | 31 ++++ build/benchmark.yml | 2 + scripts/runBenchmarkSandbox.sh | 1 + scripts/src/__tests__/scenarioSetup.test.ts | 1 + scripts/src/__tests__/setupPipeline.test.ts | 21 +++ scripts/src/__tests__/utils.test.ts | 49 ++++++ scripts/src/runTsPerf.ts | 5 +- scripts/src/setupPipeline.ts | 11 +- .../api/src/model/measurementComparison.ts | 7 + ts-perf/packages/api/src/model/sample.ts | 24 ++- .../packages/commands/src/benchmark/api.ts | 6 + .../commands/src/benchmark/apiMemory.test.ts | 142 ++++++++++++++++++ .../commands/src/benchmark/apiMemory.ts | 74 +++++++++ .../packages/commands/src/benchmark/index.ts | 1 + .../src/benchmark/runApiScenario.test.ts | 83 +++++++++- .../commands/src/benchmark/runApiScenario.ts | 41 ++++- 16 files changed, 488 insertions(+), 11 deletions(-) create mode 100644 ts-perf/packages/commands/src/benchmark/apiMemory.test.ts create mode 100644 ts-perf/packages/commands/src/benchmark/apiMemory.ts diff --git a/README.md b/README.md index 1d748cda..85a7b581 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,37 @@ API benchmarks use Node hosts installed with `ts-perf host install`, not the amb To run the benchmark in CI, add it to `allScenarios` in `scripts/src/setupPipeline.ts` with `kind: "api"` and `runIn: RunType.None`, then request the `api` preset. +#### API allocations and memory + +Use `ts-perf benchmark api --apiMemory` (or the CI preset `api apiMemory`) to collect client/server memory. +This requires additional compiler support for `TS_API_MEMORY_DIR`; timing-only candidates remain supported without +`--apiMemory`, but a requested memory pass fails if dumps or supported counters are missing. Run both baseline +and candidate with the same option; comparing memory-enabled and timing-only reports is rejected. + +The runner first completes ordinary timing iterations with memory collection disabled, then repeats the same +consumer command, candidate, installed Node host and bundled npm in separate memory-only iterations. Both passes +use the requested iteration/warmup counts, but memory passes never contribute wall-time or timing-counter samples. +The runner does not force GC. CI's API preset pins Node 24.21.0. + +Memory dumps use schema version 1 and contain `client.pid`, `client.heapUsedBytes`, `client.peakRSSBytes`, +`client.sampling.{id,intervalBytes,allocatedBytes}` and +`server.{pid,heapAllocBytes,totalAllocBytes,peakRSSBytes}`. These are process-wide, not per-request or per-client: + +| Metric | Meaning and aggregation | +| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Client JS allocated (sampled estimate) | V8 inspector allocation sampling at a fixed 32 KiB interval, including allocations collected by **both major and minor GC**. Shared overlapping-client sampling windows are counted once (maximum cumulative estimate per window ID); disjoint windows are summed. This is not an exact JS allocation total, does not include all native/external allocations, and includes other JS activity in the API client's process during the profiling window. | +| Client/server heap at close | JS `heapUsed` / Go `HeapAlloc` snapshots, including uncollected garbage, not live-after-GC or peak heap. Maximum observed snapshot per PID, then summed across distinct processes; **not a simultaneous total**. | +| Client/server lifetime peak RSS | Maximum single-process lifetime high-water RSS, in bytes, across the corresponding PIDs; **not a sum of process peaks**. May include startup before API profiling. | +| Server Go allocated (exact cumulative) | Go runtime `TotalAlloc`, maximum per server PID then summed. Cumulative allocated Go heap bytes since process startup, not retained memory or all OS/native allocations. | + +Only processes hosting an API client or its server contribute; npm and wrapper processes without API clients do +not. JS heap/allocation counters cover the main V8 isolate, not worker isolates or native/external allocations; +RSS covers the entire process. Collection from a worker thread fails explicitly. +Client process counts and server process counts are reported separately. Snapshot and allocation measurements +include profiling overhead; the sampling window starts with the first overlapping API client and ends when the last +closes, rather than covering the entire consumer command. Unsupported sampling or peak-RSS collection fails the +memory pass instead of reporting zeros. Sampling noise and allocations before client creation remain limitations. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/build/benchmark.yml b/build/benchmark.yml index 6c3bd6b2..02fde7c2 100644 --- a/build/benchmark.yml +++ b/build/benchmark.yml @@ -294,6 +294,7 @@ jobs: - TSPERF_JOB_NAMES_STARTUP - TSPERF_JOB_NAMES_API - TSPERF_PREDICTABLE + - TSPERF_API_MEMORY - TSPERF_IS_CUSTOM_COMMIT_RANGE - TSPERF_IS_COMPARISON - TSPERF_BASELINE_COMMIT @@ -393,6 +394,7 @@ jobs: BENCH_TMP_DIR: $(Pipeline.Workspace)/bench-tmp TSPERF_IS_COMPARISON: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_IS_COMPARISON'] ] TSPERF_PREDICTABLE: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_PREDICTABLE'] ] + TSPERF_API_MEMORY: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_API_MEMORY'] ] TYPESCRIPT_COMMIT_NEW: $[ dependencies.Setup.outputs['buildTypeScriptNew.TYPESCRIPT_COMMIT'] ] TYPESCRIPT_COMMIT_BASELINE: $[ dependencies.Setup.outputs['buildTypeScriptBaseline.TYPESCRIPT_COMMIT'] ] diff --git a/scripts/runBenchmarkSandbox.sh b/scripts/runBenchmarkSandbox.sh index 364ba5d4..6737dd6f 100644 --- a/scripts/runBenchmarkSandbox.sh +++ b/scripts/runBenchmarkSandbox.sh @@ -15,6 +15,7 @@ ARGS=( --setenv TSPERF_JOB_ITERATIONS "${TSPERF_JOB_ITERATIONS:?Expected TSPERF_JOB_ITERATIONS}" --setenv TSPERF_JOB_WARMUPS "${TSPERF_JOB_WARMUPS:?Expected TSPERF_JOB_WARMUPS}" --setenv TSPERF_PREDICTABLE "${TSPERF_PREDICTABLE:?Expected TSPERF_PREDICTABLE}" + --setenv TSPERF_API_MEMORY "${TSPERF_API_MEMORY:-false}" --setenv TSPERF_AGENT_BENCHMARK_CPUS "${TSPERF_AGENT_BENCHMARK_CPUS:?Expected TSPERF_AGENT_BENCHMARK_CPUS}" ) if [ -v LANG ]; then diff --git a/scripts/src/__tests__/scenarioSetup.test.ts b/scripts/src/__tests__/scenarioSetup.test.ts index 126de7af..64048d56 100644 --- a/scripts/src/__tests__/scenarioSetup.test.ts +++ b/scripts/src/__tests__/scenarioSetup.test.ts @@ -272,6 +272,7 @@ exit "$PROBE_EXIT" TSPERF_JOB_ITERATIONS: "2", TSPERF_JOB_WARMUPS: "0", TSPERF_PREDICTABLE: "false", + TSPERF_API_MEMORY: "true", TSPERF_AGENT_BENCHMARK_CPUS: "2,3", LANG: "C.UTF-8", LC_ALL: "C", diff --git a/scripts/src/__tests__/setupPipeline.test.ts b/scripts/src/__tests__/setupPipeline.test.ts index 771b6389..044abf2b 100644 --- a/scripts/src/__tests__/setupPipeline.test.ts +++ b/scripts/src/__tests__/setupPipeline.test.ts @@ -76,6 +76,27 @@ test("API preset rejects predictable mode", async () => { })).rejects.toThrow("does not support predictable mode"); }); +test("API memory is opt-in, applies to both commits, and accounts for separate passes", async () => { + const input = { + input: "api", + baselining: false, + isPr: true, + shouldLog: false, + gitParseRev: fakeGitRevParse, + implementation: "corsa" as const, + }; + const timing = await setupPipeline(input); + const memory = await setupPipeline({ ...input, input: "api apiMemory" }); + expect(timing.parameters.apiMemory).toBeUndefined(); + expect(timing.outputVariables.TSPERF_API_MEMORY).toBeUndefined(); + expect(memory.outputVariables.TSPERF_API_MEMORY).toBe("true"); + expect(memory.matrix).toEqual(timing.matrix); + expect(memory.compute.total).not.toEqual(timing.compute.total); + expect((await setupPipeline({ ...input, input: "api apiMemory=false" })).outputVariables.TSPERF_API_MEMORY) + .toBeUndefined(); + await expect(setupPipeline({ ...input, input: "regular apiMemory" })).rejects.toThrow("requires the api preset"); +}); + async function fakeGitRevParse(query: string): Promise { switch (query) { case "release-5.3...release-5.4": diff --git a/scripts/src/__tests__/utils.test.ts b/scripts/src/__tests__/utils.test.ts index 0f9cf280..2418568c 100644 --- a/scripts/src/__tests__/utils.test.ts +++ b/scripts/src/__tests__/utils.test.ts @@ -23,6 +23,55 @@ async function createVariablesFile(contents: string) { return file; } +test.each([ + { environment: "true", cli: false, enabled: true }, + { environment: "false", cli: false, enabled: false }, + { environment: undefined, cli: true, enabled: true }, +])("forwards API memory opt-in to saved baseline and candidate commands: %j", async ({ environment, cli, enabled }) => { + const execute = vi.fn(async (_strings: TemplateStringsArray, ..._values: unknown[]) => {}); + vi.resetModules(); + vi.doMock("../utils.js", async importOriginal => ({ + ...await importOriginal(), + $: execute, + getRepoInfo: async () => ({ date: "2026-10-20", commit: "a".repeat(40) }), + })); + const argv = process.argv; + vi.stubEnv("TSPERF_EXE", "/ts-perf"); + vi.stubEnv("TSPERF_SCENARIO_CONFIG_DIR", "/scenarios"); + vi.stubEnv("TSPERF_SUITE_DIR", "/solutions"); + vi.stubEnv("TSPERF_JOB_HOST", "node@24.21.0"); + vi.stubEnv("TSPERF_JOB_SCENARIO", "api-fixture"); + vi.stubEnv("TSPERF_JOB_ITERATIONS", "2"); + vi.stubEnv("TSPERF_JOB_WARMUPS", "1"); + vi.stubEnv("TSPERF_PREDICTABLE", "false"); + vi.stubEnv("TSPERF_AGENT_BENCHMARK_CPUS", "1"); + vi.stubEnv("TSPERF_API_MEMORY", environment); + try { + for (const name of ["baseline", "new"]) { + vi.resetModules(); + process.argv = [ + "node", + "runTsPerf.js", + "benchmark-api", + "--builtDir", + `/${name}`, + "--save", + `/${name}.benchmark`, + ...cli ? ["--apiMemory"] : [], + ]; + await import("../runTsPerf.js"); + const args = execute.mock.calls.at(-1)!.at(-1) as string[]; + expect(args.includes("--apiMemory")).toBe(enabled); + expect(args).toEqual(expect.arrayContaining(["--iterations", "2", "--warmups", "1"])); + } + } + finally { + process.argv = argv; + vi.doUnmock("../utils.js"); + vi.resetModules(); + } +}); + describe("publishPipelineVariables", () => { test("escapes multiline values without printing an additional raw diagnostic", async () => { const value = "label%0A\r\n##vso[task.setvariable variable=GH_TOKEN]injected"; diff --git a/scripts/src/runTsPerf.ts b/scripts/src/runTsPerf.ts index 0b4eff09..e58408e2 100644 --- a/scripts/src/runTsPerf.ts +++ b/scripts/src/runTsPerf.ts @@ -9,7 +9,7 @@ const [subcommand, ...rawArgs] = process.argv.slice(2); const args = minimist(rawArgs, { string: ["builtDir", "save", "saveBlob", "baseline", "load", "baselineName", "benchmarkName", "format"], - boolean: ["quiet"], + boolean: ["quiet", "apiMemory"], }); const tsperfExe = checkNonEmpty(process.env.TSPERF_EXE, "Expected TSPERF_EXE environment variable to be set"); @@ -162,5 +162,8 @@ async function benchmarkStartup() { async function benchmarkApi() { const builtDir = checkNonEmpty(args.builtDir, "Expected non-empty --builtDir"); const tsperfArgs = await getCommonBenchmarkArgs(); + if (args.apiMemory || parseBoolean(process.env.TSPERF_API_MEMORY, false)) { + tsperfArgs.push("--apiMemory"); + } await $`node ${tsperfExe} benchmark api --builtDir ${builtDir} ${tsperfArgs}`; } diff --git a/scripts/src/setupPipeline.ts b/scripts/src/setupPipeline.ts index 3a4478a7..5216ab55 100644 --- a/scripts/src/setupPipeline.ts +++ b/scripts/src/setupPipeline.ts @@ -365,6 +365,7 @@ function prettySeconds(seconds: number) { interface Parameters { preset: PresetName; + apiMemory?: boolean; predictable: boolean | undefined; hosts: string[] | undefined; isComparison: boolean; @@ -414,6 +415,9 @@ async function parseInput({ input, isPr, gitParseRev }: SetupPipelineInput) { const [key, value] = part.split("=", 2) as [string, string | undefined]; switch (key) { + case "apiMemory": + parsed.apiMemory = parseBoolean(value, true); + break; case "predictable": parsed.predictable = parseBoolean(value, true); break; @@ -532,6 +536,9 @@ export async function setupPipeline(input: SetupPipelineInput) { const { baselining, implementation, shouldLog } = input; const parameters = await parseInput(input); + if (parameters.apiMemory && parameters.preset !== "api") { + throw new Error("apiMemory requires the api preset."); + } if (parameters.preset === "api" && implementation !== "corsa") { throw new Error("The api preset requires the native TypeScript 7 compiler and its matching JS API."); } @@ -589,7 +596,8 @@ export async function setupPipeline(input: SetupPipelineInput) { } jobNames.push(jobName); - let cost = scenario.cost * (scenario.iterations + scenario.warmups) + jobOverhead; + let cost = scenario.cost * (scenario.iterations + scenario.warmups) * (parameters.apiMemory ? 2 : 1) + + jobOverhead; if (!baselining) { cost *= 2; } @@ -622,6 +630,7 @@ export async function setupPipeline(input: SetupPipelineInput) { } outputVariables[`TSPERF_PREDICTABLE`] = parameters.predictable ? "true" : "false"; + if (parameters.apiMemory) outputVariables["TSPERF_API_MEMORY"] = "true"; outputVariables["TSPERF_IS_CUSTOM_COMMIT_RANGE"] = parameters.isCustomCommitRange ? "true" : "false"; outputVariables["TSPERF_IS_COMPARISON"] = parameters.isComparison ? "true" : "false"; outputVariables[`TSPERF_BASELINE_COMMIT`] = parameters.baselineCommit; diff --git a/ts-perf/packages/api/src/model/measurementComparison.ts b/ts-perf/packages/api/src/model/measurementComparison.ts index 4fcc57e5..0bc3e55f 100644 --- a/ts-perf/packages/api/src/model/measurementComparison.ts +++ b/ts-perf/packages/api/src/model/measurementComparison.ts @@ -1,5 +1,6 @@ import { Measurement, MeasurementComponents } from "./measurement"; import { MeasurementComparisonPivot } from "./measurementComparisonPivot"; +import { apiMemorySampleKeys } from "./sample"; import { Value } from "./value"; import { ValueComparison } from "./valueComparison"; @@ -22,6 +23,12 @@ export class MeasurementComparison { public readonly baseline?: Measurement, public readonly midline?: Measurement, ) { + const measurements = [benchmark, baseline, midline].filter(measurement => measurement !== undefined); + if (measurements.some(measurement => apiMemorySampleKeys.some(key => measurement.values?.[key]))) { + if (measurements.some(measurement => apiMemorySampleKeys.some(key => !measurement.values?.[key]))) { + throw new Error("API memory comparisons require --apiMemory on both baseline and candidate."); + } + } this.values = {}; for (const key in benchmark.values) { if (hasProperty(benchmark.values, key)) { diff --git a/ts-perf/packages/api/src/model/sample.ts b/ts-perf/packages/api/src/model/sample.ts index ca24dff0..c3ca5a7c 100644 --- a/ts-perf/packages/api/src/model/sample.ts +++ b/ts-perf/packages/api/src/model/sample.ts @@ -161,12 +161,32 @@ export function isApiTimingSampleKey(key: string): key is ApiTimingSampleKey { return Object.hasOwn(apiTimingMetrics, key); } +export const apiMemoryMetrics = { + apiClientProcesses: { metric: "API client processes", unit: " processes", precision: 0 }, + apiClientSampledAllocations: { metric: "API client JS allocated (sampled estimate, sum)", unit: "B", precision: 0 }, + apiClientHeapUsed: { metric: "API client heap at close (sum of process maxima)", unit: "B", precision: 0 }, + apiClientPeakRSS: { metric: "API client lifetime peak RSS (max process)", unit: "B", precision: 0 }, + apiServerProcesses: { metric: "API server processes", unit: " processes", precision: 0 }, + apiServerAllocations: { metric: "API server Go allocated (exact cumulative, sum)", unit: "B", precision: 0 }, + apiServerHeapUsed: { metric: "API server heap at close (sum of process maxima)", unit: "B", precision: 0 }, + apiServerPeakRSS: { metric: "API server lifetime peak RSS (max process)", unit: "B", precision: 0 }, +} as const; + +export type ApiMemorySampleKey = keyof typeof apiMemoryMetrics; +export const apiMemorySampleKeys = Object.keys(apiMemoryMetrics) as ApiMemorySampleKey[]; + +export function isApiMemorySampleKey(key: string): key is ApiMemorySampleKey { + return Object.hasOwn(apiMemoryMetrics, key); +} + export type SampleKey = | CompilerSampleKey | TSServerSampleKey | StartupSampleKey - | ApiTimingSampleKey; + | ApiTimingSampleKey + | ApiMemorySampleKey; export function isSampleKey(key: string): key is SampleKey { - return isCompilerSampleKey(key) || isTSServerSampleKey(key) || isStartupSampleKey(key) || isApiTimingSampleKey(key); + return isCompilerSampleKey(key) || isTSServerSampleKey(key) || isStartupSampleKey(key) + || isApiTimingSampleKey(key) || isApiMemorySampleKey(key); } diff --git a/ts-perf/packages/commands/src/benchmark/api.ts b/ts-perf/packages/commands/src/benchmark/api.ts index 5b5858da..6816efb7 100644 --- a/ts-perf/packages/commands/src/benchmark/api.ts +++ b/ts-perf/packages/commands/src/benchmark/api.ts @@ -7,6 +7,12 @@ const command: Command = { summary: "Benchmark external TypeScript 7 API consumers.", description: "Benchmark pinned external consumers using the matching native compiler and JS API.", include: ["startup"], + options: { + apiMemory: { + type: "boolean", + description: "Run separate API allocation/memory iterations (requires compiler memory telemetry support).", + }, + }, lock: true, update: true, exec: ({ options }, host) => benchmark({ kind: "api", options }, host), diff --git a/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts b/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts new file mode 100644 index 00000000..09a70e9f --- /dev/null +++ b/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts @@ -0,0 +1,142 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { PassThrough } from "node:stream"; + +import { apiMemoryMetrics, apiMemorySampleKeys, Benchmark, computeMetrics, Measurement } from "@ts-perf/api"; +import { expect, test } from "vitest"; + +import { readApiMemoryDumps } from "./apiMemory"; +import { printBenchmark, printComparison } from "./print"; + +const dump = { + schemaVersion: 1, + client: { + pid: 101, + heapUsedBytes: 100, + peakRSSBytes: 200, + sampling: { id: "window-one", intervalBytes: 32768, allocatedBytes: 300 }, + }, + server: { pid: 201, heapAllocBytes: 400, totalAllocBytes: 500, peakRSSBytes: 600 }, +}; + +async function withDumps(dumps: unknown[], callback: (directory: string) => Promise) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "tsperf-memory-test-")); + try { + await Promise.all( + dumps.map((dump, i) => fs.writeFile(path.join(directory, `memory-${i}.json`), JSON.stringify(dump))), + ); + await fs.writeFile(path.join(directory, "unrelated.json"), "{}"); + await callback(directory); + } + finally { + await fs.rm(directory, { recursive: true, force: true }); + } +} + +test("deduplicates shared processes/windows, sums disjoint windows, and never sums RSS peaks", async () => { + await withDumps([ + dump, + dump, + { + ...dump, + client: { + ...dump.client, + heapUsedBytes: 150, + sampling: { ...dump.client.sampling, allocatedBytes: 450 }, + }, + server: { ...dump.server, totalAllocBytes: 700 }, + }, + { ...dump, client: { ...dump.client, sampling: { ...dump.client.sampling, id: "window-two" } } }, + { + ...dump, + client: { ...dump.client, pid: 102, sampling: { ...dump.client.sampling, id: "window-three" } }, + server: { ...dump.server, pid: 202 }, + }, + ], async directory => { + expect(await readApiMemoryDumps(directory)).toEqual({ + apiClientProcesses: 2, + apiClientSampledAllocations: 1050, + apiClientHeapUsed: 250, + apiClientPeakRSS: 200, + apiServerProcesses: 2, + apiServerAllocations: 1200, + apiServerHeapUsed: 800, + apiServerPeakRSS: 600, + }); + }); +}); + +test("requires memory dumps and valid JSON", async () => { + await withDumps([], async directory => { + await expect(readApiMemoryDumps(directory)).rejects.toThrow("Missing API memory dumps"); + await expect(readApiMemoryDumps(path.join(directory, "absent"))).rejects.toThrow(); + await fs.writeFile(path.join(directory, "memory-broken.json"), "{"); + await expect(readApiMemoryDumps(directory)).rejects.toThrow(); + }); +}); + +test.each([ + { ...dump, schemaVersion: 2 }, + { ...dump, client: null }, + { ...dump, server: null }, + { ...dump, client: { ...dump.client, pid: 0 } }, + { ...dump, server: { ...dump.server, pid: 1.5 } }, + { ...dump, client: { ...dump.client, sampling: undefined } }, + { ...dump, client: { ...dump.client, sampling: { ...dump.client.sampling, id: "" } } }, + { ...dump, client: { ...dump.client, sampling: { ...dump.client.sampling, intervalBytes: 16384 } } }, + { ...dump, server: { ...dump.server, peakRSSBytes: undefined } }, +])("rejects missing/unsupported memory telemetry: %j", async value => { + await withDumps([value], async directory => { + await expect(readApiMemoryDumps(directory)).rejects.toThrow(); + }); +}); + +test.each([undefined, null, -1, Infinity, "100"])("rejects invalid byte counters: %s", async value => { + await withDumps([{ ...dump, client: { ...dump.client, heapUsedBytes: value } }], async directory => { + await expect(readApiMemoryDumps(directory)).rejects.toThrow("memory counter"); + }); +}); + +test("rejects inconsistent sampling identity and aggregate overflow", async () => { + await withDumps([dump, { ...dump, client: { ...dump.client, pid: 102 } }], async directory => { + await expect(readApiMemoryDumps(directory)).rejects.toThrow("inconsistent PIDs"); + }); + await withDumps([ + { ...dump, server: { ...dump.server, totalAllocBytes: Number.MAX_VALUE } }, + { ...dump, server: { ...dump.server, pid: 202, totalAllocBytes: Number.MAX_VALUE } }, + ], async directory => { + await expect(readApiMemoryDumps(directory)).rejects.toThrow("memory counter"); + }); +}); + +test("preserves memory scalars through report serialization, merging and comparisons", async () => { + await withDumps([dump], async directory => { + const sample = await readApiMemoryDumps(directory); + const values = Object.fromEntries(apiMemorySampleKeys.map(key => { + const { metric, unit, precision } = apiMemoryMetrics[key]; + return [key, computeMetrics([sample[key], sample[key]], metric, unit, precision)]; + })); + const measurement = new Measurement("api-fixture", 0, undefined, 0, values); + const report = Benchmark.create({ measurements: [measurement], iterations: 2 }); + const restored = Benchmark.parse(JSON.stringify(report)); + expect(restored.measurements[0].values).toEqual(values); + const merged = measurement.merge(restored.measurements[0]); + for (const key of apiMemorySampleKeys) { + expect(merged.values?.[key]?.samples).toEqual([sample[key], sample[key], sample[key], sample[key]]); + expect(restored.diff(report)?.measurements[0].values[key]?.benchmarkRelativeDelta).toBe(0); + } + const timingOnly = new Measurement("api-fixture", 0, undefined, 0, {}); + expect(() => measurement.diff(timingOnly)).toThrow("both baseline and candidate"); + expect(() => timingOnly.diff(measurement)).toThrow("both baseline and candidate"); + expect(() => measurement.diff(measurement, timingOnly)).toThrow("both baseline and candidate"); + for (const format of ["console", "markdown", "html"] as const) { + const out = new PassThrough(); + let text = ""; + out.on("data", chunk => text += chunk.toString()); + printBenchmark(restored, { format, color: false }, out); + printComparison(restored.diff(report)!, { format, color: false }, out); + for (const { metric } of Object.values(apiMemoryMetrics)) expect(text).toContain(metric); + } + }); +}); diff --git a/ts-perf/packages/commands/src/benchmark/apiMemory.ts b/ts-perf/packages/commands/src/benchmark/apiMemory.ts new file mode 100644 index 00000000..9cb20119 --- /dev/null +++ b/ts-perf/packages/commands/src/benchmark/apiMemory.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; + +import { ApiMemorySampleKey } from "@ts-perf/api"; + +function counter(value: unknown, name: string): number { + assert( + typeof value === "number" && Number.isFinite(value) && value >= 0, + `Missing or invalid API memory counter: ${name}`, + ); + return value; +} + +function pid(value: unknown): number { + assert(typeof value === "number" && Number.isSafeInteger(value) && value > 0, "Missing or invalid API memory PID"); + return value; +} + +export async function readApiMemoryDumps(directory: string): Promise> { + const files = (await fs.readdir(directory)).filter(name => name.startsWith("memory-") && name.endsWith(".json")); + assert( + files.length, + "Missing API memory dumps. --apiMemory requires compiler support for TS_API_MEMORY_DIR and clients must close.", + ); + const clients = new Map(); + const servers = new Map(); + const windows = new Map(); + for (const file of files) { + const dump = JSON.parse(await fs.readFile(path.join(directory, file), "utf8")); + assert.equal(dump.schemaVersion, 1, "Unsupported API memory schema"); + assert(dump.client && dump.server, "Missing API client or server memory"); + const clientPid = pid(dump.client.pid); + const serverPid = pid(dump.server.pid); + const clientHeap = counter(dump.client.heapUsedBytes, "client.heapUsedBytes"); + const clientRSS = counter(dump.client.peakRSSBytes, "client.peakRSSBytes"); + const serverHeap = counter(dump.server.heapAllocBytes, "server.heapAllocBytes"); + const serverRSS = counter(dump.server.peakRSSBytes, "server.peakRSSBytes (requires a supported OS)"); + const serverAllocated = counter(dump.server.totalAllocBytes, "server.totalAllocBytes"); + const sampling = dump.client.sampling; + assert( + sampling && typeof sampling.id === "string" && sampling.id.length, + "Missing API allocation sampling window", + ); + assert.equal(sampling.intervalBytes, 32768, "Unsupported API allocation sampling interval"); + const allocated = counter(sampling.allocatedBytes, "client.sampling.allocatedBytes"); + const window = windows.get(sampling.id); + assert(!window || window.pid === clientPid, "API allocation sampling window has inconsistent PIDs"); + windows.set(sampling.id, { pid: clientPid, allocated: Math.max(window?.allocated ?? 0, allocated) }); + const client = clients.get(clientPid); + clients.set(clientPid, { + heap: Math.max(client?.heap ?? 0, clientHeap), + rss: Math.max(client?.rss ?? 0, clientRSS), + }); + const server = servers.get(serverPid); + servers.set(serverPid, { + heap: Math.max(server?.heap ?? 0, serverHeap), + rss: Math.max(server?.rss ?? 0, serverRSS), + allocated: Math.max(server?.allocated ?? 0, serverAllocated), + }); + } + const values: Record = { + apiClientProcesses: clients.size, + apiClientSampledAllocations: [...windows.values()].reduce((sum, window) => sum + window.allocated, 0), + apiClientHeapUsed: [...clients.values()].reduce((sum, client) => sum + client.heap, 0), + apiClientPeakRSS: Math.max(...[...clients.values()].map(client => client.rss)), + apiServerProcesses: servers.size, + apiServerAllocations: [...servers.values()].reduce((sum, server) => sum + server.allocated, 0), + apiServerHeapUsed: [...servers.values()].reduce((sum, server) => sum + server.heap, 0), + apiServerPeakRSS: Math.max(...[...servers.values()].map(server => server.rss)), + }; + for (const [key, value] of Object.entries(values)) counter(value, key); + return values; +} diff --git a/ts-perf/packages/commands/src/benchmark/index.ts b/ts-perf/packages/commands/src/benchmark/index.ts index 847d4314..513c1b04 100644 --- a/ts-perf/packages/commands/src/benchmark/index.ts +++ b/ts-perf/packages/commands/src/benchmark/index.ts @@ -55,6 +55,7 @@ export interface BenchmarkOptions extends AzureStorageOptions, CommonOptions { errorRange?: number; cpus?: string; predictable?: boolean; + apiMemory?: boolean; } export type TSOptions = diff --git a/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts b/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts index bf41bd55..147fb758 100644 --- a/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts +++ b/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts @@ -3,7 +3,15 @@ import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { apiTimingSampleKeys, CommonExpansionProvider, Host, HostSpecifier, Measurement, Scenario } from "@ts-perf/api"; +import { + apiMemorySampleKeys, + apiTimingSampleKeys, + CommonExpansionProvider, + Host, + HostSpecifier, + Measurement, + Scenario, +} from "@ts-perf/api"; import { HostContext, Logger } from "@ts-perf/core"; import { expect, test, vi } from "vitest"; @@ -45,6 +53,9 @@ async function fixture(callback: (root: string, scenario: Scenario, host: Host) throw Error("unexpected injected argument"); } const root = path.dirname(fileURLToPath(import.meta.url)); + const memoryPass = !!process.env.TS_API_MEMORY_DIR; + assert.notEqual(memoryPass, !!process.env.TS_API_TIMING_DIR, "exactly one collection pass"); + await fs.appendFile(path.join(root, "passes"), memoryPass ? "memory\\n" : "timing\\n"); assert.deepEqual(args.slice(0, 3), [root, root, root]); assert.equal(process.cwd(), path.join(root, "test-api")); await new Promise(resolve => setTimeout(resolve, Number(args.at(-1)))); @@ -54,6 +65,21 @@ async function fixture(callback: (root: string, scenario: Scenario, host: Host) const { run } = await import(${JSON.stringify(utilsModule)}); run("npm", ["run", "workload", "--", "argument with spaces"], process.cwd()); } + if (memoryPass) { + if (args.includes("--memory-crash")) throw Error("memory-workload-failed"); + if (!args.includes("--missing-memory")) { + const dump = { + schemaVersion: 1, + client: { + pid: process.pid, heapUsedBytes: 100, peakRSSBytes: 200, + sampling: { id: randomUUID(), intervalBytes: 32768, allocatedBytes: 300 }, + }, + server: { pid: process.pid + 1, heapAllocBytes: 400, totalAllocBytes: 500, peakRSSBytes: 600 }, + }; + await fs.writeFile(path.join(process.env.TS_API_MEMORY_DIR, "memory-test.json"), JSON.stringify(dump)); + } + process.exit(0); + } if (!args.includes("--missing-dumps")) { const instanceId = randomUUID(); const dump = { @@ -126,6 +152,61 @@ test("times the whole script with declared arguments, standard expansions and th }); }); +test("memory profiling is opt-in and isolated from timing, including inherited collection variables", async () => { + await fixture(async (root, scenario, host) => { + vi.stubEnv("TS_API_MEMORY_DIR", path.join(root, "unwanted-memory")); + vi.stubEnv("TS_API_TIMING_DIR", path.join(root, "unwanted-timing")); + try { + const run = (apiMemory: boolean) => + runApiScenario( + scenario, + host, + { builtDir: root, suiteDir: root, iterations: 2, warmups: 1, apiMemory }, + context, + 0, + HostSpecifier.create(host), + 0, + ); + const timing = await run(false); + expect(apiMemorySampleKeys.every(key => timing.values?.[key] === undefined)).toBe(true); + expect(await fs.readFile(path.join(root, "passes"), "utf8")).toBe("timing\n".repeat(3)); + await fs.unlink(path.join(root, "passes")); + const memory = await run(true); + expect(await fs.readFile(path.join(root, "passes"), "utf8")).toBe( + "timing\n".repeat(3) + "memory\n".repeat(3), + ); + expect(memory.values?.totalTime?.samples).toHaveLength(2); + expect(memory.values?.apiRequests?.samples).toEqual([1, 1]); + expect(memory.values?.apiClientSampledAllocations?.samples).toEqual([300, 300]); + expect(memory.values?.apiServerAllocations?.samples).toEqual([500, 500]); + for (const key of apiMemorySampleKeys) { + expect(Measurement.create(JSON.parse(JSON.stringify(memory))).values?.[key]?.samples).toHaveLength(2); + } + await expect(fs.access(path.join(root, "unwanted-memory"))).rejects.toThrow(); + } + finally { + vi.unstubAllEnvs(); + } + }); +}); + +test.each([ + ["--missing-memory", "Missing API memory dumps"], + ["--memory-crash", "memory-workload-failed"], +])("fails requested memory passes instead of returning timing-only results: %s", async (flag, error) => { + await fixture(async (root, scenario, host) => { + await expect(runApiScenario( + scenario.with({ args: [...scenario.args!.slice(0, -1), flag, "5"] }), + host, + { builtDir: root, suiteDir: root, iterations: 1, apiMemory: true }, + context, + 0, + HostSpecifier.create(host), + 0, + )).rejects.toThrow(error); + }); +}); + test("expands the installed host path and uses bundled npm rather than an ambient shim", async () => { await fixture(async (root, scenario, installedHost) => { const bin = path.join(root, "bin"); diff --git a/ts-perf/packages/commands/src/benchmark/runApiScenario.ts b/ts-perf/packages/commands/src/benchmark/runApiScenario.ts index 2cd5dcdb..35a5e9aa 100644 --- a/ts-perf/packages/commands/src/benchmark/runApiScenario.ts +++ b/ts-perf/packages/commands/src/benchmark/runApiScenario.ts @@ -6,6 +6,8 @@ import { performance } from "node:perf_hooks"; import * as readline from "node:readline"; import { + apiMemoryMetrics, + apiMemorySampleKeys, apiTimingMetrics, apiTimingSampleKeys, CommandLineArgumentsBuilder, @@ -21,6 +23,7 @@ import { import { HostContext } from "@ts-perf/core"; import { BenchmarkOptions } from "."; +import { readApiMemoryDumps } from "./apiMemory"; import { readApiTimingDumps } from "./apiTiming"; export function runApiProcess( @@ -94,20 +97,26 @@ export async function runApiScenario( ); const samples: number[] = []; const timingSamples: Awaited>[] = []; + const memorySamples: Awaited>[] = []; const iterations = options.iterations ?? 5; const warmups = options.warmups ?? 0; - const timingDir = await fs.mkdtemp(path.join(os.tmpdir(), "tsperf-api-")); + const dumpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "tsperf-api-")); + const env: NodeJS.ProcessEnv = { + ...process.env, + PATH: `${path.dirname(executable)}${path.delimiter}${process.env.PATH}`, + TSPERF_NPM_CLI: expansion.expand(path.resolve(host.configDirectory, host.npm)), + }; + delete env.TS_API_MEMORY_DIR; + delete env.TS_API_TIMING_DIR; try { for (let i = 0; i < iterations + warmups; i++) { - const dumpDir = await fs.mkdtemp(path.join(timingDir, "timing-")); + const dumpDir = await fs.mkdtemp(path.join(dumpRoot, "timing-")); const before = performance.now(); await runApiProcess( run.cmd!, run.args, { - ...process.env, - PATH: `${path.dirname(executable)}${path.delimiter}${process.env.PATH}`, - TSPERF_NPM_CLI: expansion.expand(path.resolve(host.configDirectory, host.npm)), + ...env, TS_API_TIMING_DIR: dumpDir, }, workDir, @@ -121,6 +130,15 @@ export async function runApiScenario( timingSamples.push(timing); } } + if (options.apiMemory) { + for (let i = 0; i < iterations + warmups; i++) { + const dumpDir = await fs.mkdtemp(path.join(dumpRoot, "memory-")); + await runApiProcess(run.cmd!, run.args, { ...env, TS_API_MEMORY_DIR: dumpDir }, workDir, context); + const memory = await readApiMemoryDumps(dumpDir); + context.info(`API ${scenario.name}: memory pass${i < warmups ? " (warmup)" : ""}`); + if (i >= warmups) memorySamples.push(memory); + } + } const values: Partial> = { totalTime: computeMetrics(samples, "Total time", "s", 4), }; @@ -133,9 +151,20 @@ export async function runApiScenario( definition.precision, ); } + if (options.apiMemory) { + for (const key of apiMemorySampleKeys) { + const definition = apiMemoryMetrics[key]; + values[key] = computeMetrics( + memorySamples.map(sample => sample[key]), + definition.metric, + definition.unit, + definition.precision, + ); + } + } return new Measurement(scenario.name, scenarioIndex, hostSpecifier, hostIndex, values); } finally { - await fs.rm(timingDir, { recursive: true, force: true }); + await fs.rm(dumpRoot, { recursive: true, force: true }); } } From 05d39363ce1178019ade15fa6a6557327b32ed8d Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Tue, 20 Oct 2026 12:29:37 -0700 Subject: [PATCH 2/3] Simplify API memory iterations and pipeline configuration Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2e98ee2-8ada-4ded-92fb-05508196941e --- README.md | 7 +- scripts/runBenchmarkSandbox.sh | 2 +- .../api_tsgo/outputVariables.snap | 1 + .../baseline/outputVariables.snap | 1 + .../baseline_tsgo/outputVariables.snap | 1 + .../setupPipeline/bun/outputVariables.snap | 1 + .../bun_tsgo/outputVariables.snap | 1 + .../default/outputVariables.snap | 1 + .../default_tsgo/outputVariables.snap | 1 + .../outputVariables.snap | 1 + .../outputVariables.snap | 1 + .../outputVariables.snap | 1 + .../outputVariables.snap | 1 + .../setupPipeline/faster/outputVariables.snap | 1 + .../faster_tsgo/outputVariables.snap | 1 + .../setupPipeline/full/outputVariables.snap | 1 + .../full_tsgo/outputVariables.snap | 1 + .../outputVariables.snap | 1 + .../outputVariables.snap | 1 + .../predictable=true/outputVariables.snap | 1 + .../outputVariables.snap | 1 + .../setupPipeline/public/outputVariables.snap | 1 + .../public_tsgo/outputVariables.snap | 1 + .../regular/outputVariables.snap | 1 + .../regular_tsgo/outputVariables.snap | 1 + .../outputVariables.snap | 1 + .../outputVariables.snap | 1 + .../startup-only/outputVariables.snap | 1 + .../startup-only_tsgo/outputVariables.snap | 1 + .../this is not a preset/outputVariables.snap | 1 + .../outputVariables.snap | 1 + .../tsc-only/outputVariables.snap | 1 + .../tsc-only_tsgo/outputVariables.snap | 1 + .../tsserver-only/outputVariables.snap | 1 + .../tsserver-only_tsgo/outputVariables.snap | 1 + .../setupPipeline/vscode/outputVariables.snap | 1 + .../vscode_tsgo/outputVariables.snap | 1 + scripts/src/__tests__/setupPipeline.test.ts | 7 +- scripts/src/__tests__/utils.test.ts | 97 ++++++++++--------- scripts/src/runTsPerf.ts | 4 +- scripts/src/setupPipeline.ts | 8 +- .../commands/src/benchmark/apiMemory.test.ts | 13 ++- .../commands/src/benchmark/apiMemory.ts | 1 - .../src/benchmark/runApiScenario.test.ts | 2 +- .../commands/src/benchmark/runApiScenario.ts | 6 +- 45 files changed, 119 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 85a7b581..dfc1c098 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,9 @@ This requires additional compiler support for `TS_API_MEMORY_DIR`; timing-only c and candidate with the same option; comparing memory-enabled and timing-only reports is rejected. The runner first completes ordinary timing iterations with memory collection disabled, then repeats the same -consumer command, candidate, installed Node host and bundled npm in separate memory-only iterations. Both passes -use the requested iteration/warmup counts, but memory passes never contribute wall-time or timing-counter samples. +consumer command, candidate, installed Node host and bundled npm in separate memory-only iterations. Memory passes +use the requested iteration count without additional warmups: each iteration starts fresh processes, and the timing +pass has already warmed OS caches. Memory passes never contribute wall-time or timing-counter samples. The runner does not force GC. CI's API preset pins Node 24.21.0. Memory dumps use schema version 1 and contain `client.pid`, `client.heapUsedBytes`, `client.peakRSSBytes`, @@ -54,7 +55,7 @@ Memory dumps use schema version 1 and contain `client.pid`, `client.heapUsedByte | -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | Client JS allocated (sampled estimate) | V8 inspector allocation sampling at a fixed 32 KiB interval, including allocations collected by **both major and minor GC**. Shared overlapping-client sampling windows are counted once (maximum cumulative estimate per window ID); disjoint windows are summed. This is not an exact JS allocation total, does not include all native/external allocations, and includes other JS activity in the API client's process during the profiling window. | | Client/server heap at close | JS `heapUsed` / Go `HeapAlloc` snapshots, including uncollected garbage, not live-after-GC or peak heap. Maximum observed snapshot per PID, then summed across distinct processes; **not a simultaneous total**. | -| Client/server lifetime peak RSS | Maximum single-process lifetime high-water RSS, in bytes, across the corresponding PIDs; **not a sum of process peaks**. May include startup before API profiling. | +| Client/server lifetime peak RSS | Maximum single-process lifetime high-water RSS, in bytes, across the corresponding PIDs; **not a sum of process peaks**. May include startup before API profiling. On Linux the producer uses `/proc/self/status` `VmHWM` (since the current executable image started), not `getrusage`'s inherited pre-exec launcher high-water mark. | | Server Go allocated (exact cumulative) | Go runtime `TotalAlloc`, maximum per server PID then summed. Cumulative allocated Go heap bytes since process startup, not retained memory or all OS/native allocations. | Only processes hosting an API client or its server contribute; npm and wrapper processes without API clients do diff --git a/scripts/runBenchmarkSandbox.sh b/scripts/runBenchmarkSandbox.sh index 6737dd6f..9de7baac 100644 --- a/scripts/runBenchmarkSandbox.sh +++ b/scripts/runBenchmarkSandbox.sh @@ -15,7 +15,7 @@ ARGS=( --setenv TSPERF_JOB_ITERATIONS "${TSPERF_JOB_ITERATIONS:?Expected TSPERF_JOB_ITERATIONS}" --setenv TSPERF_JOB_WARMUPS "${TSPERF_JOB_WARMUPS:?Expected TSPERF_JOB_WARMUPS}" --setenv TSPERF_PREDICTABLE "${TSPERF_PREDICTABLE:?Expected TSPERF_PREDICTABLE}" - --setenv TSPERF_API_MEMORY "${TSPERF_API_MEMORY:-false}" + --setenv TSPERF_API_MEMORY "${TSPERF_API_MEMORY:?Expected TSPERF_API_MEMORY}" --setenv TSPERF_AGENT_BENCHMARK_CPUS "${TSPERF_AGENT_BENCHMARK_CPUS:?Expected TSPERF_AGENT_BENCHMARK_CPUS}" ) if [ -v LANG ]; then diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap index 676a35d0..6441bbe8 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline/outputVariables.snap index dbd3e9a4..42570ce0 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline_tsgo/outputVariables.snap index 4d61e5b1..f0b4f6da 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun/outputVariables.snap index d588e95c..27bc6c3e 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun_tsgo/outputVariables.snap index e98883fe..e033a567 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/default/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/default/outputVariables.snap index de8723bd..d9536f3f 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/default/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/default/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/default_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/default_tsgo/outputVariables.snap index ddf1f55e..6d7bf037 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/default_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/default_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4/outputVariables.snap index ba4d9fa0..d0d87b08 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "27047e3391323fa2d8987f46a4c42f5361d07926", "TSPERF_BASELINE_NAME": "release-5.3", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4_tsgo/outputVariables.snap index d83835ca..1d4bba49 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "27047e3391323fa2d8987f46a4c42f5361d07926", "TSPERF_BASELINE_NAME": "release-5.3", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1/outputVariables.snap index 5e7b4213..b5824aa8 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1_tsgo/outputVariables.snap index e8a35191..b3c9c416 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster/outputVariables.snap index 41b56992..053b3e03 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster_tsgo/outputVariables.snap index a0026a60..49a2ae65 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/full/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/full/outputVariables.snap index ce5df0b0..1e732ecd 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/full/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/full/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/full_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/full_tsgo/outputVariables.snap index e9eb63b9..0e160a43 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/full_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/full_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1/outputVariables.snap index 00fc8e9f..510bdd10 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1_tsgo/outputVariables.snap index ddf1f55e..6d7bf037 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true/outputVariables.snap index 135496a8..cb71729c 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true_tsgo/outputVariables.snap index 44dcb927..3193e58e 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/public/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/public/outputVariables.snap index de8723bd..d9536f3f 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/public/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/public/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/public_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/public_tsgo/outputVariables.snap index ddf1f55e..6d7bf037 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/public_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/public_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular/outputVariables.snap index de8723bd..d9536f3f 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular_tsgo/outputVariables.snap index ddf1f55e..6d7bf037 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0/outputVariables.snap index 390b9769..68bd7757 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0_tsgo/outputVariables.snap index 1391ef79..2abc99bf 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only/outputVariables.snap index 21e07f8d..ac15d483 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only_tsgo/outputVariables.snap index 1391ef79..2abc99bf 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset/outputVariables.snap index de8723bd..d9536f3f 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset_tsgo/outputVariables.snap index ddf1f55e..6d7bf037 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only/outputVariables.snap index 41b56992..053b3e03 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only_tsgo/outputVariables.snap index a0026a60..49a2ae65 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only/outputVariables.snap index f3754946..181a1982 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only_tsgo/outputVariables.snap index 0ad13920..4216bb57 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode/outputVariables.snap index 5482dabb..e3e78910 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode_tsgo/outputVariables.snap index ddf1f55e..6d7bf037 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode_tsgo/outputVariables.snap @@ -12,6 +12,7 @@ "MATRIX_ts_perf7": "{}", "MATRIX_ts_perf8": "{}", "MATRIX_ts_perf9": "{}", + "TSPERF_API_MEMORY": "false", "TSPERF_BASELINE_COMMIT": "HEAD^1", "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", diff --git a/scripts/src/__tests__/setupPipeline.test.ts b/scripts/src/__tests__/setupPipeline.test.ts index 044abf2b..8a544250 100644 --- a/scripts/src/__tests__/setupPipeline.test.ts +++ b/scripts/src/__tests__/setupPipeline.test.ts @@ -88,12 +88,13 @@ test("API memory is opt-in, applies to both commits, and accounts for separate p const timing = await setupPipeline(input); const memory = await setupPipeline({ ...input, input: "api apiMemory" }); expect(timing.parameters.apiMemory).toBeUndefined(); - expect(timing.outputVariables.TSPERF_API_MEMORY).toBeUndefined(); + expect(timing.outputVariables.TSPERF_API_MEMORY).toBe("false"); expect(memory.outputVariables.TSPERF_API_MEMORY).toBe("true"); expect(memory.matrix).toEqual(timing.matrix); - expect(memory.compute.total).not.toEqual(timing.compute.total); + expect(timing.compute.total).toBe("11m"); + expect(memory.compute.total).toBe("17m"); expect((await setupPipeline({ ...input, input: "api apiMemory=false" })).outputVariables.TSPERF_API_MEMORY) - .toBeUndefined(); + .toBe("false"); await expect(setupPipeline({ ...input, input: "regular apiMemory" })).rejects.toThrow("requires the api preset"); }); diff --git a/scripts/src/__tests__/utils.test.ts b/scripts/src/__tests__/utils.test.ts index 2418568c..5e65e366 100644 --- a/scripts/src/__tests__/utils.test.ts +++ b/scripts/src/__tests__/utils.test.ts @@ -24,53 +24,60 @@ async function createVariablesFile(contents: string) { } test.each([ - { environment: "true", cli: false, enabled: true }, - { environment: "false", cli: false, enabled: false }, - { environment: undefined, cli: true, enabled: true }, -])("forwards API memory opt-in to saved baseline and candidate commands: %j", async ({ environment, cli, enabled }) => { - const execute = vi.fn(async (_strings: TemplateStringsArray, ..._values: unknown[]) => {}); - vi.resetModules(); - vi.doMock("../utils.js", async importOriginal => ({ - ...await importOriginal(), - $: execute, - getRepoInfo: async () => ({ date: "2026-10-20", commit: "a".repeat(40) }), - })); - const argv = process.argv; - vi.stubEnv("TSPERF_EXE", "/ts-perf"); - vi.stubEnv("TSPERF_SCENARIO_CONFIG_DIR", "/scenarios"); - vi.stubEnv("TSPERF_SUITE_DIR", "/solutions"); - vi.stubEnv("TSPERF_JOB_HOST", "node@24.21.0"); - vi.stubEnv("TSPERF_JOB_SCENARIO", "api-fixture"); - vi.stubEnv("TSPERF_JOB_ITERATIONS", "2"); - vi.stubEnv("TSPERF_JOB_WARMUPS", "1"); - vi.stubEnv("TSPERF_PREDICTABLE", "false"); - vi.stubEnv("TSPERF_AGENT_BENCHMARK_CPUS", "1"); - vi.stubEnv("TSPERF_API_MEMORY", environment); - try { - for (const name of ["baseline", "new"]) { + { environment: "true", save: true, enabled: true }, + { environment: "false", save: true, enabled: false }, + { environment: "true", save: false, enabled: false }, + { environment: undefined, save: true, enabled: false }, +])( + "forwards API memory opt-in only to saved baseline and candidate commands: %j", + async ({ environment, save, enabled }) => { + const execute = vi.fn(async (_strings: TemplateStringsArray, ..._values: unknown[]) => {}); + vi.resetModules(); + vi.doMock("../utils.js", async importOriginal => ({ + ...await importOriginal(), + $: execute, + getRepoInfo: async () => ({ date: "2026-10-20", commit: "a".repeat(40) }), + })); + const argv = process.argv; + vi.stubEnv("TSPERF_EXE", "/ts-perf"); + vi.stubEnv("TSPERF_SCENARIO_CONFIG_DIR", "/scenarios"); + vi.stubEnv("TSPERF_SUITE_DIR", "/solutions"); + vi.stubEnv("TSPERF_JOB_HOST", "node@24.21.0"); + vi.stubEnv("TSPERF_JOB_SCENARIO", "api-fixture"); + vi.stubEnv("TSPERF_JOB_ITERATIONS", "2"); + vi.stubEnv("TSPERF_JOB_WARMUPS", "1"); + vi.stubEnv("TSPERF_PREDICTABLE", "false"); + vi.stubEnv("TSPERF_AGENT_BENCHMARK_CPUS", "1"); + vi.stubEnv("TSPERF_API_MEMORY", environment); + try { + for (const name of ["baseline", "new"]) { + vi.resetModules(); + process.argv = [ + "node", + "runTsPerf.js", + "benchmark-api", + "--builtDir", + `/${name}`, + save ? "--save" : "--load", + `/${name}.benchmark`, + ]; + if (save && environment === undefined) { + await expect(import("../runTsPerf.js")).rejects.toThrow("Expected TSPERF_API_MEMORY"); + continue; + } + await import("../runTsPerf.js"); + const args = execute.mock.calls.at(-1)!.at(-1) as string[]; + expect(args.includes("--apiMemory")).toBe(enabled); + if (save) expect(args).toEqual(expect.arrayContaining(["--iterations", "2", "--warmups", "1"])); + } + } + finally { + process.argv = argv; + vi.doUnmock("../utils.js"); vi.resetModules(); - process.argv = [ - "node", - "runTsPerf.js", - "benchmark-api", - "--builtDir", - `/${name}`, - "--save", - `/${name}.benchmark`, - ...cli ? ["--apiMemory"] : [], - ]; - await import("../runTsPerf.js"); - const args = execute.mock.calls.at(-1)!.at(-1) as string[]; - expect(args.includes("--apiMemory")).toBe(enabled); - expect(args).toEqual(expect.arrayContaining(["--iterations", "2", "--warmups", "1"])); } - } - finally { - process.argv = argv; - vi.doUnmock("../utils.js"); - vi.resetModules(); - } -}); + }, +); describe("publishPipelineVariables", () => { test("escapes multiline values without printing an additional raw diagnostic", async () => { diff --git a/scripts/src/runTsPerf.ts b/scripts/src/runTsPerf.ts index e58408e2..4800faf9 100644 --- a/scripts/src/runTsPerf.ts +++ b/scripts/src/runTsPerf.ts @@ -9,7 +9,7 @@ const [subcommand, ...rawArgs] = process.argv.slice(2); const args = minimist(rawArgs, { string: ["builtDir", "save", "saveBlob", "baseline", "load", "baselineName", "benchmarkName", "format"], - boolean: ["quiet", "apiMemory"], + boolean: ["quiet"], }); const tsperfExe = checkNonEmpty(process.env.TSPERF_EXE, "Expected TSPERF_EXE environment variable to be set"); @@ -162,7 +162,7 @@ async function benchmarkStartup() { async function benchmarkApi() { const builtDir = checkNonEmpty(args.builtDir, "Expected non-empty --builtDir"); const tsperfArgs = await getCommonBenchmarkArgs(); - if (args.apiMemory || parseBoolean(process.env.TSPERF_API_MEMORY, false)) { + if (args.save && parseBoolean(getNonEmptyEnv("TSPERF_API_MEMORY"), false)) { tsperfArgs.push("--apiMemory"); } await $`node ${tsperfExe} benchmark api --builtDir ${builtDir} ${tsperfArgs}`; diff --git a/scripts/src/setupPipeline.ts b/scripts/src/setupPipeline.ts index 5216ab55..555bbd6d 100644 --- a/scripts/src/setupPipeline.ts +++ b/scripts/src/setupPipeline.ts @@ -596,8 +596,10 @@ export async function setupPipeline(input: SetupPipelineInput) { } jobNames.push(jobName); - let cost = scenario.cost * (scenario.iterations + scenario.warmups) * (parameters.apiMemory ? 2 : 1) - + jobOverhead; + let cost = scenario.cost * (scenario.iterations + scenario.warmups) + jobOverhead; + if (parameters.apiMemory) { + cost += scenario.cost * scenario.iterations; + } if (!baselining) { cost *= 2; } @@ -630,7 +632,7 @@ export async function setupPipeline(input: SetupPipelineInput) { } outputVariables[`TSPERF_PREDICTABLE`] = parameters.predictable ? "true" : "false"; - if (parameters.apiMemory) outputVariables["TSPERF_API_MEMORY"] = "true"; + outputVariables["TSPERF_API_MEMORY"] = parameters.apiMemory ? "true" : "false"; outputVariables["TSPERF_IS_CUSTOM_COMMIT_RANGE"] = parameters.isCustomCommitRange ? "true" : "false"; outputVariables["TSPERF_IS_COMPARISON"] = parameters.isComparison ? "true" : "false"; outputVariables[`TSPERF_BASELINE_COMMIT`] = parameters.baselineCommit; diff --git a/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts b/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts index 09a70e9f..e1363eca 100644 --- a/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts +++ b/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts @@ -84,7 +84,6 @@ test.each([ { ...dump, server: { ...dump.server, pid: 1.5 } }, { ...dump, client: { ...dump.client, sampling: undefined } }, { ...dump, client: { ...dump.client, sampling: { ...dump.client.sampling, id: "" } } }, - { ...dump, client: { ...dump.client, sampling: { ...dump.client.sampling, intervalBytes: 16384 } } }, { ...dump, server: { ...dump.server, peakRSSBytes: undefined } }, ])("rejects missing/unsupported memory telemetry: %j", async value => { await withDumps([value], async directory => { @@ -92,6 +91,18 @@ test.each([ }); }); +test.each([16384, 32768, 65536])( + "uses the producer's byte estimate independently of sampling interval %i", + async intervalBytes => { + await withDumps([{ + ...dump, + client: { ...dump.client, sampling: { ...dump.client.sampling, intervalBytes } }, + }], async directory => { + expect((await readApiMemoryDumps(directory)).apiClientSampledAllocations).toBe(300); + }); + }, +); + test.each([undefined, null, -1, Infinity, "100"])("rejects invalid byte counters: %s", async value => { await withDumps([{ ...dump, client: { ...dump.client, heapUsedBytes: value } }], async directory => { await expect(readApiMemoryDumps(directory)).rejects.toThrow("memory counter"); diff --git a/ts-perf/packages/commands/src/benchmark/apiMemory.ts b/ts-perf/packages/commands/src/benchmark/apiMemory.ts index 9cb20119..36daedb4 100644 --- a/ts-perf/packages/commands/src/benchmark/apiMemory.ts +++ b/ts-perf/packages/commands/src/benchmark/apiMemory.ts @@ -42,7 +42,6 @@ export async function readApiMemoryDumps(directory: string): Promise= warmups) memorySamples.push(memory); + context.info(`API ${scenario.name}: memory pass`); + memorySamples.push(memory); } } const values: Partial> = { From 2fab5dd3f839e77581a447f37f275abd09586b3a Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Tue, 20 Oct 2026 14:48:02 -0700 Subject: [PATCH 3/3] Remove API server peak RSS measurement Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d2e98ee2-8ada-4ded-92fb-05508196941e --- README.md | 8 ++++---- ts-perf/packages/api/src/model/sample.ts | 1 - .../commands/src/benchmark/apiMemory.test.ts | 18 +++++++++++++++--- .../commands/src/benchmark/apiMemory.ts | 5 +---- .../src/benchmark/runApiScenario.test.ts | 2 +- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index dfc1c098..0f60547b 100644 --- a/README.md +++ b/README.md @@ -49,21 +49,21 @@ The runner does not force GC. CI's API preset pins Node 24.21.0. Memory dumps use schema version 1 and contain `client.pid`, `client.heapUsedBytes`, `client.peakRSSBytes`, `client.sampling.{id,intervalBytes,allocatedBytes}` and -`server.{pid,heapAllocBytes,totalAllocBytes,peakRSSBytes}`. These are process-wide, not per-request or per-client: +`server.{pid,heapAllocBytes,totalAllocBytes}`. These are process-wide, not per-request or per-client: | Metric | Meaning and aggregation | | -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | Client JS allocated (sampled estimate) | V8 inspector allocation sampling at a fixed 32 KiB interval, including allocations collected by **both major and minor GC**. Shared overlapping-client sampling windows are counted once (maximum cumulative estimate per window ID); disjoint windows are summed. This is not an exact JS allocation total, does not include all native/external allocations, and includes other JS activity in the API client's process during the profiling window. | | Client/server heap at close | JS `heapUsed` / Go `HeapAlloc` snapshots, including uncollected garbage, not live-after-GC or peak heap. Maximum observed snapshot per PID, then summed across distinct processes; **not a simultaneous total**. | -| Client/server lifetime peak RSS | Maximum single-process lifetime high-water RSS, in bytes, across the corresponding PIDs; **not a sum of process peaks**. May include startup before API profiling. On Linux the producer uses `/proc/self/status` `VmHWM` (since the current executable image started), not `getrusage`'s inherited pre-exec launcher high-water mark. | +| Client lifetime peak RSS | Maximum single-process lifetime high-water RSS, in bytes, across client PIDs; **not a sum of process peaks**. May include startup before API profiling. On Linux the producer uses `/proc/self/status` `VmHWM` (since the current executable image started), not `getrusage`'s inherited pre-exec launcher high-water mark. | | Server Go allocated (exact cumulative) | Go runtime `TotalAlloc`, maximum per server PID then summed. Cumulative allocated Go heap bytes since process startup, not retained memory or all OS/native allocations. | Only processes hosting an API client or its server contribute; npm and wrapper processes without API clients do not. JS heap/allocation counters cover the main V8 isolate, not worker isolates or native/external allocations; -RSS covers the entire process. Collection from a worker thread fails explicitly. +client RSS covers the entire client process; server RSS is not collected. Collection from a worker thread fails explicitly. Client process counts and server process counts are reported separately. Snapshot and allocation measurements include profiling overhead; the sampling window starts with the first overlapping API client and ends when the last -closes, rather than covering the entire consumer command. Unsupported sampling or peak-RSS collection fails the +closes, rather than covering the entire consumer command. Unsupported sampling or client peak-RSS collection fails the memory pass instead of reporting zeros. Sampling noise and allocations before client creation remain limitations. ## Contributing diff --git a/ts-perf/packages/api/src/model/sample.ts b/ts-perf/packages/api/src/model/sample.ts index c3ca5a7c..b266ee50 100644 --- a/ts-perf/packages/api/src/model/sample.ts +++ b/ts-perf/packages/api/src/model/sample.ts @@ -169,7 +169,6 @@ export const apiMemoryMetrics = { apiServerProcesses: { metric: "API server processes", unit: " processes", precision: 0 }, apiServerAllocations: { metric: "API server Go allocated (exact cumulative, sum)", unit: "B", precision: 0 }, apiServerHeapUsed: { metric: "API server heap at close (sum of process maxima)", unit: "B", precision: 0 }, - apiServerPeakRSS: { metric: "API server lifetime peak RSS (max process)", unit: "B", precision: 0 }, } as const; export type ApiMemorySampleKey = keyof typeof apiMemoryMetrics; diff --git a/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts b/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts index e1363eca..4592b2f0 100644 --- a/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts +++ b/ts-perf/packages/commands/src/benchmark/apiMemory.test.ts @@ -17,7 +17,7 @@ const dump = { peakRSSBytes: 200, sampling: { id: "window-one", intervalBytes: 32768, allocatedBytes: 300 }, }, - server: { pid: 201, heapAllocBytes: 400, totalAllocBytes: 500, peakRSSBytes: 600 }, + server: { pid: 201, heapAllocBytes: 400, totalAllocBytes: 500 }, }; async function withDumps(dumps: unknown[], callback: (directory: string) => Promise) { @@ -62,7 +62,6 @@ test("deduplicates shared processes/windows, sums disjoint windows, and never su apiServerProcesses: 2, apiServerAllocations: 1200, apiServerHeapUsed: 800, - apiServerPeakRSS: 600, }); }); }); @@ -84,7 +83,9 @@ test.each([ { ...dump, server: { ...dump.server, pid: 1.5 } }, { ...dump, client: { ...dump.client, sampling: undefined } }, { ...dump, client: { ...dump.client, sampling: { ...dump.client.sampling, id: "" } } }, - { ...dump, server: { ...dump.server, peakRSSBytes: undefined } }, + { ...dump, client: { ...dump.client, peakRSSBytes: undefined } }, + { ...dump, server: { ...dump.server, heapAllocBytes: undefined } }, + { ...dump, server: { ...dump.server, totalAllocBytes: undefined } }, ])("rejects missing/unsupported memory telemetry: %j", async value => { await withDumps([value], async directory => { await expect(readApiMemoryDumps(directory)).rejects.toThrow(); @@ -124,12 +125,22 @@ test("rejects inconsistent sampling identity and aggregate overflow", async () = test("preserves memory scalars through report serialization, merging and comparisons", async () => { await withDumps([dump], async directory => { const sample = await readApiMemoryDumps(directory); + expect(Object.keys(sample)).toEqual([ + "apiClientProcesses", + "apiClientSampledAllocations", + "apiClientHeapUsed", + "apiClientPeakRSS", + "apiServerProcesses", + "apiServerAllocations", + "apiServerHeapUsed", + ]); const values = Object.fromEntries(apiMemorySampleKeys.map(key => { const { metric, unit, precision } = apiMemoryMetrics[key]; return [key, computeMetrics([sample[key], sample[key]], metric, unit, precision)]; })); const measurement = new Measurement("api-fixture", 0, undefined, 0, values); const report = Benchmark.create({ measurements: [measurement], iterations: 2 }); + expect(JSON.stringify(report)).not.toContain("apiServerPeakRSS"); const restored = Benchmark.parse(JSON.stringify(report)); expect(restored.measurements[0].values).toEqual(values); const merged = measurement.merge(restored.measurements[0]); @@ -148,6 +159,7 @@ test("preserves memory scalars through report serialization, merging and compari printBenchmark(restored, { format, color: false }, out); printComparison(restored.diff(report)!, { format, color: false }, out); for (const { metric } of Object.values(apiMemoryMetrics)) expect(text).toContain(metric); + expect(text).not.toContain("API server lifetime peak RSS"); } }); }); diff --git a/ts-perf/packages/commands/src/benchmark/apiMemory.ts b/ts-perf/packages/commands/src/benchmark/apiMemory.ts index 36daedb4..4a008ddd 100644 --- a/ts-perf/packages/commands/src/benchmark/apiMemory.ts +++ b/ts-perf/packages/commands/src/benchmark/apiMemory.ts @@ -24,7 +24,7 @@ export async function readApiMemoryDumps(directory: string): Promise(); - const servers = new Map(); + const servers = new Map(); const windows = new Map(); for (const file of files) { const dump = JSON.parse(await fs.readFile(path.join(directory, file), "utf8")); @@ -35,7 +35,6 @@ export async function readApiMemoryDumps(directory: string): Promise sum + server.allocated, 0), apiServerHeapUsed: [...servers.values()].reduce((sum, server) => sum + server.heap, 0), - apiServerPeakRSS: Math.max(...[...servers.values()].map(server => server.rss)), }; for (const [key, value] of Object.entries(values)) counter(value, key); return values; diff --git a/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts b/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts index c0e709db..1be8aba8 100644 --- a/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts +++ b/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts @@ -74,7 +74,7 @@ async function fixture(callback: (root: string, scenario: Scenario, host: Host) pid: process.pid, heapUsedBytes: 100, peakRSSBytes: 200, sampling: { id: randomUUID(), intervalBytes: 32768, allocatedBytes: 300 }, }, - server: { pid: process.pid + 1, heapAllocBytes: 400, totalAllocBytes: 500, peakRSSBytes: 600 }, + server: { pid: process.pid + 1, heapAllocBytes: 400, totalAllocBytes: 500 }, }; await fs.writeFile(path.join(process.env.TS_API_MEMORY_DIR, "memory-test.json"), JSON.stringify(dump)); }