diff --git a/ansible/tasks/packages.yml b/ansible/tasks/packages.yml index 618a484b..30b3b1a4 100644 --- a/ansible/tasks/packages.yml +++ b/ansible/tasks/packages.yml @@ -4,6 +4,8 @@ pkg: - ripgrep - unzip + # Used to sandbox the compiler under test; see build/benchmark.yml. + - bubblewrap # https://github.com/microsoft/vscode/blob/08631fab3a63e8439bec69dce08aa5cf95360d48/resources/linux/snap/snapcraft.yaml#L22 - ca-certificates - libasound2 diff --git a/ansible/tasks/pyperf.yml b/ansible/tasks/pyperf.yml index 7fc3c54c..7dbd4edd 100644 --- a/ansible/tasks/pyperf.yml +++ b/ansible/tasks/pyperf.yml @@ -11,11 +11,27 @@ name: pyperf become: yes -- name: Allow any user to sudo pyperf +# The compiler under test runs as the agent user. Giving that user arbitrary pyperf +# arguments would let it run any command as root, e.g. via "pyperf command". +- name: Install system tuning command + ansible.builtin.copy: + dest: /usr/local/sbin/tsperf-tune-system + owner: root + group: root + mode: '0755' + content: | + #!/bin/sh + exec /usr/local/bin/pyperf system tune --affinity {{ isolated_cpus | quote }} + become: yes + +- name: Allow the agent user to tune the system ansible.builtin.copy: dest: /etc/sudoers.d/pyperf owner: root group: root - content: 'ALL ALL=(ALL) NOPASSWD: /usr/local/bin/pyperf' + mode: '0440' + # An empty argument list prevents passing options to the tuning command. + content: | + {{ agent_user }} ALL=(root) NOPASSWD: /usr/local/sbin/tsperf-tune-system "" validate: /usr/sbin/visudo -csf %s become: yes diff --git a/build/benchmark.yml b/build/benchmark.yml index 7eaf4557..6c3bd6b2 100644 --- a/build/benchmark.yml +++ b/build/benchmark.yml @@ -158,6 +158,30 @@ jobs: - template: templates/setup.yml - template: templates/cloneAndBuildBenchmarkRepo.yml # Sets $(BENCH_SCRIPTS), $(TSPERF_EXE) + - bash: | + set -exo pipefail + sudo apt-get update + sudo apt-get install -y bubblewrap + + # Ubuntu 24.04 blocks unprivileged user namespaces unless an AppArmor profile + # allows them. Allow them for bubblewrap only, like Ubuntu's browser profiles. + if [ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns 2>/dev/null)" = "1" ]; then + sudo tee /etc/apparmor.d/bwrap > /dev/null << 'EOF' + abi , + include + + profile bwrap /usr/bin/bwrap flags=(unconfined) { + userns, + } + EOF + sudo apparmor_parser -r /etc/apparmor.d/bwrap + fi + bwrap --ro-bind / / --unshare-user --cap-drop ALL true + target: + commands: restricted + settableVariables: none + displayName: Install compiler build sandbox + - ${{ if eq(parameters.TS_GO, true) }}: - checkout: typescript-go path: typescript-go @@ -200,6 +224,9 @@ jobs: verifySha merge "$actualMergeSha" "$EXPECTED_MERGE_SHA" verifySha base "$actualBaseSha" "$EXPECTED_BASE_SHA" verifySha head "$actualHeadSha" "$EXPECTED_HEAD_SHA" + target: + commands: restricted + settableVariables: none displayName: Verify TypeScript PR snapshot workingDirectory: $(TYPESCRIPT_DIR) env: @@ -221,7 +248,12 @@ jobs: cd $(TYPESCRIPT_DIR) git fetch origin $(OLD_REF) git switch --detach FETCH_HEAD - for i in `git branch -a | grep remote | grep -v HEAD | grep -v main`; do git branch --track ${i#remotes/origin/} $i; done + for i in `git branch -a | grep remote | grep -v HEAD | grep -v main`; do + git branch --track ${i#remotes/origin/} $i + done + target: + commands: restricted + settableVariables: none displayName: Convert TypeScript clone to blobless clone # Only do this if one of the parameters looks like it's going to change the commit range. condition: and(succeeded(), contains(variables['TSPERF_PRESET'], 'commit')) @@ -229,36 +261,108 @@ jobs: - bash: | set -exo pipefail node $(BENCH_SCRIPTS)/setupPipeline.js - displayName: Setup pipeline + target: + commands: restricted + settableVariables: none + displayName: Compute pipeline setup + env: + TSPERF_VARIABLES_FILE: $(Pipeline.Workspace)/setup-variables.json + + - bash: | + set -eo pipefail + node $(BENCH_SCRIPTS)/publishPipelineVariables.js --file $(Pipeline.Workspace)/setup-variables.json --job --output + target: + commands: restricted + settableVariables: + - MATRIX_any + - MATRIX_ts_perf1 + - MATRIX_ts_perf2 + - MATRIX_ts_perf3 + - MATRIX_ts_perf4 + - MATRIX_ts_perf5 + - MATRIX_ts_perf6 + - MATRIX_ts_perf7 + - MATRIX_ts_perf8 + - MATRIX_ts_perf9 + - MATRIX_ts_perf10 + - MATRIX_ts_perf11 + - MATRIX_ts_perf12 + - TSPERF_PROCESS_KINDS + - TSPERF_JOB_NAMES_TSC + - TSPERF_JOB_NAMES_TSSERVER + - TSPERF_JOB_NAMES_LSP + - TSPERF_JOB_NAMES_STARTUP + - TSPERF_JOB_NAMES_API + - TSPERF_PREDICTABLE + - TSPERF_IS_CUSTOM_COMMIT_RANGE + - TSPERF_IS_COMPARISON + - TSPERF_BASELINE_COMMIT + - TSPERF_BASELINE_NAME + - TSPERF_NEW_COMMIT + - TSPERF_NEW_NAME + displayName: Publish pipeline setup name: setupPipeline - bash: | set -exo pipefail git switch --detach $(TSPERF_NEW_COMMIT) + target: + commands: restricted + settableVariables: none displayName: Switch to new commit condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) workingDirectory: $(TYPESCRIPT_DIR) - bash: | set -exo pipefail - node $(BENCH_SCRIPTS)/buildTypeScript.js --outputDir $(ARTIFACTS_DIR)/new + node $(BENCH_SCRIPTS)/buildTypeScript.js --outputDir $(ARTIFACTS_DIR)/new \ + --variablesFile $(Pipeline.Workspace)/new-variables.json + target: + commands: restricted + settableVariables: none displayName: Build new TypeScript condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) workingDirectory: $(TYPESCRIPT_DIR) + + - bash: | + set -eo pipefail + node $(BENCH_SCRIPTS)/publishPipelineVariables.js --file $(Pipeline.Workspace)/new-variables.json --output + target: + commands: restricted + settableVariables: + - TYPESCRIPT_COMMIT + displayName: Publish new compiler commit + condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) name: buildTypeScriptNew - bash: | set -exo pipefail git switch --detach $(TSPERF_BASELINE_COMMIT) + target: + commands: restricted + settableVariables: none displayName: Switch to baseline commit condition: and(succeeded(), not(eq(variables['TSPERF_BASELINE_COMMIT'], 'HEAD'))) workingDirectory: $(TYPESCRIPT_DIR) - bash: | set -exo pipefail - node $(BENCH_SCRIPTS)/buildTypeScript.js --baseline --outputDir $(ARTIFACTS_DIR)/baseline + node $(BENCH_SCRIPTS)/buildTypeScript.js --baseline --outputDir $(ARTIFACTS_DIR)/baseline \ + --variablesFile $(Pipeline.Workspace)/baseline-variables.json + target: + commands: restricted + settableVariables: none displayName: Build baseline TypeScript workingDirectory: $(TYPESCRIPT_DIR) + + - bash: | + set -eo pipefail + node $(BENCH_SCRIPTS)/publishPipelineVariables.js --file $(Pipeline.Workspace)/baseline-variables.json --output + target: + commands: restricted + settableVariables: + - TYPESCRIPT_COMMIT + displayName: Publish baseline compiler commit name: buildTypeScriptBaseline - publish: $(ARTIFACTS_DIR) @@ -284,6 +388,9 @@ jobs: variables: BUILT_TYPESCRIPT_DIR: $(Pipeline.Workspace)/BuiltTypeScript ARTIFACTS_DIR: $(Pipeline.Workspace)/artifacts + # Scratch directories for the sandboxed compiler; see "Run new/baseline benchmark". + BENCH_ARTIFACTS_DIR: $(Pipeline.Workspace)/bench-artifacts + BENCH_TMP_DIR: $(Pipeline.Workspace)/bench-tmp TSPERF_IS_COMPARISON: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_IS_COMPARISON'] ] TSPERF_PREDICTABLE: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_PREDICTABLE'] ] TYPESCRIPT_COMMIT_NEW: $[ dependencies.Setup.outputs['buildTypeScriptNew.TYPESCRIPT_COMMIT'] ] @@ -300,68 +407,232 @@ jobs: - bash: | set -exo pipefail find $(BUILT_TYPESCRIPT_DIR) -type f \( -name 'tsgo' -o -name 'tsc' \) -exec chmod +x {} + + target: + commands: restricted + settableVariables: none displayName: Make native TypeScript executable - # This is provided by the agent. + # Installed by ansible/tasks/pyperf.yml. - bash: | set -exo pipefail - sudo pyperf system tune --affinity "${TSPERF_AGENT_ISOLATED_CPUS:-$TSPERF_AGENT_BENCHMARK_CPUS}" + sudo -n /usr/local/sbin/tsperf-tune-system + target: + commands: restricted + settableVariables: none displayName: Tune system + # Install hosts before entering the network-isolated benchmark sandbox. + # Read-only mounts protect these binaries from writes by the compiler. - bash: | set -exo pipefail node $(BENCH_SCRIPTS)/runTsPerf.js install-hosts + target: + commands: restricted + settableVariables: none displayName: Install hosts retryCountOnTaskFailure: 3 + + # Set up the first benchmark before constructing the sandbox, so its repository + # exists and we can protect its Git metadata. Third-party installers are sandboxed + # separately (see cases/common.sh's run_sandboxed). - bash: | - set -exo pipefail + set -eo pipefail # Special case for self benchmark - export TYPESCRIPT_COMMIT=$(TYPESCRIPT_COMMIT_NEW) - SETUP=$(TSPERF_SCENARIO_CONFIG_DIR)/$(TSPERF_JOB_SCENARIO)/setup.sh - if test -f $SETUP; then - echo "Running $SETUP" - bash $SETUP + if [ "$(TSPERF_IS_COMPARISON)" = "true" ]; then + export TYPESCRIPT_COMMIT="$TYPESCRIPT_COMMIT_NEW" + else + export TYPESCRIPT_COMMIT="$TYPESCRIPT_COMMIT_BASELINE" fi - displayName: Set up scenario for new benchmark - condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) + bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/setupScenario.sh + target: + commands: restricted + settableVariables: none + displayName: Set up scenario for first benchmark retryCountOnTaskFailure: 3 + env: + TYPESCRIPT_COMMIT_NEW: $(TYPESCRIPT_COMMIT_NEW) + TYPESCRIPT_COMMIT_BASELINE: $(TYPESCRIPT_COMMIT_BASELINE) + TSPERF_VARIABLES_FILE: $(Pipeline.Workspace)/first-scenario-variables.json + - bash: | + set -eo pipefail + node $(BENCH_SCRIPTS)/publishPipelineVariables.js --file $(Pipeline.Workspace)/first-scenario-variables.json --job + target: + commands: restricted + settableVariables: + - SCENARIO_REF + displayName: Publish scenario ref for first benchmark + + # Maintainers approve compiler revisions; this sandbox adds defense in depth if + # harmful compiler or dependency behavior is missed during review. + # The compiler runs as the agent account, not a separate low-privilege user. + # Bubblewrap restricts filesystem access to the mounts below, isolates process + # visibility and networking, and ties sandbox lifetime to its parent process. + # Only the runner's allowlisted environment enters the sandbox. Bubblewrap sets + # no_new_privs, preventing privilege elevation through setuid binaries like sudo. + # + # Interleaved host and sandbox runs on a benchmark agent showed no meaningful + # runtime difference. Startup deltas are tiny and in both directions: + # + # host bwrap + # c-tsc/TotalTime 958.5m ± 0% 959.5m ± 1% ~ (p=0.963 n=72) + # s-tsc/TotalTime 8.690 ± 0% 8.680 ± 0% ~ (p=0.821 n=36) + # c-api/TotalTime 790.6m ± 1% 787.8m ± 0% ~ (p=0.099 n=36) + # c-lsp/Req2Geterr 346.0 ± 0% 347.0 ± 0% ~ (p=0.367 n=72) + # s-tsserver/Req2Geterr 2.018k ± 11% 2.089k ± 15% ~ (p=0.831 n=36) + # c-tsgo-startup/ExecutionTime 160.0µ ± 6% 151.0µ ± 4% -5.62% (p=0.000 n=7200) + # s-tsc-startup/ExecutionTime 185.7 ± 0% 185.6 ± 0% -0.03% (p=0.028 n=3600) + # s-tsserver-startup/ExecutionTime 286.6 ± 0% 286.8 ± 0% +0.07% (p=0.001 n=3600) + # c-tsc/MemoryUsed 113.5k ± 0% 113.4k ± 0% ~ (p=0.294 n=72) + # + # Keep a persistent directory bound at /tmp rather than a per-run tmpfs. + # TypeScript calls module.enableCompileCache(), so /tmp holds V8 bytecode for the + # compiler. Discarding it on every run cost about 5% in tsc measurements. - bash: | set -exo pipefail - node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-$(TSPERF_JOB_KIND) \ + mkdir -p "$BENCH_ARTIFACTS_DIR" "$BENCH_TMP_DIR" "$TSPERF_SUITE_DIR" "$HOME/.tsperf" + + # Expose these mounts instead of the entire host filesystem. Files in the + # mounted trees remain readable; this is not a blanket credential boundary. + # Writable binds follow and override read-only binds of their parent paths. + A="--ro-bind /usr /usr --ro-bind /etc /etc --proc /proc --dev /dev" + A="$A --symlink usr/lib /lib --symlink usr/lib64 /lib64" + A="$A --symlink usr/bin /bin --symlink usr/sbin /sbin" + # node, installed by UseNode@1. + A="$A --ro-bind $AGENT_TOOLSDIRECTORY $AGENT_TOOLSDIRECTORY" + # Prevent compiler writes to the installed host runtimes. + A="$A --ro-bind $HOME/.tsperf $HOME/.tsperf" + # ts-perf itself, and the compiler under test. + A="$A --ro-bind $PIPELINE_WORKSPACE/typescript-benchmarking $PIPELINE_WORKSPACE/typescript-benchmarking" + A="$A --ro-bind $BUILT_TYPESCRIPT_DIR $BUILT_TYPESCRIPT_DIR" + # Keep the suite root read-only so repositories cannot be replaced or renamed. + # Individual source trees are writable for scenarios that emit in place, but + # Git metadata must remain read-only: host-side setup uses it on the next run, + # and git clean/reset do not remove hooks or restore repository configuration. + A="$A --ro-bind $TSPERF_SUITE_DIR $TSPERF_SUITE_DIR" + for SCENARIO_DIR in "$TSPERF_SUITE_DIR"/*; do + if [ -d "$SCENARIO_DIR" ] && [ ! -L "$SCENARIO_DIR" ]; then + A="$A --bind $SCENARIO_DIR $SCENARIO_DIR" + if [ -e "$SCENARIO_DIR/.git" ] || [ -L "$SCENARIO_DIR/.git" ]; then + if [ ! -d "$SCENARIO_DIR/.git" ] || [ -L "$SCENARIO_DIR/.git" ]; then + echo "Unexpected Git metadata at $SCENARIO_DIR/.git" >&2 + exit 1 + fi + A="$A --ro-bind $SCENARIO_DIR/.git $SCENARIO_DIR/.git" + fi + fi + done + A="$A --bind $BENCH_ARTIFACTS_DIR $BENCH_ARTIFACTS_DIR" + A="$A --bind $BENCH_TMP_DIR /tmp" + # Unprivileged bubblewrap already does this; make it explicit. + A="$A --unshare-user --cap-drop ALL" + A="$A --unshare-pid --unshare-ipc --unshare-uts --unshare-net" + A="$A --die-with-parent --new-session" + + echo "##vso[task.setvariable variable=BWRAP_ARGS;isReadOnly=true]$A" + target: + commands: restricted + settableVariables: + - BWRAP_ARGS + displayName: Create benchmark sandbox + + - bash: | + set -exo pipefail + bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/runBenchmarkSandbox.sh \ + $(BWRAP_ARGS) node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-$(TSPERF_JOB_KIND) \ --builtDir $(BUILT_TYPESCRIPT_DIR)/new \ - --save $(ARTIFACTS_DIR)/new_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark + --save $(BENCH_ARTIFACTS_DIR)/new_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark + target: + commands: restricted + settableVariables: none displayName: Run new $(TSPERF_JOB_KIND) benchmark condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) - bash: | - set -exo pipefail + set -eo pipefail # Special case for self benchmark - export TYPESCRIPT_COMMIT=$(TYPESCRIPT_COMMIT_BASELINE) - SETUP=$(TSPERF_SCENARIO_CONFIG_DIR)/$(TSPERF_JOB_SCENARIO)/setup.sh - if test -f $SETUP; then - echo "Running $SETUP" - bash $SETUP - fi + export TYPESCRIPT_COMMIT="$TYPESCRIPT_COMMIT_BASELINE" + bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/setupScenario.sh + target: + commands: restricted + settableVariables: none displayName: Set up scenario for baseline benchmark + condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) retryCountOnTaskFailure: 3 + env: + TYPESCRIPT_COMMIT_BASELINE: $(TYPESCRIPT_COMMIT_BASELINE) + TSPERF_VARIABLES_FILE: $(Pipeline.Workspace)/baseline-scenario-variables.json + + - bash: | + set -eo pipefail + node $(BENCH_SCRIPTS)/publishPipelineVariables.js --file $(Pipeline.Workspace)/baseline-scenario-variables.json --job + target: + commands: restricted + settableVariables: + - SCENARIO_REF + displayName: Publish scenario ref for baseline benchmark + condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) - bash: | set -exo pipefail - node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-$(TSPERF_JOB_KIND) \ + bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/runBenchmarkSandbox.sh \ + $(BWRAP_ARGS) node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-$(TSPERF_JOB_KIND) \ --builtDir $(BUILT_TYPESCRIPT_DIR)/baseline \ - --save $(ARTIFACTS_DIR)/baseline_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark + --save $(BENCH_ARTIFACTS_DIR)/baseline_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark + target: + commands: restricted + settableVariables: none displayName: Run baseline $(TSPERF_JOB_KIND) benchmark + # Publish only scheduled filenames, not everything in $(BENCH_ARTIFACTS_DIR). + # This limits artifact selection; it does not authenticate result contents. + - bash: | + set -exo pipefail + + mkdir -p $(ARTIFACTS_DIR) + + copy_result() { + SRC=$(BENCH_ARTIFACTS_DIR)/$1 + DEST=$(ARTIFACTS_DIR)/$1 + # This host-side copy must not follow a result symlink to an otherwise + # inaccessible file. Copy the link itself, then reject nonregular results. + cp --no-dereference --no-target-directory "$SRC" "$DEST" + if [ -L "$DEST" ] || [ ! -f "$DEST" ]; then + rm -f "$DEST" + echo "Unexpected benchmark result at $SRC" >&2 + exit 1 + fi + } + + if [ "$(TSPERF_IS_COMPARISON)" = "true" ]; then + copy_result new_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark + fi + copy_result baseline_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark + target: + commands: restricted + settableVariables: none + displayName: Collect benchmark results + - bash: | set -exo pipefail node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-$(TSPERF_JOB_KIND) \ --builtDir $(BUILT_TYPESCRIPT_DIR)/baseline \ --baseline $(ARTIFACTS_DIR)/baseline_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark \ --load $(ARTIFACTS_DIR)/new_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark + target: + commands: restricted + settableVariables: none displayName: Compare new to baseline benchmark condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) + - bash: | + set -eo pipefail + node $(BENCH_SCRIPTS)/validateBenchmarkResults.js artifacts + target: + commands: restricted + settableVariables: none + displayName: Validate benchmark artifacts + - publish: $(ARTIFACTS_DIR) artifact: PartialBenchmark_$(TSPERF_JOB_NAME) displayName: Publish benchmarks @@ -383,8 +654,14 @@ jobs: clean: all # Always start with a clean slate. variables: + GH_TOKEN: '' TSPERF_IS_COMPARISON: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_IS_COMPARISON'] ] TSPERF_PROCESS_KINDS: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_PROCESS_KINDS'] ] + TSPERF_JOB_NAMES_TSC: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_JOB_NAMES_TSC'] ] + TSPERF_JOB_NAMES_TSSERVER: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_JOB_NAMES_TSSERVER'] ] + TSPERF_JOB_NAMES_LSP: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_JOB_NAMES_LSP'] ] + TSPERF_JOB_NAMES_STARTUP: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_JOB_NAMES_STARTUP'] ] + TSPERF_JOB_NAMES_API: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_JOB_NAMES_API'] ] TSPERF_NEW_NAME: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_NEW_NAME'] ] TSPERF_BASELINE_NAME: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_BASELINE_NAME'] ] BUILT_TYPESCRIPT_DIR: $(Pipeline.Workspace)/BuiltTypeScript @@ -394,21 +671,15 @@ jobs: steps: - template: templates/setup.yml - - download: current - patterns: '**/*.benchmark' + # Restrict the artifact namespace as well as the extension. Compiler builds may + # contain arbitrary files, but only scheduled partial results belong in the merge. + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + itemPattern: 'PartialBenchmark_*/**/*.benchmark' + targetPath: $(Pipeline.Workspace) displayName: Download benchmarks - # Azure Pipelines haphazardly drops all artifacts into $(Pipeline.Workspace) with no - # way to customize where they're placed. To deal with this, we download the benchmarks - # first and move them somewhere else, before doing any other steps that would download - # further artifacts. - - bash: | - set -exo pipefail - shopt -s globstar - mkdir -p $(BENCHMARKS_DIR) - mv $(Pipeline.Workspace)/**/*.benchmark $(BENCHMARKS_DIR) - displayName: Move artifacts - # We only need this because ts-perf requires it even for loading existing benchmarks - download: current artifact: BuiltTypeScript @@ -416,37 +687,48 @@ jobs: - template: templates/cloneAndBuildBenchmarkRepo.yml # Sets $(BENCH_SCRIPTS), $(TSPERF_EXE) - - task: AzureCLI@2 - inputs: - azureSubscription: $(azureSubscription) - scriptType: bash - scriptLocation: inlineScript - inlineScript: node $(CREATE_TOKEN_SCRIPT) - displayName: Create GitHub App token - env: - APP_CLIENT_ID: $(TYPESCRIPT_AUTOMATION_GITHUB_APP_CLIENT_ID) - KEY_ID: $(TYPESCRIPT_AUTOMATION_GITHUB_APP_KEY_ID) - OWNER: microsoft - REPOSITORIES: $(REPO) - PERMISSIONS: contents:read,issues:write,pull_requests:write - OUTPUT: azure-pipelines - AZURE_TOKEN_VARIABLE: GH_TOKEN + - bash: | + set -eo pipefail + node $(BENCH_SCRIPTS)/collectBenchmarkResults.js + target: + commands: restricted + settableVariables: none + displayName: Collect scheduled benchmark results - bash: | set -exo pipefail mkdir -p $(MERGED_DIR) + target: + commands: restricted + settableVariables: none displayName: Create merged output directory - bash: | - set -exo pipefail + set -eo pipefail for KIND in $(TSPERF_PROCESS_KINDS); do echo "Merging ${KIND} benchmarks" - node $(TSPERF_EXE) merge --output $(MERGED_DIR)/baseline.${KIND}.benchmark $(BENCHMARKS_DIR)/baseline_*.${KIND}.benchmark + # Merge scheduled results by name rather than globbing downloaded files. + # Unexpected artifact files must not become additional report inputs. + JOB_NAMES_VAR="TSPERF_JOB_NAMES_${KIND^^}" + JOB_NAMES="${!JOB_NAMES_VAR}" + if [[ -z "${JOB_NAMES}" ]]; then + echo "No job names for ${KIND} benchmarks" >&2 + exit 1 + fi + + BASELINE_FILES=() + NEW_FILES=() + for JOB_NAME in ${JOB_NAMES}; do + BASELINE_FILES+=("$(BENCHMARKS_DIR)/baseline_${JOB_NAME}.${KIND}.benchmark") + NEW_FILES+=("$(BENCHMARKS_DIR)/new_${JOB_NAME}.${KIND}.benchmark") + done + + node $(TSPERF_EXE) merge --output $(MERGED_DIR)/baseline.${KIND}.benchmark "${BASELINE_FILES[@]}" if [[ $(TSPERF_IS_COMPARISON) == "true" ]]; then - node $(TSPERF_EXE) merge --output $(MERGED_DIR)/new.${KIND}.benchmark $(BENCHMARKS_DIR)/new_*.${KIND}.benchmark + node $(TSPERF_EXE) merge --output $(MERGED_DIR)/new.${KIND}.benchmark "${NEW_FILES[@]}" # Print summary for logs node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-${KIND} \ @@ -457,8 +739,8 @@ jobs: echo "

${KIND}

" >> $(Pipeline.Workspace)/comment.html node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-${KIND} \ --builtDir $(BUILT_TYPESCRIPT_DIR)/baseline \ - --baseline $(MERGED_DIR)/baseline.${KIND}.benchmark --baselineName $(TSPERF_BASELINE_NAME) \ - --load $(MERGED_DIR)/new.${KIND}.benchmark --benchmarkName $(TSPERF_NEW_NAME) \ + --baseline $(MERGED_DIR)/baseline.${KIND}.benchmark --baselineName "$TSPERF_BASELINE_NAME" \ + --load $(MERGED_DIR)/new.${KIND}.benchmark --benchmarkName "$TSPERF_NEW_NAME" \ --format html-fragment \ --quiet >> $(Pipeline.Workspace)/comment.html else @@ -472,15 +754,53 @@ jobs: --quiet >> $(Pipeline.Workspace)/comment.html fi done + target: + commands: restricted + settableVariables: none displayName: Merge benchmarks + env: + TSPERF_NEW_NAME: $(TSPERF_NEW_NAME) + TSPERF_BASELINE_NAME: $(TSPERF_BASELINE_NAME) + + - bash: | + set -eo pipefail + node $(BENCH_SCRIPTS)/validateBenchmarkResults.js processed + target: + commands: restricted + settableVariables: none + displayName: Validate processed results - publish: $(MERGED_DIR) artifact: Benchmarks displayName: Publish benchmark artifacts + - task: AzureCLI@2 + inputs: + azureSubscription: $(azureSubscription) + scriptType: bash + scriptLocation: inlineScript + inlineScript: node $(CREATE_TOKEN_SCRIPT) + target: + commands: restricted + settableVariables: + - GH_TOKEN + displayName: Create GitHub App token + condition: and(succeeded(), eq(variables['IS_PR'], 'true')) + env: + APP_CLIENT_ID: $(TYPESCRIPT_AUTOMATION_GITHUB_APP_CLIENT_ID) + KEY_ID: $(TYPESCRIPT_AUTOMATION_GITHUB_APP_KEY_ID) + OWNER: microsoft + REPOSITORIES: $(REPO) + PERMISSIONS: pull_requests:write + OUTPUT: azure-pipelines + AZURE_TOKEN_VARIABLE: GH_TOKEN + - bash: | set -exo pipefail node $(BENCH_SCRIPTS)/postPerfResult.js --fragment $(Pipeline.Workspace)/comment.html + target: + commands: restricted + settableVariables: none displayName: Publish PR comment condition: and(succeeded(), eq(variables['IS_PR'], 'true')) env: @@ -490,14 +810,92 @@ jobs: STATUS_COMMENT: ${{ parameters.STATUS_COMMENT }} GH_TOKEN: $(GH_TOKEN) - # Sets $(TSPERF_BLOB_LATEST) + - script: node $(CREATE_TOKEN_SCRIPT) + target: + commands: restricted + settableVariables: none + displayName: Revoke GitHub App token + condition: and(always(), ne(variables['GH_TOKEN'], '')) + env: + REVOKE_TOKEN: $(GH_TOKEN) + + # Keep blob-write credentials out of ProcessResults, which parses and renders results + # produced while running the approved compiler revisions. Upload eligibility comes from + # Setup, not runtime variables in ProcessResults. ProcessResults still supplies the merged + # data and must succeed before this job runs; separation does not authenticate that data. + - job: UploadResults + dependsOn: + - Setup + - ProcessResults + condition: and(succeeded(), eq(variables['SHOULD_UPLOAD'], 'true')) + pool: + name: TypeScript-1ES-Deploys + demands: + - ImageOverride -equals azure-linux-3 + workspace: + clean: all # Always start with a clean slate. + + variables: + GH_TOKEN: '' + TSPERF_PROCESS_KINDS: $[ dependencies.Setup.outputs['setupPipeline.TSPERF_PROCESS_KINDS'] ] + BUILT_TYPESCRIPT_DIR: $(Pipeline.Workspace)/BuiltTypeScript + # The merged benchmarks, as published by ProcessResults. + MERGED_DIR: $(Pipeline.Workspace)/Benchmarks + + steps: + - template: templates/setup.yml + + - download: current + artifact: Benchmarks + displayName: Download merged benchmarks + + # We only need this because ts-perf requires it even for loading existing benchmarks + - download: current + artifact: BuiltTypeScript + displayName: Download built TypeScript + + - template: templates/cloneAndBuildBenchmarkRepo.yml # Sets $(BENCH_SCRIPTS), $(TSPERF_EXE) + + - task: AzureCLI@2 + inputs: + azureSubscription: $(azureSubscription) + scriptType: bash + scriptLocation: inlineScript + inlineScript: node $(CREATE_TOKEN_SCRIPT) + target: + commands: restricted + settableVariables: + - GH_TOKEN + displayName: Create GitHub App token + env: + APP_CLIENT_ID: $(TYPESCRIPT_AUTOMATION_GITHUB_APP_CLIENT_ID) + KEY_ID: $(TYPESCRIPT_AUTOMATION_GITHUB_APP_KEY_ID) + OWNER: microsoft + REPOSITORIES: $(REPO) + # This job only reads a ref to see if it is the latest commit. + PERMISSIONS: contents:read + OUTPUT: azure-pipelines + AZURE_TOKEN_VARIABLE: GH_TOKEN + - bash: | set -exo pipefail node $(BENCH_SCRIPTS)/checkLatestCommitForRef.js --builtDir $(BUILT_TYPESCRIPT_DIR)/baseline + target: + commands: restricted + settableVariables: none displayName: Check if commit is latest for ref - condition: and(succeeded(), eq(variables['SHOULD_UPLOAD'], 'true')) env: GH_TOKEN: $(GH_TOKEN) + TSPERF_VARIABLES_FILE: $(Pipeline.Workspace)/upload-variables.json + + - bash: | + set -eo pipefail + node $(BENCH_SCRIPTS)/publishPipelineVariables.js --file $(Pipeline.Workspace)/upload-variables.json --job + target: + commands: restricted + settableVariables: + - TSPERF_BLOB_LATEST + displayName: Publish upload metadata - task: AzureCLI@2 inputs: @@ -532,12 +930,17 @@ jobs: --load $(MERGED_DIR)/baseline.${KIND}.benchmark \ --saveBlob ${NAME} done + target: + commands: restricted + settableVariables: none displayName: Upload benchmarks to blob store - condition: and(succeeded(), eq(variables['SHOULD_UPLOAD'], 'true')) - script: node $(CREATE_TOKEN_SCRIPT) + target: + commands: restricted + settableVariables: none displayName: Revoke GitHub App token - condition: always() + condition: and(always(), ne(variables['GH_TOKEN'], '')) env: REVOKE_TOKEN: $(GH_TOKEN) @@ -555,6 +958,9 @@ jobs: workspace: clean: all # Always start with a clean slate. + variables: + GH_TOKEN: '' + steps: - template: templates/setup.yml - template: templates/cloneAndBuildBenchmarkRepo.yml # Sets $(BENCH_SCRIPTS), $(TSPERF_EXE) @@ -565,19 +971,26 @@ jobs: scriptType: bash scriptLocation: inlineScript inlineScript: node $(CREATE_TOKEN_SCRIPT) + target: + commands: restricted + settableVariables: + - GH_TOKEN displayName: Create GitHub App token env: APP_CLIENT_ID: $(TYPESCRIPT_AUTOMATION_GITHUB_APP_CLIENT_ID) KEY_ID: $(TYPESCRIPT_AUTOMATION_GITHUB_APP_KEY_ID) OWNER: microsoft REPOSITORIES: $(REPO) - PERMISSIONS: contents:read,issues:write,pull_requests:write + PERMISSIONS: pull_requests:write OUTPUT: azure-pipelines AZURE_TOKEN_VARIABLE: GH_TOKEN - bash: | set -exo pipefail node $(BENCH_SCRIPTS)/postPerfResult.js --failed + target: + commands: restricted + settableVariables: none displayName: Publish PR comment env: DISTINCT_ID: ${{ parameters.DISTINCT_ID }} @@ -587,7 +1000,10 @@ jobs: GH_TOKEN: $(GH_TOKEN) - script: node $(CREATE_TOKEN_SCRIPT) + target: + commands: restricted + settableVariables: none displayName: Revoke GitHub App token - condition: always() + condition: and(always(), ne(variables['GH_TOKEN'], '')) env: REVOKE_TOKEN: $(GH_TOKEN) diff --git a/build/templates/cloneAndBuildBenchmarkRepo.yml b/build/templates/cloneAndBuildBenchmarkRepo.yml index 16ea8076..d2c9b937 100644 --- a/build/templates/cloneAndBuildBenchmarkRepo.yml +++ b/build/templates/cloneAndBuildBenchmarkRepo.yml @@ -10,6 +10,9 @@ steps: set -exo pipefail cd $(Pipeline.Workspace)/typescript-benchmarking pnpm install + target: + commands: restricted + settableVariables: none displayName: Install typescript-benchmarking retryCountOnTaskFailure: 3 @@ -17,15 +20,26 @@ steps: set -exo pipefail cd $(Pipeline.Workspace)/typescript-benchmarking pnpm build + target: + commands: restricted + settableVariables: none displayName: Build typescript-benchmarking - bash: | - echo "##vso[task.setvariable variable=BENCH_SCRIPTS]$(Pipeline.Workspace)/typescript-benchmarking/scripts/dist" + echo "##vso[task.setvariable variable=BENCH_SCRIPTS;isReadOnly=true]$(Pipeline.Workspace)/typescript-benchmarking/scripts/dist" # TODO(jakebailey): we can resolve this directly from the scripts dir now - echo "##vso[task.setvariable variable=TSPERF_EXE]$(Pipeline.Workspace)/typescript-benchmarking/ts-perf/bin/ts-perf" + echo "##vso[task.setvariable variable=TSPERF_EXE;isReadOnly=true]$(Pipeline.Workspace)/typescript-benchmarking/ts-perf/bin/ts-perf" # TODO(jakebailey): actually create these dirs, move them to more reasonable names, clone external repos as needed - echo "##vso[task.setvariable variable=TSPERF_SCENARIO_CONFIG_DIR]$(Pipeline.Workspace)/typescript-benchmarking/cases/scenarios" - echo "##vso[task.setvariable variable=TSPERF_SUITE_DIR]$(Pipeline.Workspace)/typescript-benchmarking/cases/solutions" - echo "##vso[task.setvariable variable=CREATE_TOKEN_SCRIPT]$(Pipeline.Workspace)/typescript-benchmarking/scripts/create-github-app-token-from-keyvault.cjs" + echo "##vso[task.setvariable variable=TSPERF_SCENARIO_CONFIG_DIR;isReadOnly=true]$(Pipeline.Workspace)/typescript-benchmarking/cases/scenarios" + echo "##vso[task.setvariable variable=TSPERF_SUITE_DIR;isReadOnly=true]$(Pipeline.Workspace)/typescript-benchmarking/cases/solutions" + echo "##vso[task.setvariable variable=CREATE_TOKEN_SCRIPT;isReadOnly=true]$(Pipeline.Workspace)/typescript-benchmarking/scripts/create-github-app-token-from-keyvault.cjs" + target: + commands: restricted + settableVariables: + - BENCH_SCRIPTS + - TSPERF_EXE + - TSPERF_SCENARIO_CONFIG_DIR + - TSPERF_SUITE_DIR + - CREATE_TOKEN_SCRIPT displayName: Set typescript-benchmarking variables diff --git a/build/templates/setup.yml b/build/templates/setup.yml index 2ff9e395..37976585 100644 --- a/build/templates/setup.yml +++ b/build/templates/setup.yml @@ -16,6 +16,9 @@ steps: - bash: | set -exo pipefail git config --global core.longpaths true + target: + commands: restricted + settableVariables: none displayName: Enable git long paths - script: | @@ -23,4 +26,7 @@ steps: corepack enable corepack enable npm corepack prepare pnpm@latest-11 --activate + target: + commands: restricted + settableVariables: none displayName: 'Setup pnpm' diff --git a/cases/common.sh b/cases/common.sh index ce64b671..3828a05d 100755 --- a/cases/common.sh +++ b/cases/common.sh @@ -11,19 +11,23 @@ function clone_scenario() { SOLUTION_DIR=../../solutions/$SCENARIO_NAME if ! test -d $SOLUTION_DIR; then - git clone --filter blob:none $SCENARIO_REPO $SOLUTION_DIR + git -c core.hooksPath=/dev/null clone --filter blob:none $SCENARIO_REPO $SOLUTION_DIR fi cd $SOLUTION_DIR - git clean -fdx - git reset --hard HEAD + git -c core.hooksPath=/dev/null -c core.fsmonitor=false clean -fdx + git -c core.hooksPath=/dev/null -c core.fsmonitor=false reset --hard HEAD if test -z "$SCENARIO_REF"; then - SCENARIO_REF=$(git rev-parse HEAD) - echo "##vso[task.setvariable variable=SCENARIO_REF]$SCENARIO_REF" + SCENARIO_REF=$(git -c core.hooksPath=/dev/null rev-parse HEAD) + if [ -n "${TSPERF_SCENARIO_REF_FILE:-}" ]; then + printf '%s\n' "$SCENARIO_REF" > "$TSPERF_SCENARIO_REF_FILE" + else + echo "##vso[task.setvariable variable=SCENARIO_REF]$SCENARIO_REF" + fi else - git fetch origin $SCENARIO_REF - git switch --detach FETCH_HEAD + git -c core.hooksPath=/dev/null fetch origin $SCENARIO_REF + git -c core.hooksPath=/dev/null -c core.fsmonitor=false switch --detach FETCH_HEAD fi } @@ -41,6 +45,15 @@ function clone_strada_scenario() { } function run_sandboxed() { + local -a GIT_MOUNT=() + if [ -e "$PWD/.git" ] || [ -L "$PWD/.git" ]; then + if [ ! -d "$PWD/.git" ] || [ -L "$PWD/.git" ]; then + echo "Unexpected Git metadata at $PWD/.git" >&2 + return 1 + fi + GIT_MOUNT=(--volume "$PWD/.git:/sandbox/.git:ro") + fi + USER_DOCKER_SOCK=/run/user/$(id -u)/docker.sock if [ -e $USER_DOCKER_SOCK ]; then @@ -121,13 +134,16 @@ function run_sandboxed() { --name $SANDBOX_CONTAINER \ --rm \ --network $NO_INTERNET \ - --volume $PWD:/sandbox \ + --volume "$PWD:/sandbox" \ + "${GIT_MOUNT[@]}" \ --workdir /sandbox \ --env HTTP_PROXY=$PROXY_ADDR \ --env HTTPS_PROXY=$PROXY_ADDR \ --env YARN_HTTP_PROXY=$PROXY_ADDR \ --env YARN_HTTPS_PROXY=$PROXY_ADDR \ --env COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + --env HUSKY=0 \ + --env HUSKY_SKIP_INSTALL=1 \ --env CHANGE_USER_ID=$CHANGE_USER_ID \ $NODE_IMAGE \ sh -c ' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2935f2be..95778413 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -55,7 +55,7 @@ importers: version: 8.65.0(eslint@10.8.0(supports-color@2.0.0))(supports-color@2.0.0)(typescript@6.0.3) vitest: specifier: ^4.1.10 - version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(yaml@2.9.0)) + version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(yaml@2.9.1)) scripts: devDependencies: @@ -98,6 +98,9 @@ importers: sort-keys: specifier: ^6.0.1 version: 6.0.1 + yaml: + specifier: ^2.9.1 + version: 2.9.1 ts-perf: {} @@ -2105,6 +2108,11 @@ packages: engines: {node: '>= 14.6'} hasBin: true + yaml@2.9.1: + resolution: {integrity: sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==} + engines: {node: '>= 14.6'} + hasBin: true + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -2773,13 +2781,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.1 - '@vitest/mocker@4.1.10(vite@8.1.5(@types/node@22.20.1)(yaml@2.9.0))': + '@vitest/mocker@4.1.10(vite@8.1.5(@types/node@22.20.1)(yaml@2.9.1))': dependencies: '@vitest/spy': 4.1.10 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.1.5(@types/node@22.20.1)(yaml@2.9.0) + vite: 8.1.5(@types/node@22.20.1)(yaml@2.9.1) '@vitest/pretty-format@4.1.10': dependencies: @@ -3857,7 +3865,7 @@ snapshots: util-deprecate@1.0.2: {} - vite@8.1.5(@types/node@22.20.1)(yaml@2.9.0): + vite@8.1.5(@types/node@22.20.1)(yaml@2.9.1): dependencies: lightningcss: 1.33.0 picomatch: 4.0.5 @@ -3867,12 +3875,12 @@ snapshots: optionalDependencies: '@types/node': 22.20.1 fsevents: 2.3.3 - yaml: 2.9.0 + yaml: 2.9.1 - vitest@4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(yaml@2.9.0)): + vitest@4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(yaml@2.9.1)): dependencies: '@vitest/expect': 4.1.10 - '@vitest/mocker': 4.1.10(vite@8.1.5(@types/node@22.20.1)(yaml@2.9.0)) + '@vitest/mocker': 4.1.10(vite@8.1.5(@types/node@22.20.1)(yaml@2.9.1)) '@vitest/pretty-format': 4.1.10 '@vitest/runner': 4.1.10 '@vitest/snapshot': 4.1.10 @@ -3889,7 +3897,7 @@ snapshots: tinyexec: 1.2.4 tinyglobby: 0.2.17 tinyrainbow: 3.1.1 - vite: 8.1.5(@types/node@22.20.1)(yaml@2.9.0) + vite: 8.1.5(@types/node@22.20.1)(yaml@2.9.1) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 22.20.1 @@ -3928,6 +3936,8 @@ snapshots: yaml@2.9.0: {} + yaml@2.9.1: {} + yocto-queue@0.1.0: {} yoctocolors@2.2.0: {} diff --git a/scripts/buildTypeScriptSandbox.sh b/scripts/buildTypeScriptSandbox.sh new file mode 100644 index 00000000..965a55d9 --- /dev/null +++ b/scripts/buildTypeScriptSandbox.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -eo pipefail + +: "${AGENT_TOOLSDIRECTORY:?Expected the agent tool directory}" + +SOURCE_DIR=$1 +BUILD_HOME=$2 +BUILD_TMP=$3 +shift 3 + +if [ ! -d "$SOURCE_DIR/.git" ] || [ -L "$SOURCE_DIR/.git" ]; then + echo "Unexpected Git metadata in compiler checkout" >&2 + exit 1 +fi + +# Builds need network access for public package/module downloads. Expose the compiler +# checkout and scratch directories, not the agent's home or benchmarking checkout. +# Clear inherited environment variables; files in the mounted trees remain readable. +ARGS=( + --ro-bind /usr /usr --ro-bind /etc /etc --proc /proc --dev /dev + --symlink usr/lib /lib --symlink usr/lib64 /lib64 + --symlink usr/bin /bin --symlink usr/sbin /sbin + --ro-bind "$AGENT_TOOLSDIRECTORY" "$AGENT_TOOLSDIRECTORY" + --bind "$SOURCE_DIR" "$SOURCE_DIR" + --ro-bind "$SOURCE_DIR/.git" "$SOURCE_DIR/.git" + # Bind /tmp first; the scratch home may live under the host's /tmp and must stay visible. + --bind "$BUILD_TMP" /tmp --bind "$BUILD_HOME" "$BUILD_HOME" + --clearenv --setenv HOME "$BUILD_HOME" --setenv PATH "$PATH" + --setenv CI true --setenv COREPACK_ENABLE_DOWNLOAD_PROMPT 0 + --setenv COREPACK_ENABLE_STRICT 0 --setenv GIT_OPTIONAL_LOCKS 0 + --setenv HUSKY 0 --setenv HUSKY_SKIP_INSTALL 1 + --unshare-user --unshare-pid --unshare-ipc --unshare-uts + --cap-drop ALL --die-with-parent --new-session --chdir "$SOURCE_DIR" +) +# Some images keep resolv.conf's target under /run, which is otherwise not exposed. +RESOLV_CONF=$(readlink -f /etc/resolv.conf) +ARGS+=(--ro-bind "$RESOLV_CONF" "$RESOLV_CONF") +if [ -n "${GOROOT:-}" ]; then + ARGS+=(--setenv GOROOT "$GOROOT") +fi + +exec bwrap "${ARGS[@]}" "$@" diff --git a/scripts/package.json b/scripts/package.json index cb5faea0..373c4e0f 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -21,7 +21,8 @@ "octokit": "^5.0.5", "ora": "^9.4.1", "pretty-ms": "^9.3.0", - "sort-keys": "^6.0.1" + "sort-keys": "^6.0.1", + "yaml": "^2.9.1" }, "scripts": { "build": "tsc", diff --git a/scripts/runBenchmarkSandbox.sh b/scripts/runBenchmarkSandbox.sh new file mode 100644 index 00000000..364ba5d4 --- /dev/null +++ b/scripts/runBenchmarkSandbox.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -eo pipefail + +ARGS=( + --clearenv + --setenv HOME "${HOME:?Expected HOME}" + --setenv PATH "${PATH:?Expected PATH}" + --setenv CI true + --setenv TSPERF_EXE "${TSPERF_EXE:?Expected TSPERF_EXE}" + --setenv TSPERF_SCENARIO_CONFIG_DIR "${TSPERF_SCENARIO_CONFIG_DIR:?Expected TSPERF_SCENARIO_CONFIG_DIR}" + --setenv TSPERF_SUITE_DIR "${TSPERF_SUITE_DIR:?Expected TSPERF_SUITE_DIR}" + --setenv TSPERF_JOB_HOST "${TSPERF_JOB_HOST:?Expected TSPERF_JOB_HOST}" + --setenv TSPERF_JOB_SCENARIO "${TSPERF_JOB_SCENARIO:?Expected TSPERF_JOB_SCENARIO}" + --setenv TSPERF_JOB_ITERATIONS "${TSPERF_JOB_ITERATIONS:?Expected TSPERF_JOB_ITERATIONS}" + --setenv TSPERF_JOB_WARMUPS "${TSPERF_JOB_WARMUPS:?Expected TSPERF_JOB_WARMUPS}" + --setenv TSPERF_PREDICTABLE "${TSPERF_PREDICTABLE:?Expected TSPERF_PREDICTABLE}" + --setenv TSPERF_AGENT_BENCHMARK_CPUS "${TSPERF_AGENT_BENCHMARK_CPUS:?Expected TSPERF_AGENT_BENCHMARK_CPUS}" +) +if [ -v LANG ]; then + ARGS+=(--setenv LANG "$LANG") +fi +if [ -v LC_ALL ]; then + ARGS+=(--setenv LC_ALL "$LC_ALL") +fi +if [ -v LC_CTYPE ]; then + ARGS+=(--setenv LC_CTYPE "$LC_CTYPE") +fi +if [ -v TZ ]; then + ARGS+=(--setenv TZ "$TZ") +fi + +exec bwrap "${ARGS[@]}" "$@" diff --git a/scripts/setupScenario.sh b/scripts/setupScenario.sh new file mode 100644 index 00000000..5a3df5ae --- /dev/null +++ b/scripts/setupScenario.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -eo pipefail + +: "${TSPERF_VARIABLES_FILE:?Expected a scenario variables file}" + +if [[ ! "${TYPESCRIPT_COMMIT:-}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Unexpected compiler commit for scenario setup" >&2 + exit 1 +fi + +# This handoff stays outside the solution mounted into third-party installers. +SCENARIO_REF_FILE=$(mktemp) +trap 'rm -f "$SCENARIO_REF_FILE"' EXIT +export TSPERF_SCENARIO_REF_FILE=$SCENARIO_REF_FILE + +SETUP=$TSPERF_SCENARIO_CONFIG_DIR/$TSPERF_JOB_SCENARIO/setup.sh +if test -f "$SETUP"; then + echo "Running $SETUP" + bash "$SETUP" +fi + +# The pipeline-controlled setup script hands off a commit ID outside the installer mount. +# Validate its format before handing it to the separate variable publisher. +if test -s "$SCENARIO_REF_FILE"; then + SCENARIO_REF=$(cat "$SCENARIO_REF_FILE") + if [[ ! "$SCENARIO_REF" =~ ^[0-9a-f]{40}$ ]]; then + echo "Unexpected scenario commit from setup" >&2 + exit 1 + fi + printf '{"SCENARIO_REF":"%s"}\n' "$SCENARIO_REF" > "$TSPERF_VARIABLES_FILE" +else + printf '{}\n' > "$TSPERF_VARIABLES_FILE" +fi diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap index b1239892..676a35d0 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "api_node_24_21_0_typescript_eslint_api_floating api_node_24_21_0_typescript_eslint_api_recommended api_node_24_21_0_vscode_languageserver_api_metamodel", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline/outputVariables.snap index 5015af22..dbd3e9a4 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "false", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_node_18_15_0_tsc_startup startup_node_18_15_0_tsserver_startup startup_node_18_15_0_tsserverlibrary_startup startup_node_18_15_0_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_15_0_Compiler_Unions tsc_node_18_15_0_angular_1 tsc_node_18_15_0_arktype tsc_node_18_15_0_mui_docs_1 tsc_node_18_15_0_self_build_src_public_api tsc_node_18_15_0_sentry tsc_node_18_15_0_social_app tsc_node_18_15_0_strada_build_src tsc_node_18_15_0_strada_compiler tsc_node_18_15_0_t3code_server tsc_node_18_15_0_ts_pre_modules tsc_node_18_15_0_vscode tsc_node_18_15_0_webpack_1 tsc_node_18_15_0_xstate_main_1 tsc_node_20_5_1_Compiler_Unions tsc_node_20_5_1_angular_1 tsc_node_20_5_1_arktype tsc_node_20_5_1_mui_docs_1 tsc_node_20_5_1_self_build_src_public_api tsc_node_20_5_1_sentry tsc_node_20_5_1_social_app tsc_node_20_5_1_strada_build_src tsc_node_20_5_1_strada_compiler tsc_node_20_5_1_t3code_server tsc_node_20_5_1_ts_pre_modules tsc_node_20_5_1_vscode tsc_node_20_5_1_webpack_1 tsc_node_20_5_1_xstate_main_1", + "TSPERF_JOB_NAMES_TSSERVER": "tsserver_node_18_15_0_CompilerTSServer tsserver_node_18_15_0_Compiler_UnionsTSServer tsserver_node_18_15_0_xstate_main_1_tsserver", "TSPERF_NEW_COMMIT": "", "TSPERF_NEW_NAME": "", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline_tsgo/outputVariables.snap index 05dc85c8..4d61e5b1 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/baseline_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "false", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "lsp_native_CompilerLSP lsp_native_Compiler_UnionsLSP lsp_native_xstate_main_1_LSP", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "", "TSPERF_NEW_NAME": "", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun/outputVariables.snap index f7537893..d588e95c 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_bun_1_1_24_tsc_startup startup_bun_1_1_24_tsserverlibrary_startup startup_bun_1_1_24_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_bun_1_1_24_Compiler_Unions tsc_bun_1_1_24_angular_1 tsc_bun_1_1_24_arktype tsc_bun_1_1_24_mui_docs tsc_bun_1_1_24_self_build_src_public_api tsc_bun_1_1_24_sentry tsc_bun_1_1_24_social_app tsc_bun_1_1_24_strada_build_src tsc_bun_1_1_24_strada_compiler tsc_bun_1_1_24_t3code_server tsc_bun_1_1_24_ts_pre_modules tsc_bun_1_1_24_vscode tsc_bun_1_1_24_webpack tsc_bun_1_1_24_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun_tsgo/outputVariables.snap index acfc7898..e98883fe 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/bun_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/default/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/default/outputVariables.snap index b1048d38..de8723bd 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/default/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/default/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_node_18_15_0_tsc_startup startup_node_18_15_0_tsserver_startup startup_node_18_15_0_tsserverlibrary_startup startup_node_18_15_0_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_15_0_Compiler_Unions tsc_node_18_15_0_angular_1 tsc_node_18_15_0_arktype tsc_node_18_15_0_mui_docs tsc_node_18_15_0_self_build_src_public_api tsc_node_18_15_0_sentry tsc_node_18_15_0_social_app tsc_node_18_15_0_strada_build_src tsc_node_18_15_0_strada_compiler tsc_node_18_15_0_t3code_server tsc_node_18_15_0_ts_pre_modules tsc_node_18_15_0_vscode tsc_node_18_15_0_webpack tsc_node_18_15_0_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "tsserver_node_18_15_0_CompilerTSServer tsserver_node_18_15_0_Compiler_UnionsTSServer tsserver_node_18_15_0_xstate_main_1_tsserver", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/default_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/default_tsgo/outputVariables.snap index 31e00fe8..ddf1f55e 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/default_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/default_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "lsp_native_CompilerLSP lsp_native_Compiler_UnionsLSP lsp_native_xstate_main_1_LSP", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4/outputVariables.snap index fb0b08db..ba4d9fa0 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "release-5.3", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "true", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_15_0_Compiler_Unions tsc_node_18_15_0_angular_1 tsc_node_18_15_0_arktype tsc_node_18_15_0_mui_docs tsc_node_18_15_0_self_build_src_public_api tsc_node_18_15_0_sentry tsc_node_18_15_0_social_app tsc_node_18_15_0_strada_build_src tsc_node_18_15_0_strada_compiler tsc_node_18_15_0_t3code_server tsc_node_18_15_0_ts_pre_modules tsc_node_18_15_0_vscode tsc_node_18_15_0_webpack tsc_node_18_15_0_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "6ea273cdcca99db809074d2b2d38d0e5b59ee81b", "TSPERF_NEW_NAME": "release-5.4", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4_tsgo/outputVariables.snap index 9f325432..d83835ca 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster commits=release-5.3...release-5.4_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "release-5.3", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "true", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "6ea273cdcca99db809074d2b2d38d0e5b59ee81b", "TSPERF_NEW_NAME": "release-5.4", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1/outputVariables.snap index 426df8d3..5e7b4213 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_5_1_Compiler_Unions tsc_node_18_5_1_angular_1 tsc_node_18_5_1_arktype tsc_node_18_5_1_mui_docs tsc_node_18_5_1_self_build_src_public_api tsc_node_18_5_1_sentry tsc_node_18_5_1_social_app tsc_node_18_5_1_strada_build_src tsc_node_18_5_1_strada_compiler tsc_node_18_5_1_t3code_server tsc_node_18_5_1_ts_pre_modules tsc_node_18_5_1_vscode tsc_node_18_5_1_webpack tsc_node_18_5_1_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1_tsgo/outputVariables.snap index 5fe38583..e8a35191 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster predictable host=node@18.5.1_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster/outputVariables.snap index fa704cb4..41b56992 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_15_0_Compiler_Unions tsc_node_18_15_0_angular_1 tsc_node_18_15_0_arktype tsc_node_18_15_0_mui_docs tsc_node_18_15_0_self_build_src_public_api tsc_node_18_15_0_sentry tsc_node_18_15_0_social_app tsc_node_18_15_0_strada_build_src tsc_node_18_15_0_strada_compiler tsc_node_18_15_0_t3code_server tsc_node_18_15_0_ts_pre_modules tsc_node_18_15_0_vscode tsc_node_18_15_0_webpack tsc_node_18_15_0_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster_tsgo/outputVariables.snap index 4f56d1d0..a0026a60 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/faster_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/full/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/full/outputVariables.snap index 5d198cf2..ce5df0b0 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/full/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/full/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_node_18_15_0_tsc_startup startup_node_18_15_0_tsserver_startup startup_node_18_15_0_tsserverlibrary_startup startup_node_18_15_0_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_15_0_Compiler_Unions tsc_node_18_15_0_angular_1 tsc_node_18_15_0_arktype tsc_node_18_15_0_mui_docs tsc_node_18_15_0_self_build_src_public_api tsc_node_18_15_0_sentry tsc_node_18_15_0_social_app tsc_node_18_15_0_strada_build_src tsc_node_18_15_0_strada_compiler tsc_node_18_15_0_t3code_server tsc_node_18_15_0_ts_pre_modules tsc_node_18_15_0_vscode tsc_node_18_15_0_webpack tsc_node_18_15_0_xstate_main tsc_node_20_5_1_Compiler_Unions tsc_node_20_5_1_angular_1 tsc_node_20_5_1_arktype tsc_node_20_5_1_mui_docs tsc_node_20_5_1_self_build_src_public_api tsc_node_20_5_1_sentry tsc_node_20_5_1_social_app tsc_node_20_5_1_strada_build_src tsc_node_20_5_1_strada_compiler tsc_node_20_5_1_t3code_server tsc_node_20_5_1_ts_pre_modules tsc_node_20_5_1_vscode tsc_node_20_5_1_webpack tsc_node_20_5_1_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "tsserver_node_18_15_0_CompilerTSServer tsserver_node_18_15_0_Compiler_UnionsTSServer tsserver_node_18_15_0_xstate_main_1_tsserver", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/full_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/full_tsgo/outputVariables.snap index 0735bcfd..e9eb63b9 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/full_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/full_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "lsp_native_CompilerLSP lsp_native_Compiler_UnionsLSP lsp_native_xstate_main_1_LSP", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1/outputVariables.snap index 36435eaa..00fc8e9f 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_bun_1_1_3_tsc_startup startup_bun_1_1_3_tsserver_startup startup_bun_1_1_3_tsserverlibrary_startup startup_bun_1_1_3_typescript_startup startup_vscode_1_88_1_tsc_startup startup_vscode_1_88_1_tsserver_startup startup_vscode_1_88_1_tsserverlibrary_startup startup_vscode_1_88_1_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_bun_1_1_3_Compiler_Unions tsc_bun_1_1_3_angular_1 tsc_bun_1_1_3_arktype tsc_bun_1_1_3_mui_docs tsc_bun_1_1_3_self_build_src_public_api tsc_bun_1_1_3_sentry tsc_bun_1_1_3_social_app tsc_bun_1_1_3_strada_build_src tsc_bun_1_1_3_strada_compiler tsc_bun_1_1_3_t3code_server tsc_bun_1_1_3_ts_pre_modules tsc_bun_1_1_3_vscode tsc_bun_1_1_3_webpack tsc_bun_1_1_3_xstate_main tsc_vscode_1_88_1_Compiler_Unions tsc_vscode_1_88_1_angular_1 tsc_vscode_1_88_1_arktype tsc_vscode_1_88_1_mui_docs tsc_vscode_1_88_1_self_build_src_public_api tsc_vscode_1_88_1_sentry tsc_vscode_1_88_1_social_app tsc_vscode_1_88_1_strada_build_src tsc_vscode_1_88_1_strada_compiler tsc_vscode_1_88_1_t3code_server tsc_vscode_1_88_1_ts_pre_modules tsc_vscode_1_88_1_vscode tsc_vscode_1_88_1_webpack tsc_vscode_1_88_1_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "tsserver_bun_1_1_3_CompilerTSServer tsserver_bun_1_1_3_Compiler_UnionsTSServer tsserver_bun_1_1_3_xstate_main_1_tsserver tsserver_vscode_1_88_1_CompilerTSServer tsserver_vscode_1_88_1_Compiler_UnionsTSServer tsserver_vscode_1_88_1_xstate_main_1_tsserver", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1_tsgo/outputVariables.snap index 31e00fe8..ddf1f55e 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/hosts=bun@1.1.3,vscode@1.88.1_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "lsp_native_CompilerLSP lsp_native_Compiler_UnionsLSP lsp_native_xstate_main_1_LSP", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true/outputVariables.snap index 2d9c0b29..135496a8 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_node_18_15_0_tsc_startup startup_node_18_15_0_tsserver_startup startup_node_18_15_0_tsserverlibrary_startup startup_node_18_15_0_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_15_0_Compiler_Unions tsc_node_18_15_0_angular_1 tsc_node_18_15_0_arktype tsc_node_18_15_0_mui_docs tsc_node_18_15_0_self_build_src_public_api tsc_node_18_15_0_sentry tsc_node_18_15_0_social_app tsc_node_18_15_0_strada_build_src tsc_node_18_15_0_strada_compiler tsc_node_18_15_0_t3code_server tsc_node_18_15_0_ts_pre_modules tsc_node_18_15_0_vscode tsc_node_18_15_0_webpack tsc_node_18_15_0_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "tsserver_node_18_15_0_CompilerTSServer tsserver_node_18_15_0_Compiler_UnionsTSServer tsserver_node_18_15_0_xstate_main_1_tsserver", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true_tsgo/outputVariables.snap index 60dcb96d..44dcb927 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/predictable=true_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "lsp_native_CompilerLSP lsp_native_Compiler_UnionsLSP lsp_native_xstate_main_1_LSP", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "true", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/public/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/public/outputVariables.snap index b1048d38..de8723bd 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/public/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/public/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_node_18_15_0_tsc_startup startup_node_18_15_0_tsserver_startup startup_node_18_15_0_tsserverlibrary_startup startup_node_18_15_0_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_15_0_Compiler_Unions tsc_node_18_15_0_angular_1 tsc_node_18_15_0_arktype tsc_node_18_15_0_mui_docs tsc_node_18_15_0_self_build_src_public_api tsc_node_18_15_0_sentry tsc_node_18_15_0_social_app tsc_node_18_15_0_strada_build_src tsc_node_18_15_0_strada_compiler tsc_node_18_15_0_t3code_server tsc_node_18_15_0_ts_pre_modules tsc_node_18_15_0_vscode tsc_node_18_15_0_webpack tsc_node_18_15_0_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "tsserver_node_18_15_0_CompilerTSServer tsserver_node_18_15_0_Compiler_UnionsTSServer tsserver_node_18_15_0_xstate_main_1_tsserver", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/public_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/public_tsgo/outputVariables.snap index 31e00fe8..ddf1f55e 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/public_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/public_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "lsp_native_CompilerLSP lsp_native_Compiler_UnionsLSP lsp_native_xstate_main_1_LSP", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular/outputVariables.snap index b1048d38..de8723bd 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_node_18_15_0_tsc_startup startup_node_18_15_0_tsserver_startup startup_node_18_15_0_tsserverlibrary_startup startup_node_18_15_0_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_15_0_Compiler_Unions tsc_node_18_15_0_angular_1 tsc_node_18_15_0_arktype tsc_node_18_15_0_mui_docs tsc_node_18_15_0_self_build_src_public_api tsc_node_18_15_0_sentry tsc_node_18_15_0_social_app tsc_node_18_15_0_strada_build_src tsc_node_18_15_0_strada_compiler tsc_node_18_15_0_t3code_server tsc_node_18_15_0_ts_pre_modules tsc_node_18_15_0_vscode tsc_node_18_15_0_webpack tsc_node_18_15_0_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "tsserver_node_18_15_0_CompilerTSServer tsserver_node_18_15_0_Compiler_UnionsTSServer tsserver_node_18_15_0_xstate_main_1_tsserver", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular_tsgo/outputVariables.snap index 31e00fe8..ddf1f55e 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/regular_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "lsp_native_CompilerLSP lsp_native_Compiler_UnionsLSP lsp_native_xstate_main_1_LSP", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0/outputVariables.snap index 3c3f26c6..390b9769 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_node_22_8_0_tsc_startup startup_node_22_8_0_tsserver_startup startup_node_22_8_0_tsserverlibrary_startup startup_node_22_8_0_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0_tsgo/outputVariables.snap index 81bde7a9..1391ef79 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only hosts=node@22.8.0_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only/outputVariables.snap index d55b47c4..21e07f8d 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_node_18_15_0_tsc_startup startup_node_18_15_0_tsserver_startup startup_node_18_15_0_tsserverlibrary_startup startup_node_18_15_0_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only_tsgo/outputVariables.snap index 81bde7a9..1391ef79 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/startup-only_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset/outputVariables.snap index b1048d38..de8723bd 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_node_18_15_0_tsc_startup startup_node_18_15_0_tsserver_startup startup_node_18_15_0_tsserverlibrary_startup startup_node_18_15_0_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_15_0_Compiler_Unions tsc_node_18_15_0_angular_1 tsc_node_18_15_0_arktype tsc_node_18_15_0_mui_docs tsc_node_18_15_0_self_build_src_public_api tsc_node_18_15_0_sentry tsc_node_18_15_0_social_app tsc_node_18_15_0_strada_build_src tsc_node_18_15_0_strada_compiler tsc_node_18_15_0_t3code_server tsc_node_18_15_0_ts_pre_modules tsc_node_18_15_0_vscode tsc_node_18_15_0_webpack tsc_node_18_15_0_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "tsserver_node_18_15_0_CompilerTSServer tsserver_node_18_15_0_Compiler_UnionsTSServer tsserver_node_18_15_0_xstate_main_1_tsserver", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset_tsgo/outputVariables.snap index 31e00fe8..ddf1f55e 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/this is not a preset_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "lsp_native_CompilerLSP lsp_native_Compiler_UnionsLSP lsp_native_xstate_main_1_LSP", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only/outputVariables.snap index fa704cb4..41b56992 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "tsc_node_18_15_0_Compiler_Unions tsc_node_18_15_0_angular_1 tsc_node_18_15_0_arktype tsc_node_18_15_0_mui_docs tsc_node_18_15_0_self_build_src_public_api tsc_node_18_15_0_sentry tsc_node_18_15_0_social_app tsc_node_18_15_0_strada_build_src tsc_node_18_15_0_strada_compiler tsc_node_18_15_0_t3code_server tsc_node_18_15_0_ts_pre_modules tsc_node_18_15_0_vscode tsc_node_18_15_0_webpack tsc_node_18_15_0_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only_tsgo/outputVariables.snap index 4f56d1d0..a0026a60 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsc-only_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only/outputVariables.snap index 8f3f7d20..f3754946 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "", + "TSPERF_JOB_NAMES_TSSERVER": "tsserver_node_18_15_0_CompilerTSServer tsserver_node_18_15_0_Compiler_UnionsTSServer tsserver_node_18_15_0_xstate_main_1_tsserver", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only_tsgo/outputVariables.snap index 74a4133d..0ad13920 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/tsserver-only_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "", + "TSPERF_JOB_NAMES_TSC": "", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode/outputVariables.snap index 0c0f1797..5482dabb 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "", + "TSPERF_JOB_NAMES_STARTUP": "startup_vscode_1_82_1_tsc_startup startup_vscode_1_82_1_tsserver_startup startup_vscode_1_82_1_tsserverlibrary_startup startup_vscode_1_82_1_typescript_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_vscode_1_82_1_Compiler_Unions tsc_vscode_1_82_1_angular_1 tsc_vscode_1_82_1_arktype tsc_vscode_1_82_1_mui_docs tsc_vscode_1_82_1_self_build_src_public_api tsc_vscode_1_82_1_sentry tsc_vscode_1_82_1_social_app tsc_vscode_1_82_1_strada_build_src tsc_vscode_1_82_1_strada_compiler tsc_vscode_1_82_1_t3code_server tsc_vscode_1_82_1_ts_pre_modules tsc_vscode_1_82_1_vscode tsc_vscode_1_82_1_webpack tsc_vscode_1_82_1_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "tsserver_vscode_1_82_1_CompilerTSServer tsserver_vscode_1_82_1_Compiler_UnionsTSServer tsserver_vscode_1_82_1_xstate_main_1_tsserver", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode_tsgo/outputVariables.snap index 31e00fe8..ddf1f55e 100644 --- a/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode_tsgo/outputVariables.snap +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/vscode_tsgo/outputVariables.snap @@ -16,6 +16,11 @@ "TSPERF_BASELINE_NAME": "baseline", "TSPERF_IS_COMPARISON": "true", "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_JOB_NAMES_API": "", + "TSPERF_JOB_NAMES_LSP": "lsp_native_CompilerLSP lsp_native_Compiler_UnionsLSP lsp_native_xstate_main_1_LSP", + "TSPERF_JOB_NAMES_STARTUP": "startup_native_lsp_startup startup_native_tsgo_startup", + "TSPERF_JOB_NAMES_TSC": "tsc_native_Compiler_Unions tsc_native_angular_1 tsc_native_arktype tsc_native_mui_docs tsc_native_sentry tsc_native_social_app tsc_native_strada_build_src tsc_native_strada_compiler tsc_native_t3code_server tsc_native_ts_pre_modules tsc_native_vscode tsc_native_webpack tsc_native_xstate_main", + "TSPERF_JOB_NAMES_TSSERVER": "", "TSPERF_NEW_COMMIT": "HEAD", "TSPERF_NEW_NAME": "pr", "TSPERF_PREDICTABLE": "false", diff --git a/scripts/src/__tests__/apiArtifact.test.ts b/scripts/src/__tests__/apiArtifact.test.ts index b086cfb7..8510a927 100644 --- a/scripts/src/__tests__/apiArtifact.test.ts +++ b/scripts/src/__tests__/apiArtifact.test.ts @@ -106,6 +106,29 @@ test("rejects registry binary fallback", async () => { await expect(f.verify()).rejects.toThrow("Registry binary fallback"); }); +test.each(["typescript", platformName])("rejects symlinks in the %s package before copying", async name => { + const f = await fixture(); + await fs.symlink("/nonexistent", path.join(f.source, "built/npm", name, "link")); + await expect(f.preserve()).rejects.toThrow("Unexpected build output"); + await expect(fs.access(f.output)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +test.each(["typescript", platformName])("rejects logging-command filenames in the %s package", async name => { + const f = await fixture(); + await fs.writeFile(path.join(f.source, "built/npm", name, "##vso[task.debug]output"), ""); + await expect(f.preserve()).rejects.toThrow("Unsafe build output filename"); + await expect(fs.access(f.output)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +test("rejects a symlinked parent of the packaged API outputs", async () => { + const f = await fixture(); + const packages = path.join(f.source, "built/npm"); + await fs.rename(packages, `${packages}-original`); + await fs.symlink("npm-original", packages); + await expect(f.preserve()).rejects.toThrow("Expected a regular build output parent directory"); + await expect(fs.access(f.output)).rejects.toMatchObject({ code: "ENOENT" }); +}); + test("rejects a missing packaged native binary", async () => { const f = await fixture(); await f.preserve(); diff --git a/scripts/src/__tests__/buildTypeScript.test.ts b/scripts/src/__tests__/buildTypeScript.test.ts new file mode 100644 index 00000000..db949fb0 --- /dev/null +++ b/scripts/src/__tests__/buildTypeScript.test.ts @@ -0,0 +1,172 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, expect, test } from "vitest"; + +import { moveBuildOutput, removeScratch } from "../buildTypeScript.js"; + +const sandboxScript = fileURLToPath(new URL("../../buildTypeScriptSandbox.sh", import.meta.url)); +const tempDirs: string[] = []; + +afterEach(() => { + for (const tempDir of tempDirs.splice(0)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +function createBuild() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-build-")); + tempDirs.push(root); + const source = path.join(root, "source"); + const output = path.join(root, "output"); + const home = path.join(root, "home"); + const tmp = path.join(root, "tmp"); + for (const directory of [source, home, tmp]) { + fs.mkdirSync(directory); + } + return { root, source, output, home, tmp }; +} + +test("collects regular compiler outputs, including nested files", async () => { + const build = createBuild(); + fs.mkdirSync(path.join(build.source, "nested")); + fs.writeFileSync(path.join(build.source, "nested/tsc.js"), "compiler"); + await moveBuildOutput(build.source, build.output); + expect(fs.readFileSync(path.join(build.output, "nested/tsc.js"), "utf8")).toBe("compiler"); + expect(fs.existsSync(build.source)).toBe(false); +}); + +test("removes scratch trees containing read-only directories", async () => { + const build = createBuild(); + const moduleDir = path.join(build.home, "go/pkg/mod/example.com/m@v1.0.0"); + fs.mkdirSync(moduleDir, { recursive: true }); + fs.writeFileSync(path.join(moduleDir, "go.mod"), "module example.com/m\n"); + fs.chmodSync(path.join(moduleDir, "go.mod"), 0o444); + fs.chmodSync(moduleDir, 0o555); + await removeScratch(build.root); + expect(fs.existsSync(build.root)).toBe(false); +}); + +test("rejects a symlinked compiler output directory", async () => { + const build = createBuild(); + const link = path.join(build.root, "lib"); + fs.symlinkSync(build.source, link); + await expect(moveBuildOutput(link, build.output)).rejects.toThrow("Expected a regular build output directory"); + expect(fs.existsSync(build.output)).toBe(false); +}); + +test("rejects a symlinked parent of the compiler output", async () => { + const build = createBuild(); + fs.mkdirSync(path.join(build.source, "local")); + const link = path.join(build.root, "built"); + fs.symlinkSync(build.source, link); + await expect(moveBuildOutput(path.join(link, "local"), build.output, build.root)) + .rejects.toThrow("Expected a regular build output parent directory"); + expect(fs.existsSync(build.output)).toBe(false); +}); + +test.each(["file", "directory"])("rejects nested %s symlinks in compiler outputs", async kind => { + const build = createBuild(); + const target = path.join(build.root, "target"); + if (kind === "directory") { + fs.mkdirSync(target); + } + else { + fs.writeFileSync(target, "host-only data"); + } + fs.symlinkSync(target, path.join(build.source, "link")); + await expect(moveBuildOutput(build.source, build.output)).rejects.toThrow("Unexpected build output"); + expect(fs.existsSync(build.output)).toBe(false); +}); + +test("rejects nonregular build outputs without copying or opening them", async () => { + const build = createBuild(); + execFileSync("mkfifo", [path.join(build.source, "pipe")]); + await expect(moveBuildOutput(build.source, build.output)).rejects.toThrow("Unexpected build output"); + expect(fs.existsSync(build.output)).toBe(false); +}); + +test("neutralizes logging commands in rejected build filenames", async () => { + const build = createBuild(); + const filename = "##vso[task.debug]output"; + fs.symlinkSync("/nonexistent", path.join(build.source, filename)); + await expect(moveBuildOutput(build.source, build.output)).rejects.toThrow("# #vso[task.debug]output"); +}); + +test.each([ + "##vso[task.setvariable variable=TSPERF_NEW_NAME]$(id) #.benchmark", + "prefix##[warning]compiler.js", + "with\nnewline.js", + "with\u007Fdelete.js", + "with\u0085control.js", +])("rejects unsafe regular build filename %j", async filename => { + const build = createBuild(); + fs.writeFileSync(path.join(build.source, filename), "compiler"); + await expect(moveBuildOutput(build.source, build.output)).rejects.toThrow("Unsafe build output filename"); + expect(fs.existsSync(build.output)).toBe(false); +}); + +test("rejects logging-command markers in directory names", async () => { + const build = createBuild(); + const directory = path.join(build.source, "##vso[task.debug]nested"); + fs.mkdirSync(directory); + fs.writeFileSync(path.join(directory, "tsc.js"), "compiler"); + await expect(moveBuildOutput(build.source, build.output)).rejects.toThrow("Unsafe build output filename"); + expect(fs.existsSync(build.output)).toBe(false); +}); + +test.each([0, 37])("passes build isolation arguments and preserves exit code %i", exitCode => { + const build = createBuild(); + fs.mkdirSync(path.join(build.source, ".git")); + const bin = path.join(build.root, "bin"); + const capture = path.join(build.root, "bwrap-args"); + fs.mkdirSync(bin); + fs.writeFileSync( + path.join(bin, "bwrap"), + `#!/usr/bin/env bash +printf '%s\\0' "$@" > "$BWRAP_CAPTURE" +echo 'prefix ##vso[task.setvariable variable=TYPESCRIPT_COMMIT]untrusted' +echo '##[warning]compiler stderr' >&2 +exit "$PROBE_EXIT" +`, + { mode: 0o755 }, + ); + const result = spawnSync("bash", [sandboxScript, build.source, build.home, build.tmp, "npm", "ci"], { + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + AGENT_TOOLSDIRECTORY: bin, + BWRAP_CAPTURE: capture, + PROBE_EXIT: String(exitCode), + SYNTHETIC_SECRET: "must-not-be-forwarded", + }, + encoding: "utf8", + timeout: 10_000, + }); + expect(result.status).toBe(exitCode); + expect(result.stdout).toContain("prefix ##vso[task.setvariable variable=TYPESCRIPT_COMMIT]untrusted"); + expect(result.stderr).toContain("##[warning]compiler stderr"); + const args = fs.readFileSync(capture, "utf8").split("\0").slice(0, -1); + expect(args).toContain("--clearenv"); + expect(args).toContain("--unshare-user"); + expect(args).toContain("--unshare-pid"); + expect(args).toContain("--die-with-parent"); + expect(args).toContain("--new-session"); + const metadataIndex = args.indexOf(`${build.source}/.git`); + expect(args.slice(metadataIndex - 1, metadataIndex + 2)).toEqual([ + "--ro-bind", + `${build.source}/.git`, + `${build.source}/.git`, + ]); + // The home bind must follow the /tmp bind, or a home under the host's /tmp is hidden. + const tmpIndex = args.indexOf("/tmp"); + expect(args.slice(tmpIndex - 2, tmpIndex + 1)).toEqual(["--bind", build.tmp, "/tmp"]); + const homeBindIndex = args.findIndex((arg, i) => arg === "--bind" && args[i + 1] === build.home); + expect(homeBindIndex).toBeGreaterThan(tmpIndex); + expect(args.slice(-2)).toEqual(["npm", "ci"]); + expect(args).not.toContain("SYNTHETIC_SECRET"); + expect(args).not.toContain("must-not-be-forwarded"); +}); diff --git a/scripts/src/__tests__/collectBenchmarkResults.test.ts b/scripts/src/__tests__/collectBenchmarkResults.test.ts new file mode 100644 index 00000000..5c29097f --- /dev/null +++ b/scripts/src/__tests__/collectBenchmarkResults.test.ts @@ -0,0 +1,250 @@ +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, expect, test, vi } from "vitest"; +import { parse } from "yaml"; + +import { collectBenchmarkResults } from "../collectBenchmarkResults.js"; +import { JobKind, setupPipeline } from "../setupPipeline.js"; +import { validateBenchmarkArtifacts, validateProcessedResults } from "../validateBenchmarkResults.js"; + +const tempDirs: string[] = []; + +function* scriptSteps(value: unknown): Generator> { + if (Array.isArray(value)) { + for (const item of value) { + yield* scriptSteps(item); + } + } + else if (typeof value === "object" && value !== null) { + if ("bash" in value || "script" in value || "task" in value && value.task === "AzureCLI@2") { + yield value as Record; + } + for (const child of Object.values(value)) { + yield* scriptSteps(child); + } + } +} + +function readYaml(relativePath: string): unknown { + return parse(fs.readFileSync(fileURLToPath(new URL(relativePath, import.meta.url)), "utf8")); +} + +afterEach(() => { + vi.restoreAllMocks(); + for (const tempDir of tempDirs.splice(0)) { + fs.rmSync(tempDir, { recursive: true }); + } +}); + +function createWorkspace() { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-results-")); + tempDirs.push(workspace); + const outputDir = path.join(workspace, "benchmarks"); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + function createJob(kind: JobKind, jobName: string, isComparison = true) { + const artifactDir = path.join(workspace, `PartialBenchmark_${jobName}`); + fs.mkdirSync(artifactDir); + for (const prefix of isComparison ? ["baseline", "new"] : ["baseline"]) { + fs.writeFileSync(path.join(artifactDir, `${prefix}_${jobName}.${kind}.benchmark`), prefix); + } + return artifactDir; + } + + return { workspace, outputDir, createJob, log }; +} + +test("collects exact scheduled results without reading or logging compiler and extra artifact files", async () => { + const fixture = createWorkspace(); + const tsc = "tsc_node_probe"; + const startup = "startup_native_probe"; + const artifactDir = fixture.createJob("tsc", tsc); + fixture.createJob("startup", startup); + fixture.createJob("tsc", "tsc_node_unscheduled"); + const filename = "##vso[task.setvariable variable=TSPERF_NEW_NAME]$(id) #.benchmark"; + fs.writeFileSync(path.join(artifactDir, filename), "untrusted"); + const compilerDir = path.join(fixture.workspace, "BuiltTypeScript/new"); + fs.mkdirSync(compilerDir, { recursive: true }); + fs.writeFileSync(path.join(compilerDir, filename), "untrusted"); + fs.writeFileSync(path.join(compilerDir, `baseline_${tsc}.tsc.benchmark`), "compiler"); + + await collectBenchmarkResults({ + workspace: fixture.workspace, + outputDir: fixture.outputDir, + isComparison: true, + jobNamesByKind: { tsc: [tsc], startup: [startup] }, + }); + + expect(fs.readdirSync(fixture.outputDir).sort()).toEqual([ + `baseline_${startup}.startup.benchmark`, + `baseline_${tsc}.tsc.benchmark`, + `new_${startup}.startup.benchmark`, + `new_${tsc}.tsc.benchmark`, + ]); + expect(fs.readFileSync(path.join(fixture.outputDir, `baseline_${tsc}.tsc.benchmark`), "utf8")).toBe("baseline"); + expect(fixture.log.mock.calls.flat().join("\n")).not.toContain("##vso["); + expect(fs.existsSync(path.join(artifactDir, filename))).toBe(true); + expect(fs.existsSync(path.join(compilerDir, filename))).toBe(true); +}); + +test.each(["lsp", "api"] as const)("collects only baseline files for a baseline-only %s run", async kind => { + const fixture = createWorkspace(); + const jobName = `${kind}_node_probe`; + fixture.createJob(kind, jobName, false); + await collectBenchmarkResults({ + workspace: fixture.workspace, + outputDir: fixture.outputDir, + isComparison: false, + jobNamesByKind: { [kind]: [jobName] }, + }); + expect(fs.readdirSync(fixture.outputDir)).toEqual([`baseline_${jobName}.${kind}.benchmark`]); +}); + +test("fails on a missing scheduled result rather than using a compiler artifact with the same name", async () => { + const fixture = createWorkspace(); + const jobName = "tsc_native_probe"; + const artifactDir = fixture.createJob("tsc", jobName, false); + const filename = `baseline_${jobName}.tsc.benchmark`; + fs.unlinkSync(path.join(artifactDir, filename)); + const compilerDir = path.join(fixture.workspace, "BuiltTypeScript/baseline"); + fs.mkdirSync(compilerDir, { recursive: true }); + fs.writeFileSync(path.join(compilerDir, filename), "compiler"); + await expect(collectBenchmarkResults({ + workspace: fixture.workspace, + outputDir: fixture.outputDir, + isComparison: false, + jobNamesByKind: { tsc: [jobName] }, + })).rejects.toMatchObject({ code: "ENOENT" }); + expect(fs.readdirSync(fixture.outputDir)).toEqual([]); +}); + +test.each(["symlink", "fifo"])("rejects a scheduled %s rather than following or opening it", async kind => { + const fixture = createWorkspace(); + const jobName = "tsserver_node_probe"; + const artifactDir = fixture.createJob("tsserver", jobName, false); + const source = path.join(artifactDir, `baseline_${jobName}.tsserver.benchmark`); + fs.unlinkSync(source); + if (kind === "symlink") { + fs.symlinkSync("/nonexistent", source); + } + else { + execFileSync("mkfifo", [source]); + } + await expect(collectBenchmarkResults({ + workspace: fixture.workspace, + outputDir: fixture.outputDir, + isComparison: false, + jobNamesByKind: { tsserver: [jobName] }, + })).rejects.toThrow("Expected a regular benchmark result"); +}); + +test("rejects symlinked artifact directories", async () => { + const fixture = createWorkspace(); + const jobName = "tsc_node_probe"; + const artifactDir = fixture.createJob("tsc", jobName, false); + fs.renameSync(artifactDir, `${artifactDir}_original`); + fs.symlinkSync(`${artifactDir}_original`, artifactDir); + await expect(collectBenchmarkResults({ + workspace: fixture.workspace, + outputDir: fixture.outputDir, + isComparison: false, + jobNamesByKind: { tsc: [jobName] }, + })).rejects.toThrow("Expected a regular partial artifact directory"); +}); + +test("rejects path traversal in scheduled names", async () => { + const fixture = createWorkspace(); + await expect(collectBenchmarkResults({ + workspace: fixture.workspace, + outputDir: fixture.outputDir, + isComparison: false, + jobNamesByKind: { tsc: ["../outside"] }, + })).rejects.toThrow("Unexpected benchmark job name"); +}); + +test("does not overwrite an existing destination", async () => { + const fixture = createWorkspace(); + const jobName = "tsc_node_probe"; + fixture.createJob("tsc", jobName, false); + fs.mkdirSync(fixture.outputDir); + const destination = path.join(fixture.outputDir, `baseline_${jobName}.tsc.benchmark`); + fs.writeFileSync(destination, "existing"); + await expect(collectBenchmarkResults({ + workspace: fixture.workspace, + outputDir: fixture.outputDir, + isComparison: false, + jobNamesByKind: { tsc: [jobName] }, + })).rejects.toMatchObject({ code: "EEXIST" }); + expect(fs.readFileSync(destination, "utf8")).toBe("existing"); +}); + +test.each(["baseline", "default", "api"])("allows the actual %s setup output variables", async input => { + const result = await setupPipeline({ + input, + baselining: input === "baseline", + isPr: input !== "baseline", + shouldLog: false, + implementation: "corsa", + gitParseRev: async () => { + throw new Error("Unexpected revision query"); + }, + }); + const publisher = [...scriptSteps(readYaml("../../../build/benchmark.yml"))] + .find(step => step.name === "setupPipeline"); + expect(publisher?.target).toEqual({ + commands: "restricted", + settableVariables: expect.arrayContaining(Object.keys(result.outputVariables)), + }); +}); + +test.each([false, true])("independently requires every processed output (comparison: %s)", async isComparison => { + const fixture = createWorkspace(); + fs.mkdirSync(fixture.outputDir); + const report = path.join(fixture.workspace, "comment.html"); + const validate = () => + validateProcessedResults({ mergedDir: fixture.outputDir, report, kinds: ["tsc", "api"], isComparison }); + await expect(validate()).rejects.toThrow("Missing or nonregular merged benchmark result"); + for (const kind of ["tsc", "api"]) { + for (const prefix of isComparison ? ["baseline", "new"] : ["baseline"]) { + fs.writeFileSync(path.join(fixture.outputDir, `${prefix}.${kind}.benchmark`), "{}"); + } + } + await expect(validate()).rejects.toThrow("Missing or nonregular benchmark report"); + fs.writeFileSync(report, "
"); + await validate(); + fs.unlinkSync(report); + fs.symlinkSync(path.join(fixture.outputDir, "baseline.api.benchmark"), report); + await expect(validate()).rejects.toThrow("Missing or nonregular benchmark report"); +}); + +test.each([false, true])( + "publishes only expected benchmark artifact filenames (comparison: %s)", + async isComparison => { + const fixture = createWorkspace(); + fs.mkdirSync(fixture.outputDir); + const validate = () => + validateBenchmarkArtifacts({ + artifactsDir: fixture.outputDir, + jobName: "api_node_probe", + jobKind: "api", + isComparison, + }); + await expect(validate()).rejects.toThrow("Unexpected benchmark artifact files"); + for (const prefix of isComparison ? ["baseline", "new"] : ["baseline"]) { + fs.writeFileSync(path.join(fixture.outputDir, `${prefix}_api_node_probe.api.benchmark`), "{}"); + } + await validate(); + const unexpected = "##vso[task.setvariable variable=GH_TOKEN]injected.benchmark"; + fs.writeFileSync(path.join(fixture.outputDir, unexpected), "{}"); + await expect(validate()).rejects.toThrow(/^Unexpected benchmark artifact files$/); + fs.unlinkSync(path.join(fixture.outputDir, unexpected)); + const baseline = path.join(fixture.outputDir, "baseline_api_node_probe.api.benchmark"); + fs.unlinkSync(baseline); + fs.symlinkSync("/nonexistent", baseline); + await expect(validate()).rejects.toThrow("Missing or nonregular benchmark artifact"); + }, +); diff --git a/scripts/src/__tests__/scenarioSetup.test.ts b/scripts/src/__tests__/scenarioSetup.test.ts new file mode 100644 index 00000000..126de7af --- /dev/null +++ b/scripts/src/__tests__/scenarioSetup.test.ts @@ -0,0 +1,321 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, expect, test } from "vitest"; + +const commonScript = fileURLToPath(new URL("../../../cases/common.sh", import.meta.url)); +const setupScript = fileURLToPath(new URL("../../setupScenario.sh", import.meta.url)); +const benchmarkScript = fileURLToPath(new URL("../../runBenchmarkSandbox.sh", import.meta.url)); +const tempDirs: string[] = []; + +afterEach(() => { + for (const tempDir of tempDirs.splice(0)) { + fs.rmSync(tempDir, { recursive: true }); + } +}); + +function createScenario() { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-setup-")); + tempDirs.push(tempDir); + const scenarioDir = path.join(tempDir, "cases/scenarios/probe"); + const repoDir = path.join(tempDir, "cases/solutions/probe"); + fs.mkdirSync(scenarioDir, { recursive: true }); + fs.mkdirSync(repoDir, { recursive: true }); + + const git = (...args: string[]) => + execFileSync("git", args, { cwd: repoDir, encoding: "utf8", stdio: "pipe" }).trim(); + git("init", "--quiet"); + git( + "-c", + "user.name=Setup Test", + "-c", + "user.email=setup@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + "commit", + "--quiet", + "--allow-empty", + "-m", + "Initial scenario", + ); + git("remote", "add", "origin", repoDir); + const commit = git("rev-parse", "HEAD"); + const refPathCapture = path.join(tempDir, "ref-path"); + const variablesFile = path.join(tempDir, "scenario-variables.json"); + fs.writeFileSync( + path.join(scenarioDir, "setup.sh"), + `set -exo pipefail +cd "\${0%/*}" +source "$COMMON_SCRIPT" +printf '%s' "$TSPERF_SCENARIO_REF_FILE" > "$REF_PATH_CAPTURE" +clone_scenario "$SCENARIO_REPO" "\${SCENARIO_REF:-}" +echo 'prefix ##vso[task.setvariable variable=BWRAP_ARGS]untrusted' +echo '##[warning]installer stderr' >&2 +if [ "\${INVALID_REF:-}" = "true" ]; then + echo '##vso[task.setvariable variable=BWRAP_ARGS]untrusted' > "$TSPERF_SCENARIO_REF_FILE" +fi +if [ "\${FAIL_SETUP:-}" = "true" ]; then + exit 37 +fi +`, + ); + + function run(env: Record = {}) { + return spawnSync("bash", [setupScript], { + env: { + ...process.env, + SCENARIO_REF: "", + COMMON_SCRIPT: commonScript, + SCENARIO_REPO: repoDir, + REF_PATH_CAPTURE: refPathCapture, + TSPERF_SCENARIO_CONFIG_DIR: path.dirname(scenarioDir), + TSPERF_JOB_SCENARIO: "probe", + TYPESCRIPT_COMMIT: commit, + TSPERF_VARIABLES_FILE: variablesFile, + ...env, + }, + encoding: "utf8", + timeout: 10_000, + }); + } + + return { commit, tempDir, repoDir, scenarioDir, refPathCapture, variablesFile, run, git }; +} + +function runInstaller(scenario: ReturnType) { + const binDir = path.join(scenario.tempDir, "bin"); + const argsFile = path.join(scenario.tempDir, "docker-args"); + fs.mkdirSync(binDir); + fs.mkdirSync(path.join(scenario.tempDir, "cases/sandbox")); + fs.writeFileSync( + path.join(binDir, "docker"), + `#!/usr/bin/env bash +if [ "$1" = "inspect" ]; then printf '172.18.0.2\\n'; fi +if [ "$1" = "run" ]; then printf '%s\\n' "$@" >> "$DOCKER_ARGS"; fi +`, + { mode: 0o755 }, + ); + const result = spawnSync("bash", ["-c", 'set -eo pipefail; source "$COMMON_SCRIPT"; run_sandboxed true'], { + cwd: scenario.repoDir, + env: { + ...process.env, + COMMON_SCRIPT: commonScript, + DOCKER_ARGS: argsFile, + PATH: `${binDir}:${process.env.PATH}`, + }, + encoding: "utf8", + timeout: 10_000, + }); + return { result, args: fs.existsSync(argsFile) ? fs.readFileSync(argsFile, "utf8").split("\n") : [] }; +} + +test("passes a read-only Git metadata mount and disables Husky during installation", () => { + const scenario = createScenario(); + const { result, args } = runInstaller(scenario); + expect(result.status).toBe(0); + expect(args).toContain(`${scenario.repoDir}:/sandbox`); + expect(args).toContain(`${scenario.repoDir}/.git:/sandbox/.git:ro`); + expect(args).toContain("HUSKY=0"); + expect(args).toContain("HUSKY_SKIP_INSTALL=1"); +}); + +test("rejects symlinked installer Git metadata before running any containers", () => { + const scenario = createScenario(); + fs.renameSync(path.join(scenario.repoDir, ".git"), path.join(scenario.repoDir, ".git-original")); + fs.symlinkSync(".git-original", path.join(scenario.repoDir, ".git")); + const { result, args } = runInstaller(scenario); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Unexpected Git metadata"); + expect(args).toEqual([]); +}); + +test("preserves setup output and hands off only the validated scenario ref as metadata", () => { + const scenario = createScenario(); + const filename = "##vso[task.setvariable variable=BWRAP_ARGS]--help; printf ESCAPED; #"; + fs.writeFileSync(path.join(scenario.repoDir, filename), ""); + const result = scenario.run(); + expect(result.status).toBe(0); + expect(result.stdout).toContain("Removing ##vso[task.setvariable variable=BWRAP_ARGS]"); + expect(result.stdout).toContain("prefix ##vso[task.setvariable variable=BWRAP_ARGS]untrusted"); + expect(result.stderr).toContain("##[warning]installer stderr"); + expect(result.stdout).not.toContain("##vso[task.setvariable variable=SCENARIO_REF]"); + expect(JSON.parse(fs.readFileSync(scenario.variablesFile, "utf8"))).toEqual({ SCENARIO_REF: scenario.commit }); + expect(fs.existsSync(path.join(scenario.repoDir, filename))).toBe(false); + expect(fs.existsSync(fs.readFileSync(scenario.refPathCapture, "utf8"))).toBe(false); +}); + +test("preserves a pinned scenario ref without emitting another agent command", () => { + const scenario = createScenario(); + const filename = "##vso[task.setvariable variable=BWRAP_ARGS]--help; printf ESCAPED; #"; + fs.writeFileSync(path.join(scenario.repoDir, filename), ""); + const result = scenario.run({ SCENARIO_REF: scenario.commit }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("Removing ##vso[task.setvariable variable=BWRAP_ARGS]"); + expect(result.stdout).not.toContain("##vso[task.setvariable variable=SCENARIO_REF]"); + expect(JSON.parse(fs.readFileSync(scenario.variablesFile, "utf8"))).toEqual({}); + expect(fs.existsSync(path.join(scenario.repoDir, filename))).toBe(false); + expect(fs.existsSync(fs.readFileSync(scenario.refPathCapture, "utf8"))).toBe(false); +}); + +test("does not execute scenario Git hooks during host-side baseline setup", () => { + const scenario = createScenario(); + const hookMarker = path.join(scenario.repoDir, "hook-ran"); + fs.writeFileSync( + path.join(scenario.repoDir, ".git/hooks/post-checkout"), + `#!/usr/bin/env bash\nprintf 'hook ran' > "${hookMarker}"\n`, + { mode: 0o755 }, + ); + const result = scenario.run({ SCENARIO_REF: scenario.commit }); + expect(result.status).toBe(0); + expect(fs.existsSync(hookMarker)).toBe(false); +}); + +test("preserves setup failures and cleans up the ref handoff", () => { + const scenario = createScenario(); + const result = scenario.run({ FAIL_SETUP: "true" }); + expect(result.status).toBe(37); + expect(fs.existsSync(scenario.variablesFile)).toBe(false); + expect(fs.existsSync(fs.readFileSync(scenario.refPathCapture, "utf8"))).toBe(false); +}); + +test("rejects invalid ref handoffs without publishing metadata", () => { + const scenario = createScenario(); + const result = scenario.run({ INVALID_REF: "true" }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Unexpected scenario commit from setup"); + expect(fs.existsSync(scenario.variablesFile)).toBe(false); + expect(fs.existsSync(fs.readFileSync(scenario.refPathCapture, "utf8"))).toBe(false); +}); + +test("rejects shell syntax in compiler commits without executing or logging it", () => { + const scenario = createScenario(); + const result = scenario.run({ TYPESCRIPT_COMMIT: "##vso[task.debug]invalid; printf ESCAPED" }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Unexpected compiler commit"); + expect(result.stdout).toBe(""); + expect(result.stderr).not.toContain("##vso["); + expect(fs.existsSync(scenario.refPathCapture)).toBe(false); +}); + +test("supports scenarios without a setup script", () => { + const scenario = createScenario(); + fs.unlinkSync(path.join(scenario.scenarioDir, "setup.sh")); + const result = scenario.run(); + expect(result.status).toBe(0); + expect(result.stdout).toBe(""); +}); + +test.each(["floating", "pinned", "no setup"])("hands off %s scenario metadata without publishing it", kind => { + const scenario = createScenario(); + const variablesFile = path.join(scenario.tempDir, "variables.json"); + if (kind === "no setup") fs.unlinkSync(path.join(scenario.scenarioDir, "setup.sh")); + const result = scenario.run({ + TSPERF_VARIABLES_FILE: variablesFile, + SCENARIO_REF: kind === "pinned" ? scenario.commit : "", + }); + expect(result.status).toBe(0); + expect(result.stdout).not.toContain("##vso[task.setvariable variable=SCENARIO_REF]"); + expect(JSON.parse(fs.readFileSync(variablesFile, "utf8"))).toEqual( + kind === "floating" ? { SCENARIO_REF: scenario.commit } : {}, + ); +}); + +test("preserves the ref output for callers without the pipeline metadata wrapper", () => { + const scenario = createScenario(); + const result = spawnSync("bash", [ + "-c", + 'set -eo pipefail; source "$COMMON_SCRIPT"; clone_scenario "$SCENARIO_REPO" ""', + ], { + cwd: scenario.scenarioDir, + env: { + ...process.env, + COMMON_SCRIPT: commonScript, + SCENARIO_REPO: scenario.repoDir, + TSPERF_SCENARIO_REF_FILE: "", + }, + encoding: "utf8", + timeout: 10_000, + }); + expect(result.status).toBe(0); + expect(result.stdout).toContain(`##vso[task.setvariable variable=SCENARIO_REF]${scenario.commit}`); +}); + +test.each([0, 37])("allowlists benchmark environment and preserves exit code %i", exitCode => { + const scenario = createScenario(); + const binDir = path.join(scenario.tempDir, "bin"); + const capture = path.join(scenario.tempDir, "bwrap-args"); + fs.mkdirSync(binDir); + fs.writeFileSync( + path.join(binDir, "bwrap"), + `#!/usr/bin/env bash +printf '%s\\0' "$@" > "$BWRAP_CAPTURE" +echo 'prefix ##vso[task.debug]compiler' +echo '##[warning]compiler stderr' >&2 +exit "$PROBE_EXIT" +`, + { mode: 0o755 }, + ); + const allowed = { + HOME: scenario.tempDir, + PATH: `${binDir}:${process.env.PATH}`, + CI: "true", + TSPERF_EXE: "/trusted/ts-perf", + TSPERF_SCENARIO_CONFIG_DIR: "/trusted/scenario configs", + TSPERF_SUITE_DIR: "/trusted/solutions", + TSPERF_JOB_HOST: "node@24", + TSPERF_JOB_SCENARIO: "probe", + TSPERF_JOB_ITERATIONS: "2", + TSPERF_JOB_WARMUPS: "0", + TSPERF_PREDICTABLE: "false", + TSPERF_AGENT_BENCHMARK_CPUS: "2,3", + LANG: "C.UTF-8", + LC_ALL: "C", + LC_CTYPE: "", + TZ: "UTC", + }; + const secrets = { + GH_TOKEN: "synthetic-github-token", + SYSTEM_ACCESSTOKEN: "synthetic-job-token", + AZURE_CLIENT_SECRET: "synthetic-azure-secret", + ENDPOINT_AUTH_PARAMETER_TEST_ACCESSTOKEN: "synthetic-endpoint-token", + NODE_OPTIONS: "--title=synthetic-node-option", + TSPERF_UNEXPECTED_SECRET: "synthetic-prefixed-secret", + }; + const command = ["--unshare-net", "node", "/trusted/runner with spaces.js"]; + const result = spawnSync("bash", [benchmarkScript, ...command], { + env: { ...process.env, ...allowed, ...secrets, BWRAP_CAPTURE: capture, PROBE_EXIT: String(exitCode) }, + encoding: "utf8", + timeout: 10_000, + }); + expect(result.status).toBe(exitCode); + expect(result.stdout).toContain("prefix ##vso[task.debug]compiler"); + expect(result.stderr).toContain("##[warning]compiler stderr"); + const args = fs.readFileSync(capture, "utf8").split("\0").slice(0, -1); + expect(args[0]).toBe("--clearenv"); + const environment: Record = {}; + for (let index = 1; index < args.length - command.length; index += 3) { + expect(args[index]).toBe("--setenv"); + environment[args[index + 1]] = args[index + 2]; + } + expect(environment).toEqual(allowed); + expect(args.slice(-command.length)).toEqual(command); + for (const [name, value] of Object.entries(secrets)) { + expect(args).not.toContain(name); + expect(args).not.toContain(value); + } +}); + +test("requires benchmark configuration before starting the sandbox", () => { + const result = spawnSync("bash", [benchmarkScript, "true"], { + env: { ...process.env, TSPERF_EXE: "" }, + encoding: "utf8", + timeout: 10_000, + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("Expected TSPERF_EXE"); +}); diff --git a/scripts/src/__tests__/setupPipeline.test.ts b/scripts/src/__tests__/setupPipeline.test.ts index 75e25e58..771b6389 100644 --- a/scripts/src/__tests__/setupPipeline.test.ts +++ b/scripts/src/__tests__/setupPipeline.test.ts @@ -1,10 +1,11 @@ import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import url from "node:url"; import filenamify from "filenamify"; import { globSync } from "glob"; -import { expect, test } from "vitest"; +import { expect, test, vi } from "vitest"; import { allPresetNames, GitParseRevResult, setupPipeline } from "../setupPipeline.js"; @@ -174,3 +175,32 @@ test("abandoned snapshots", () => { ); } }); + +test("reports setup failures without rewriting compiler manifest diagnostics", async () => { + const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-pipeline-")); + const exitCode = process.exitCode; + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.resetModules(); + vi.doMock("es-main", () => ({ default: () => true })); + vi.stubEnv("TSPERF_PRESET", "default"); + vi.stubEnv("TYPESCRIPT_DIR", repoDir); + fs.writeFileSync( + path.join(repoDir, "package.json"), + "##vso[task.setvariable variable=BENCH_SCRIPTS]/untrusted/compiler/repo", + ); + try { + await import("../setupPipeline.js"); + expect(process.exitCode).toBe(1); + expect(log).toHaveBeenCalledOnce(); + const diagnostic = log.mock.calls.flat().join("\n"); + expect(diagnostic).toContain("##vso[task"); + } + finally { + process.exitCode = exitCode; + fs.rmSync(repoDir, { recursive: true }); + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + vi.doUnmock("es-main"); + vi.resetModules(); + } +}); diff --git a/scripts/src/__tests__/typeScriptRepo.test.ts b/scripts/src/__tests__/typeScriptRepo.test.ts index 1a6ecd8f..684858a7 100644 --- a/scripts/src/__tests__/typeScriptRepo.test.ts +++ b/scripts/src/__tests__/typeScriptRepo.test.ts @@ -1,6 +1,8 @@ +import { execFileSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { inspect } from "node:util"; import { afterEach, expect, test } from "vitest"; @@ -37,3 +39,45 @@ test("detects the legacy typescript-go repository as Corsa", () => { test("detects the migrated Corsa implementation from its root package name", () => { expect(detectTypeScriptImplementation(createRepo("@typescript/repo"))).toBe("corsa"); }); + +test("returns undefined when the compiler manifest is missing", () => { + const repoDir = createRepo("typescript"); + fs.unlinkSync(path.join(repoDir, "package.json")); + expect(detectTypeScriptImplementation(repoDir)).toBeUndefined(); +}); + +test.each(["symlink", "broken symlink", "directory", "fifo"])("rejects a %s compiler manifest", kind => { + const repoDir = createRepo("typescript"); + const packageJson = path.join(repoDir, "package.json"); + fs.unlinkSync(packageJson); + switch (kind) { + case "symlink": + fs.symlinkSync(path.join(createRepo("typescript"), "package.json"), packageJson); + break; + case "broken symlink": + fs.symlinkSync("/nonexistent", packageJson); + break; + case "directory": + fs.mkdirSync(packageJson); + break; + case "fifo": + execFileSync("mkfifo", [packageJson]); + break; + } + expect(() => detectTypeScriptImplementation(repoDir)).toThrow("Expected a regular compiler package.json"); +}); + +test.each(["##vso[task.debug]x", "##[warning]x"])("preserves malformed compiler manifest diagnostics for %j", text => { + const repoDir = createRepo("typescript"); + fs.writeFileSync(path.join(repoDir, "package.json"), text); + let error: unknown; + try { + detectTypeScriptImplementation(repoDir); + } + catch (caught) { + error = caught; + } + expect(error).toBeInstanceOf(SyntaxError); + const diagnostic = inspect(new Error("Failed to detect compiler implementation", { cause: error })); + expect(diagnostic).toContain(text); +}); diff --git a/scripts/src/__tests__/utils.test.ts b/scripts/src/__tests__/utils.test.ts new file mode 100644 index 00000000..0f9cf280 --- /dev/null +++ b/scripts/src/__tests__/utils.test.ts @@ -0,0 +1,172 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, test, vi } from "vitest"; + +import { publishPipelineVariables } from "../publishPipelineVariables.js"; +import { defangLoggingCommands } from "../utils.js"; + +const tempDirs: string[] = []; + +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await Promise.all(tempDirs.splice(0).map(directory => fs.rm(directory, { recursive: true }))); +}); + +async function createVariablesFile(contents: string) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "pipeline-variables-")); + tempDirs.push(directory); + const file = path.join(directory, "variables.json"); + await fs.writeFile(file, contents); + return file; +} + +describe("publishPipelineVariables", () => { + test("escapes multiline values without printing an additional raw diagnostic", async () => { + const value = "label%0A\r\n##vso[task.setvariable variable=GH_TOKEN]injected"; + const file = await createVariablesFile(JSON.stringify({ TSPERF_NEW_NAME: value })); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + await publishPipelineVariables(file, { job: true, output: true }); + const escaped = "label%AZP250A%0D%0A##vso[task.setvariable variable=GH_TOKEN]injected"; + expect(log.mock.calls).toEqual([ + [`##vso[task.setvariable variable=TSPERF_NEW_NAME;isOutput=true]${escaped}`], + [`##vso[task.setvariable variable=TSPERF_NEW_NAME;isReadOnly=true]${escaped}`], + ]); + }); + + test.each([ + { job: true, output: false, property: "isReadOnly" }, + { job: false, output: true, property: "isOutput" }, + ])("publishes only the requested scope: $property", async ({ job, output, property }) => { + const file = await createVariablesFile('{"TYPESCRIPT_COMMIT":"commit"}'); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + await publishPipelineVariables(file, { job, output }); + expect(log.mock.calls).toEqual([ + [`##vso[task.setvariable variable=TYPESCRIPT_COMMIT;${property}=true]commit`], + ]); + }); + + test("does not republish a job variable already set to the same value", async () => { + vi.stubEnv("SCENARIO_REF", "commit"); + const file = await createVariablesFile('{"SCENARIO_REF":"commit"}'); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + await publishPipelineVariables(file, { job: true, output: false }); + expect(log).not.toHaveBeenCalled(); + }); + + test("rejects a job variable already set to a different value", async () => { + vi.stubEnv("SCENARIO_REF", "first"); + const file = await createVariablesFile('{"OTHER":"value","SCENARIO_REF":"second"}'); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + await expect(publishPipelineVariables(file, { job: true, output: false })) + .rejects.toThrow("Pipeline variable SCENARIO_REF is already set"); + expect(log).not.toHaveBeenCalled(); + }); + + test.each([ + '{"VALID":"first","BAD;isOutput=true":"second"}', + '{"VALID":"first","INVALID":true}', + "[]", + "null", + '{"VALID":', + ])("rejects malformed metadata before publishing any variables: %s", async contents => { + const file = await createVariablesFile(contents); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + await expect(publishPipelineVariables(file, { job: true, output: false })).rejects.toThrow(); + expect(log).not.toHaveBeenCalled(); + }); + + test("fails when a producer exits without writing its required metadata", async () => { + const file = await createVariablesFile("{}"); + await fs.unlink(file); + await expect(publishPipelineVariables(file, { job: true, output: false })) + .rejects.toMatchObject({ code: "ENOENT" }); + }); + + test("rejects symlinked metadata files", async () => { + const file = await createVariablesFile("{}"); + const link = `${file}.link`; + await fs.symlink(file, link); + await expect(publishPipelineVariables(link, { job: true, output: false })) + .rejects.toThrow("Expected a regular pipeline variables file"); + }); +}); + +describe("defangLoggingCommands", () => { + test.each([ + [ + "##vso[task.setvariable variable=SHOULD_UPLOAD]true", + "# #vso[task.setvariable variable=SHOULD_UPLOAD]true", + ], + [" ##vso[task.prependpath]/evil", " # #vso[task.prependpath]/evil"], + ["##[warning]hello", "# #[warning]hello"], + ["
  • ##vso[task.setvariable variable=X]y
  • ", "
  • # #vso[task.setvariable variable=X]y
  • "], + ["prefix ##[warning]hello", "prefix # #[warning]hello"], + ["##vso[task.debug]a ##vso[task.debug]b", "# #vso[task.debug]a # #vso[task.debug]b"], + ])("defangs %j", (input, expected) => { + const output = defangLoggingCommands(input); + expect(output).toBe(expected); + expect(output).not.toContain("##vso["); + expect(output).not.toContain("##["); + expect(defangLoggingCommands(output)).toBe(output); + }); + + test("preserves report labels when forwarding them as CLI arguments", async () => { + const execute = vi.fn(async (_strings: TemplateStringsArray, ..._values: unknown[]) => {}); + vi.resetModules(); + vi.doMock("../utils.js", async importOriginal => ({ + ...await importOriginal(), + $: execute, + })); + const argv = process.argv; + process.argv = [ + "node", + "runTsPerf.js", + "benchmark-tsc", + "--builtDir", + "/compiler", + "--load", + "/result.benchmark", + "--baselineName", + "baseline ##vso[task.debug]label", + "--benchmarkName", + "new ##[warning]label", + ]; + vi.stubEnv("TSPERF_EXE", "/ts-perf"); + vi.stubEnv("TSPERF_SCENARIO_CONFIG_DIR", "/scenarios"); + vi.stubEnv("TSPERF_SUITE_DIR", "/solutions"); + try { + await import("../runTsPerf.js"); + expect(execute).toHaveBeenCalledOnce(); + expect(execute.mock.calls[0].at(-1)).toEqual(expect.arrayContaining([ + "--baselineName", + "baseline ##vso[task.debug]label", + "--benchmarkName", + "new ##[warning]label", + ])); + } + finally { + process.argv = argv; + vi.unstubAllEnvs(); + vi.doUnmock("../utils.js"); + vi.resetModules(); + } + }); + + test("defangs commands on any line", () => { + const fragment = "\n##vso[task.setvariable variable=BASH_ENV]/evil\n
    "; + expect(defangLoggingCommands(fragment)).toBe( + "\n# #vso[task.setvariable variable=BASH_ENV]/evil\n
    ", + ); + }); + + test.each([ + "# heading", + "a ## b", + "##notacommand", + ])("leaves %j alone", input => { + expect(defangLoggingCommands(input)).toBe(input); + }); +}); diff --git a/scripts/src/apiArtifact.ts b/scripts/src/apiArtifact.ts index 81fa63b5..c4449054 100644 --- a/scripts/src/apiArtifact.ts +++ b/scripts/src/apiArtifact.ts @@ -2,6 +2,8 @@ import assert from "node:assert"; import fs from "node:fs/promises"; import path from "node:path"; +import { validateBuildOutput } from "./utils.js"; + export async function preserveApiArtifact( outputDir: string, commit: string, @@ -12,6 +14,8 @@ export async function preserveApiArtifact( const modules = path.join(apiDir, "node_modules"); const packageDir = path.join(modules, "typescript"); const platformName = `typescript-${platform}`; + await validateBuildOutput(path.join(sourceDir, "built/npm/typescript"), sourceDir); + await validateBuildOutput(path.join(sourceDir, `built/npm/${platformName}`), sourceDir); await fs.cp(path.join(sourceDir, "built/npm/typescript"), packageDir, { recursive: true }); await fs.cp(path.join(sourceDir, `built/npm/${platformName}`), path.join(modules, "@typescript", platformName), { recursive: true, diff --git a/scripts/src/buildTypeScript.ts b/scripts/src/buildTypeScript.ts index cddce257..8baca3d0 100644 --- a/scripts/src/buildTypeScript.ts +++ b/scripts/src/buildTypeScript.ts @@ -1,107 +1,143 @@ import assert from "node:assert"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; +import { fileURLToPath } from "node:url"; +import esMain from "es-main"; import minimist from "minimist"; import { Octokit } from "octokit"; import { preserveApiArtifact } from "./apiArtifact.js"; import { detectTypeScriptImplementation } from "./typeScriptRepo.js"; -import { $, $pipe, getNonEmptyEnv, parseBoolean, RepoInfo, retry, setOutputVariable } from "./utils.js"; +import { $, $pipe, getNonEmptyEnv, parseBoolean, RepoInfo, retry, validateBuildOutput } from "./utils.js"; -const { stdout: commit } = await $pipe`git rev-parse HEAD`; -const { stdout: commitShort } = await $pipe`git rev-parse --short HEAD`; -const { stdout: date } = await $pipe`git log -1 --format=%cI`; -const { stdout: timestampDir } = await $pipe`date -d ${date} -u +%Y/%m/%d`; - -const args = minimist(process.argv.slice(2), { - string: ["outputDir"], - boolean: ["baseline"], -}); - -const outputDir = args.outputDir; -assert(outputDir, "Expected output path as first argument"); - -const implementation = detectTypeScriptImplementation("."); -assert(implementation, "Expected to be run from the TypeScript repo"); -const isCorsa = implementation === "corsa"; -const buildApi = process.env.TSPERF_PROCESS_KINDS?.split(/\s+/).includes("api") ?? false; -assert(!buildApi || isCorsa, "The api preset requires native TypeScript 7"); - -await $`mkdir -p ${path.dirname(outputDir)}`; +export async function moveBuildOutput(source: string, output: string, root = path.dirname(path.resolve(source))) { + await validateBuildOutput(source, root); + await $`mv -- ${source} ${output}`; +} -await retry(() => $`npm ci`); +// Go's module cache is read-only, so make the tree writable before removing it. +export async function removeScratch(scratch: string) { + await $`chmod -R u+w -- ${scratch}`; + await fs.promises.rm(scratch, { recursive: true }); +} -if (fs.existsSync("Herebyfile.mjs")) { - if (isCorsa) { - await $`npx hereby build`; - if (buildApi) { - await $`npx hereby typescript:build-main`; - await $`npx hereby typescript:build-platform --platform ${`${process.platform}-${process.arch}`}`; +async function main() { + const { stdout: commit } = await $pipe`git rev-parse HEAD`; + const { stdout: commitShort } = await $pipe`git rev-parse --short HEAD`; + const { stdout: date } = await $pipe`git log -1 --format=%cI`; + const { stdout: timestampDir } = await $pipe`date -d ${date} -u +%Y/%m/%d`; + + const args = minimist(process.argv.slice(2), { + string: ["outputDir", "variablesFile"], + boolean: ["baseline"], + }); + + const outputDir = args.outputDir; + assert(outputDir, "Expected output path as first argument"); + assert(typeof args.variablesFile === "string" && args.variablesFile, "Expected --variablesFile"); + + const implementation = detectTypeScriptImplementation("."); + assert(implementation, "Expected to be run from the TypeScript repo"); + const isCorsa = implementation === "corsa"; + const buildApi = process.env.TSPERF_PROCESS_KINDS?.split(/\s+/).includes("api") ?? false; + assert(!buildApi || isCorsa, "The api preset requires native TypeScript 7"); + + const sourceDir = process.cwd(); + const sandboxScript = fileURLToPath(new URL("../buildTypeScriptSandbox.sh", import.meta.url)); + const scratch = await fs.promises.mkdtemp(path.join(os.tmpdir(), "tsperf-build-")); + const buildHome = path.join(scratch, "home"); + const buildTmp = path.join(scratch, "tmp"); + + try { + await fs.promises.mkdir(path.dirname(outputDir), { recursive: true }); + await fs.promises.mkdir(buildHome); + await fs.promises.mkdir(buildTmp); + await retry(() => $`bash ${sandboxScript} ${sourceDir} ${buildHome} ${buildTmp} npm ci`); + + if (fs.existsSync("Herebyfile.mjs")) { + if (isCorsa) { + await $`bash ${sandboxScript} ${sourceDir} ${buildHome} ${buildTmp} npx hereby build`; + if (buildApi) { + await $`bash ${sandboxScript} ${sourceDir} ${buildHome} ${buildTmp} npx hereby typescript:build-main`; + await $`bash ${sandboxScript} ${sourceDir} ${buildHome} ${buildTmp} npx hereby typescript:build-platform --platform ${`${process.platform}-${process.arch}`}`; + } + await moveBuildOutput("built/local", outputDir, sourceDir); + if (buildApi) await preserveApiArtifact(outputDir, commit); + } + else { + await $`bash ${sandboxScript} ${sourceDir} ${buildHome} ${buildTmp} npx hereby lkg`; + await moveBuildOutput("lib", outputDir, sourceDir); + } + } + else { + await $`bash ${sandboxScript} ${sourceDir} ${buildHome} ${buildTmp} npm run build:compiler`; + await moveBuildOutput("built/local", outputDir, sourceDir); } - await $`mv built/local ${outputDir}`; - if (buildApi) await preserveApiArtifact(outputDir, commit); } - else { - await $`npx hereby lkg`; - await $`mv lib ${outputDir}`; + finally { + await removeScratch(scratch); } -} -else { - await $`npm run build:compiler`; - await $`mv built/local ${outputDir}`; -} -await $`git clean -fddx`; -await $`git reset --hard HEAD`; - -let branch: string | undefined; - -const isPR = parseBoolean(getNonEmptyEnv("IS_PR"), false); -const ref = getNonEmptyEnv("REF"); -const isCustomCommitRange = parseBoolean(getNonEmptyEnv("TSPERF_IS_CUSTOM_COMMIT_RANGE"), false); - -// If this is a custom commit range, don't bother trying to figure out what the branch names are. -if (!isCustomCommitRange) { - if (isPR) { - // This is a PR run. Pull the branch info from the PR. - const prefix = "refs/pull/"; - assert(ref.startsWith(prefix), `Expected ref to start with ${prefix}, instead got ${ref}`); - - if (args.baseline) { - const prNumber = ref.slice(prefix.length).split("/")[0]; - - const octokit = new Octokit(); - const pr = await octokit.rest.pulls.get({ - owner: "microsoft", - repo: getNonEmptyEnv("REPO"), - pull_number: +prNumber, - }); - branch = pr.data.base.ref; + await $`git -c core.hooksPath=/dev/null -c core.fsmonitor=false clean -fddx`; + await $`git -c core.hooksPath=/dev/null -c core.fsmonitor=false reset --hard HEAD`; + + let branch: string | undefined; + + const isPR = parseBoolean(getNonEmptyEnv("IS_PR"), false); + const ref = getNonEmptyEnv("REF"); + const isCustomCommitRange = parseBoolean(getNonEmptyEnv("TSPERF_IS_CUSTOM_COMMIT_RANGE"), false); + + // If this is a custom commit range, don't bother trying to figure out what the branch names are. + if (!isCustomCommitRange) { + if (isPR) { + // This is a PR run. Pull the branch info from the PR. + const prefix = "refs/pull/"; + assert(ref.startsWith(prefix), `Expected ref to start with ${prefix}, instead got ${ref}`); + + if (args.baseline) { + const prNumber = ref.slice(prefix.length).split("/")[0]; + + const octokit = new Octokit(); + const pr = await octokit.rest.pulls.get({ + owner: "microsoft", + repo: getNonEmptyEnv("REPO"), + pull_number: +prNumber, + }); + branch = pr.data.base.ref; + } + else { + branch = ref; + } } else { - branch = ref; + const prefix = "refs/heads/"; + assert(ref.startsWith(prefix), `Expected ref to start with ${prefix}`); + branch = ref.slice(prefix.length); } } - else { - const prefix = "refs/heads/"; - assert(ref.startsWith(prefix), `Expected ref to start with ${prefix}`); - branch = ref.slice(prefix.length); - } -} -const info: RepoInfo = { - commit, - commitShort, - branch, - date, - timestampDir, -}; + const info: RepoInfo = { + commit, + commitShort, + branch, + date, + timestampDir, + }; -const outputInfoPath = path.join(outputDir, "info.json"); -const outputInfo = JSON.stringify(info, undefined, 4); + const outputInfoPath = path.join(outputDir, "info.json"); + const outputInfo = JSON.stringify(info, undefined, 4); -console.log(`Writing ${outputInfoPath} with contents:\n${outputInfo}}`); -await fs.promises.writeFile(outputInfoPath, outputInfo); + console.log(`Writing ${outputInfoPath} with contents:\n${outputInfo}}`); + await fs.promises.writeFile(outputInfoPath, outputInfo); -setOutputVariable("TYPESCRIPT_COMMIT", commit); + await fs.promises.writeFile(args.variablesFile, JSON.stringify({ TYPESCRIPT_COMMIT: commit }), { flag: "wx" }); +} + +if (esMain(import.meta)) { + await main().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/scripts/src/checkLatestCommitForRef.ts b/scripts/src/checkLatestCommitForRef.ts index 28c323da..b88e17b1 100644 --- a/scripts/src/checkLatestCommitForRef.ts +++ b/scripts/src/checkLatestCommitForRef.ts @@ -1,9 +1,10 @@ import assert from "node:assert"; +import fs from "node:fs/promises"; import minimist from "minimist"; import { Octokit } from "octokit"; -import { getNonEmptyEnv, getRepoInfo, setJobVariable } from "./utils.js"; +import { getNonEmptyEnv, getRepoInfo } from "./utils.js"; const auth = getNonEmptyEnv("GH_TOKEN"); @@ -49,4 +50,8 @@ const info = await getRepoInfo(args.builtDir); const ref = getNonEmptyEnv("REF"); const isLatest = await isLatestCommitForRef(ref, info.commit); -setJobVariable("TSPERF_BLOB_LATEST", isLatest); +await fs.writeFile( + getNonEmptyEnv("TSPERF_VARIABLES_FILE"), + JSON.stringify({ TSPERF_BLOB_LATEST: String(isLatest) }), + { flag: "wx" }, +); diff --git a/scripts/src/collectBenchmarkResults.ts b/scripts/src/collectBenchmarkResults.ts new file mode 100644 index 00000000..34fdb1a2 --- /dev/null +++ b/scripts/src/collectBenchmarkResults.ts @@ -0,0 +1,71 @@ +import assert from "node:assert"; +import fs from "node:fs"; +import path from "node:path"; + +import esMain from "es-main"; + +import { allJobKinds, isJobKind, JobKind } from "./setupPipeline.js"; +import { getNonEmptyEnv, parseBoolean } from "./utils.js"; + +export interface CollectBenchmarkResultsOptions { + workspace: string; + outputDir: string; + isComparison: boolean; + jobNamesByKind: Partial>; +} + +export async function collectBenchmarkResults(options: CollectBenchmarkResultsOptions) { + assert(Object.keys(options.jobNamesByKind).length > 0, "Expected scheduled benchmark jobs"); + await fs.promises.mkdir(options.outputDir, { recursive: true }); + + for (const kind of allJobKinds) { + const jobNames = options.jobNamesByKind[kind]; + if (!jobNames) { + continue; + } + assert(jobNames.length > 0, `Expected scheduled ${kind} jobs`); + for (const jobName of jobNames) { + assert( + /^[a-zA-Z0-9_]+$/.test(jobName) && jobName.startsWith(`${kind}_`), + "Unexpected benchmark job name", + ); + const artifactDir = path.join(options.workspace, `PartialBenchmark_${jobName}`); + assert( + (await fs.promises.lstat(artifactDir)).isDirectory(), + "Expected a regular partial artifact directory", + ); + for (const prefix of options.isComparison ? ["baseline", "new"] : ["baseline"]) { + const filename = `${prefix}_${jobName}.${kind}.benchmark`; + const source = path.join(artifactDir, filename); + assert((await fs.promises.lstat(source)).isFile(), `Expected a regular benchmark result: ${filename}`); + await fs.promises.copyFile( + source, + path.join(options.outputDir, filename), + fs.constants.COPYFILE_EXCL, + ); + console.log(`Collected ${filename}`); + } + } + } +} + +async function main() { + const jobNamesByKind: Partial> = {}; + for (const kind of getNonEmptyEnv("TSPERF_PROCESS_KINDS").split(/\s+/)) { + assert(isJobKind(kind), "Unexpected benchmark process kind"); + jobNamesByKind[kind] = getNonEmptyEnv(`TSPERF_JOB_NAMES_${kind.toUpperCase()}`).split(/\s+/); + } + await collectBenchmarkResults({ + workspace: getNonEmptyEnv("PIPELINE_WORKSPACE"), + outputDir: getNonEmptyEnv("BENCHMARKS_DIR"), + isComparison: parseBoolean(getNonEmptyEnv("TSPERF_IS_COMPARISON"), false), + jobNamesByKind, + }); +} + +if (esMain(import.meta)) { + await main().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/scripts/src/publishPipelineVariables.ts b/scripts/src/publishPipelineVariables.ts new file mode 100644 index 00000000..aca8cf6c --- /dev/null +++ b/scripts/src/publishPipelineVariables.ts @@ -0,0 +1,45 @@ +import assert from "node:assert"; +import fs from "node:fs/promises"; +import { inspect } from "node:util"; + +import esMain from "es-main"; +import minimist from "minimist"; + +import { defangLoggingCommands, setJobVariable, setOutputVariable } from "./utils.js"; + +export async function publishPipelineVariables(file: string, options: { job: boolean; output: boolean; }) { + assert(options.job || options.output, "Expected a pipeline variable scope"); + assert((await fs.lstat(file)).isFile(), "Expected a regular pipeline variables file"); + const variables: unknown = JSON.parse(await fs.readFile(file, "utf8")); + assert( + typeof variables === "object" && variables !== null && !Array.isArray(variables), + "Expected a pipeline variables object", + ); + const entries = Object.entries(variables); + for (const [name, value] of entries) { + assert(/^[A-Za-z_][A-Za-z0-9_]*$/.test(name), "Unexpected pipeline variable name"); + assert(typeof value === "string", "Expected a string pipeline variable value"); + // Job variables are read-only once set, so a repeated setup step cannot republish + // them. Azure exposes job variables to later steps as uppercase environment names. + const existing = options.job ? process.env[name.toUpperCase()] : undefined; + assert(existing === undefined || existing === value, `Pipeline variable ${name} is already set`); + } + for (const [name, value] of entries) { + assert(typeof value === "string"); + if (options.output) setOutputVariable(name, value); + if (options.job && process.env[name.toUpperCase()] === undefined) setJobVariable(name, value); + } +} + +async function main() { + const args = minimist(process.argv.slice(2), { string: ["file"], boolean: ["job", "output"] }); + assert(typeof args.file === "string" && args.file, "Expected --file"); + await publishPipelineVariables(args.file, { job: args.job, output: args.output }); +} + +if (esMain(import.meta)) { + await main().catch((error: unknown) => { + console.error(defangLoggingCommands(inspect(error))); + process.exitCode = 1; + }); +} diff --git a/scripts/src/setupPipeline.ts b/scripts/src/setupPipeline.ts index 0eeffbe5..3a4478a7 100644 --- a/scripts/src/setupPipeline.ts +++ b/scripts/src/setupPipeline.ts @@ -1,4 +1,5 @@ import assert from "node:assert"; +import fs from "node:fs/promises"; import { isDeepStrictEqual } from "node:util"; import esMain from "es-main"; @@ -6,7 +7,7 @@ import prettyMilliseconds from "pretty-ms"; import sortKeys from "sort-keys"; import { detectTypeScriptImplementation, TypeScriptImplementation } from "./typeScriptRepo.js"; -import { $pipe, getNonEmptyEnv, parseBoolean, setJobVariable, setOutputVariable } from "./utils.js"; +import { $pipe, getNonEmptyEnv, parseBoolean } from "./utils.js"; // Keep in sync with inventory.yml and benchmark.yml. const allAgents = [ @@ -55,8 +56,12 @@ const hosts = { native: "native", // Native Go binary; no host runtime needed. } as const satisfies Record; -const allJobKinds = ["tsc", "tsserver", "lsp", "startup", "api"] as const; -type JobKind = typeof allJobKinds[number]; +export const allJobKinds = ["tsc", "tsserver", "lsp", "startup", "api"] as const; +export type JobKind = typeof allJobKinds[number]; + +export function isJobKind(value: string): value is JobKind { + return allJobKinds.some(kind => kind === value); +} const enum RunType { None = 0, @@ -559,6 +564,7 @@ export async function setupPipeline(input: SetupPipelineInput) { }; const processKinds = new Set(); + const jobNamesByKind = new Map(); const jobOverhead = 40; // Time taken per benchmark job to clone, build, etc let totalCost = 0; @@ -577,6 +583,11 @@ export async function setupPipeline(input: SetupPipelineInput) { TSPERF_JOB_WARMUPS: scenario.warmups, }; processKinds.add(scenario.kind); + let jobNames = jobNamesByKind.get(scenario.kind); + if (!jobNames) { + jobNamesByKind.set(scenario.kind, jobNames = []); + } + jobNames.push(jobName); let cost = scenario.cost * (scenario.iterations + scenario.warmups) + jobOverhead; if (!baselining) { @@ -599,11 +610,17 @@ export async function setupPipeline(input: SetupPipelineInput) { } } - // These are outputs for the ProcessResults job, specifying which results were - // produced previously and need to be processed. This is a space separated list, - // iterated in the pipeline in bash. + // Tell ProcessResults which benchmark kinds are scheduled, as a space-separated + // list for the pipeline's Bash loop. outputVariables[`TSPERF_PROCESS_KINDS`] = kindOrder.filter(kind => processKinds.has(kind)).join(" "); + // Give ProcessResults exact scheduled names so unexpected artifact files do not enter + // the merge. This constrains file selection, not the authenticity of their contents. + for (const kind of kindOrder) { + const jobNames = jobNamesByKind.get(kind) ?? []; + outputVariables[`TSPERF_JOB_NAMES_${kind.toUpperCase()}`] = jobNames.sort().join(" "); + } + outputVariables[`TSPERF_PREDICTABLE`] = parameters.predictable ? "true" : "false"; outputVariables["TSPERF_IS_CUSTOM_COMMIT_RANGE"] = parameters.isCustomCommitRange ? "true" : "false"; outputVariables["TSPERF_IS_COMPARISON"] = parameters.isComparison ? "true" : "false"; @@ -633,7 +650,7 @@ export async function setupPipeline(input: SetupPipelineInput) { }; } -if (esMain(import.meta)) { +async function main() { async function gitParseRev(query: string): Promise { const cwd = getNonEmptyEnv("TYPESCRIPT_DIR"); @@ -698,9 +715,12 @@ if (esMain(import.meta)) { shouldLog: true, gitParseRev, }); - for (const [key, value] of Object.entries(outputVariables)) { - setOutputVariable(key, value); - // Also set them as environment variables for the job. - setJobVariable(key, value); - } + await fs.writeFile(getNonEmptyEnv("TSPERF_VARIABLES_FILE"), JSON.stringify(outputVariables), { flag: "wx" }); +} + +if (esMain(import.meta)) { + await main().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; + }); } diff --git a/scripts/src/typeScriptRepo.ts b/scripts/src/typeScriptRepo.ts index 60449b05..cfec2798 100644 --- a/scripts/src/typeScriptRepo.ts +++ b/scripts/src/typeScriptRepo.ts @@ -1,3 +1,4 @@ +import assert from "node:assert"; import fs from "node:fs"; import path from "node:path"; @@ -5,12 +6,18 @@ export type TypeScriptImplementation = "strada" | "corsa"; export function detectTypeScriptImplementation(repoDir: string): TypeScriptImplementation | undefined { const packageJsonPath = path.join(repoDir, "package.json"); - if (!fs.existsSync(packageJsonPath)) { + const stat = fs.lstatSync(packageJsonPath, { throwIfNoEntry: false }); + if (!stat) { return undefined; } + assert(stat.isFile(), "Expected a regular compiler package.json"); - const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); - return typeof packageJson.name === "string" + const packageJson: unknown = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); + assert( + typeof packageJson === "object" && packageJson !== null && !Array.isArray(packageJson), + "Expected a compiler package.json object", + ); + return "name" in packageJson && typeof packageJson.name === "string" ? packageJson.name === "typescript" ? "strada" : "corsa" : undefined; } diff --git a/scripts/src/utils.ts b/scripts/src/utils.ts index 835c1095..1fa7fc4d 100644 --- a/scripts/src/utils.ts +++ b/scripts/src/utils.ts @@ -63,17 +63,61 @@ export async function getRepoInfo(builtDir: string) { return RepoInfo.parse(parsed); } +function escapeVariableValue(value: string | number | boolean) { + return String(value).replaceAll("%", "%AZP25").replaceAll("\r", "%0D").replaceAll("\n", "%0A"); +} + export function setOutputVariable(name: string, value: string | number | boolean) { - console.log(`setting output ${name}=${value}`); - console.log(`##vso[task.setvariable variable=${name};isOutput=true]${value}`); + assert(/^[A-Za-z_][A-Za-z0-9_]*$/.test(name), "Unexpected pipeline variable name"); + console.log(`##vso[task.setvariable variable=${name};isOutput=true]${escapeVariableValue(value)}`); } export function setJobVariable(name: string, value: string | number | boolean) { - console.log(`setting variable ${name}=${value}`); - console.log(`##vso[task.setvariable variable=${name}]${value}`); + assert(/^[A-Za-z_][A-Za-z0-9_]*$/.test(name), "Unexpected pipeline variable name"); + console.log(`##vso[task.setvariable variable=${name};isReadOnly=true]${escapeVariableValue(value)}`); } export function parseBoolean(value: string | undefined, defaultValue: boolean): boolean { if (!value) return defaultValue; return value.toUpperCase() === "TRUE"; } + +// Azure Pipelines recognizes logging commands anywhere in a line. Break their markers +// rather than prefixing output, so diagnostic and artifact text remains data. +const loggingCommandRegExp = /##(?=\[|vso\[)/g; + +export function defangLoggingCommands(text: string): string { + return text.replace(loggingCommandRegExp, "# #"); +} + +export async function validateBuildOutput(source: string, root: string) { + const relative = path.relative(root, source); + assert( + relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative), + "Expected build output inside the compiler checkout", + ); + let directory = root; + for (const component of relative.split(path.sep).slice(0, -1)) { + directory = path.join(directory, component); + assert((await fs.promises.lstat(directory)).isDirectory(), "Expected a regular build output parent directory"); + } + + async function validate(file: string): Promise { + const stat = await fs.promises.lstat(file); + if (stat.isDirectory()) { + for (const name of await fs.promises.readdir(file)) { + assert( + !/[\u0000-\u001F\u007F-\u009F]/.test(name) && defangLoggingCommands(name) === name, + `Unsafe build output filename ${JSON.stringify(defangLoggingCommands(name))}`, + ); + await validate(path.join(file, name)); + } + } + else { + assert(stat.isFile(), `Unexpected build output at ${defangLoggingCommands(file)}`); + } + } + + assert((await fs.promises.lstat(source)).isDirectory(), "Expected a regular build output directory"); + await validate(source); +} diff --git a/scripts/src/validateBenchmarkResults.ts b/scripts/src/validateBenchmarkResults.ts new file mode 100644 index 00000000..f8961625 --- /dev/null +++ b/scripts/src/validateBenchmarkResults.ts @@ -0,0 +1,96 @@ +import assert from "node:assert"; +import fs from "node:fs"; +import path from "node:path"; + +import esMain from "es-main"; + +import { isJobKind, JobKind } from "./setupPipeline.js"; +import { getNonEmptyEnv, parseBoolean } from "./utils.js"; + +function resultPrefixes(isComparison: boolean) { + return isComparison ? ["baseline", "new"] : ["baseline"]; +} + +// Messages are fixed because artifact names are untrusted and may contain logging commands. +async function assertNonEmptyFile(file: string, message: string) { + const stat = await fs.promises.lstat(file).catch(() => undefined); + assert(stat?.isFile() && stat.size > 0, message); +} + +export interface ValidateBenchmarkArtifactsOptions { + artifactsDir: string; + jobName: string; + jobKind: JobKind; + isComparison: boolean; +} + +export async function validateBenchmarkArtifacts(options: ValidateBenchmarkArtifactsOptions) { + const expected = resultPrefixes(options.isComparison) + .map(prefix => `${prefix}_${options.jobName}.${options.jobKind}.benchmark`); + const stat = await fs.promises.lstat(options.artifactsDir).catch(() => undefined); + assert(stat?.isDirectory(), "Unexpected benchmark artifact files"); + const actual = await fs.promises.readdir(options.artifactsDir); + assert( + actual.length === expected.length && expected.every(name => actual.includes(name)), + "Unexpected benchmark artifact files", + ); + for (const name of expected) { + await assertNonEmptyFile(path.join(options.artifactsDir, name), "Missing or nonregular benchmark artifact"); + } +} + +export interface ValidateProcessedResultsOptions { + mergedDir: string; + report: string; + kinds: readonly JobKind[]; + isComparison: boolean; +} + +export async function validateProcessedResults(options: ValidateProcessedResultsOptions) { + assert(options.kinds.length > 0, "Expected processed benchmark kinds"); + for (const kind of options.kinds) { + for (const prefix of resultPrefixes(options.isComparison)) { + await assertNonEmptyFile( + path.join(options.mergedDir, `${prefix}.${kind}.benchmark`), + "Missing or nonregular merged benchmark result", + ); + } + } + await assertNonEmptyFile(options.report, "Missing or nonregular benchmark report"); +} + +function getJobKind(value: string) { + assert(isJobKind(value), "Unexpected benchmark kind"); + return value; +} + +async function main() { + const isComparison = parseBoolean(getNonEmptyEnv("TSPERF_IS_COMPARISON"), false); + switch (process.argv[2]) { + case "artifacts": + await validateBenchmarkArtifacts({ + artifactsDir: getNonEmptyEnv("ARTIFACTS_DIR"), + jobName: getNonEmptyEnv("TSPERF_JOB_NAME"), + jobKind: getJobKind(getNonEmptyEnv("TSPERF_JOB_KIND")), + isComparison, + }); + break; + case "processed": + await validateProcessedResults({ + mergedDir: getNonEmptyEnv("MERGED_DIR"), + report: path.join(getNonEmptyEnv("PIPELINE_WORKSPACE"), "comment.html"), + kinds: getNonEmptyEnv("TSPERF_PROCESS_KINDS").split(/\s+/).map(getJobKind), + isComparison, + }); + break; + default: + throw new Error("Usage: validateBenchmarkResults.js artifacts|processed"); + } +} + +if (esMain(import.meta)) { + await main().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/ts-perf/packages/api/test/benchmark.test.ts b/ts-perf/packages/api/test/benchmark.test.ts new file mode 100644 index 00000000..98584d4e --- /dev/null +++ b/ts-perf/packages/api/test/benchmark.test.ts @@ -0,0 +1,19 @@ +import { expect, test } from "vitest"; + +import { Benchmark } from "../src/model/benchmark.js"; + +test("preserves control characters and logging-command markers in benchmark string values", () => { + const scenarioName = "scenario\n##vso[task.setvariable variable=SHOULD_UPLOAD]true"; + const hostName = "inline ##vso[task.debug]host"; + const measurementName = "measurement\r\n##[warning]measurement"; + const benchmark = Benchmark.parse(JSON.stringify({ + version: "2", + scenarios: [{ name: scenarioName }], + hosts: [{ name: hostName }], + measurements: [{ scenarioName: measurementName, metric: "Total Time" }], + })); + + expect(benchmark.scenarios[0].name).toBe(scenarioName); + expect(benchmark.hosts[0].name).toBe(hostName); + expect(benchmark.measurements[0].scenarioName).toBe(measurementName); +}); diff --git a/ts-perf/packages/commands/src/merge/index.ts b/ts-perf/packages/commands/src/merge/index.ts index 00c11e34..ff1519f0 100644 --- a/ts-perf/packages/commands/src/merge/index.ts +++ b/ts-perf/packages/commands/src/merge/index.ts @@ -1,24 +1,21 @@ import { Benchmark, Command, CommandMap } from "@ts-perf/api"; -import { HostContext } from "@ts-perf/core"; export interface MergeOptions { files: string[]; output: string; } -export async function merge(options: MergeOptions, host: HostContext) { +export async function merge(options: MergeOptions) { let benchmarks; try { benchmarks = await Promise.all(options.files.map(file => Benchmark.loadAsync(file))); } catch (e) { - host.error(`Failed to load benchmark files: ${e}`); - return; + throw new Error(`Failed to load benchmark files`, { cause: e }); } if (benchmarks.length < 1) { - host.error(`Expected at least one benchmark file`); - return; + throw new Error(`Expected at least one benchmark file`); } try { @@ -29,8 +26,7 @@ export async function merge(options: MergeOptions, host: HostContext) { await result.saveAsync(options.output); } catch (e) { - host.error(`Failed to merge benchmark files: ${e}`); - return; + throw new Error(`Failed to merge benchmark files`, { cause: e }); } } @@ -52,7 +48,7 @@ const command: Command = { description: "Where to store the merged benchmark file.", }, }, - exec: ({ options }, host) => merge(options, host), + exec: ({ options }) => merge(options), }; export function registerCommands(commands: CommandMap) {