Harden benchmark runners and pipelines

ansible/tasks/packages.yml

@@ -4,6 +4,8 @@
     pkg:
       - ripgrep
       - unzip
+      # Used to sandbox the compiler under test; see build/benchmark.yml.
+      - bubblewrap
       # https://github.com/microsoft/vscode/blob/08631fab3a63e8439bec69dce08aa5cf95360d48/resources/linux/snap/snapcraft.yaml#L22
       - ca-certificates
       - libasound2

ansible/tasks/pyperf.yml

@@ -11,11 +11,27 @@
     name: pyperf
   become: yes
 
-- name: Allow any user to sudo pyperf
+# The compiler under test runs as the agent user. Giving that user arbitrary pyperf
+# arguments would let it run any command as root, e.g. via "pyperf command".
+- name: Install system tuning command
+  ansible.builtin.copy:
+    dest: /usr/local/sbin/tsperf-tune-system
+    owner: root
+    group: root
+    mode: '0755'
+    content: |
+      #!/bin/sh
+      exec /usr/local/bin/pyperf system tune --affinity {{ isolated_cpus | quote }}
+  become: yes
+
+- name: Allow the agent user to tune the system
   ansible.builtin.copy:
     dest: /etc/sudoers.d/pyperf
     owner: root
     group: root
-    content: 'ALL ALL=(ALL) NOPASSWD: /usr/local/bin/pyperf'
+    mode: '0440'
+    # An empty argument list prevents passing options to the tuning command.
+    content: |
+      {{ agent_user }} ALL=(root) NOPASSWD: /usr/local/sbin/tsperf-tune-system ""
     validate: /usr/sbin/visudo -csf %s
   become: yes

build/templates/cloneAndBuildBenchmarkRepo.yml

@@ -10,22 +10,36 @@ steps:
       set -exo pipefail
       cd $(Pipeline.Workspace)/typescript-benchmarking
       pnpm install
+    target:
+      commands: restricted
+      settableVariables: none
     displayName: Install typescript-benchmarking
     retryCountOnTaskFailure: 3
 
   - bash: |
       set -exo pipefail
       cd $(Pipeline.Workspace)/typescript-benchmarking
       pnpm build
+    target:
+      commands: restricted
+      settableVariables: none
     displayName: Build typescript-benchmarking
 
   - bash: |
-      echo "##vso[task.setvariable variable=BENCH_SCRIPTS]$(Pipeline.Workspace)/typescript-benchmarking/scripts/dist"
+      echo "##vso[task.setvariable variable=BENCH_SCRIPTS;isReadOnly=true]$(Pipeline.Workspace)/typescript-benchmarking/scripts/dist"
       # TODO(jakebailey): we can resolve this directly from the scripts dir now
-      echo "##vso[task.setvariable variable=TSPERF_EXE]$(Pipeline.Workspace)/typescript-benchmarking/ts-perf/bin/ts-perf"
+      echo "##vso[task.setvariable variable=TSPERF_EXE;isReadOnly=true]$(Pipeline.Workspace)/typescript-benchmarking/ts-perf/bin/ts-perf"
 
       # TODO(jakebailey): actually create these dirs, move them to more reasonable names, clone external repos as needed
-      echo "##vso[task.setvariable variable=TSPERF_SCENARIO_CONFIG_DIR]$(Pipeline.Workspace)/typescript-benchmarking/cases/scenarios"
-      echo "##vso[task.setvariable variable=TSPERF_SUITE_DIR]$(Pipeline.Workspace)/typescript-benchmarking/cases/solutions"
-      echo "##vso[task.setvariable variable=CREATE_TOKEN_SCRIPT]$(Pipeline.Workspace)/typescript-benchmarking/scripts/create-github-app-token-from-keyvault.cjs"
+      echo "##vso[task.setvariable variable=TSPERF_SCENARIO_CONFIG_DIR;isReadOnly=true]$(Pipeline.Workspace)/typescript-benchmarking/cases/scenarios"
+      echo "##vso[task.setvariable variable=TSPERF_SUITE_DIR;isReadOnly=true]$(Pipeline.Workspace)/typescript-benchmarking/cases/solutions"
+      echo "##vso[task.setvariable variable=CREATE_TOKEN_SCRIPT;isReadOnly=true]$(Pipeline.Workspace)/typescript-benchmarking/scripts/create-github-app-token-from-keyvault.cjs"
+    target:
+      commands: restricted
+      settableVariables:
+        - BENCH_SCRIPTS
+        - TSPERF_EXE
+        - TSPERF_SCENARIO_CONFIG_DIR
+        - TSPERF_SUITE_DIR
+        - CREATE_TOKEN_SCRIPT
     displayName: Set typescript-benchmarking variables

build/templates/setup.yml

@@ -16,11 +16,17 @@ steps:
   - bash: |
       set -exo pipefail
       git config --global core.longpaths true
+    target:
+      commands: restricted
+      settableVariables: none
     displayName: Enable git long paths
 
   - script: |
       COREPACK_ENABLE_STRICT=0 npm install -g corepack@latest
       corepack enable
       corepack enable npm
       corepack prepare pnpm@latest-11 --activate
+    target:
+      commands: restricted
+      settableVariables: none
     displayName: 'Setup pnpm'

cases/common.sh

@@ -11,19 +11,23 @@ function clone_scenario() {
     SOLUTION_DIR=../../solutions/$SCENARIO_NAME
 
     if ! test -d $SOLUTION_DIR; then
-        git clone --filter blob:none $SCENARIO_REPO $SOLUTION_DIR
+        git -c core.hooksPath=/dev/null clone --filter blob:none $SCENARIO_REPO $SOLUTION_DIR
     fi
 
     cd $SOLUTION_DIR
-    git clean -fdx
-    git reset --hard HEAD
+    git -c core.hooksPath=/dev/null -c core.fsmonitor=false clean -fdx
+    git -c core.hooksPath=/dev/null -c core.fsmonitor=false reset --hard HEAD
 
     if test -z "$SCENARIO_REF"; then
-        SCENARIO_REF=$(git rev-parse HEAD)
-        echo "##vso[task.setvariable variable=SCENARIO_REF]$SCENARIO_REF"
+        SCENARIO_REF=$(git -c core.hooksPath=/dev/null rev-parse HEAD)
+        if [ -n "${TSPERF_SCENARIO_REF_FILE:-}" ]; then
+            printf '%s\n' "$SCENARIO_REF" > "$TSPERF_SCENARIO_REF_FILE"
+        else
+            echo "##vso[task.setvariable variable=SCENARIO_REF]$SCENARIO_REF"
+        fi
     else
-        git fetch origin $SCENARIO_REF
-        git switch --detach FETCH_HEAD
+        git -c core.hooksPath=/dev/null fetch origin $SCENARIO_REF
+        git -c core.hooksPath=/dev/null -c core.fsmonitor=false switch --detach FETCH_HEAD
     fi
 }
 
@@ -41,6 +45,15 @@ function clone_strada_scenario() {
 }
 
 function run_sandboxed() {
+    local -a GIT_MOUNT=()
+    if [ -e "$PWD/.git" ] || [ -L "$PWD/.git" ]; then
+        if [ ! -d "$PWD/.git" ] || [ -L "$PWD/.git" ]; then
+            echo "Unexpected Git metadata at $PWD/.git" >&2
+            return 1
+        fi
+        GIT_MOUNT=(--volume "$PWD/.git:/sandbox/.git:ro")
+    fi
+
     USER_DOCKER_SOCK=/run/user/$(id -u)/docker.sock
 
     if [ -e $USER_DOCKER_SOCK ]; then
@@ -121,13 +134,16 @@ function run_sandboxed() {
         --name $SANDBOX_CONTAINER \
         --rm \
         --network $NO_INTERNET \
-        --volume $PWD:/sandbox \
+        --volume "$PWD:/sandbox" \
+        "${GIT_MOUNT[@]}" \
         --workdir /sandbox \
         --env HTTP_PROXY=$PROXY_ADDR \
         --env HTTPS_PROXY=$PROXY_ADDR \
         --env YARN_HTTP_PROXY=$PROXY_ADDR \
         --env YARN_HTTPS_PROXY=$PROXY_ADDR \
         --env COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
+        --env HUSKY=0 \
+        --env HUSKY_SKIP_INSTALL=1 \
         --env CHANGE_USER_ID=$CHANGE_USER_ID \
         $NODE_IMAGE \
         sh -c '

scripts/buildTypeScriptSandbox.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+: "${AGENT_TOOLSDIRECTORY:?Expected the agent tool directory}"
+
+SOURCE_DIR=$1
+BUILD_HOME=$2
+BUILD_TMP=$3
+shift 3
+
+if [ ! -d "$SOURCE_DIR/.git" ] || [ -L "$SOURCE_DIR/.git" ]; then
+    echo "Unexpected Git metadata in compiler checkout" >&2
+    exit 1
+fi
+
+# Builds need network access for public package/module downloads. Expose the compiler
+# checkout and scratch directories, not the agent's home or benchmarking checkout.
+# Clear inherited environment variables; files in the mounted trees remain readable.
+ARGS=(
+    --ro-bind /usr /usr --ro-bind /etc /etc --proc /proc --dev /dev
+    --symlink usr/lib /lib --symlink usr/lib64 /lib64
+    --symlink usr/bin /bin --symlink usr/sbin /sbin
+    --ro-bind "$AGENT_TOOLSDIRECTORY" "$AGENT_TOOLSDIRECTORY"
+    --bind "$SOURCE_DIR" "$SOURCE_DIR"
+    --ro-bind "$SOURCE_DIR/.git" "$SOURCE_DIR/.git"
+    # Bind /tmp first; the scratch home may live under the host's /tmp and must stay visible.
+    --bind "$BUILD_TMP" /tmp --bind "$BUILD_HOME" "$BUILD_HOME"
+    --clearenv --setenv HOME "$BUILD_HOME" --setenv PATH "$PATH"
+    --setenv CI true --setenv COREPACK_ENABLE_DOWNLOAD_PROMPT 0
+    --setenv COREPACK_ENABLE_STRICT 0 --setenv GIT_OPTIONAL_LOCKS 0
+    --setenv HUSKY 0 --setenv HUSKY_SKIP_INSTALL 1
+    --unshare-user --unshare-pid --unshare-ipc --unshare-uts
+    --cap-drop ALL --die-with-parent --new-session --chdir "$SOURCE_DIR"
+)
+# Some images keep resolv.conf's target under /run, which is otherwise not exposed.
+RESOLV_CONF=$(readlink -f /etc/resolv.conf)
+ARGS+=(--ro-bind "$RESOLV_CONF" "$RESOLV_CONF")
+if [ -n "${GOROOT:-}" ]; then
+    ARGS+=(--setenv GOROOT "$GOROOT")
+fi
+
+exec bwrap "${ARGS[@]}" "$@"

scripts/package.json

@@ -21,7 +21,8 @@
         "octokit": "^5.0.5",
         "ora": "^9.4.1",
         "pretty-ms": "^9.3.0",
-        "sort-keys": "^6.0.1"
+        "sort-keys": "^6.0.1",
+        "yaml": "^2.9.1"
     },
     "scripts": {
         "build": "tsc",

scripts/runBenchmarkSandbox.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+ARGS=(
+    --clearenv
+    --setenv HOME "${HOME:?Expected HOME}"
+    --setenv PATH "${PATH:?Expected PATH}"
+    --setenv CI true
+    --setenv TSPERF_EXE "${TSPERF_EXE:?Expected TSPERF_EXE}"
+    --setenv TSPERF_SCENARIO_CONFIG_DIR "${TSPERF_SCENARIO_CONFIG_DIR:?Expected TSPERF_SCENARIO_CONFIG_DIR}"
+    --setenv TSPERF_SUITE_DIR "${TSPERF_SUITE_DIR:?Expected TSPERF_SUITE_DIR}"
+    --setenv TSPERF_JOB_HOST "${TSPERF_JOB_HOST:?Expected TSPERF_JOB_HOST}"
+    --setenv TSPERF_JOB_SCENARIO "${TSPERF_JOB_SCENARIO:?Expected TSPERF_JOB_SCENARIO}"
+    --setenv TSPERF_JOB_ITERATIONS "${TSPERF_JOB_ITERATIONS:?Expected TSPERF_JOB_ITERATIONS}"
+    --setenv TSPERF_JOB_WARMUPS "${TSPERF_JOB_WARMUPS:?Expected TSPERF_JOB_WARMUPS}"
+    --setenv TSPERF_PREDICTABLE "${TSPERF_PREDICTABLE:?Expected TSPERF_PREDICTABLE}"
+    --setenv TSPERF_AGENT_BENCHMARK_CPUS "${TSPERF_AGENT_BENCHMARK_CPUS:?Expected TSPERF_AGENT_BENCHMARK_CPUS}"
+)
+if [ -v LANG ]; then
+    ARGS+=(--setenv LANG "$LANG")
+fi
+if [ -v LC_ALL ]; then
+    ARGS+=(--setenv LC_ALL "$LC_ALL")
+fi
+if [ -v LC_CTYPE ]; then
+    ARGS+=(--setenv LC_CTYPE "$LC_CTYPE")
+fi
+if [ -v TZ ]; then
+    ARGS+=(--setenv TZ "$TZ")
+fi
+
+exec bwrap "${ARGS[@]}" "$@"

scripts/setupScenario.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+: "${TSPERF_VARIABLES_FILE:?Expected a scenario variables file}"
+
+if [[ ! "${TYPESCRIPT_COMMIT:-}" =~ ^[0-9a-f]{40}$ ]]; then
+    echo "Unexpected compiler commit for scenario setup" >&2
+    exit 1
+fi
+
+# This handoff stays outside the solution mounted into third-party installers.
+SCENARIO_REF_FILE=$(mktemp)
+trap 'rm -f "$SCENARIO_REF_FILE"' EXIT
+export TSPERF_SCENARIO_REF_FILE=$SCENARIO_REF_FILE
+
+SETUP=$TSPERF_SCENARIO_CONFIG_DIR/$TSPERF_JOB_SCENARIO/setup.sh
+if test -f "$SETUP"; then
+    echo "Running $SETUP"
+    bash "$SETUP"
+fi
+
+# The pipeline-controlled setup script hands off a commit ID outside the installer mount.
+# Validate its format before handing it to the separate variable publisher.
+if test -s "$SCENARIO_REF_FILE"; then
+    SCENARIO_REF=$(cat "$SCENARIO_REF_FILE")
+    if [[ ! "$SCENARIO_REF" =~ ^[0-9a-f]{40}$ ]]; then
+        echo "Unexpected scenario commit from setup" >&2
+        exit 1
+    fi
+    printf '{"SCENARIO_REF":"%s"}\n' "$SCENARIO_REF" > "$TSPERF_VARIABLES_FILE"
+else
+    printf '{}\n' > "$TSPERF_VARIABLES_FILE"
+fi

scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap

@@ -16,6 +16,11 @@
   "TSPERF_BASELINE_NAME": "baseline",
   "TSPERF_IS_COMPARISON": "true",
   "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false",
+  "TSPERF_JOB_NAMES_API": "api_node_24_21_0_typescript_eslint_api_floating api_node_24_21_0_typescript_eslint_api_recommended api_node_24_21_0_vscode_languageserver_api_metamodel",
+  "TSPERF_JOB_NAMES_LSP": "",
+  "TSPERF_JOB_NAMES_STARTUP": "",
+  "TSPERF_JOB_NAMES_TSC": "",
+  "TSPERF_JOB_NAMES_TSSERVER": "",
   "TSPERF_NEW_COMMIT": "HEAD",
   "TSPERF_NEW_NAME": "pr",
   "TSPERF_PREDICTABLE": "false",