From 9c278d652a1e6e06b8c195c947eab1a64dcbd709 Mon Sep 17 00:00:00 2001 From: Okechi Jones-Williams Date: Mon, 12 Oct 2026 19:47:11 +0100 Subject: [PATCH 1/3] Updated run_local_isolated script to use the Supabase CLI installed inside the project --- scripts/run_local_isolated.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run_local_isolated.sh b/scripts/run_local_isolated.sh index c803a885..6a0b825a 100755 --- a/scripts/run_local_isolated.sh +++ b/scripts/run_local_isolated.sh @@ -35,13 +35,13 @@ export $(cat .env.local | grep -v '^#' | xargs) export $(cat .env.test | grep -v '^#' | xargs) # Ensure Supabase local stack is running; if not, reset/start it -STATUS_JSON=$(supabase status --output json 2>/dev/null || echo '') +STATUS_JSON=$(npx supabase status --output json 2>/dev/null || echo '') API_URL=$(echo "$STATUS_JSON" | jq -r '.API_URL // empty') if [ -z "$API_URL" ]; then echo "Supabase is not running. Bootstrapping local stack with: yarn test:db:reset" yarn test:db:reset - STATUS_JSON=$(supabase status --output json) + STATUS_JSON=$(npx supabase status --output json) fi export NEXT_PUBLIC_SUPABASE_URL=$(echo "$STATUS_JSON" | jq -r '.API_URL') From 23c377b1042e1c3e8dd67210b767670ef20fcd4d Mon Sep 17 00:00:00 2001 From: Okechi Jones-Williams Date: Tue, 13 Oct 2026 14:48:51 +0100 Subject: [PATCH 2/3] Updated the running the app section of README.md, to be easier to understand --- README.md | 76 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 45cff17e..f353e731 100644 --- a/README.md +++ b/README.md @@ -196,45 +196,33 @@ Then, install the dependencies for this project: yarn install --frozen-lockfile ``` -### Tests +### Running the app -Make sure the Jest tests pass: +There are two ways to run Compass locally: -```bash -yarn test -``` +| | Quick start (`yarn dev`) | Full isolation (`yarn dev:isolated`) | +| --------------- | ------------------------------------- | ---------------------------------------------------- | +| Database & auth | Shared remote dev environment | Local Supabase + Firebase emulators | +| Extra setup | None | Docker, Java 21+, Firebase CLI | +| Best for | UI changes, wording fixes, first look | Database, auth, or anything that writes lots of data | -If they don't and you can't find out why, simply raise an issue! Sometimes it's something on our end that we overlooked. +If you're unsure, start with the quick start. You can switch to full isolation later. -### Running the Development Server +After pulling new changes, run `yarn install --frozen-lockfile` before launching either one. -Start the development server: +#### Option A: Quick start (shared dev database) ```bash yarn dev ``` -[//]: # 'Note: the dev remote server is not available until further notice (to save money). The above command will start a local' -[//]: # "server instead (same as `yarn dev:isolated` below). You'll have to set up Firebase and Supabase first (see below)." -[//]: # 'Contact us if you need help; we may spin up the dev remote server for you.' - -Once the server is running, visit http://localhost:3000 to start using the app. You can sign up and visit the profiles; -you should see a few synthetic profiles. - -Note: it's normal if page loading locally is much slower than the deployed version. It can take up to 10 seconds, it -would be great to improve that though! +> ⚠️ The remote dev database is shared by all contributors. Don't reset or wipe it. If you need a clean or disposable +> database, use full isolation. -If you update the database schema (in the SQL files) or the database is not super clean or too large (e.g., someone -clogged it up), you can reset it to a clean state by running: +#### Option B: Full isolation (local database) -```bash -yarn dev:db:reset -``` - -#### Full isolation - -Running `yarn dev:isolated` spins up a local Supabase and Firebase emulator instead of pointing at the shared remote -database. This is strongly recommended for day-to-day development: +`yarn dev:isolated` runs a local Supabase database and Firebase emulators instead of using the shared remote +environment: - **Freedom** — Reset, wipe, and reseed your local database as many times as you want without affecting other contributors. @@ -242,7 +230,7 @@ database. This is strongly recommended for day-to-day development: - **Works offline** — No internet required once services are started locally. - **Faster** — No network latency on every database query. -However, running in full isolation requires installing several heavy dependencies: +The trade-off is heavier setup: - **Docker** (~500MB) — runs the Supabase Postgres container - **Supabase CLI** — manages the local Supabase stack (10+ Docker containers, ~1-2GB RAM when running) @@ -252,11 +240,7 @@ However, running in full isolation requires installing several heavy dependencie First startup is slow (30-60s) and the stack uses significant memory. If your machine has less than 8GB RAM, you may notice slowdowns. -If this feels like too much, you can skip isolation entirely — `yarn dev` works out of the box against the shared remote -and is perfectly fine for most contributions, especially UI changes, wording fixes, or anything that doesn't touch the -database or authentication. - -###### Setup instructions +##### Prerequisites As always, don't hesitate to raise an issue if you run into any problems! @@ -305,14 +289,34 @@ npm install -g firebase-tools firebase --version ``` -Run in isolation +##### Run + +Start each session with a fresh local database that has all the latest migrations: ```bash +yarn test:db:reset yarn dev:isolated ``` -Visit http://localhost:3000 as usual. Your local database comes preloaded with synthetic test profiles so the app -looks and feels like the real thing. +`yarn test:db:reset` only affects your local database and never touches the shared dev one. The Firebase emulator +doesn't keep users between runs, so resetting the database each time keeps the two in sync. + +#### What to expect + +Visit http://localhost:3000. You can sign up and browse a few synthetic profiles. + +Local page loads can be much slower than the deployed site, up to 10 seconds. That's expected (though improvements are +welcome!). + +### Tests + +Before opening a pull request, make sure the Jest tests pass: + +```bash +yarn test +``` + +If they fail and you can't work out why, raise an issue. Sometimes it's something on our end that we overlooked. ### Contributing From 5e72f4fec84ecd7465ccb3a88f1b66e3803f058e Mon Sep 17 00:00:00 2001 From: Okechi Jones-Williams Date: Fri, 23 Oct 2026 18:09:51 +0100 Subject: [PATCH 3/3] This commit is for adding load test func Updated the testing.md with load test info Added load-testing.md focused on explaining load testing and how to interact with it Added JMeter test plan with tests for a singular endpoint Added accounts.csv so that tests can be carried out locally Added workflows to cover smoke and full load tests Added check_results.py, which fails the CI run when error rate or p95 responce time exceeds its limits Updated gitignore to cover the files created by JMeter that wont be commited --- .github/workflows/load-full-test.yml | 22 + .github/workflows/load-smoke-test.yml | 28 + .github/workflows/load-test.yml | 122 +++ .gitignore | 7 + CLAUDE.md | 1 + docs/load-testing.md | 1166 +++++++++++++++++++++++++ docs/testing.md | 10 +- tests/load/accounts.csv | 200 +++++ tests/load/check_results.py | 83 ++ tests/load/compass-testPlan.jmx | 519 +++++++++++ 10 files changed, 2157 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/load-full-test.yml create mode 100644 .github/workflows/load-smoke-test.yml create mode 100644 .github/workflows/load-test.yml create mode 100644 docs/load-testing.md create mode 100644 tests/load/accounts.csv create mode 100644 tests/load/check_results.py create mode 100644 tests/load/compass-testPlan.jmx diff --git a/.github/workflows/load-full-test.yml b/.github/workflows/load-full-test.yml new file mode 100644 index 00000000..7448cd90 --- /dev/null +++ b/.github/workflows/load-full-test.yml @@ -0,0 +1,22 @@ +name: Load test (full) + +# The full load test against the dev API, once a week: 100 users for 120 seconds. +# 100 users is Firebase's limit of 100 new accounts per hour from one IP address. Raising it needs a +# higher sign-up quota in the Firebase console first. +on: + schedule: + - cron: '0 6 * * 1' # Mondays at 06:00 UTC + workflow_dispatch: # adds a "Run workflow" button in the Actions tab + +permissions: + contents: read + +jobs: + full: + uses: ./.github/workflows/load-test.yml + with: + users: 100 + duration: 120 + max_error_rate: 1 + max_p95: 800 + report_name: load-full-report diff --git a/.github/workflows/load-smoke-test.yml b/.github/workflows/load-smoke-test.yml new file mode 100644 index 00000000..d50f59d3 --- /dev/null +++ b/.github/workflows/load-smoke-test.yml @@ -0,0 +1,28 @@ +name: Load test (smoke) + +# A small JMeter run when a pull request changes API-related files, to check the test plan still works. +# It tests the API deployed to dev, not the pull request's code: dev is only updated by running +# backend/api/deploy-api.sh dev by hand. The weekly full run is in load-full-test.yml. +on: + pull_request: + branches: [main] + paths: + - 'backend/api/**' + - 'backend/shared/**' + - 'common/src/api/**' + - 'tests/load/**' + - '.github/workflows/load-*.yml' + workflow_dispatch: # adds a "Run workflow" button in the Actions tab + +permissions: + contents: read + +jobs: + smoke: + uses: ./.github/workflows/load-test.yml + with: + users: 3 + duration: 30 + max_error_rate: 0 + max_p95: 2000 + report_name: load-smoke-report diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml new file mode 100644 index 00000000..1794f814 --- /dev/null +++ b/.github/workflows/load-test.yml @@ -0,0 +1,122 @@ +name: Load test + +# The steps shared by the smoke and full load tests. It doesn't run on its own: load-smoke-test.yml and +# load-full-test.yml call it with their own settings. See docs/load-testing.md. +# +# It tests the API deployed to dev (the plan's default apiHost), not the code in this checkout. +on: + workflow_call: + inputs: + users: + description: Threads for account creation and the load test (one account each) + type: number + required: true + duration: + description: Seconds the load test runs + type: number + required: true + max_error_rate: + description: Highest error rate allowed for any request, in percent + type: number + required: true + max_p95: + description: Highest p95 response time allowed for TEST requests, in milliseconds + type: number + required: true + report_name: + description: Name of the uploaded report artifact + type: string + required: true + +env: + JMETER_VERSION: '5.6.3' + PLAN: compass-testPlan.jmx + USERS: ${{ inputs.users }} + DURATION: ${{ inputs.duration }} + MAX_ERROR_RATE: ${{ inputs.max_error_rate }} + MAX_P95: ${{ inputs.max_p95 }} + +jobs: + load-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + # Never run two load tests at once: they hit the same dev API, so each would skew the other's results. + concurrency: + group: load-test + cancel-in-progress: false + defaults: + run: + working-directory: tests/load # folder holding the .jmx and check_results.py + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + + - name: Cache JMeter + id: jmeter-cache + uses: actions/cache@v4 + with: + path: ~/jmeter + key: jmeter-${{ env.JMETER_VERSION }} + + - name: Install JMeter + if: steps.jmeter-cache.outputs.cache-hit != 'true' + run: | + cd "$HOME" + base="https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-${JMETER_VERSION}.tgz" + curl -fsSLO "$base" + # make sure the download wasn't corrupted or tampered with + echo "$(curl -fsSL "$base.sha512" | cut -d' ' -f1) apache-jmeter-${JMETER_VERSION}.tgz" | sha512sum -c - + mkdir -p jmeter + tar -xzf "apache-jmeter-${JMETER_VERSION}.tgz" -C jmeter --strip-components=1 + + - name: Add JMeter to PATH + run: echo "$HOME/jmeter/bin" >> "$GITHUB_PATH" + + - name: Make test accounts for this run + run: | + # one row per user: email,password,username,name (no header, no commas inside values) + # The run ID and attempt number make the accounts unique, including when a failed job is re-run. + id="${GITHUB_RUN_ID: -5}${GITHUB_RUN_ATTEMPT}" + rm -f accounts.csv + for i in $(seq 1 "$USERS"); do + echo "loadTest${id}x${i}@example.com,$(openssl rand -hex 8),loadUser${id}x${i},load Name${id}x${i}" >> accounts.csv + done + echo "Made $(wc -l < accounts.csv) accounts" + + - name: Run load test + id: jmeter + run: | + mkdir -p results + jmeter -n -t "$PLAN" \ + -JsetupThreads="$USERS" -Jthreads="$USERS" -Jduration="$DURATION" \ + -l results/results.jtl -j jmeter.log -e -o report + + - name: Check results + run: python3 check_results.py results/results.jtl --max-error-rate "$MAX_ERROR_RATE" --max-p95 "$MAX_P95" + + # tearDown normally deletes the accounts. This only runs if JMeter crashed or the job was cancelled. + - name: Clean up accounts if the test didn't finish + if: always() && steps.jmeter.outcome != 'success' + run: | + [ -s created_accounts.txt ] || { echo "No accounts to clean up"; exit 0; } + jmeter -n -t "$PLAN" \ + -JresetFile=0 -JsetupThreads=0 -Jthreads=0 \ + -l results/cleanup.jtl -j cleanup.log + + - name: Save report + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.report_name }} + path: | + tests/load/report + tests/load/results + tests/load/errors.xml + tests/load/jmeter.log + tests/load/cleanup.log + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 34ff2b9f..b9497a10 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,13 @@ /tests/reports/playwright-report /tests/e2e/web/.auth/ +# JMeter load tests (generated by each run, see docs/load-testing.md) +/tests/load/created_accounts.txt +/tests/load/errors.xml +/tests/load/jmeter.log +/tests/load/results/ +/tests/load/report/ + # next.js /.next/ /out/ diff --git a/CLAUDE.md b/CLAUDE.md index 1c8f4ee2..14da4eea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,6 +132,7 @@ Cross-cutting: [`docs/compatibility-questions.md`](docs/compatibility-questions. compatibility prompts), [`docs/feed.md`](docs/feed.md), [`docs/internationalization.md`](docs/internationalization.md), [`docs/profile-fields.md`](docs/profile-fields.md), [`docs/testing.md`](docs/testing.md), +[`docs/load-testing.md`](docs/load-testing.md) (JMeter load tests), [`docs/logging-monitoring.md`](docs/logging-monitoring.md), [`docs/troubleshooting.md`](docs/troubleshooting.md). Mobile: [`android/README.md`](android/README.md), [`ios/README.md`](ios/README.md), diff --git a/docs/load-testing.md b/docs/load-testing.md new file mode 100644 index 00000000..7f5fb1f0 --- /dev/null +++ b/docs/load-testing.md @@ -0,0 +1,1166 @@ +# JMeter (Load) Testing Guide + +Load tests check how the API holds up when many people use it at the same time. They measure response times, +throughput (requests per second) and error rate, so slowdowns and failures under load show up before real users find +them. They don't check that features work correctly; unit, integration and E2E tests do that (see +[testing.md](testing.md)). + +Load tests use [Apache JMeter](https://jmeter.apache.org/). The test plan, `tests/load/compass-testPlan.jmx`, load +tests the `/me` endpoint with real, temporary accounts. Each run has three stages: + +1. **Create** a set of temporary accounts. +2. **Load test:** all the accounts call the API at the same time, for a set amount of time. +3. **Delete** the accounts again. + +**In this guide:** + +- [Placeholders](#placeholders) and [Prerequisites](#prerequisites), including + [setups for Windows, WSL, Linux and macOS](#choose-your-setup) +- [JMeter basics](#jmeter-basics): the terms and rules the rest of the guide relies on +- [How the plan works](#how-the-plan-works): what happens, stage by stage +- [accounts.csv](#accountscsv): the accounts the plan creates +- [Running the test](#running-the-test): commands, stopping early, flags and properties +- [Why it is structured this way](#why-it-is-structured-this-way) +- [Element reference](#element-reference): what each element does, why it's used and how to change it +- [Adding a new test to this plan](#adding-a-new-test-to-this-plan) +- [Reading the results](#reading-the-results) +- [Automated runs](#automated-runs-github-actions): the smoke and weekly full tests in GitHub Actions +- [Troubleshooting](#troubleshooting) + +--- + +## Placeholders + +Values in angle brackets are placeholders. Replace them with the real values when you copy a command. + +| Placeholder | What it is | Where to find the real value | +| -------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| `` | The API host the plan sends requests to | `apiHost` in the plan's User Defined Variables | +| `` | The Firebase key used to sign up and sign in | `firebaseApiKey` in the plan's User Defined Variables (the dev key is also in `common/src/envs/dev.ts`) | +| `` | The email domain of the test accounts | `tests/load/accounts.csv` | +| `` | The password of the test accounts | `tests/load/accounts.csv` | + +--- + +## Prerequisites + +- **Java 8 or newer** (17+ recommended), installed on the system where JMeter runs. JMeter runs on Java. +- **[Apache JMeter 5.6+](https://jmeter.apache.org/download_jmeter.cgi)**, set up for your system as described in + [Choose your setup](#choose-your-setup). + +The plan's files live in `tests/load/`: + +| File | What it is | +| ---------------------- | ------------------------------------------------------------------------------------------- | +| `compass-testPlan.jmx` | The test plan | +| `accounts.csv` | The accounts to create (see [accounts.csv](#accountscsv)) | +| `check_results.py` | Checks a results file against limits, used by CI (see [check_results.py](#check_resultspy)) | + +A run creates these files. The [commands](#commands) are run from `tests/load/`, so they all land there. They are +generated output and are git-ignored: + +| File or folder | What it is | +| ---------------------- | ------------------------------------------------------------------- | +| `created_accounts.txt` | The accounts created in this run, passed between stages | +| `errors.xml` | Details of failed requests | +| `jmeter.log` | JMeter's own log, including the warnings the plan's scripts write | +| `results/` | The raw results of each run ([`.jtl` files](#the-results-file-jtl)) | +| `report/` | The HTML report | + +### Choose your setup + +The [commands](#commands) use short paths such as `compass-testPlan.jmx` and `results/smoke.jtl`, which only work when +JMeter runs **inside `tests/load/`** on the same system as the project. Find the row that matches your machine, then +follow its section below. + +| Your setup | Run the commands in | Open the GUI with | +| ------------------------------------------- | ---------------------------------------------------------- | ----------------------------------------------- | +| Project and JMeter on the same system | A terminal on that system (PowerShell on Windows) | JMeter on that system | +| Project in WSL, JMeter installed on Windows | A WSL terminal, using the Windows install's Linux launcher | JMeter on Windows, with the plan's Windows path | +| Project in WSL, JMeter installed in WSL | A WSL terminal | JMeter in WSL, shown as a window on Windows | + +Common to every setup: + +- **A WSL terminal** (needed for both WSL setups) is a terminal running Linux inside WSL. Open one with the **Ubuntu** + app from the Start menu, an Ubuntu tab in Windows Terminal, or `wsl` in PowerShell. If the project is open in VS Code + through WSL (the bottom-left corner shows **WSL: Ubuntu**), VS Code's own terminal (**Terminal → New Terminal**) is + already one. +- **Java** must be installed on the system that runs the commands. For both WSL setups that means **in WSL**, even if + Java is also installed on Windows. Check with `java -version`. To install it in WSL, use the full package (the + `-headless` one can't show the GUI): + + ```bash + sudo apt install openjdk-21-jre + ``` + +- **Check your setup** by running `jmeter --version` in the terminal you'll run the commands from. +- **Commands** are then run exactly as written under [Commands](#commands), from `tests/load/`. +- **To open the plan in the GUI:** start JMeter, choose **File → Open**, paste the full path of + `compass-testPlan.jmx` into the file name box and open it. When JMeter runs on the same system as the project, you + can instead run `jmeter -t compass-testPlan.jmx` from `tests/load/`, which opens the plan straight away. + +#### Project and JMeter on the same system + +For example, everything on Linux, on macOS, or on Windows without WSL. + +1. Install JMeter. +2. Add JMeter's `bin/` folder to your PATH, so the `jmeter` command works from any folder. + +To start the GUI, run `jmeter`. On Windows, you can also double-click `jmeter.bat` in JMeter's `bin/` folder. + +#### Project in WSL, JMeter installed on Windows + +The JMeter download is the same for every system, so its `bin/` folder has a launcher for each: `jmeter.bat` for +Windows and `jmeter` (a shell script) for Linux and macOS. WSL sees your Windows drive under `/mnt/c`, so it can run the +Linux launcher from your Windows install, using WSL's Java. + +Put JMeter's `bin/` folder on your WSL PATH in **one** of these two ways. Either way, JMeter's `shutdown.sh` also +becomes available, which you need to [stop a run early](#stopping-a-run-early). + +**Option A: add it to your Windows PATH.** WSL adds the Windows PATH to its own, so this makes `jmeter` work in WSL +**and** in PowerShell. + +1. In the Start menu, search for **Edit environment variables for your account** and open it. +2. Select **Path**, click **Edit**, then **New**, and paste the `bin/` folder's Windows path, e.g. + `C:\tools\apache-jmeter-5.6.3\bin`. Click **OK** in both windows. +3. Close and reopen your terminals and VS Code, because WSL only sees the new PATH in a fresh session. If `jmeter` still + isn't found in WSL, run `wsl --shutdown` in PowerShell and open WSL again. + +This doesn't work if WSL is set not to include the Windows PATH (`appendWindowsPath = false` in `/etc/wsl.conf`). In +that case, use option B. + +**Option B: add it to your WSL PATH only.** Run these in a WSL terminal, from any folder. + +1. Find the WSL path of your JMeter folder, using the Windows path where you unpacked it: + + ```bash + wslpath 'C:\tools\apache-jmeter-5.6.3' + ``` + + This prints e.g. `/mnt/c/tools/apache-jmeter-5.6.3`. + +2. Add its `bin/` folder to your WSL PATH, using the path from step 1: + + ```bash + echo 'export PATH="$PATH:/mnt/c/tools/apache-jmeter-5.6.3/bin"' >> ~/.bashrc && source ~/.bashrc + ``` + +In this setup, `jmeter` starts about a second slower than with JMeter installed in WSL (the very first start of the day +can take a few seconds), because its files are read from the Windows drive. Only start-up is affected: a running test +is just as fast. + +**GUI:** start JMeter on Windows (double-click `jmeter.bat` in its `bin/` folder, or type `jmeter` in PowerShell if you +used option A) and open the plan with its Windows path. It looks like `\\wsl.localhost\\home\\…\tests\load\compass-testPlan.jmx`. To get it, run this from +`tests/load/` in WSL and copy the output: + +```bash +wslpath -w compass-testPlan.jmx +``` + +You can also open the folder in File Explorer with `explorer.exe .` and copy the path from the address bar. + +> **Don't run the commands with `jmeter.bat`.** Batch files run in `cmd.exe`, which can't use a WSL folder (a +> `\\wsl.localhost\…` path) as its working folder. It switches to `C:\Windows` instead, so the short paths in the +> commands point to the wrong place and the run fails. + +#### Project in WSL, JMeter installed in WSL + +Everything runs inside WSL. JMeter starts faster than in the setup above, and its GUI opens as a normal window on +Windows through WSLg, which is built into current versions of WSL. + +Run these in a WSL terminal, from any folder (the first command moves to your home folder itself): + +1. Download and unpack JMeter into your WSL home folder: + + ```bash + cd ~ && curl -LO https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-5.6.3.tgz && tar -xzf apache-jmeter-5.6.3.tgz + ``` + +2. Add its `bin/` folder to the **front** of your PATH: + + ```bash + echo 'export PATH="$HOME/apache-jmeter-5.6.3/bin:$PATH"' >> ~/.bashrc && source ~/.bashrc + ``` + + Putting it first means this JMeter is used even if another one is on your Windows PATH, because WSL places the + Windows PATH in the middle of its own. + +3. Check which JMeter runs with `command -v jmeter`. It should print a path in your home folder, e.g. + `/home//apache-jmeter-5.6.3/bin/jmeter`. + +**GUI:** run `jmeter`. For **File → Open**, use the plan's WSL path, e.g. +`/home//…/tests/load/compass-testPlan.jmx`. To print it, run `realpath compass-testPlan.jmx` in `tests/load/`. + +--- + +## JMeter basics + +A JMeter test plan is a **tree** of elements. Each element type has one job: + +| Term | What it means | Example in this plan | +| ---------------------- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | +| **Test plan** | The whole file and its top-level settings | [`compass-testPlan.jmx`](#test-plan) | +| **Thread group** | A group of threads that all run the same steps | [`1 - Create accounts`](#thread-groups) | +| **Thread** | One simulated user (see the note below) | One of the 100 threads in a thread group | +| **Sampler** | Something a thread _does_, usually one request | [`SETUP \| Firebase Signup POST`](#http-request) | +| **Config element** | Settings or data used by samplers | [CSV Data Set Config](#csv-data-set-config), [HTTP Header Manager](#http-header-manager) | +| **Pre/Post-processor** | Runs before/after a sampler, e.g. to pull a value out of a response | [JSON Extractor](#json-extractor) | +| **Assertion** | Checks a response and marks the request as failed if it's wrong | [Response Assertion](#response-assertion) (code must be `200`) | +| **Timer** | Adds a pause before requests | [Think time](#constant-timer) | +| **Logic controller** | Controls _how often_ or _whether_ its children run | [Runtime Controller](#runtime-controller) | +| **Listener** | Collects or shows results | [Simple Data Writer](#listeners) | + +> **Words used in this guide:** +> +> - **Thread:** one simulated user that JMeter runs. JMeter's own screens sometimes call these "users", e.g. +> "Number of Threads (users)". +> - **Account:** a login (email, password, token) that a thread uses. +> - **User:** only ever means a real person using the app. + +### Variables and properties + +- **Variables** belong to **one thread**. You read them with `${name}`, e.g. `${authToken}`. Each thread has its own + copy, so 100 threads can each hold a different token. A variable that a thread sets while it runs (like + `authToken`) **doesn't carry over** to other thread groups. +- The exception is variables defined in the Test Plan or in [User Defined Variables](#user-defined-variables). JMeter + gives every thread its own copy of those when the thread starts, so threads in every group can read them. That's why + `${apiHost}` works everywhere. +- **Properties** are **global** to the whole run. You read them with `${__P(name,default)}` and set them from the + command line with `-Jname=value`. This plan uses properties for anything you might want to change per run (thread + count, duration, host). See [Properties](#properties) and [Functions](#functions). + +### Scope: where an element applies + +An element applies to its **parent and everything under its parent** in the tree: + +- A Header Manager placed **under one request** only affects that request. +- A timer placed under a controller applies before **every** request in that controller. +- A CSV Data Set placed under a thread group feeds **every thread** in that group. + +### Order of execution for each request + +For every sampler, JMeter runs the elements in scope in this order: + +```text +config elements → pre-processors → timers → the sampler (request) → post-processors → assertions → listeners +``` + +This matters when one element depends on another. For example, post-processors run **before** assertions, so a script +in a post-processor can't see whether an assertion passed. The [Save account script](#jsr223-scripts-groovy) is an +example. + +--- + +## How the plan works + +The plan has one thread group per stage: + +```text +compass-testPlan.jmx +├── User Defined Variables hosts and endpoints shared by all requests +├── 0 - Reset shared file (setUp) empties created_accounts.txt +├── 1 - Create accounts (setUp) creates accounts, saves them to created_accounts.txt +├── 2 - /me load test (Thread Group) the load test, reads created_accounts.txt +├── 3 - Delete accounts (tearDown) reads created_accounts.txt, deletes each account +└── Listeners Summary Report, Simple Data Writer, View Results Tree +``` + +Account data moves through the plan like this: + +```text +accounts.csv ──► 1 - Create accounts ──► created_accounts.txt ──► 2 - /me load test + └────────► 3 - Delete accounts +``` + +### Stage by stage + +The stages run one after another (see [Test Plan](#test-plan)). The elements in brackets are explained in the +[Element reference](#element-reference). + +**0 - Reset shared file** (setUp, 1 thread) + +- `SETUP | Reset created_accounts.txt` ([JSR223 Sampler](#jsr223-scripts-groovy)) empties `created_accounts.txt`, so + only this run's accounts end up in it. The stage is skipped when `resetFile=0` (used by + [Cleanup only](#cleanup-only)). + +**1 - Create accounts** (setUp, `setupThreads` threads, one account each) + +Each thread takes the next row of `accounts.csv` ([CSV Data Set Config](#csv-data-set-config)), then: + +1. `SETUP | Firebase Signup POST` creates the Firebase login. A [JSON Extractor](#json-extractor) saves the sign-in + token from the response as `authToken`. +2. `SETUP | create-user-and-profile POST` creates the account's app user and profile, signed in with `authToken`. +3. **Save account for test and teardown** ([JSR223 PostProcessor](#jsr223-scripts-groovy)) adds the account to + `created_accounts.txt` as one line: `email|password|token|name`. + +A disabled [Debug Sampler](#debug-sampler) sits at the end of this stage, for checking the thread's variables while +debugging. + +**2 - /me load test** (Thread Group, `meThreads` threads) + +Each thread takes one line of `created_accounts.txt`, so it gets one account. A +[Runtime Controller](#runtime-controller) then repeats this request for `meDuration` seconds: + +- `TEST | /me GET` calls `/me` with that account's token. It passes if the response code is `200` + ([Response Assertion](#response-assertion)) **and** the returned `name` matches the account + ([JSON Assertion](#json-assertion)). + +**Think time** ([Constant Timer](#constant-timer)) can add a pause before each request. It's 0 by default. + +**3 - Delete accounts** (tearDown, 1 thread) + +The thread goes through `created_accounts.txt` line by line until the file runs out. For each account: + +1. `TEARDOWN | Firebase Sign in POST` signs in again to get a fresh token (saved by a JSON Extractor). Tokens expire + after 1 hour, so the one from setUp may be too old. +2. `TEARDOWN | Delete app user` calls `/me/delete`, which deletes the app user, their files and their Firebase login. + It sends the reason `jmeter-load-test`, which the API stores in the `deleted_users` table, so load-test deletions + can be told apart from real users' deletions. + +--- + +## accounts.csv + +### Why it's needed + +The API only accepts most requests from **signed-in accounts**, so the test needs real accounts. Each thread needs its +**own** account: + +- Each row holds one account, so each thread acts like a separate person using the app, as in real traffic. +- Sign-up rejects an email that's already in use, so every row needs its own email. + +### How the plan uses it + +- The file is read by the **CSV - Accounts** element in `1 - Create accounts`. +- Each thread in that group takes the **next unused row**. The sharing mode is _All threads_, so no two threads get + the same row. +- The four columns become the variables `${userEmail}`, `${userPassword}`, `${username}` and `${name}`. These are used + in the bodies of the sign-up and profile requests. +- If there are more threads than rows, the extra threads **stop** without doing anything ("Stop thread on EOF"). + +### Format + +Four columns, separated by commas, with **no header row**: + +```text +email,password,username,name +``` + +Example row: + +```text +loadtest001@,,loadtest001,Load Test 001 +``` + +Rules for each column: + +| Column | Rule | +| ---------- | ------------------------------------------------------------------------------------------------------------------------ | +| `email` | Unique, and not already used by an account in the environment you're testing | +| `password` | At least 6 characters (Firebase's minimum) | +| `username` | Unique, 3-25 letters, numbers or underscores. If it's already taken, the API quietly picks a similar one | +| `name` | At most 30 characters. Stick to letters, numbers and single spaces, and avoid staff words such as "admin" or "moderator" | + +**No commas or `|` inside any value.** A comma always starts a new column, and the accounts are saved to +`created_accounts.txt` with `|` between values. + +**Why the name rules matter:** the API cleans names before saving them. It squeezes repeated spaces, removes emoji and +symbols, cuts names to 30 characters, and rejects names that look like staff. The `/me` test checks that the returned +name matches `accounts.csv` exactly, so if the API changes a name, every `/me` request for that account fails. + +### How to change it + +- **Support more threads:** you need **at least as many rows as `setupThreads`** (100 by default). To make a new file + with more rows, run this from `tests/load/`, using the domain and password from the current file: + + ```bash + domain=''; password=''; count=200 + for i in $(seq -f '%03g' 1 "$count"); do echo "loadtest$i@$domain,$password,loadtest$i,Load Test $i"; done > accounts.csv + ``` + + ```powershell + $domain = ''; $password = ''; $count = 200 + 1..$count | ForEach-Object { $n = '{0:D3}' -f $_; "loadtest$n@$domain,$password,loadtest$n,Load Test $n" } | Set-Content -Encoding ascii accounts.csv + ``` + +- **Keep values unique:** no two rows can share an email or username. +- **Reusing rows is fine:** the accounts are deleted at the end of every run, so the same file works run after run. If + a run fails to clean up, see [Troubleshooting](#sign-up-fails-with-email_exists). +- **Adding, removing or reordering columns:** the columns map to variables **by position**. If you change them, also + update: + 1. **CSV - Accounts → Variable Names**, so the names are in the same order as the columns. + 2. Every request body that uses the variables (e.g. `SETUP | Firebase Signup POST`). + 3. The **Save account for test and teardown** script, if the new value should be passed on to later stages. + +--- + +## Running the test + +### Why non-GUI mode + +Always run load tests from the command line (**non-GUI mode**, the `-n` flag). Use the GUI only to build and debug the +plan. + +- **It uses far less CPU and memory.** The GUI draws results on screen and keeps them in memory while the test runs. + That takes resources away from sending requests, so the machine running JMeter can become the bottleneck, and the + results get less accurate. +- **It works in CI and scripts,** since no screen is needed. + +JMeter's own docs give the same advice, and JMeter prints a warning about it whenever the GUI opens. + +### Commands + +Run these from `tests/load/`, in the terminal for your [setup](#choose-your-setup) (a WSL terminal if the project is in +WSL): + +```bash +cd tests/load +``` + +The results (`results/…`) and report (`report`) paths are relative to that folder. + +#### Smoke test + +1 thread for 10 seconds. Run this 1hr(`Firebase creation limit resets every hour`) before every full run to check that each step works. + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/smoke.jtl -e -o report -Jthreads=1 -JsetupRamp=1 -Jramp=1 -Jduration=10 +``` + +#### Full run with HTML report + +Uses the defaults (100 threads, 120 seconds). + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report +``` + +#### Report with only the load test requests + +Hides the account setup and cleanup requests from the report. The filter is a regular expression that keeps only the +labels starting with `TEST |`. + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report '-Jjmeter.reportgenerator.sample_filter=^TEST \|.*' +``` + +```powershell +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -e -o report "-Jjmeter.reportgenerator.sample_filter=^TEST \|.*" +``` + +#### Run against a different environment + +The plan tests the dev environment by default. An API only accepts sign-in tokens from its own Firebase project, so +when you change the API host, also pass the Firebase key of the same environment: + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl -JapiHost= -JfirebaseApiKey= +``` + +```powershell +jmeter -n -f -t compass-testPlan.jmx -l results/results.jtl "-JapiHost=" -JfirebaseApiKey= +``` + +> **PowerShell note:** wrap any `-J` argument whose value contains a dot in double quotes, as above. Without the +> quotes, PowerShell can split the argument at the dot, and JMeter gets a broken value. + +#### Cleanup only + +Deletes the accounts still listed in `created_accounts.txt`, without creating accounts or running the test. Use it +when a run stopped before cleanup finished. + +```bash +jmeter -n -f -t compass-testPlan.jmx -l results/cleanup.jtl -JresetFile=0 -Jthreads=0 +``` + +- `-JresetFile=0` gives `0 - Reset shared file` zero threads, so the file isn't emptied. +- `-Jthreads=0` gives account creation and the load test zero threads, so they're skipped. +- `3 - Delete accounts` always runs with 1 thread, so cleanup is all that runs. + +#### Rebuild a report from an existing results file + +Builds the HTML report again without running the test. The `report` folder must be empty or not exist yet. + +```bash +jmeter -g results/results.jtl -o report +``` + +### Stopping a run early + +To stop a run before it finishes **and still delete the accounts**, stop it gracefully. From another terminal of the +same kind as the one running the test (a WSL terminal if the project is in WSL), run JMeter's shutdown script. It's +in JMeter's `bin/` folder, so it's on your PATH: + +```bash +shutdown.sh +``` + +```powershell +shutdown.cmd +``` + +Type the full name: plain `shutdown` is your operating system's command to turn the computer off. + +JMeter lets the requests in progress finish, then runs `3 - Delete accounts` as usual. In the GUI, use +**Run → Shutdown** for the same effect. + +Pressing **Ctrl+C**, running `stoptest.sh`/`stoptest.cmd`, or using **Run → Stop** in the GUI stops JMeter straight +away, and cleanup **doesn't** run. Afterwards, run [Cleanup only](#cleanup-only) to delete the accounts that were +created. + +### Command flags + +| Flag | Meaning | +| ------------- | --------------------------------------------------------------------------- | +| `-n` | Non-GUI mode | +| `-t ` | The test plan to run | +| `-l ` | Where to save the [raw results](#the-results-file-jtl) (`.jtl`, a CSV file) | +| `-e -o ` | After the run, build an HTML report into this folder | +| `-f` | Delete the old results file and report folder first | +| `-g ` | Build a report from an existing `.jtl` without running a test | +| `-J=` | Set a property the plan reads (see below) | + +Without `-f`, `-l` **adds** to an existing results file (mixing runs together), and `-e -o` **fails** if the report +folder isn't empty. + +### Properties + +The plan reads its settings from properties, so you can change a run without editing the `.jmx`. + +| Property | Default | What it controls | +| ---------------- | -------------------- | -------------------------------------------------------------------------------------- | +| `threads` | `100` | Threads for account creation and every load test, unless overridden | +| `ramp` | `10` | Seconds to start all threads in each load test, unless overridden | +| `duration` | `120` | Seconds each load test runs, unless overridden | +| `setupThreads` | `threads` | Number of accounts to create | +| `setupRamp` | `10` | Seconds to start all account-creation threads | +| `meThreads` | `threads` | Threads in the `/me` test | +| `meRamp` | `ramp` | Seconds to start all threads in the `/me` test | +| `meDuration` | `duration` | Seconds the `/me` test runs | +| `thinkTimeMs` | `0` | Pause before each load test request, per thread (0 = none) | +| `apiHost` | `` | API host to test against | +| `firebaseApiKey` | `` | Firebase key for sign-up and sign-in. Must belong to the same environment as `apiHost` | +| `resetFile` | `1` | `0` keeps `created_accounts.txt` instead of emptying it | + +**Specific settings override general ones.** `-Jthreads=50` sets account creation and every load test to 50 threads. +Adding `-JmeThreads=200` changes only the `/me` test. The reset and delete stages always use 1 thread. + +Each load test has its own property prefix (`me` for the `/me` test). +A new test gets its own `Threads`, `Ramp` and `Duration`. + +**Keep `setupThreads` at least as high as the largest load test.** The tests reuse the accounts created in setUp, and +a thread with no account stops. + +> **Ramp-up** is how long JMeter takes to start all the threads. With 100 threads and a 10-second ramp-up, a new +> thread starts every 0.1 seconds. Starting them all at once is less realistic and can overload sign-in. + +--- + +## Why it is structured this way + +- **Stages allow tests to be the main focus.** Creating accounts is slow and only happens once per account. Doing it in a + separate stage keeps it out of the load test results, so they only measure what's being tested. +- **setUp and tearDown thread groups run automatically.** JMeter runs setUp groups before the main test and tearDown + groups after it. tearDown also runs after a [graceful shutdown](#stopping-a-run-early), so test accounts don't pile + up when you stop a run early that way. +- **Stages run in order.** The test plan has **Run Thread Groups consecutively** turned on, so each group waits for the + one before it to finish (see [Test Plan](#test-plan)). +- **One file, one command.** All stages live in one `.jmx`, so the order is guaranteed and CI only needs one command. +- **A shared file passes accounts between stages.** Variables belong to one thread in one thread group, so later stages + can't see what setUp created. setUp writes each account to `created_accounts.txt`, and later stages read it. +- **Properties instead of fixed values.** The same plan works as a smoke test, a full run or a cleanup, just by + changing the command. No edits to the file needed. +- **Paths are relative to the `.jmx`.** The plan looks for `accounts.csv`, `created_accounts.txt` and `errors.xml` + next to the `.jmx`, so JMeter can be started from any folder, on any machine, and still find them. +- **Clear labels.** Every request name starts with `SETUP |`, `TEST |` or `TEARDOWN |`. This makes the stages easy to + tell apart in the results, and lets the report [filter](#report-with-only-the-load-test-requests) to `TEST |` only. + +--- + +## Element reference + +Each element type used in the plan, with: + +- **What it does:** how the element works. +- **Why it's used here:** what the plan needs it for. +- **How it's set up here:** its key settings in this plan, and why. +- **How to change it:** where to edit it, what else has to change with it, and how to add a new one. + +**GUI basics:** to **add** an element, right-click the element it should go under, choose **Add**, then pick the +category and the element. To **change** an element, click it in the tree and edit the fields on the right. To turn an +element off without deleting it, right-click it → **Disable**. Save with **Ctrl+S**. + +### Test Plan + +**What it does:** The top of the tree. It holds settings that apply to the whole run. + +**Why it's used here:** Its settings make the stages run in order, and make cleanup run after a graceful early stop. + +**How it's set up here:** + +- **Run Thread Groups consecutively (i.e. one at a time): on.** Each stage waits for the one before it to finish. + Without it, groups of the same type start together: `0 - Reset shared file` could empty `created_accounts.txt` while + `1 - Create accounts` is saving accounts to it. +- **Run tearDown Thread Groups after shutdown of main threads: on.** Accounts are still deleted after a + [graceful shutdown](#stopping-a-run-early). It doesn't help after Ctrl+C or **Stop**. + +**How to change it:** Click the top element of the tree and tick or untick the boxes. + +### Thread groups + +**What they do:** A thread group is a set of threads that all run the steps under it. There are three types: + +- **setUp Thread Group:** runs **before** all normal thread groups. +- **Thread Group:** the normal kind, used for the load tests. +- **tearDown Thread Group:** runs **after** all normal thread groups. + +**Why they're used here:** Each stage is its own thread group. That keeps the slow, one-off account creation out of the +measured load, and lets JMeter run setUp and cleanup at the right time without any extra logic. + +**How they're set up here:** + +| Thread group | Type | Threads | Loops | +| ----------------------- | ------------ | --------------------------------------- | ------------------------------------------------------ | +| `0 - Reset shared file` | setUp | `resetFile` (1, or 0 to skip the group) | 1 | +| `1 - Create accounts` | setUp | `setupThreads` | 1 (one account per thread) | +| `2 - /me load test` | Thread Group | `meThreads` | 1 (the Runtime Controller inside repeats the requests) | +| `3 - Delete accounts` | tearDown | 1 | Infinite, stops when `created_accounts.txt` runs out | + +- **Number of Threads (users):** how many threads to run. Here it's a property with a fallback, e.g. + `${__P(setupThreads,${__P(threads,100)})}` (see [Functions](#functions)). A group with 0 threads is skipped, which + is how `resetFile=0` and `threads=0` turn stages off. +- **Ramp-up period (seconds):** how long it takes to start all the threads (see the ramp-up note under + [Properties](#properties)). +- **Loop Count:** how many times each thread runs the steps. "Infinite" means until something stops it (here, the end + of `created_accounts.txt`). +- **Action to be taken after a Sampler error:** **Continue** in all groups, so one failed request doesn't stop a + thread. + +**How to change it:** Click the thread group and edit the fields. To change the numbers for one run, use +[properties](#properties) instead of editing the fields. To add a group, right-click the Test Plan → **Add → Threads +(Users)** → pick the type. Within each type, groups run in their order in the tree. + +### User Defined Variables + +**What it does:** Defines named values once, so every element can use them with `${name}`. Every thread gets its own +copy when it starts (see [Variables and properties](#variables-and-properties)). + +**Why it's used here:** Hosts, endpoint paths and the Firebase key are used by many requests. Keeping them in one place +means you change them once instead of in every request. + +**How it's set up here:** + +| Name | Value | +| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `baseAuthUrl` | Firebase Auth host | +| `apiHost` | `${__P(apiHost,)}` (can be overridden with `-JapiHost`) | +| `firebaseApiKey` | `${__P(firebaseApiKey,)}` (can be overridden with `-JfirebaseApiKey`) | +| `signupUrl`, `signinUrl`, `deleteAccountUrl`, `getUserDataUrl` | Firebase Auth endpoint paths. `deleteAccountUrl` and `getUserDataUrl` aren't used by any request at the moment | + +**How to change it:** Click **User Defined Variables** and edit the table, or add a row for a new value. To make a +value changeable from the command line, wrap it in `__P`, e.g. `${__P(myValue,default)}`, and run with +`-JmyValue=...`. To add another set: right-click the Test Plan → **Add → Config Element → User Defined Variables**. + +### CSV Data Set Config + +**What it does:** Reads a text file line by line. Each line is split into columns, and the columns become variables +for the thread that read it. + +**Why it's used here:** Each thread needs its own account. The CSV configs hand every thread a different line, from +`accounts.csv` in setUp and from `created_accounts.txt` (which passes the accounts on) in the later stages. + +**How it's set up here:** + +| Element | Thread group | File | Variables | Delimiter | Sharing mode | +| ------------------------ | --------------------- | ---------------------- | --------------------------------------- | --------- | -------------------- | +| `CSV - Accounts` | `1 - Create accounts` | `accounts.csv` | `userEmail,userPassword,username,name` | `,` | All threads | +| `CSV - Created accounts` | `2 - /me load test` | `created_accounts.txt` | `userEmail,userPassword,authToken,name` | `\|` | Current thread group | +| `CSV - created accounts` | `3 - Delete accounts` | `created_accounts.txt` | `userEmail,userPassword,authToken,name` | `\|` | Current thread group | + +- **Filename:** a plain file name is looked up **next to the `.jmx`**. +- **Variable Names:** one name per column, in column order, separated by commas. +- **Delimiter:** the character between columns. `created_accounts.txt` uses `|`, because that's how the + [Save account script](#jsr223-scripts-groovy) writes it. +- **Recycle on EOF?:** **False**. When the file runs out, don't start again from the top, because each account is + used once per group. +- **Stop thread on EOF?:** **True**. A thread with no line left stops, instead of running with empty values. +- **Sharing mode:** which threads share one position in the file. + - **All threads:** one shared reader for the whole run, so every thread in every group gets a different line. Used + for `accounts.csv`. + - **Current thread group:** each thread group gets its own reader, starting from line 1. Used for + `created_accounts.txt`, so every load test **and** the cleanup each go through all the accounts. + +**When threads read a line:** each thread reads the next line at the **start of every loop** of its thread group. In +`2 - /me load test`, each thread loops once, so it keeps one account for the whole test. In `3 - Delete accounts`, the +single thread loops until the file runs out, taking the next account each time. + +**How to change it:** Click the CSV config and edit the fields. If you change the columns of either file, update the +Variable Names to match (see [accounts.csv → How to change it](#how-to-change-it), and the +[Save account script](#jsr223-scripts-groovy) for `created_accounts.txt`). To add one: right-click the thread group → +**Add → Config Element → CSV Data Set Config**. + +### HTTP Request + +**What it does:** Sends one HTTP request and records how long the response took. + +**Why it's used here:** Every step of the plan is an HTTP request: signing up, creating the profile, the load test +itself and the cleanup. The plan's requests, in run order, are listed in [Stage by stage](#stage-by-stage). + +**How it's set up here** (Basic tab), using `SETUP | Firebase Signup POST` as the example: + +- **Name** (at the top): the label shown in results. It starts with `SETUP |`, `TEST |` or `TEARDOWN |` (see + [Clear labels](#why-it-is-structured-this-way)). +- **Protocol:** `https`. +- **Server Name or IP:** a variable, here `${baseAuthUrl}` (requests to the API use `${apiHost}`). Don't include + `https://` here. +- **HTTP Request (method):** `POST`. +- **Path:** `${signupUrl}${firebaseApiKey}`. Variables can be combined. +- **Body Data tab:** the JSON body, with variables inside: + `{ "email": "${userEmail}", "password": "${userPassword}", "returnSecureToken": true }`. + +**How to change it:** Click the request and edit the Basic tab. To add one: right-click the thread group or controller +→ **Add → Sampler → HTTP Request**. Give it a name with the right prefix, plus a +[Header Manager](#http-header-manager) and a [Response Assertion](#response-assertion), like the existing requests. + +### HTTP Header Manager + +**What it does:** Adds HTTP headers to the requests in its scope. + +**Why it's used here:** The API and Firebase expect JSON bodies, and signed-in requests must send a token to prove +which account they come from. + +**How it's set up here:** Each request has its own Header Manager underneath it: + +- `Content-Type: application/json` on every request. +- `Authorization: Bearer ${authToken}` on requests that need a signed-in account. + +**How to change it:** Click the Header Manager and edit its table of header **Name** and **Value**. Values can use +variables. To add one: right-click the **request** → **Add → Config Element → HTTP Header Manager**. Placing it under +the request means it only applies to that request. Placing it under a thread group would apply it to **every** request +in the group (see [Scope](#scope-where-an-element-applies)). + +### JSON Extractor + +**What it does:** Reads a value out of a JSON response and saves it as a variable. + +**Why it's used here:** The sign-in token only exists once Firebase sends it back. The extractor catches it, so later +requests can send it in the `Authorization` header. + +**How it's set up here:** There's one under `SETUP | Firebase Signup POST` and one under +`TEARDOWN | Firebase Sign in POST`. Both take `idToken` from the response and save it as `authToken`. + +| Field | Value | Meaning | +| -------------------------- | ----------- | ------------------------------------------------------- | +| Names of created variables | `authToken` | The variable to save into | +| JSON Path expressions | `$.idToken` | Where to find the value. `$` is the top of the response | +| Match No. (0 for Random) | `1` | Use the first match | +| Default Values | (empty) | What to save if nothing is found | + +**How to change it:** Click the extractor and edit the fields. To capture more values, list more variable names and +JSON paths, separated by `;`. To add one: right-click the **request** → **Add → Post Processors → JSON Extractor**. + +### JSR223 scripts (Groovy) + +**What they do:** Run a short script. A **JSR223 Sampler** is a step of its own and shows up in the results. A +**JSR223 PostProcessor** runs right after the request it sits under. This plan uses **Groovy**, the recommended (and +fastest) language for JMeter scripts. + +Objects a script can use: + +| Object | What it is | +| ------- | ------------------------------------------------------------------------ | +| `vars` | This thread's variables: `vars.get("name")`, `vars.put("name", "value")` | +| `props` | Global properties | +| `prev` | The result of the previous request, e.g. `prev.isSuccessful()` | +| `log` | Writes to `jmeter.log`, e.g. `log.warn("...")` | + +**Why they're used here:** Emptying `created_accounts.txt` and adding accounts to it are easiest with a few lines of +script. + +**How they're set up here:** + +- **Language:** `groovy`. +- **Cache compiled script if available:** **on**. The script is compiled once, which matters when it runs thousands + of times. + +`SETUP | Reset created_accounts.txt` (JSR223 Sampler in `0 - Reset shared file`): + +```groovy +import org.apache.jmeter.services.FileServer +new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt").text = "" +``` + +- `FileServer.getFileServer().getBaseDir()` is the folder containing the `.jmx`. Using it means the file is always + found next to the plan, wherever JMeter was started from. +- `.text = ""` empties the file. + +**Save account for test and teardown** (JSR223 PostProcessor under `SETUP | create-user-and-profile POST`): + +```groovy +import org.apache.jmeter.services.FileServer + +def token = vars.get("authToken") +if (!token) { + log.warn("No token for ${vars.get('userEmail')}, not saving") + return +} +if (!prev.isSuccessful()) { + log.warn("Profile creation failed for ${vars.get('userEmail')}, saving anyway so it gets cleaned up") +} + +def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") +synchronized (FileServer.class) { + file.append([vars.get("userEmail"), vars.get("userPassword"), token, vars.get("name")].join("|") + "\n", "UTF-8") +} +``` + +- **No token:** sign-up failed, so there's no account to save. It logs a warning and stops. +- **Profile failed:** `prev` is the profile request. `prev.isSuccessful()` is false when the response code is 4xx or + 5xx. The script runs before the Response Assertion, so the assertion's result isn't included (see + [Order of execution](#order-of-execution-for-each-request)). The Firebase login exists either way, so the account + is **saved anyway**, and cleanup deletes it. +- **`synchronized (FileServer.class)`:** many threads write to the same file at the same time. This lets only one + thread write at a time, so lines never get mixed together. +- **`join("|")`:** joins the values into one line in the format `created_accounts.txt` expects. + +**How to change them:** Click the element and edit the **Script** box. If you add a value to `created_accounts.txt`, +add it to the `join` list here **and** to the Variable Names of both CSV configs that read the file, in the same +position. To add one: right-click a thread group → **Add → Sampler → JSR223 Sampler**, or right-click a request → +**Add → Post Processors → JSR223 PostProcessor**. Check that the language is `groovy`. + +### Response Assertion + +**What it does:** Checks part of a response against a pattern. If it doesn't match, the request is marked as +**failed** in the results. + +**Why it's used here:** On its own, JMeter only marks a request as failed when the response code is 4xx or 5xx. The +assertion makes the check strict: anything other than exactly `200` counts as a failure. + +**How it's set up here:** Every request has one: + +- **Field to Test:** `Response Code` +- **Pattern Matching Rules:** `Equals` +- **Patterns to Test:** `200` + +**How to change it:** Click the assertion and edit **Patterns to Test**, e.g. if a new request should return `201`. +To add one: right-click the **request** → **Add → Assertions → Response Assertion**. + +### JSON Assertion + +**What it does:** Checks that a value exists in a JSON response, and optionally that it equals an expected value. + +**Why it's used here:** A `200` only says the request worked. Checking the returned `name` proves the API sent back +**this** account's data, not someone else's. + +**How it's set up here:** On `TEST | /me GET`: + +- **Assert JSON Path exists:** `$.name` +- **Additionally assert value:** on +- **Expected Value:** `${name}`, the account's name from `created_accounts.txt` +- **Match as regular expression:** **off**. When it's on, characters like `.` or `(` in a name are treated as pattern + symbols, and valid names can fail. + +Because of this check, names in `accounts.csv` must follow the [name rules](#format). + +**How to change it:** Click the assertion and change the JSON path and expected value to something that proves the +response is right. To add one: right-click the **request** → **Add → Assertions → JSON Assertion**. + +### Runtime Controller + +**What it does:** Repeats the elements under it, over and over, until a set number of seconds has passed. + +**Why it's used here:** It makes the load test run for a fixed time, so a test takes the same time however fast or slow +the API responds. + +**How it's set up here:** It wraps `TEST | /me GET`, so each thread keeps sending requests for `meDuration` seconds. + +- **Runtime (seconds):** `${__P(meDuration,${__P(duration,120)})}` + +**How to change it:** Change the time per run with `-JmeDuration` or `-Jduration` (see [Properties](#properties)) +rather than editing the field. To test a sequence of requests, put more requests under the controller. To add one: +right-click the thread group → **Add → Logic Controller → Runtime Controller**, then drag the requests onto it. + +### Constant Timer + +**What it does:** Pauses each thread for a fixed time **before every request in its scope**. + +**Why it's used here:** As **think time**: an optional pause between requests, so threads can act more like real +people. It defaults to `0` (no pause, maximum load). + +**How it's set up here:** It's called **Think time** and sits under the Runtime Controller. + +- **Thread Delay (in milliseconds):** `${__P(thinkTimeMs,0)}` + +**How to change it:** Set the pause per run with `-JthinkTimeMs=`. To add one: right-click the +controller or thread group → **Add → Timer → Constant Timer**. + +### Listeners + +**What they do:** Collect and show results. They apply to every request in their scope; here they sit at the top +level, so they see everything. + +**Why they're used here:** To keep the details of failed requests without slowing the run, and to show results live +when debugging in the GUI. + +**How they're set up here:** + +| Listener | Status | What it does | +| ---------------------- | -------- | ---------------------------------------------------------------------------------------- | +| **Summary Report** | Enabled | A live results table, only useful in the GUI | +| **Simple Data Writer** | Enabled | Saves **failed requests only** to `errors.xml`, with the response body showing why | +| **View Results Tree** | Disabled | Shows every request and response in detail. Great for debugging, too heavy for load runs | + +Simple Data Writer settings: + +- **Filename:** `errors.xml` (relative to the `.jmx`). +- **Log/Display Only: Errors:** ticked, so only failed requests are saved. +- **Configure** button: + - **Save Response Data (XML):** on, to see why a request failed. + - **Save Sampler Data (XML):** **off**. Request bodies contain passwords and tokens. + +The results file from `-l` is separate from these listeners. It always records every request. + +**How to change them:** To debug a smoke test in the GUI, enable **View Results Tree** (right-click → **Enable**), and +disable it again before a load run. To add one: right-click the Test Plan → **Add → Listener**. + +### Debug Sampler + +**What it does:** A sampler that doesn't send a request. Instead, it records the thread's variables at that point in +the steps, so you can read them in **View Results Tree**. + +**Why it's used here:** To check what a thread holds while building or fixing the plan, e.g. whether `authToken` was +extracted, or which row of `accounts.csv` the thread read. It's **disabled** normally, because it adds a row per thread +to the results and isn't needed during a load run. + +**How it's set up here:** It's the last step of `1 - Create accounts`, after `SETUP | create-user-and-profile POST`, so +it shows each thread's variables once its account exists. It shows **JMeter variables** only; JMeter properties and +system properties are turned off. + +**How to change it:** Right-click it → **Enable**, enable **View Results Tree** as well, and run a smoke test in the +GUI. Select the **Debug Sampler** row and open the **Response data** tab to see the variables, which include the +account's password and token. Disable both again before a load run. To check variables somewhere else, add one there: +right-click the thread group → **Add → Sampler → Debug Sampler**, and drag it to the step you want to check. + +### Functions + +**What they do:** Special `${__name(...)}` values that JMeter works out at run time. + +**Why they're used here:** `__P` lets the command line change the plan's settings without editing the `.jmx`. + +**How they're set up here:** The plan only uses `__P`, which reads a property: + +- `${__P(duration,120)}` means the `duration` property, or `120` if it isn't set. +- `${__P(setupThreads,${__P(threads,100)})}` means `setupThreads`; if that isn't set, `threads`; if that isn't set + either, `100`. This is how specific settings override general ones. + +**How to change them:** Edit the function where it's used, e.g. to change a default value. **Tools → Function Helper +Dialog** in the GUI lists every function and builds the syntax for you. + +--- + +## Adding a new test to this plan + +New load tests **reuse the accounts** created by setUp and deleted by tearDown, so you only need to build the test +itself. + +1. **Duplicate the existing test group.** Right-click `2 - /me load test` → **Duplicate**. Drag the copy so it sits + **after the last load test and before `3 - Delete accounts`**. +2. **Rename it** `N - `, where `N` is its position in the run order. +3. **Choose a property prefix** for the new test, e.g. ``, and set these fields: + - Number of Threads: `${__P(Threads,${__P(threads,100)})}` + - Ramp-up period: `${__P(Ramp,${__P(ramp,10)})}` + - Runtime Controller → Runtime: `${__P(Duration,${__P(duration,120)})}` +4. **Keep the `CSV - Created accounts` element** as it is. Its sharing mode, _Current thread group_, gives this test + its own pass through all the accounts. +5. **Change the HTTP Request:** + - Name: `TEST | ` + - Method, Path and Body Data for the new request + - Keep the Header Manager with `Authorization: Bearer ${authToken}` if the request needs a signed-in account +6. **Update the assertions:** + - Keep the Response Assertion if `200` is the expected code, or change it. + - Change or remove the JSON Assertion so it checks something in **this** response. +7. **Optional:** adjust or remove the think time timer, or add more requests inside the Runtime Controller to test a + sequence of requests. +8. **Check account numbers:** `setupThreads` must cover the new test's thread count (see [Properties](#properties)). +9. **Save, then run a [smoke test](#smoke-test).** Check that the new `TEST |` rows show up and pass. + +**Things to keep in mind:** + +- **Tests run one after another and share the same accounts.** Total run time is the sum of all test durations. +- **Long runs need fresh tokens.** All tests use the tokens created in setUp, which expire after 1 hour. If the tests + together run longer than that, later tests start failing with `401`. In that case, add a sign-in request (like + `TEARDOWN | Firebase Sign in POST` and its JSON Extractor) at the start of the new test group, before the Runtime + Controller. +- **Tests that change account data affect later tests.** For example, a test that edits the profile name makes a later + test that checks the name fail. Put such tests last, or make later checks not depend on that data. + +--- + +## Reading the results + +A run gives you three outputs: the results file (`.jtl`), the HTML report built from it, and `errors.xml`. + +### The results file (`.jtl`) + +The file you name with `-l` (e.g. `results/results.jtl`) holds the **raw results** of a run. JMeter writes one row for +every request it sends, in every stage, with how long it took and whether it passed. The HTML report and its statistics +are all calculated from this file. + +For example, a smoke test gives about 50 rows: one for each setup and cleanup request, plus one for every `/me` call +the thread made in its 10 seconds. + +It's a plain CSV file with a header row, so it opens in any spreadsheet. The most useful columns: + +| Column | Meaning | +| ------------------------------------------- | ------------------------------------------------------------------------------ | +| `timeStamp` | When the request was sent (milliseconds since 1970) | +| `elapsed` | Response time in ms, the main number in a load test | +| `label` | Which request it was, e.g. `TEST \| /me GET` | +| `responseCode`, `success`, `failureMessage` | Whether it passed, and why not if it failed | +| `Latency` | Time until the first byte of the response arrived | +| `Connect` | Time spent opening the connection | +| `allThreads` | How many threads were running at that moment, which shows the load at the time | +| `threadName` | Which thread sent it, e.g. `1 - Create accounts 2-1` | + +What it's used for: + +- **Building the HTML report.** `-e -o report` builds it at the end of a run, and + [Rebuild a report](#rebuild-a-report-from-an-existing-results-file) builds it again later from the same file. +- **Looking at single requests.** Sort by `elapsed` to find the slowest ones, or filter on `success` = `false` to see + what failed and when. +- **Comparing runs.** Keep the file from each run, e.g. before and after a change to the API. Because the commands use + `-f`, a run overwrites the file with the same name, so copy or rename the ones you want to keep. + +It doesn't contain passwords or tokens. The `URL` column does include the Firebase API key, but that key isn't secret +(the web app sends it to every browser), and `results/` is git-ignored anyway. + +### The HTML report + +- **Check the run matches what you asked for.** In the **Statistics** table, `#Samples` should fit the threads and + duration you used, e.g. 100 sign-up samples for 100 threads. +- **Error %** shows the share of failed requests per label. Any failures in `SETUP |` rows mean fewer accounts, and so + less load, than intended. +- **Response times:** _Average_ can hide slow outliers. The **90th/95th/99th pct** columns show the time that + 90/95/99% of requests finished within. + +### errors.xml + +Holds only the failed requests, each with the response body that shows why it failed. Unlike the `.jtl`, which `-f` replaces, it +**adds to itself on every run**, so delete it before a run if you only want that run's errors. + +--- + +## Automated runs (GitHub Actions) + +Two GitHub Actions workflows run the plan automatically. Both run the same steps, kept in +`.github/workflows/load-test.yml`, with different settings: + +| | Smoke test | Full test | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | +| Workflow | `load-smoke-test.yml` | `load-full-test.yml` | +| Runs when | A pull request into `main` changes API-related files: `backend/api/`, `backend/shared/`, `common/src/api/`, `tests/load/` or the load test workflows | Every Monday at 06:00 UTC | +| Size | 3 threads for 30 seconds | 100 threads for 120 seconds | +| Passes when | No request fails, and the p95 of `TEST \|` requests is at most 2000 ms | At most 1% of each request fails, and the p95 of `TEST \|` requests is at most 800 ms | +| Report artifact | `load-smoke-report` | `load-full-report` | + +You can also start either one by hand: open the **Actions** tab, pick the workflow and click **Run workflow**. + +> **They test the API deployed to dev, not the pull request's code.** Nothing deploys to dev automatically: dev +> only changes when someone runs `./deploy-api.sh dev` in `backend/api/`. To load test a change, deploy it to dev +> first, then start the smoke or full test by hand. + +### What a run does + +1. Installs JMeter (cached between runs) and checks the download against Apache's SHA-512 checksum. +2. Makes a fresh `accounts.csv` with one account per thread, each with a random password. The run's ID and attempt + number make every account unique, including when a failed run is re-run. The committed `accounts.csv` isn't used. +3. Runs the plan with the workflow's size, and builds the HTML report. +4. Checks the results with [check_results.py](#check_resultspy). This step fails the run when a limit is broken. +5. If JMeter didn't finish (it crashed, or the run was cancelled or timed out), runs [Cleanup only](#cleanup-only), + so the accounts don't stay on dev. +6. Uploads the report, the results, `errors.xml` and the logs. To get them, open the run in the **Actions** tab and + download the artifact under **Artifacts**. + +Only one load test runs at a time, because two tests on the same dev API would skew each other's results. A run that +starts while another is going waits for it to finish; if several are waiting, only the newest one runs. + +The full test's 100 threads match Firebase's limit of 100 new accounts per hour from one IP address. To use more, +raise the sign-up quota in the Firebase console first. + +To change a workflow's size or limits, edit the values under `with:` in `load-smoke-test.yml` or +`load-full-test.yml`. + +### check_results.py + +JMeter exits as if all went well even when requests fail, so CI needs something else to decide pass or fail. +`tests/load/check_results.py` reads a results file and checks two limits for each request label: + +- **Error rate**, for every request: `SETUP |`, `TEST |` and `TEARDOWN |`. A failed setup request means less load than + asked for, and a failed cleanup request leaves accounts behind on dev. +- **p95 response time**, for `TEST |` requests only, because setup and cleanup times aren't what the test measures. + It's only checked when a label has at least 20 samples, since fewer is too little to judge. + +It also fails when there are no `TEST |` results at all, which means no accounts were created. + +You can run it on a local run's results too, from `tests/load/` (on Windows, use `python` instead of `python3`): + +```bash +python3 check_results.py results/results.jtl --max-error-rate 1 --max-p95 800 +``` + +It prints each request's sample count, error rate and p95, then either `PASS` or one `FAIL:` line per broken limit. +It exits with `0` for a pass and `1` for a fail. Without the options, the limits are 1% errors and 800 ms. + +--- + +## Troubleshooting + +### Sign-up fails with `EMAIL_EXISTS` + +A previous run didn't finish cleaning up, so accounts from `accounts.csv` still exist. + +- If `created_accounts.txt` still lists them, run [Cleanup only](#cleanup-only). +- If it doesn't (every run empties it at the start), rebuild it from `accounts.csv`, then run + [Cleanup only](#cleanup-only). Cleanup signs in again with each email and password, so the token column can be any + text: + + ```bash + awk -F, '{print $1 "|" $2 "|x|" $4}' accounts.csv > created_accounts.txt + ``` + + ```powershell + Import-Csv accounts.csv -Header email,password,username,name | ForEach-Object { "$($_.email)|$($_.password)|x|$($_.name)" } | Set-Content -Encoding ascii created_accounts.txt + ``` + + Accounts that no longer exist just show up as failed `TEARDOWN |` requests. + +Don't delete the accounts in the Firebase console instead. That only removes the Firebase login. The app user stays in +the database, and nothing warns you: the next run still works, because the API quietly gives the new accounts +different usernames, so the leftover users pile up. + +### Fewer load test threads than expected + +`created_accounts.txt` has fewer lines than the test has threads, so some threads stop right away. Check the `SETUP |` +rows in the results for failures. Also make sure `setupThreads` is high enough (see [Properties](#properties)), and +that `accounts.csv` has enough rows. + +### `/me` requests fail the JSON Assertion + +The `name` the API returned doesn't match the one in `accounts.csv`. The API cleans names when it saves them, so a +name with emoji, symbols, double spaces or more than 30 characters comes back changed. Fix the names (see the +[name rules](#format)) and run again. + +### `${something}` appears literally in a request + +The variable was never set. Common causes: + +- The name is spelled differently (names are case-sensitive). +- The element that sets it is out of scope (e.g. it's in another thread group). +- The extractor found nothing. + +Enable **View Results Tree** and run a smoke test in the GUI to see the actual requests sent. To see every variable a +thread holds, also enable the [Debug Sampler](#debug-sampler). + +### Report folder error + +JMeter won't write a report into a folder that isn't empty. Add `-f`, or delete the report folder first. diff --git a/docs/testing.md b/docs/testing.md index db10ae48..47f601b6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -22,7 +22,7 @@ How we apply it here ### Test types at a glance -This project uses three complementary test types. Use the right level for the job: +This project uses four complementary test types. Use the right level for the job: - Unit tests - Purpose: Verify a single function/module in isolation; fast, deterministic. @@ -48,6 +48,14 @@ This project uses three complementary test types. Use the right level for the jo - Naming: `*.e2e.spec.ts`. - When to use: Critical journeys (signup, login, checkout), cross-service interactions, smoke tests for deployments. +- Load tests + - Purpose: Check how the API behaves when many users hit it at once (response times, throughput, error rate). + - Where: `tests/load/` (`compass-testPlan.jmx`). + - Runner: Apache JMeter 5.6+. + - When to use: Before releases or after changes to frequently used endpoints. Not for checking correctness; + unit, integration and E2E tests cover that. + - Guide: [Load testing guide](load-testing.md). + Quick commands ```bash diff --git a/tests/load/accounts.csv b/tests/load/accounts.csv new file mode 100644 index 00000000..28353650 --- /dev/null +++ b/tests/load/accounts.csv @@ -0,0 +1,200 @@ +compassLoadTest001@testExample.com,password123,LoadTestUser001,LoadTestName001 +compassLoadTest002@testExample.com,password123,LoadTestUser002,LoadTestName002 +compassLoadTest003@testExample.com,password123,LoadTestUser003,LoadTestName003 +compassLoadTest004@testExample.com,password123,LoadTestUser004,LoadTestName004 +compassLoadTest005@testExample.com,password123,LoadTestUser005,LoadTestName005 +compassLoadTest006@testExample.com,password123,LoadTestUser006,LoadTestName006 +compassLoadTest007@testExample.com,password123,LoadTestUser007,LoadTestName007 +compassLoadTest008@testExample.com,password123,LoadTestUser008,LoadTestName008 +compassLoadTest009@testExample.com,password123,LoadTestUser009,LoadTestName009 +compassLoadTest010@testExample.com,password123,LoadTestUser010,LoadTestName010 +compassLoadTest011@testExample.com,password123,LoadTestUser011,LoadTestName011 +compassLoadTest012@testExample.com,password123,LoadTestUser012,LoadTestName012 +compassLoadTest013@testExample.com,password123,LoadTestUser013,LoadTestName013 +compassLoadTest014@testExample.com,password123,LoadTestUser014,LoadTestName014 +compassLoadTest015@testExample.com,password123,LoadTestUser015,LoadTestName015 +compassLoadTest016@testExample.com,password123,LoadTestUser016,LoadTestName016 +compassLoadTest017@testExample.com,password123,LoadTestUser017,LoadTestName017 +compassLoadTest018@testExample.com,password123,LoadTestUser018,LoadTestName018 +compassLoadTest019@testExample.com,password123,LoadTestUser019,LoadTestName019 +compassLoadTest020@testExample.com,password123,LoadTestUser020,LoadTestName020 +compassLoadTest021@testExample.com,password123,LoadTestUser021,LoadTestName021 +compassLoadTest022@testExample.com,password123,LoadTestUser022,LoadTestName022 +compassLoadTest023@testExample.com,password123,LoadTestUser023,LoadTestName023 +compassLoadTest024@testExample.com,password123,LoadTestUser024,LoadTestName024 +compassLoadTest025@testExample.com,password123,LoadTestUser025,LoadTestName025 +compassLoadTest026@testExample.com,password123,LoadTestUser026,LoadTestName026 +compassLoadTest027@testExample.com,password123,LoadTestUser027,LoadTestName027 +compassLoadTest028@testExample.com,password123,LoadTestUser028,LoadTestName028 +compassLoadTest029@testExample.com,password123,LoadTestUser029,LoadTestName029 +compassLoadTest030@testExample.com,password123,LoadTestUser030,LoadTestName030 +compassLoadTest031@testExample.com,password123,LoadTestUser031,LoadTestName031 +compassLoadTest032@testExample.com,password123,LoadTestUser032,LoadTestName032 +compassLoadTest033@testExample.com,password123,LoadTestUser033,LoadTestName033 +compassLoadTest034@testExample.com,password123,LoadTestUser034,LoadTestName034 +compassLoadTest035@testExample.com,password123,LoadTestUser035,LoadTestName035 +compassLoadTest036@testExample.com,password123,LoadTestUser036,LoadTestName036 +compassLoadTest037@testExample.com,password123,LoadTestUser037,LoadTestName037 +compassLoadTest038@testExample.com,password123,LoadTestUser038,LoadTestName038 +compassLoadTest039@testExample.com,password123,LoadTestUser039,LoadTestName039 +compassLoadTest040@testExample.com,password123,LoadTestUser040,LoadTestName040 +compassLoadTest041@testExample.com,password123,LoadTestUser041,LoadTestName041 +compassLoadTest042@testExample.com,password123,LoadTestUser042,LoadTestName042 +compassLoadTest043@testExample.com,password123,LoadTestUser043,LoadTestName043 +compassLoadTest044@testExample.com,password123,LoadTestUser044,LoadTestName044 +compassLoadTest045@testExample.com,password123,LoadTestUser045,LoadTestName045 +compassLoadTest046@testExample.com,password123,LoadTestUser046,LoadTestName046 +compassLoadTest047@testExample.com,password123,LoadTestUser047,LoadTestName047 +compassLoadTest048@testExample.com,password123,LoadTestUser048,LoadTestName048 +compassLoadTest049@testExample.com,password123,LoadTestUser049,LoadTestName049 +compassLoadTest050@testExample.com,password123,LoadTestUser050,LoadTestName050 +compassLoadTest051@testExample.com,password123,LoadTestUser051,LoadTestName051 +compassLoadTest052@testExample.com,password123,LoadTestUser052,LoadTestName052 +compassLoadTest053@testExample.com,password123,LoadTestUser053,LoadTestName053 +compassLoadTest054@testExample.com,password123,LoadTestUser054,LoadTestName054 +compassLoadTest055@testExample.com,password123,LoadTestUser055,LoadTestName055 +compassLoadTest056@testExample.com,password123,LoadTestUser056,LoadTestName056 +compassLoadTest057@testExample.com,password123,LoadTestUser057,LoadTestName057 +compassLoadTest058@testExample.com,password123,LoadTestUser058,LoadTestName058 +compassLoadTest059@testExample.com,password123,LoadTestUser059,LoadTestName059 +compassLoadTest060@testExample.com,password123,LoadTestUser060,LoadTestName060 +compassLoadTest061@testExample.com,password123,LoadTestUser061,LoadTestName061 +compassLoadTest062@testExample.com,password123,LoadTestUser062,LoadTestName062 +compassLoadTest063@testExample.com,password123,LoadTestUser063,LoadTestName063 +compassLoadTest064@testExample.com,password123,LoadTestUser064,LoadTestName064 +compassLoadTest065@testExample.com,password123,LoadTestUser065,LoadTestName065 +compassLoadTest066@testExample.com,password123,LoadTestUser066,LoadTestName066 +compassLoadTest067@testExample.com,password123,LoadTestUser067,LoadTestName067 +compassLoadTest068@testExample.com,password123,LoadTestUser068,LoadTestName068 +compassLoadTest069@testExample.com,password123,LoadTestUser069,LoadTestName069 +compassLoadTest070@testExample.com,password123,LoadTestUser070,LoadTestName070 +compassLoadTest071@testExample.com,password123,LoadTestUser071,LoadTestName071 +compassLoadTest072@testExample.com,password123,LoadTestUser072,LoadTestName072 +compassLoadTest073@testExample.com,password123,LoadTestUser073,LoadTestName073 +compassLoadTest074@testExample.com,password123,LoadTestUser074,LoadTestName074 +compassLoadTest075@testExample.com,password123,LoadTestUser075,LoadTestName075 +compassLoadTest076@testExample.com,password123,LoadTestUser076,LoadTestName076 +compassLoadTest077@testExample.com,password123,LoadTestUser077,LoadTestName077 +compassLoadTest078@testExample.com,password123,LoadTestUser078,LoadTestName078 +compassLoadTest079@testExample.com,password123,LoadTestUser079,LoadTestName079 +compassLoadTest080@testExample.com,password123,LoadTestUser080,LoadTestName080 +compassLoadTest081@testExample.com,password123,LoadTestUser081,LoadTestName081 +compassLoadTest082@testExample.com,password123,LoadTestUser082,LoadTestName082 +compassLoadTest083@testExample.com,password123,LoadTestUser083,LoadTestName083 +compassLoadTest084@testExample.com,password123,LoadTestUser084,LoadTestName084 +compassLoadTest085@testExample.com,password123,LoadTestUser085,LoadTestName085 +compassLoadTest086@testExample.com,password123,LoadTestUser086,LoadTestName086 +compassLoadTest087@testExample.com,password123,LoadTestUser087,LoadTestName087 +compassLoadTest088@testExample.com,password123,LoadTestUser088,LoadTestName088 +compassLoadTest089@testExample.com,password123,LoadTestUser089,LoadTestName089 +compassLoadTest090@testExample.com,password123,LoadTestUser090,LoadTestName090 +compassLoadTest091@testExample.com,password123,LoadTestUser091,LoadTestName091 +compassLoadTest092@testExample.com,password123,LoadTestUser092,LoadTestName092 +compassLoadTest093@testExample.com,password123,LoadTestUser093,LoadTestName093 +compassLoadTest094@testExample.com,password123,LoadTestUser094,LoadTestName094 +compassLoadTest095@testExample.com,password123,LoadTestUser095,LoadTestName095 +compassLoadTest096@testExample.com,password123,LoadTestUser096,LoadTestName096 +compassLoadTest097@testExample.com,password123,LoadTestUser097,LoadTestName097 +compassLoadTest098@testExample.com,password123,LoadTestUser098,LoadTestName098 +compassLoadTest099@testExample.com,password123,LoadTestUser099,LoadTestName099 +compassLoadTest100@testExample.com,password123,LoadTestUser100,LoadTestName100 +compassLoadTest101@testExample.com,password123,LoadTestUser101,LoadTestName101 +compassLoadTest102@testExample.com,password123,LoadTestUser102,LoadTestName102 +compassLoadTest103@testExample.com,password123,LoadTestUser103,LoadTestName103 +compassLoadTest104@testExample.com,password123,LoadTestUser104,LoadTestName104 +compassLoadTest105@testExample.com,password123,LoadTestUser105,LoadTestName105 +compassLoadTest106@testExample.com,password123,LoadTestUser106,LoadTestName106 +compassLoadTest107@testExample.com,password123,LoadTestUser107,LoadTestName107 +compassLoadTest108@testExample.com,password123,LoadTestUser108,LoadTestName108 +compassLoadTest109@testExample.com,password123,LoadTestUser109,LoadTestName109 +compassLoadTest110@testExample.com,password123,LoadTestUser110,LoadTestName110 +compassLoadTest111@testExample.com,password123,LoadTestUser111,LoadTestName111 +compassLoadTest112@testExample.com,password123,LoadTestUser112,LoadTestName112 +compassLoadTest113@testExample.com,password123,LoadTestUser113,LoadTestName113 +compassLoadTest114@testExample.com,password123,LoadTestUser114,LoadTestName114 +compassLoadTest115@testExample.com,password123,LoadTestUser115,LoadTestName115 +compassLoadTest116@testExample.com,password123,LoadTestUser116,LoadTestName116 +compassLoadTest117@testExample.com,password123,LoadTestUser117,LoadTestName117 +compassLoadTest118@testExample.com,password123,LoadTestUser118,LoadTestName118 +compassLoadTest119@testExample.com,password123,LoadTestUser119,LoadTestName119 +compassLoadTest120@testExample.com,password123,LoadTestUser120,LoadTestName120 +compassLoadTest121@testExample.com,password123,LoadTestUser121,LoadTestName121 +compassLoadTest122@testExample.com,password123,LoadTestUser122,LoadTestName122 +compassLoadTest123@testExample.com,password123,LoadTestUser123,LoadTestName123 +compassLoadTest124@testExample.com,password123,LoadTestUser124,LoadTestName124 +compassLoadTest125@testExample.com,password123,LoadTestUser125,LoadTestName125 +compassLoadTest126@testExample.com,password123,LoadTestUser126,LoadTestName126 +compassLoadTest127@testExample.com,password123,LoadTestUser127,LoadTestName127 +compassLoadTest128@testExample.com,password123,LoadTestUser128,LoadTestName128 +compassLoadTest129@testExample.com,password123,LoadTestUser129,LoadTestName129 +compassLoadTest130@testExample.com,password123,LoadTestUser130,LoadTestName130 +compassLoadTest131@testExample.com,password123,LoadTestUser131,LoadTestName131 +compassLoadTest132@testExample.com,password123,LoadTestUser132,LoadTestName132 +compassLoadTest133@testExample.com,password123,LoadTestUser133,LoadTestName133 +compassLoadTest134@testExample.com,password123,LoadTestUser134,LoadTestName134 +compassLoadTest135@testExample.com,password123,LoadTestUser135,LoadTestName135 +compassLoadTest136@testExample.com,password123,LoadTestUser136,LoadTestName136 +compassLoadTest137@testExample.com,password123,LoadTestUser137,LoadTestName137 +compassLoadTest138@testExample.com,password123,LoadTestUser138,LoadTestName138 +compassLoadTest139@testExample.com,password123,LoadTestUser139,LoadTestName139 +compassLoadTest140@testExample.com,password123,LoadTestUser140,LoadTestName140 +compassLoadTest141@testExample.com,password123,LoadTestUser141,LoadTestName141 +compassLoadTest142@testExample.com,password123,LoadTestUser142,LoadTestName142 +compassLoadTest143@testExample.com,password123,LoadTestUser143,LoadTestName143 +compassLoadTest144@testExample.com,password123,LoadTestUser144,LoadTestName144 +compassLoadTest145@testExample.com,password123,LoadTestUser145,LoadTestName145 +compassLoadTest146@testExample.com,password123,LoadTestUser146,LoadTestName146 +compassLoadTest147@testExample.com,password123,LoadTestUser147,LoadTestName147 +compassLoadTest148@testExample.com,password123,LoadTestUser148,LoadTestName148 +compassLoadTest149@testExample.com,password123,LoadTestUser149,LoadTestName149 +compassLoadTest150@testExample.com,password123,LoadTestUser150,LoadTestName150 +compassLoadTest151@testExample.com,password123,LoadTestUser151,LoadTestName151 +compassLoadTest152@testExample.com,password123,LoadTestUser152,LoadTestName152 +compassLoadTest153@testExample.com,password123,LoadTestUser153,LoadTestName153 +compassLoadTest154@testExample.com,password123,LoadTestUser154,LoadTestName154 +compassLoadTest155@testExample.com,password123,LoadTestUser155,LoadTestName155 +compassLoadTest156@testExample.com,password123,LoadTestUser156,LoadTestName156 +compassLoadTest157@testExample.com,password123,LoadTestUser157,LoadTestName157 +compassLoadTest158@testExample.com,password123,LoadTestUser158,LoadTestName158 +compassLoadTest159@testExample.com,password123,LoadTestUser159,LoadTestName159 +compassLoadTest160@testExample.com,password123,LoadTestUser160,LoadTestName160 +compassLoadTest161@testExample.com,password123,LoadTestUser161,LoadTestName161 +compassLoadTest162@testExample.com,password123,LoadTestUser162,LoadTestName162 +compassLoadTest163@testExample.com,password123,LoadTestUser163,LoadTestName163 +compassLoadTest164@testExample.com,password123,LoadTestUser164,LoadTestName164 +compassLoadTest165@testExample.com,password123,LoadTestUser165,LoadTestName165 +compassLoadTest166@testExample.com,password123,LoadTestUser166,LoadTestName166 +compassLoadTest167@testExample.com,password123,LoadTestUser167,LoadTestName167 +compassLoadTest168@testExample.com,password123,LoadTestUser168,LoadTestName168 +compassLoadTest169@testExample.com,password123,LoadTestUser169,LoadTestName169 +compassLoadTest170@testExample.com,password123,LoadTestUser170,LoadTestName170 +compassLoadTest171@testExample.com,password123,LoadTestUser171,LoadTestName171 +compassLoadTest172@testExample.com,password123,LoadTestUser172,LoadTestName172 +compassLoadTest173@testExample.com,password123,LoadTestUser173,LoadTestName173 +compassLoadTest174@testExample.com,password123,LoadTestUser174,LoadTestName174 +compassLoadTest175@testExample.com,password123,LoadTestUser175,LoadTestName175 +compassLoadTest176@testExample.com,password123,LoadTestUser176,LoadTestName176 +compassLoadTest177@testExample.com,password123,LoadTestUser177,LoadTestName177 +compassLoadTest178@testExample.com,password123,LoadTestUser178,LoadTestName178 +compassLoadTest179@testExample.com,password123,LoadTestUser179,LoadTestName179 +compassLoadTest180@testExample.com,password123,LoadTestUser180,LoadTestName180 +compassLoadTest181@testExample.com,password123,LoadTestUser181,LoadTestName181 +compassLoadTest182@testExample.com,password123,LoadTestUser182,LoadTestName182 +compassLoadTest183@testExample.com,password123,LoadTestUser183,LoadTestName183 +compassLoadTest184@testExample.com,password123,LoadTestUser184,LoadTestName184 +compassLoadTest185@testExample.com,password123,LoadTestUser185,LoadTestName185 +compassLoadTest186@testExample.com,password123,LoadTestUser186,LoadTestName186 +compassLoadTest187@testExample.com,password123,LoadTestUser187,LoadTestName187 +compassLoadTest188@testExample.com,password123,LoadTestUser188,LoadTestName188 +compassLoadTest189@testExample.com,password123,LoadTestUser189,LoadTestName189 +compassLoadTest190@testExample.com,password123,LoadTestUser190,LoadTestName190 +compassLoadTest191@testExample.com,password123,LoadTestUser191,LoadTestName191 +compassLoadTest192@testExample.com,password123,LoadTestUser192,LoadTestName192 +compassLoadTest193@testExample.com,password123,LoadTestUser193,LoadTestName193 +compassLoadTest194@testExample.com,password123,LoadTestUser194,LoadTestName194 +compassLoadTest195@testExample.com,password123,LoadTestUser195,LoadTestName195 +compassLoadTest196@testExample.com,password123,LoadTestUser196,LoadTestName196 +compassLoadTest197@testExample.com,password123,LoadTestUser197,LoadTestName197 +compassLoadTest198@testExample.com,password123,LoadTestUser198,LoadTestName198 +compassLoadTest199@testExample.com,password123,LoadTestUser199,LoadTestName199 +compassLoadTest200@testExample.com,password123,LoadTestUser200,LoadTestName200 \ No newline at end of file diff --git a/tests/load/check_results.py b/tests/load/check_results.py new file mode 100644 index 00000000..480936a0 --- /dev/null +++ b/tests/load/check_results.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Fail the build when a JMeter results file (.jtl) breaks the limits. + +JMeter exits with 0 even when requests fail, so CI runs this afterwards. + +Usage: + python3 check_results.py results.jtl --max-error-rate 1 --max-p95 800 +""" +import argparse +import csv +import math +import sys + +# Requests in the plan are labelled "SETUP | ...", "TEST | ..." or "TEARDOWN | ...". +# The error limit applies to all of them: a failed SETUP request means less load than asked for, and a +# failed TEARDOWN request leaves test accounts behind on the server. The p95 limit only applies to TEST +# requests, because setup and cleanup times aren't what the test measures. +TEST_LABEL_PREFIX = "TEST |" + + +def load_samples(path): + with open(path, newline="", encoding="utf-8") as f: + return list(csv.DictReader(f)) + + +def percentile(values, pct): + # nearest-rank: the value that pct% of samples are at or below + ordered = sorted(values) + rank = math.ceil(pct / 100 * len(ordered)) + return ordered[max(rank - 1, 0)] + + +def summarise(samples): + """Group samples by label (request name) and work out the numbers for each.""" + stats = {} + for label in sorted({s["label"] for s in samples}): + group = [s for s in samples if s["label"] == label] + errors = sum(1 for s in group if s["success"] != "true") + stats[label] = { + "count": len(group), + "error_rate": errors / len(group) * 100, # percent + "p95": percentile([int(s["elapsed"]) for s in group], 95), # ms + } + return stats + + +def check_limits(stats, max_error_rate, max_p95): + """Return one message per broken limit. An empty list means the run passed.""" + problems = [] + for label, s in stats.items(): + if s["error_rate"] > max_error_rate: + problems.append(f"{label}: error rate {s['error_rate']:.1f}% is over {max_error_rate}%") + if label.startswith(TEST_LABEL_PREFIX) and s["count"] >= 20 and s["p95"] > max_p95: + problems.append(f"{label}: p95 {s['p95']} ms is over {max_p95} ms") + return problems + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("jtl", help="results file written by jmeter -l") + parser.add_argument("--max-error-rate", type=float, default=1.0, help="percent, default 1") + parser.add_argument("--max-p95", type=int, default=800, help="milliseconds, default 800") + args = parser.parse_args() + + samples = load_samples(args.jtl) + if not any(s["label"].startswith(TEST_LABEL_PREFIX) for s in samples): + print("FAIL: no samples from the main test. Did setUp create any accounts?") + return 1 + + stats = summarise(samples) + for label, s in stats.items(): + print(f"{label}: {s['count']} samples, {s['error_rate']:.1f}% errors, p95 {s['p95']} ms") + + problems = check_limits(stats, args.max_error_rate, args.max_p95) + for p in problems: + print("FAIL:", p) + if not problems: + print("PASS") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/load/compass-testPlan.jmx b/tests/load/compass-testPlan.jmx new file mode 100644 index 00000000..6ff5560f --- /dev/null +++ b/tests/load/compass-testPlan.jmx @@ -0,0 +1,519 @@ + + + + + true + true + + + + false + + + + ${__P(resetFile,1)} + 1 + true + continue + + 1 + false + + + + + true + + + import org.apache.jmeter.services.FileServer +new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt").text = "" + groovy + + + + + + + baseAuthUrl + identitytoolkit.googleapis.com + + = + + + apiHost + ${__P(apiHost,api.dev.compassmeet.com)} + = + + + firebaseApiKey + ${__P(firebaseApiKey,AIzaSyBspL9glBXWbMsjmtt36dgb2yU0YGGhzKo)} + = + + + signupUrl + /v1/accounts:signUp?key= + = + + + signinUrl + /v1/accounts:signInWithPassword?key= + = + + + deleteAccountUrl + /v1/accounts:delete?key= + = + + + getUserDataUrl + /v1/accounts:lookup?key= + = + + + + + + ${__P(setupThreads,${__P(threads,100)})} + ${__P(setupRamp,10)} + true + continue + + 1 + false + + + + + accounts.csv + UTF-8 + userEmail,userPassword,username,name + false + , + false + false + true + shareMode.all + + + + ${baseAuthUrl} + https + ${signupUrl}${firebaseApiKey} + true + POST + true + true + + + + false + { "email": "${userEmail}", "password": "${userPassword}", "returnSecureToken": true } + = + + + + + + + + + Content-Type + application/json + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + authToken + $.idToken + 1 + + + + + ${apiHost} + https + /create-user-and-profile + true + POST + true + true + + + + false + {"username": "${username}", "name": "${name}", "profile": {"visibility": "public"}} + = + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + groovy + + + true + import org.apache.jmeter.services.FileServer + +def token = vars.get("authToken") +if (!token) { + log.warn("No token for ${vars.get('userEmail')}, not saving") + return +} +if (!prev.isSuccessful()) { + log.warn("Profile creation failed for ${vars.get('userEmail')}, saving anyway so it gets cleaned up") +} + +def file = new File(FileServer.getFileServer().getBaseDir(), "created_accounts.txt") +synchronized (FileServer.class) { + file.append([vars.get("userEmail"), vars.get("userPassword"), token, vars.get("name")].join("|") + "\n", "UTF-8") +} + + + + + false + true + false + + + + + ${__P(meThreads,${__P(threads,100)})} + ${__P(meRamp,${__P(ramp,10)})} + true + continue + + 1 + false + + + + + created_accounts.txt + UTF-8 + userEmail,userPassword,authToken,name + false + | + false + false + true + shareMode.group + + + + ${__P(meDuration,${__P(duration,120)})} + + + + ${apiHost} + https + /me + true + GET + true + false + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + $.name + ${name} + true + false + false + false + + + + + ${__P(thinkTimeMs,0)} + + + + + + 1 + 1 + true + continue + + -1 + false + + + + + created_accounts.txt + UTF-8 + userEmail,userPassword,authToken,name + false + | + false + false + true + shareMode.group + + + + ${baseAuthUrl} + https + ${signinUrl}${firebaseApiKey} + true + POST + true + true + + + + false + {"email": "${userEmail}", "password": "${userPassword}", "returnSecureToken": true} + = + + + + + + + + + Content-Type + application/json + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + authToken + $.idToken + 1 + + + + + ${apiHost} + https + /me/delete + true + POST + true + true + + + + false + {"reasonCategory": "jmeter-load-test"} + = + + + + + + + + + Content-Type + application/json + + + Authorization + Bearer ${authToken} + + + + + + + 200 + + + Assertion.response_code + false + 8 + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + true + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + true + false + true + true + false + false + false + true + 0 + true + true + true + true + true + true + + + errors.xml + + + + +