T-22072 [CI] Name the test in the E2E Linear issue and include the failure summary

.github/workflows/e2e.yml

@@ -177,6 +177,30 @@ jobs:
       - name: "[Test] Read the lines back through the SQL API connection"
         run: |
           set -o pipefail
+          mkdir -p out
+          fail() { FAILURE="$1"; echo "::error::$1"; [ -z "${2:-}" ] || echo "$2"; exit 1; }
+          # Every way this step fails still leaves a summary: what arrived against what was expected.
+          summarize_failure() {
+            local status=$?
+            [ "$status" = 0 ] && return
+            {
+              echo "### AWS CloudWatch Lambda E2E \`$RUN_ID\`"
+              echo
+              echo "❌ ${FAILURE:-The read-back stopped on an error, see the log of this step}"
+              if [ -n "${TOTAL:-}" ]; then
+                echo
+                echo "| read back through \`$CONNECT_HOST\` | arrived | expected |"
+                echo "|---|---|---|"
+                echo "| rows | $TOTAL | 20 |"
+                echo "| level error | $ERRORS | 5 |"
+                echo "| level info | $INFOS | 15 |"
+                echo "| lambda name \`$FUNCTION_NAME\` | $NAMED | 20 |"
+              fi
+              [ -z "${PROBLEMS:-}" ] || { echo; echo '```'; head -20 <<<"$PROBLEMS"; echo '```'; }
+            } > out/summary.md
+            exit "$status"
+          }
+          trap summarize_failure EXIT
           sql() {
             CODE=$(curl -sS -o response.json -w '%{http_code}' -u "$TEST_TEAM_CONNECTION_USERNAME:$TEST_TEAM_CONNECTION_PASSWORD" -H 'Content-Type: plain/text' --data-binary "$1 FORMAT JSONEachRow" "https://$CONNECT_HOST/")
             # A source's collection exists only from its first stored row on; until then the query is "no rows yet".
@@ -196,14 +220,13 @@ jobs:
           while :; do
             counts
             [ "$TOTAL" -ge 20 ] && break
-            [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::only $TOTAL of 20 lines arrived within 5 min"; exit 1; }
+            [ "$SECONDS" -lt "$DEADLINE" ] || fail "Only $TOTAL of 20 lines arrived within 5 min"
             sleep 15
           done
           # Everything arrived; a last look 30 s later catches duplicate delivery.
           sleep 30
           counts
-          [ "$TOTAL" = 20 ] && [ "$ERRORS" = 5 ] && [ "$INFOS" = 15 ] && [ "$NAMED" = 20 ] || { echo "::error::unexpected rows: total $TOTAL (20), error $ERRORS (5), info $INFOS (15), lambda name $NAMED (20)"; exit 1; }
-          mkdir -p out
+          [ "$TOTAL" = 20 ] && [ "$ERRORS" = 5 ] && [ "$INFOS" = 15 ] && [ "$NAMED" = 20 ] || fail "Unexpected rows: total $TOTAL (20), error $ERRORS (5), info $INFOS (15), lambda name $NAMED (20)"
           sql "SELECT * FROM remote(${COLLECTION}_logs) WHERE $SINCE AND position(raw, '$MARKER') > 0 ORDER BY dt LIMIT 2" > out/sample_log.json
           # Every stored row, field by field, against the event that produced it (events.json from the write step):
           # the level prefix stripped from the message, the level, the CloudWatch timestamp as dt, and the context
@@ -229,7 +252,7 @@ jobs:
                   ] as $problems
                 | select($problems | length > 0) | "n=\($n): \($problems | join(", "))")
             | .[]' rows.jsonl)
-          [ -z "$PROBLEMS" ] || { echo "::error::stored rows differ from what the Lambda should have produced:"; echo "$PROBLEMS"; exit 1; }
+          [ -z "$PROBLEMS" ] || fail "Stored rows differ from what the Lambda should have produced" "$PROBLEMS"
           {
             echo "### AWS CloudWatch Lambda E2E \`$RUN_ID\`"
             echo
@@ -243,7 +266,11 @@ jobs:
       - name: "[Test] The Lambda logged no errors"
         run: |
           ERRORS=$(aws logs filter-log-events --log-group-name "/aws/lambda/$FUNCTION_NAME" --filter-pattern '?ERROR ?"Task timed out" ?Unhandled' --query 'events[].message' --output text)
-          [ -z "$ERRORS" ] || { echo "::error::the Lambda logged errors:"; echo "$ERRORS"; exit 1; }
+          if [ -n "$ERRORS" ]; then
+            echo "::error::the Lambda logged errors:"; echo "$ERRORS"
+            { echo; echo "❌ The Lambda logged errors:"; echo; echo '```'; head -20 <<<"$ERRORS"; echo '```'; } >> out/summary.md
+            exit 1
+          fi
           echo "no errors in /aws/lambda/$FUNCTION_NAME"
 
       - name: "[Cleanup] Delete this run's resources in Better Stack and AWS (and stale ones)"
@@ -279,15 +306,44 @@ jobs:
 
       - name: "[Report] Job summary"
         if: ${{ always() }}
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: |
-          if [ -f out/summary.md ]; then cat out/summary.md >> "$GITHUB_STEP_SUMMARY"; else echo "No summary, the run failed before verification." >> "$GITHUB_STEP_SUMMARY"; fi
+          mkdir -p report
+          if [ -f out/summary.md ]; then
+            cp out/summary.md report/summary.md
+          else
+            # The read-back writes the summary however it ends, so a step before it failed (or the job was cancelled).
+            FAILED_STEP=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs" \
+              --jq ".jobs[] | select(.runner_name == \"$RUNNER_NAME\") | .steps[] | select(.conclusion == \"failure\") | .name" | head -1) || true
+            { echo "### AWS CloudWatch Lambda E2E \`$RUN_ID\`"; echo; echo "❌ Stopped${FAILED_STEP:+ at **$FAILED_STEP**}, before anything was read back."; } > report/summary.md
+          fi
+          cat report/summary.md >> "$GITHUB_STEP_SUMMARY"
+          # The Linear issue gets the cleanup only when it left something behind.
+          grep -q '❌' out/cleanup.md 2>/dev/null && { echo; cat out/cleanup.md; } >> report/summary.md
           # Rows come back as JSONEachRow with the stored event as a JSON string in raw; render them as a pretty array.
           [ -f out/sample_log.json ] && { echo; echo '<details><summary>sample_log.json</summary>'; echo; echo '```json'; jq -s 'map(if has("raw") then .raw |= (fromjson? // .) else . end)' out/sample_log.json; echo '```'; echo '</details>'; echo; } >> "$GITHUB_STEP_SUMMARY"
           [ -f out/cleanup.md ] && { echo; cat out/cleanup.md; } >> "$GITHUB_STEP_SUMMARY"
           true
 
+      - name: "[Report] Hand the summary to the Linear issue"
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: e2e-summary-CloudWatch round trip
+          path: report/summary.md
+          retention-days: 7
+          overwrite: true
+
   notify-linear:
     if: ${{ failure() && (github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) }}
     needs: [test]
+    permissions:
+      contents: read
+      actions: read
+      checks: read
     uses: ./.github/workflows/notify_linear_failure.yml
+    with:
+      test_name: AWS Lambda via CloudWatch
+      job_summaries: true
     secrets: inherit

.github/workflows/notify_linear_failure.yml

@@ -1,34 +1,71 @@
 name: "Notify Linear on failure"
 
+# A caller that sets neither input gets the plain issue: "E2E failure: <workflow name>" listing the failed jobs and steps.
 on:
   workflow_call:
+    inputs:
+      test_name:
+        description: "Names the test in the issue title instead of the workflow name"
+        type: string
+        default: ""
+      job_summaries:
+        description: >-
+          Adds each failed job's errors and summary to the issue. A job hands its summary over as an artifact named
+          "e2e-summary-<job name>" holding summary.md.
+        type: boolean
+        default: false
 
 jobs:
   create-linear-issue:
     runs-on: ubuntu-latest
+    # A caller must grant checks: read as well (the errors come from the annotations), or this job cannot start.
     permissions:
       contents: read
       actions: read
+      checks: read
     steps:
       - name: "Gather failure context"
         id: ctx
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           RUN_ID: ${{ github.event.workflow_run.id || github.run_id }}
           RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt || github.run_attempt }}
+          JOB_SUMMARIES: ${{ inputs.job_summaries }}
         run: |
-          FAILED=$(gh api \
+          JOBS=$(gh api \
             "repos/${{ github.repository }}/actions/runs/$RUN_ID/attempts/$RUN_ATTEMPT/jobs" \
             --paginate \
-            --jq '[.jobs[]
+            --jq '.jobs[]
                     | select(.conclusion == "failure")
-                    | "- **\(.name)** — failed at _\([.steps[] | select(.conclusion == "failure") | .name] | join(", "))_"]
-                  | join("\n")')
-
+                    | {id, name, steps: ([.steps[] | select(.conclusion == "failure") | .name] | join(", "))}
+                    | @json')
+
+          FAILED=""
+          while read -r job; do
+            [ -n "$job" ] || continue
+            NAME=$(jq -r .name <<<"$job")
+            STEPS=$(jq -r .steps <<<"$job")
+            if [ "$JOB_SUMMARIES" != true ]; then
+              FAILED="$FAILED- **$NAME** — failed at _${STEPS}_"$'\n'
+              continue
+            fi
+            # The job's ::error:: lines, without the runner's closing "Process completed with exit code 1."
+            ERRORS=$(gh api "repos/${{ github.repository }}/check-runs/$(jq -r .id <<<"$job")/annotations" \
+              --jq '.[] | select(.annotation_level == "failure" and (.message | startswith("Process completed with exit code") | not)) | "- \(.message | gsub("\n"; " "))"') || ERRORS=""
+            FAILED="$FAILED### $NAME"$'\n\n'"Failed at **${STEPS}**"$'\n\n'"${ERRORS:+$ERRORS$'\n\n'}"
+            # A job that failed before writing its summary has no artifact and gets the errors alone.
+            SUMMARY_DIR="summaries/$(jq -r .id <<<"$job")"
+            gh run download "$RUN_ID" -R "${{ github.repository }}" -n "e2e-summary-$NAME" -D "$SUMMARY_DIR" >/dev/null 2>&1 || true
+            [ ! -f "$SUMMARY_DIR/summary.md" ] || FAILED="$FAILED$(cat "$SUMMARY_DIR/summary.md")"$'\n\n'
+          done <<<"$JOBS"
+
+          while [[ "$FAILED" == *$'\n' ]]; do FAILED="${FAILED%$'\n'}"; done
           if [ -z "$FAILED" ]; then
             FAILED="_(no failed jobs reported by the API yet)_"
+          elif [ "${#FAILED}" -gt 20000 ]; then
+            FAILED="${FAILED:0:20000}"$'\n\n'"_(cut at 20,000 characters, the rest is in the run)_"
           fi
-          
+
           {
             echo "failed_jobs<<CTXEOF"
             echo "$FAILED"
@@ -41,7 +78,7 @@ jobs:
           LINEAR_STATUS_PLAN_ID: ${{ secrets.LINEAR_STATUS_PLAN_ID }}
           LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID: ${{ secrets.LINEAR_PROJECT_INGESTION_FOR_EVERYTHING_ID }}
           LINEAR_ASSIGNEE_PETR_ID: ${{ secrets.LINEAR_ASSIGNEE_PETR_ID }}
-          WORKFLOW: ${{ github.event.workflow_run.name || github.workflow }}
+          WORKFLOW: ${{ inputs.test_name || github.event.workflow_run.name || github.workflow }}
           REF_NAME: ${{ github.event.workflow_run.head_branch || github.ref_name }}
           RUN_URL: ${{ github.event.workflow_run.html_url || format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }}
           RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt || github.run_attempt }}
@@ -51,32 +88,32 @@ jobs:
           FAILED_JOBS: ${{ steps.ctx.outputs.failed_jobs }}
         run: |
           SHORT_SHA="${SHA:0:7}"
-          
+
           # head_commit is only populated on push events; skip the line otherwise
           COMMIT_LINE="**Commit:** \`$SHORT_SHA\`"
           if [ -n "$COMMIT_MESSAGE" ]; then
             COMMIT_SUBJECT="$(printf '%s' "$COMMIT_MESSAGE" | head -n1)"
             COMMIT_LINE="**Commit:** \`$SHORT_SHA\` — $COMMIT_SUBJECT"
           fi
-          
+
           ATTEMPT_NOTE=""
           if [ "$RUN_ATTEMPT" != "1" ]; then
             ATTEMPT_NOTE=" (attempt #$RUN_ATTEMPT)"
           fi
-          
+
           TITLE="E2E failure: $WORKFLOW"
           DESCRIPTION=$(cat <<EOF
           **$WORKFLOW** failed on \`$REF_NAME\` via \`$EVENT_NAME\`$ATTEMPT_NOTE
-          
+
           **Failing jobs:**
           $FAILED_JOBS
-          
+
           $COMMIT_LINE
-          
+
           [View failing run]($RUN_URL)
           EOF
           )
-          
+
           PAYLOAD=$(jq -n \
             --arg title "$TITLE" \
             --arg description "$DESCRIPTION" \
@@ -98,7 +135,7 @@ jobs:
                 }
               }
             }')
-          
+
           curl -sSf -X POST https://api.linear.app/graphql \
             -H "Authorization: $LINEAR_API_KEY" \
             -H 'Content-Type: application/json' \