Send test results to an API for analysis (#6261)

* Send test results to an API

* Include platform

* Include target

* Return earlier

* Include actual commit SHA

* Include PR number

* Rename variables for clarity

.github/workflows/e2e-tests.yml

@@ -231,6 +231,11 @@ jobs:
         env:
           token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
           snapshottoken: ${{ secrets.KITTYCAD_API_TOKEN }}
+          TAB_API_URL: ${{ secrets.TAB_API_URL }}
+          TAB_API_KEY: ${{ secrets.TAB_API_KEY }}
+          CI_COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+          CI_PR_NUMBER: ${{ github.event.pull_request.number }}
+          TARGET: web
 
       - uses: actions/upload-artifact@v4
         if: ${{ needs.conditions.outputs.should-run == 'true' && !cancelled() && (success() || failure()) }}
@@ -365,6 +370,11 @@ jobs:
         env:
           FAIL_ON_CONSOLE_ERRORS: true
           token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
+          TAB_API_URL: ${{ secrets.TAB_API_URL }}
+          TAB_API_KEY: ${{ secrets.TAB_API_KEY }}
+          CI_COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+          CI_PR_NUMBER: ${{ github.event.pull_request.number }}
+          TARGET: desktop
 
       - uses: actions/upload-artifact@v4
         if: ${{ needs.conditions.outputs.should-run == 'true' && always() }}

Makefile

@@ -17,7 +17,7 @@ WASM_PACK ?= $(USERPROFILE)/.cargo/bin/wasm-pack.exe
 else
 CARGO ?= ~/.cargo/bin/cargo
 WASM_PACK ?= ~/.cargo/bin/wasm-pack
-endif 
+endif
 
 .PHONY: install
 install: node_modules/.yarn-integrity $(CARGO) $(WASM_PACK) ## Install dependencies

e2e/playwright/lib/api-reporter.ts

@@ -0,0 +1,63 @@
+import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter'
+
+class MyAPIReporter implements Reporter {
+  onTestEnd(test: TestCase, result: TestResult): void {
+    if (!process.env.TAB_API_URL || !process.env.TAB_API_KEY) {
+      return
+    }
+
+    const payload = {
+      // Required information
+      project: 'https://github.com/KittyCAD/modeling-app',
+      branch: process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF || '',
+      commit: process.env.CI_COMMIT_SHA || process.env.GITHUB_SHA || '',
+      test: test.titlePath().slice(2).join(' › '),
+      status: result.status,
+      // Optional information
+      duration: result.duration / 1000,
+      message: result.error?.stack,
+      target: process.env.TARGET || null,
+      platform: process.env.RUNNER_OS || process.platform,
+      // Extra test and result data
+      annotations: test.annotations.map((a) => a.type),
+      retries: result.retry,
+      // Extra environment variables
+      CI_COMMIT_SHA: process.env.CI_COMMIT_SHA || null,
+      CI_PR_NUMBER: process.env.CI_PR_NUMBER || null,
+      GITHUB_BASE_REF: process.env.GITHUB_BASE_REF || null,
+      GITHUB_EVENT_NAME: process.env.GITHUB_EVENT_NAME || null,
+      GITHUB_HEAD_REF: process.env.GITHUB_HEAD_REF || null,
+      GITHUB_REF_NAME: process.env.GITHUB_REF_NAME || null,
+      GITHUB_REF: process.env.GITHUB_REF || null,
+      GITHUB_SHA: process.env.GITHUB_SHA || null,
+      GITHUB_WORKFLOW: process.env.GITHUB_WORKFLOW || null,
+      RUNNER_ARCH: process.env.RUNNER_ARCH || null,
+    }
+
+    void (async () => {
+      try {
+        const response = await fetch(`${process.env.TAB_API_URL}/api/results`, {
+          method: 'POST',
+          headers: new Headers({
+            'Content-Type': 'application/json',
+            'X-API-Key': process.env.TAB_API_KEY || '',
+          }),
+          body: JSON.stringify(payload),
+        })
+
+        if (!response.ok && !process.env.CI) {
+          console.error(
+            'TAB API - Failed to send test result:',
+            await response.text()
+          )
+        }
+      } catch {
+        if (!process.env.CI) {
+          console.error('TAB API - Unable to send test result')
+        }
+      }
+    })()
+  }
+}
+
+export default MyAPIReporter

playwright.config.ts

@@ -45,6 +45,7 @@ export default defineConfig({
     [process.env.CI ? 'dot' : 'list'],
     ['json', { outputFile: './test-results/report.json' }],
     ['html'],
+    ['./e2e/playwright/lib/api-reporter.ts'],
   ],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {

playwright.electron.config.ts

@@ -45,6 +45,7 @@ export default defineConfig({
     ['dot'],
     ['json', { outputFile: './test-results/report.json' }],
     ['html'],
+    ['./e2e/playwright/lib/api-reporter.ts'],
   ],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {