send failing test to axiom (#2996)

* send failing test to axiom (ubuntu)

* forgot always

* rename

* Update .github/workflows/playwright.yml

Co-authored-by: Adam Sunderland <adam@kittycad.io>

* update to indivdual lines of json

* another fix

* tweak output

* log macos too

---------

Co-authored-by: Adam Sunderland <adam@kittycad.io>

.github/workflows/playwright.yml

@@ -83,6 +83,20 @@ jobs:
       uses: Swatinem/rust-cache@v2
       with:
         workspaces: './src/wasm-lib'
+    - name: Install vector
+      run: |
+        curl --proto '=https' --tlsv1.2 -sSfL https://sh.vector.dev > /tmp/vector.sh
+        chmod +x /tmp/vector.sh
+        /tmp/vector.sh -y -no-modify-path
+        mkdir -p /tmp/vector
+        cp .github/workflows/vector.toml /tmp/vector.toml
+        sed -i "s#GITHUB_WORKFLOW#${GITHUB_WORKFLOW}#g" /tmp/vector.toml
+        sed -i "s#GITHUB_REPOSITORY#${GITHUB_REPOSITORY}#g" /tmp/vector.toml
+        sed -i "s#GITHUB_SHA#${GITHUB_SHA}#g" /tmp/vector.toml
+        sed -i "s#GITHUB_REF_NAME#${GITHUB_REF_NAME}#g" /tmp/vector.toml
+        sed -i "s#GH_ACTIONS_AXIOM_TOKEN#${{secrets.GH_ACTIONS_AXIOM_TOKEN}}#g" /tmp/vector.toml
+        cat /tmp/vector.toml
+        ${HOME}/.vector/bin/vector --config /tmp/vector.toml &
     - name: Build Wasm (because rust diff)
       if: needs.check-rust-changes.outputs.rust-changed == 'true'
       run: yarn build:wasm
@@ -160,6 +174,11 @@ jobs:
       env:
         CI: true
         token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
+    - name: send to axiom
+      if: always()
+      shell: bash
+      run: |
+        node playwrightProcess.mjs | tee /tmp/github-actions.log
     - uses: actions/upload-artifact@v4
       if: always()
       with:
@@ -226,6 +245,20 @@ jobs:
       uses: Swatinem/rust-cache@v2
       with:
         workspaces: './src/wasm-lib'
+    - name: Install vector
+      run: |
+        curl --proto '=https' --tlsv1.2 -sSfL https://sh.vector.dev > /tmp/vector.sh
+        chmod +x /tmp/vector.sh
+        /tmp/vector.sh -y -no-modify-path
+        mkdir -p /tmp/vector
+        cp .github/workflows/vector.toml /tmp/vector.toml
+        sed -i "" "s#GITHUB_WORKFLOW#${GITHUB_WORKFLOW}#g" /tmp/vector.toml
+        sed -i "" "s#GITHUB_REPOSITORY#${GITHUB_REPOSITORY}#g" /tmp/vector.toml
+        sed -i "" "s#GITHUB_SHA#${GITHUB_SHA}#g" /tmp/vector.toml
+        sed -i "" "s#GITHUB_REF_NAME#${GITHUB_REF_NAME}#g" /tmp/vector.toml
+        sed -i "" "s#GH_ACTIONS_AXIOM_TOKEN#${{secrets.GH_ACTIONS_AXIOM_TOKEN}}#g" /tmp/vector.toml
+        cat /tmp/vector.toml
+        ${HOME}/.vector/bin/vector --config /tmp/vector.toml &
     - name: Build Wasm (because rust diff)
       if: needs.check-rust-changes.outputs.rust-changed == 'true'
       run: yarn build:wasm
@@ -264,6 +297,11 @@ jobs:
       env:
         CI: true
         token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
+    - name: send to axiom
+      if: always()
+      shell: bash
+      run: |
+        node playwrightProcess.mjs | tee /tmp/github-actions.log
     - uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:

playwright.config.ts

@@ -20,7 +20,10 @@ export default defineConfig({
   /* Different amount of parallelism on CI and local. */
   workers: process.env.CI ? 4 : 4,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'html',
+  reporter: [
+    [process.env.CI ? 'dot' : 'list'],
+    ['json', { outputFile: './test-results/report.json' }],
+  ],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Base URL to use in actions like `await page.goto('/')`. */

playwrightProcess.mjs

@@ -0,0 +1,65 @@
+import { readFileSync } from 'fs'
+
+const data = readFileSync('./test-results/report.json', 'utf8')
+
+// types, but was easier to store and run as normal js
+// interface FailedTest {
+//     name: string;
+//     projectName: string;
+//     error: string;
+// }
+
+// interface Spec {
+//     title: string;
+//     tests: Test[];
+// }
+
+// interface Test {
+//     expectedStatus: 'passed' | 'failed' | 'pending';
+//     projectName: string;
+//     title: string;
+//     results: {
+//         status: 'passed' | 'failed' | 'pending';
+//         error: {stack: string}
+//     }[]
+// }
+
+// interface Suite {
+//     title: string
+//     suites: Suite[];
+//     specs: Spec[];
+// }
+
+// const processReport = (suites: Suite[]): FailedTest[] => {
+//     const failedTests: FailedTest[] = []
+//     const loopSuites = (suites: Suite[], previousName = '') => {
+const processReport = (suites) => {
+  const failedTests = []
+  const loopSuites = (suites, previousName = '') => {
+    if (!suites) return
+    for (const suite of suites) {
+      const name = (previousName ? `${previousName} -- ` : '') + suite.title
+      for (const spec of suite.specs) {
+        for (const test of spec.tests) {
+          for (const result of test.results) {
+            if ((result.status !== 'passed') && test.expectedStatus === 'passed') {
+              failedTests.push({
+                name: (name + ' -- ' + spec.title) + (test.title ? ` -- ${test.title}` : ''),
+                status: result.status,
+                projectName: test.projectName,
+                error: result.error?.stack,
+              })
+            }
+          }
+        }
+      }
+      loopSuites(suite.suites, name)
+    }
+  }
+  loopSuites(suites)
+  return failedTests.map(line => JSON.stringify(line)).join('\n')
+}
+const failedTests = processReport(JSON.parse(data).suites)
+// log to stdout to be piped to axiom
+console.log(failedTests)
+