From 08e4c03ca71f04fb1f71bbc0060f93490123c963 Mon Sep 17 00:00:00 2001 From: Kurt Hutten Date: Thu, 11 Jul 2024 14:32:36 +1000 Subject: [PATCH] send failing test to axiom (#2996) * send failing test to axiom (ubuntu) * forgot always * rename * Update .github/workflows/playwright.yml Co-authored-by: Adam Sunderland * update to indivdual lines of json * another fix * tweak output * log macos too --------- Co-authored-by: Adam Sunderland --- .github/workflows/playwright.yml | 38 +++++++++++++++++++ playwright.config.ts | 5 ++- playwrightProcess.mjs | 65 ++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 playwrightProcess.mjs diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 3490ce17c..5733afa4a 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -83,6 +83,20 @@ jobs: uses: Swatinem/rust-cache@v2 with: workspaces: './src/wasm-lib' + - name: Install vector + run: | + curl --proto '=https' --tlsv1.2 -sSfL https://sh.vector.dev > /tmp/vector.sh + chmod +x /tmp/vector.sh + /tmp/vector.sh -y -no-modify-path + mkdir -p /tmp/vector + cp .github/workflows/vector.toml /tmp/vector.toml + sed -i "s#GITHUB_WORKFLOW#${GITHUB_WORKFLOW}#g" /tmp/vector.toml + sed -i "s#GITHUB_REPOSITORY#${GITHUB_REPOSITORY}#g" /tmp/vector.toml + sed -i "s#GITHUB_SHA#${GITHUB_SHA}#g" /tmp/vector.toml + sed -i "s#GITHUB_REF_NAME#${GITHUB_REF_NAME}#g" /tmp/vector.toml + sed -i "s#GH_ACTIONS_AXIOM_TOKEN#${{secrets.GH_ACTIONS_AXIOM_TOKEN}}#g" /tmp/vector.toml + cat /tmp/vector.toml + ${HOME}/.vector/bin/vector --config /tmp/vector.toml & - name: Build Wasm (because rust diff) if: needs.check-rust-changes.outputs.rust-changed == 'true' run: yarn build:wasm @@ -160,6 +174,11 @@ jobs: env: CI: true token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }} + - name: send to axiom + if: always() + shell: bash + run: | + node playwrightProcess.mjs | tee /tmp/github-actions.log - uses: actions/upload-artifact@v4 if: always() with: @@ -226,6 +245,20 @@ jobs: uses: Swatinem/rust-cache@v2 with: workspaces: './src/wasm-lib' + - name: Install vector + run: | + curl --proto '=https' --tlsv1.2 -sSfL https://sh.vector.dev > /tmp/vector.sh + chmod +x /tmp/vector.sh + /tmp/vector.sh -y -no-modify-path + mkdir -p /tmp/vector + cp .github/workflows/vector.toml /tmp/vector.toml + sed -i "" "s#GITHUB_WORKFLOW#${GITHUB_WORKFLOW}#g" /tmp/vector.toml + sed -i "" "s#GITHUB_REPOSITORY#${GITHUB_REPOSITORY}#g" /tmp/vector.toml + sed -i "" "s#GITHUB_SHA#${GITHUB_SHA}#g" /tmp/vector.toml + sed -i "" "s#GITHUB_REF_NAME#${GITHUB_REF_NAME}#g" /tmp/vector.toml + sed -i "" "s#GH_ACTIONS_AXIOM_TOKEN#${{secrets.GH_ACTIONS_AXIOM_TOKEN}}#g" /tmp/vector.toml + cat /tmp/vector.toml + ${HOME}/.vector/bin/vector --config /tmp/vector.toml & - name: Build Wasm (because rust diff) if: needs.check-rust-changes.outputs.rust-changed == 'true' run: yarn build:wasm @@ -264,6 +297,11 @@ jobs: env: CI: true token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }} + - name: send to axiom + if: always() + shell: bash + run: | + node playwrightProcess.mjs | tee /tmp/github-actions.log - uses: actions/upload-artifact@v4 if: ${{ always() }} with: diff --git a/playwright.config.ts b/playwright.config.ts index 4b5c7c43d..95136d8ae 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -20,7 +20,10 @@ export default defineConfig({ /* Different amount of parallelism on CI and local. */ workers: process.env.CI ? 4 : 4, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', + reporter: [ + [process.env.CI ? 'dot' : 'list'], + ['json', { outputFile: './test-results/report.json' }], + ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ diff --git a/playwrightProcess.mjs b/playwrightProcess.mjs new file mode 100644 index 000000000..c614e38fd --- /dev/null +++ b/playwrightProcess.mjs @@ -0,0 +1,65 @@ +import { readFileSync } from 'fs' + +const data = readFileSync('./test-results/report.json', 'utf8') + +// types, but was easier to store and run as normal js +// interface FailedTest { +// name: string; +// projectName: string; +// error: string; +// } + +// interface Spec { +// title: string; +// tests: Test[]; +// } + +// interface Test { +// expectedStatus: 'passed' | 'failed' | 'pending'; +// projectName: string; +// title: string; +// results: { +// status: 'passed' | 'failed' | 'pending'; +// error: {stack: string} +// }[] +// } + +// interface Suite { +// title: string +// suites: Suite[]; +// specs: Spec[]; +// } + +// const processReport = (suites: Suite[]): FailedTest[] => { +// const failedTests: FailedTest[] = [] +// const loopSuites = (suites: Suite[], previousName = '') => { +const processReport = (suites) => { + const failedTests = [] + const loopSuites = (suites, previousName = '') => { + if (!suites) return + for (const suite of suites) { + const name = (previousName ? `${previousName} -- ` : '') + suite.title + for (const spec of suite.specs) { + for (const test of spec.tests) { + for (const result of test.results) { + if ((result.status !== 'passed') && test.expectedStatus === 'passed') { + failedTests.push({ + name: (name + ' -- ' + spec.title) + (test.title ? ` -- ${test.title}` : ''), + status: result.status, + projectName: test.projectName, + error: result.error?.stack, + }) + } + } + } + } + loopSuites(suite.suites, name) + } + } + loopSuites(suites) + return failedTests.map(line => JSON.stringify(line)).join('\n') +} +const failedTests = processReport(JSON.parse(data).suites) +// log to stdout to be piped to axiom +console.log(failedTests) +