diff --git a/.github/ci-cd-scripts/playwright-electron.sh b/.github/ci-cd-scripts/playwright-electron.sh index 931653f4c..d8f66898e 100755 --- a/.github/ci-cd-scripts/playwright-electron.sh +++ b/.github/ci-cd-scripts/playwright-electron.sh @@ -26,8 +26,8 @@ max_retries=1 # Retry failed tests, doing our own retries because using inbuilt Playwright retries causes connection issues while [[ $retry -le $max_retries ]]; do if [[ -f "test-results/.last-run.json" ]]; then - failed_tests=$(jq '.failedTests | length' test-results/.last-run.json) - if [[ $failed_tests -gt 0 ]]; then + status=$(jq -r '.status' test-results/.last-run.json) + if [[ "$status" == "failed" ]]; then echo "retried=true" >>$GITHUB_OUTPUT echo "run playwright with last failed tests and retry $retry" if [[ "$3" == *ubuntu* ]]; then @@ -56,10 +56,11 @@ done echo "retried=false" >>$GITHUB_OUTPUT if [[ -f "test-results/.last-run.json" ]]; then - failed_tests=$(jq '.failedTests | length' test-results/.last-run.json) - if [[ $failed_tests -gt 0 ]]; then - # If it still fails after 3 retries, then fail the job + status=$(jq -r '.status' test-results/.last-run.json) + if [[ "$status" == "failed" ]]; then + # If it still fails after retries, then fail the job exit 1 fi fi + exit 0 diff --git a/e2e/playwright/lib/api-reporter.ts b/e2e/playwright/lib/api-reporter.ts index 7474d63b3..395761930 100644 --- a/e2e/playwright/lib/api-reporter.ts +++ b/e2e/playwright/lib/api-reporter.ts @@ -1,6 +1,25 @@ -import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter' +import type { + Reporter, + TestCase, + TestResult, + FullResult, +} from '@playwright/test/reporter' class MyAPIReporter implements Reporter { + private pendingRequests: Promise[] = [] + private allResults: Record[] = [] + private blockingResults: Record[] = [] + + async onEnd(result: FullResult): Promise { + await Promise.all(this.pendingRequests) + if (this.allResults.length > 0 && this.blockingResults.length === 0) { + result.status = 'passed' + if (!process.env.CI) { + console.error('TAB API - Marked failures as non-blocking') + } + } + } + onTestEnd(test: TestCase, result: TestResult): void { if (!process.env.TAB_API_URL || !process.env.TAB_API_KEY) { return @@ -20,6 +39,7 @@ class MyAPIReporter implements Reporter { platform: process.env.RUNNER_OS || process.platform, // Extra test and result data annotations: test.annotations.map((a) => a.type), // e.g. 'fail' or 'fixme' + id: test.id, // computed file/test/project ID used for reruns retry: result.retry, tags: test.tags, // e.g. '@snapshot' or '@skipWin' // Extra environment variables @@ -35,7 +55,7 @@ class MyAPIReporter implements Reporter { RUNNER_ARCH: process.env.RUNNER_ARCH || null, } - void (async () => { + const request = (async () => { try { const response = await fetch(`${process.env.TAB_API_URL}/api/results`, { method: 'POST', @@ -46,18 +66,27 @@ class MyAPIReporter implements Reporter { body: JSON.stringify(payload), }) - if (!response.ok && !process.env.CI) { - console.error( - 'TAB API - Failed to send test result:', - await response.text() - ) + if (response.ok) { + const result = await response.json() + this.allResults.push(result) + if (result.block) { + this.blockingResults.push(result) + } + } else { + const error = await response.json() + if (!process.env.CI) { + console.error('TAB API - Failed to send test result:', error) + } } - } catch { + } catch (error) { + const message = error instanceof Error ? error.message : String(error) if (!process.env.CI) { - console.error('TAB API - Unable to send test result') + console.error('TAB API - Unable to send test result:', message) } } })() + + this.pendingRequests.push(request) } } diff --git a/e2e/playwright/testing-settings.spec.ts b/e2e/playwright/testing-settings.spec.ts index 82fa1f081..dfd80d2b6 100644 --- a/e2e/playwright/testing-settings.spec.ts +++ b/e2e/playwright/testing-settings.spec.ts @@ -181,7 +181,6 @@ test.describe('Testing settings', () => { }) test('Project and user settings can be reset', async ({ page, homePage }) => { - test.fixme(orRunWhenFullSuiteEnabled()) const u = await getUtils(page) await test.step(`Setup`, async () => { await page.setBodyDimensions({ width: 1200, height: 500 })