Use TAB API to determine when merges are blocked (#6405)

* Use TAB API to determine when merges are blocked

* Enable a test that is known to fail

.github/ci-cd-scripts/playwright-electron.sh

@@ -26,8 +26,8 @@ max_retries=1
 # Retry failed tests, doing our own retries because using inbuilt Playwright retries causes connection issues
 while [[ $retry -le $max_retries ]]; do
     if [[ -f "test-results/.last-run.json" ]]; then
-        failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
+        status=$(jq -r '.status' test-results/.last-run.json)
-        if [[ $failed_tests -gt 0 ]]; then
+        if [[ "$status" == "failed" ]]; then
             echo "retried=true" >>$GITHUB_OUTPUT
             echo "run playwright with last failed tests and retry $retry"
             if [[ "$3" == *ubuntu* ]]; then
@@ -56,10 +56,11 @@ done
 echo "retried=false" >>$GITHUB_OUTPUT
 
 if [[ -f "test-results/.last-run.json" ]]; then
-    failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
+    status=$(jq -r '.status' test-results/.last-run.json)
-    if [[ $failed_tests -gt 0 ]]; then
+    if [[ "$status" == "failed" ]]; then
-        # If it still fails after 3 retries, then fail the job
+        # If it still fails after retries, then fail the job
         exit 1
     fi
 fi
+
 exit 0

e2e/playwright/lib/api-reporter.ts

@@ -1,6 +1,25 @@
-import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter'
+import type {
+  Reporter,
+  TestCase,
+  TestResult,
+  FullResult,
+} from '@playwright/test/reporter'
 
 class MyAPIReporter implements Reporter {
+  private pendingRequests: Promise<void>[] = []
+  private allResults: Record<string, any>[] = []
+  private blockingResults: Record<string, any>[] = []
+
+  async onEnd(result: FullResult): Promise<void> {
+    await Promise.all(this.pendingRequests)
+    if (this.allResults.length > 0 && this.blockingResults.length === 0) {
+      result.status = 'passed'
+      if (!process.env.CI) {
+        console.error('TAB API - Marked failures as non-blocking')
+      }
+    }
+  }
+
   onTestEnd(test: TestCase, result: TestResult): void {
     if (!process.env.TAB_API_URL || !process.env.TAB_API_KEY) {
       return
@@ -20,6 +39,7 @@ class MyAPIReporter implements Reporter {
       platform: process.env.RUNNER_OS || process.platform,
       // Extra test and result data
       annotations: test.annotations.map((a) => a.type), // e.g. 'fail' or 'fixme'
+      id: test.id, // computed file/test/project ID used for reruns
       retry: result.retry,
       tags: test.tags, // e.g. '@snapshot' or '@skipWin'
       // Extra environment variables
@@ -35,7 +55,7 @@ class MyAPIReporter implements Reporter {
       RUNNER_ARCH: process.env.RUNNER_ARCH || null,
     }
 
-    void (async () => {
+    const request = (async () => {
       try {
         const response = await fetch(`${process.env.TAB_API_URL}/api/results`, {
           method: 'POST',
@@ -46,18 +66,27 @@ class MyAPIReporter implements Reporter {
           body: JSON.stringify(payload),
         })
 
-        if (!response.ok && !process.env.CI) {
+        if (response.ok) {
-          console.error(
+          const result = await response.json()
-            'TAB API - Failed to send test result:',
+          this.allResults.push(result)
-            await response.text()
+          if (result.block) {
-          )
+            this.blockingResults.push(result)
+          }
+        } else {
+          const error = await response.json()
+          if (!process.env.CI) {
+            console.error('TAB API - Failed to send test result:', error)
+          }
         }
-      } catch {
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error)
         if (!process.env.CI) {
-          console.error('TAB API - Unable to send test result')
+          console.error('TAB API - Unable to send test result:', message)
         }
       }
     })()
+
+    this.pendingRequests.push(request)
   }
 }
 

e2e/playwright/testing-settings.spec.ts

@@ -181,7 +181,6 @@ test.describe('Testing settings', () => {
   })
 
   test('Project and user settings can be reset', async ({ page, homePage }) => {
-    test.fixme(orRunWhenFullSuiteEnabled())
     const u = await getUtils(page)
     await test.step(`Setup`, async () => {
       await page.setBodyDimensions({ width: 1200, height: 500 })