github actions Playwright shard execution (#3199)

* add @snapshot tag to all snapshot-tests

* set workers to 1 on CI

* try sharding on google chrome only

* add retry when navigating to the app (on CI)

* reduce ubuntu-cores to 2

* revert runner back to 8 cores

* re-add retry + enable macos

* Reduce timeouts to 30 minutes

* ensure retry is triggered

* removed sharding when retrying failures

* added --pass-with-no-tests

* ensure failure occurs

* revert failure

* use smaller sized runners

* revert back to supported runner size

* revert to og version

* minimize changes

* yarn fmt

* ensure failure

* undo failure

---------

Co-authored-by: ryanrosello-og <ry@zoo.dev>

.github/workflows/playwright.yml

@@ -13,7 +13,7 @@ permissions:
   contents: write
   pull-requests: write
   actions: read
-  
+
 
 jobs:
 
@@ -34,8 +34,13 @@ jobs:
               - 'src/wasm-lib/**'
 
   playwright-ubuntu:
-    timeout-minutes: 60
+    timeout-minutes: 30
     runs-on: ubuntu-latest-8-cores
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
     needs: check-rust-changes
     steps:
     - name: Tune GitHub-hosted runner network
@@ -107,7 +112,7 @@ jobs:
       run: yarn build:local
     - name: Run ubuntu/chrome snapshots
       run: |
-        yarn playwright test --project="Google Chrome" --retries="3" --update-snapshots e2e/playwright/snapshot-tests.spec.ts
+        yarn playwright test --project="Google Chrome" --retries="3" --update-snapshots --grep=@snapshot  --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
       env:
         CI: true
         token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
@@ -115,7 +120,7 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: always()
       with:
-        name: playwright-report-ubuntu-snapshot-${{ github.sha }}
+        name: playwright-report-ubuntu-snapshot-${{ matrix.shardIndex }}-${{ github.sha }}
         path: playwright-report/
         retention-days: 30
         overwrite: true
@@ -149,7 +154,7 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: steps.git-check.outputs.modified == 'true'
       with:
-        name: playwright-report-ubuntu-${{ github.sha }}
+        name: playwright-report-ubuntu-${{ matrix.shardIndex }}-${{ github.sha }}
         path: playwright-report/
         retention-days: 30
     # if have previous run results, use them
@@ -157,7 +162,7 @@ jobs:
       if: always()
       continue-on-error: true
       with:
-        name: test-results-ubuntu-${{ github.sha }}
+        name: test-results-ubuntu-${{ matrix.shardIndex }}-${{ github.sha }}
         path: test-results/
     - name: Run ubuntu/chrome flow (with retries)
       id: retry
@@ -166,14 +171,14 @@ jobs:
         if [[ ! -f "test-results/.last-run.json" ]]; then
             # if no last run artifact, than run plawright normally
             echo "run playwright normally"
-            yarn playwright test --project="Google Chrome" e2e/playwright/flow-tests.spec.ts || true
+            yarn playwright test --project="Google Chrome" --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --grep-invert=@snapshot || true
             # # send to axiom
             node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
         fi
-        
+
         retry=1
         max_retrys=4
-        
+
         # retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues
         while [[ $retry -le $max_retrys ]]; do
             if [[ -f "test-results/.last-run.json" ]]; then
@@ -181,7 +186,7 @@ jobs:
                 if [[ $failed_tests -gt 0 ]]; then
                     echo "retried=true" >>$GITHUB_OUTPUT
                     echo "run playwright with last failed tests and retry $retry"
-                    yarn playwright test --project="Google Chrome" --last-failed e2e/playwright/flow-tests.spec.ts || true
+                    yarn playwright test --project="Google Chrome" --last-failed --grep-invert=@snapshot || true
                     # send to axiom
                     node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
                     retry=$((retry + 1))
@@ -194,9 +199,9 @@ jobs:
                 exit 0
             fi
         done
-        
+
         echo "retried=false" >>$GITHUB_OUTPUT
-        
+
         if [[ -f "test-results/.last-run.json" ]]; then
             failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
             if [[ $failed_tests -gt 0 ]]; then
@@ -216,21 +221,26 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: always()
       with:
-        name: test-results-ubuntu-${{ github.sha }}
+        name: test-results-ubuntu-${{ matrix.shardIndex }}-${{ github.sha }}
         path: test-results/
         retention-days: 30
         overwrite: true
     - uses: actions/upload-artifact@v4
       if: always()
       with:
-        name: playwright-report-ubuntu-${{ github.sha }}
+        name: playwright-report-ubuntu-${{ matrix.shardIndex }}-${{ github.sha }}
         path: playwright-report/
         retention-days: 30
         overwrite: true
 
   playwright-macos:
-    timeout-minutes: 60
-    runs-on: macos-14-large
+    timeout-minutes: 30
+    runs-on: macos-14
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
     needs: check-rust-changes
     steps:
     - name: Tune GitHub-hosted runner network
@@ -306,7 +316,7 @@ jobs:
       if: ${{ always() }}
       continue-on-error: true
       with:
-        name: test-results-macos-${{ github.sha }}
+        name: test-results-macos-${{ matrix.shardIndex }}-${{ github.sha }}
         path: test-results/
     - name: Run macos/safari flow (with retries)
       id: retry
@@ -315,14 +325,14 @@ jobs:
         if [[ ! -f "test-results/.last-run.json" ]]; then
             # if no last run artifact, than run plawright normally
             echo "run playwright normally"
-            yarn playwright test --project="webkit" e2e/playwright/flow-tests.spec.ts || true
+            yarn playwright test --project="webkit" --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --grep-invert=@snapshot || true
             # # send to axiom
             node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
         fi
-        
+
         retry=1
         max_retrys=4
-        
+
         # retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues
         while [[ $retry -le $max_retrys ]]; do
             if [[ -f "test-results/.last-run.json" ]]; then
@@ -330,7 +340,7 @@ jobs:
                 if [[ $failed_tests -gt 0 ]]; then
                     echo "retried=true" >>$GITHUB_OUTPUT
                     echo "run playwright with last failed tests and retry $retry"
-                    yarn playwright test --project="webkit" --last-failed e2e/playwright/flow-tests.spec.ts || true
+                    yarn playwright test --project="webkit" --last-failed --grep-invert=@snapshot || true
                     # send to axiom
                     node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
                     retry=$((retry + 1))
@@ -343,9 +353,9 @@ jobs:
                 exit 0
             fi
         done
-        
+
         echo "retried=false" >>$GITHUB_OUTPUT
-        
+
         if [[ -f "test-results/.last-run.json" ]]; then
             failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
             if [[ $failed_tests -gt 0 ]]; then
@@ -360,15 +370,14 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: test-results-macos-${{ github.sha }}
+        name: test-results-macos-${{ matrix.shardIndex }}-${{ github.sha }}
         path: test-results/
         retention-days: 30
         overwrite: true
     - uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: playwright-report-macos-${{ github.sha }}
+        name: playwright-report-macos-${{ matrix.shardIndex }}-${{ github.sha }}
         path: playwright-report/
         retention-days: 30
         overwrite: true
-

e2e/playwright/test-utils.ts

@@ -15,6 +15,23 @@ export const TEST_COLORS = {
   BLUE: [0, 0, 255] as TestColor,
 } as const
 
+async function waitForPageLoadWithRetry(page: Page) {
+  await expect(async () => {
+    await page.goto('/')
+    const errorMessage = 'App failed to load - 🔃 Retrying ...'
+    await expect(page.getByTestId('loading'), errorMessage).not.toBeAttached({
+      timeout: 20_000,
+    })
+
+    await expect(
+      page.getByRole('button', { name: 'Start Sketch' }),
+      errorMessage
+    ).toBeEnabled({
+      timeout: 20_000,
+    })
+  }).toPass({ timeout: 70_000, intervals: [1_000] })
+}
+
 async function waitForPageLoad(page: Page) {
   // wait for all spinners to be gone
   await expect(page.getByTestId('loading')).not.toBeAttached({
@@ -218,9 +235,12 @@ async function waitForAuthAndLsp(page: Page) {
     }
     return false
   })
-
-  await page.goto('/')
-  await waitForPageLoad(page)
+  if (process.env.CI) {
+    await waitForPageLoadWithRetry(page)
+  } else {
+    await page.goto('/')
+    await waitForPageLoad(page)
+  }
 
   return waitForLspPromise
 }
@@ -234,6 +254,7 @@ export async function getUtils(page: Page) {
   return {
     waitForAuthSkipAppStart: () => waitForAuthAndLsp(page),
     waitForPageLoad: () => waitForPageLoad(page),
+    waitForPageLoadWithRetry: () => waitForPageLoadWithRetry(page),
     removeCurrentCode: () => removeCurrentCode(page),
     sendCustomCmd: (cmd: EngineCommand) => sendCustomCmd(page, cmd),
     updateCamPosition: async (xyz: [number, number, number]) => {

playwright.config.ts

@@ -18,7 +18,7 @@ export default defineConfig({
   /* Do not retry */
   retries: process.env.CI ? 0 : 0,
   /* Different amount of parallelism on CI and local. */
-  workers: process.env.CI ? 4 : 4,
+  workers: process.env.CI ? 1 : 4,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: [
     [process.env.CI ? 'dot' : 'list'],