github actions Playwright shard execution (#3199)

* add @snapshot tag to all snapshot-tests

* set workers to 1 on CI

* try sharding on google chrome only

* add retry when navigating to the app (on CI)

* reduce ubuntu-cores to 2

* revert runner back to 8 cores

* re-add retry + enable macos

* Reduce timeouts to 30 minutes

* ensure retry is triggered

* removed sharding when retrying failures

* added --pass-with-no-tests

* ensure failure occurs

* revert failure

* use smaller sized runners

* revert back to supported runner size

* revert to og version

* minimize changes

* yarn fmt

* ensure failure

* undo failure

---------

Co-authored-by: ryanrosello-og <ry@zoo.dev>

.github/workflows/playwright.yml

@@ -34,8 +34,13 @@ jobs:
               - 'src/wasm-lib/**'
 
   playwright-ubuntu:
-    timeout-minutes: 60
+    timeout-minutes: 30
     runs-on: ubuntu-latest-8-cores
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
     needs: check-rust-changes
     steps:
     - name: Tune GitHub-hosted runner network
@@ -107,7 +112,7 @@ jobs:
       run: yarn build:local
     - name: Run ubuntu/chrome snapshots
       run: |
-        yarn playwright test --project="Google Chrome" --retries="3" --update-snapshots e2e/playwright/snapshot-tests.spec.ts
+        yarn playwright test --project="Google Chrome" --retries="3" --update-snapshots --grep=@snapshot  --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
       env:
         CI: true
         token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
@@ -115,7 +120,7 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: always()
       with:
-        name: playwright-report-ubuntu-snapshot-${{ github.sha }}
+        name: playwright-report-ubuntu-snapshot-${{ matrix.shardIndex }}-${{ github.sha }}
         path: playwright-report/
         retention-days: 30
         overwrite: true
@@ -149,7 +154,7 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: steps.git-check.outputs.modified == 'true'
       with:
-        name: playwright-report-ubuntu-${{ github.sha }}
+        name: playwright-report-ubuntu-${{ matrix.shardIndex }}-${{ github.sha }}
         path: playwright-report/
         retention-days: 30
     # if have previous run results, use them
@@ -157,7 +162,7 @@ jobs:
       if: always()
       continue-on-error: true
       with:
-        name: test-results-ubuntu-${{ github.sha }}
+        name: test-results-ubuntu-${{ matrix.shardIndex }}-${{ github.sha }}
         path: test-results/
     - name: Run ubuntu/chrome flow (with retries)
       id: retry
@@ -166,7 +171,7 @@ jobs:
         if [[ ! -f "test-results/.last-run.json" ]]; then
             # if no last run artifact, than run plawright normally
             echo "run playwright normally"
-            yarn playwright test --project="Google Chrome" e2e/playwright/flow-tests.spec.ts || true
+            yarn playwright test --project="Google Chrome" --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --grep-invert=@snapshot || true
             # # send to axiom
             node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
         fi
@@ -181,7 +186,7 @@ jobs:
                 if [[ $failed_tests -gt 0 ]]; then
                     echo "retried=true" >>$GITHUB_OUTPUT
                     echo "run playwright with last failed tests and retry $retry"
-                    yarn playwright test --project="Google Chrome" --last-failed e2e/playwright/flow-tests.spec.ts || true
+                    yarn playwright test --project="Google Chrome" --last-failed --grep-invert=@snapshot || true
                     # send to axiom
                     node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
                     retry=$((retry + 1))
@@ -216,21 +221,26 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: always()
       with:
-        name: test-results-ubuntu-${{ github.sha }}
+        name: test-results-ubuntu-${{ matrix.shardIndex }}-${{ github.sha }}
         path: test-results/
         retention-days: 30
         overwrite: true
     - uses: actions/upload-artifact@v4
       if: always()
       with:
-        name: playwright-report-ubuntu-${{ github.sha }}
+        name: playwright-report-ubuntu-${{ matrix.shardIndex }}-${{ github.sha }}
         path: playwright-report/
         retention-days: 30
         overwrite: true
 
   playwright-macos:
-    timeout-minutes: 60
-    runs-on: macos-14-large
+    timeout-minutes: 30
+    runs-on: macos-14
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
     needs: check-rust-changes
     steps:
     - name: Tune GitHub-hosted runner network
@@ -306,7 +316,7 @@ jobs:
       if: ${{ always() }}
       continue-on-error: true
       with:
-        name: test-results-macos-${{ github.sha }}
+        name: test-results-macos-${{ matrix.shardIndex }}-${{ github.sha }}
         path: test-results/
     - name: Run macos/safari flow (with retries)
       id: retry
@@ -315,7 +325,7 @@ jobs:
         if [[ ! -f "test-results/.last-run.json" ]]; then
             # if no last run artifact, than run plawright normally
             echo "run playwright normally"
-            yarn playwright test --project="webkit" e2e/playwright/flow-tests.spec.ts || true
+            yarn playwright test --project="webkit" --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --grep-invert=@snapshot || true
             # # send to axiom
             node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
         fi
@@ -330,7 +340,7 @@ jobs:
                 if [[ $failed_tests -gt 0 ]]; then
                     echo "retried=true" >>$GITHUB_OUTPUT
                     echo "run playwright with last failed tests and retry $retry"
-                    yarn playwright test --project="webkit" --last-failed e2e/playwright/flow-tests.spec.ts || true
+                    yarn playwright test --project="webkit" --last-failed --grep-invert=@snapshot || true
                     # send to axiom
                     node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
                     retry=$((retry + 1))
@@ -360,15 +370,14 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: test-results-macos-${{ github.sha }}
+        name: test-results-macos-${{ matrix.shardIndex }}-${{ github.sha }}
         path: test-results/
         retention-days: 30
         overwrite: true
     - uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: playwright-report-macos-${{ github.sha }}
+        name: playwright-report-macos-${{ matrix.shardIndex }}-${{ github.sha }}
         path: playwright-report/
         retention-days: 30
         overwrite: true
-

e2e/playwright/snapshot-tests.spec.ts

@@ -42,7 +42,10 @@ test.beforeEach(async ({ page }) => {
 
 test.setTimeout(60_000)
 
-test('exports of each format should work', async ({ page, context }) => {
+test(
+  'exports of each format should work',
+  { tag: '@snapshot' },
+  async ({ page, context }) => {
     // FYI this test doesn't work with only engine running locally
     // And you will need to have the KittyCAD CLI installed
     const u = await getUtils(page)
@@ -296,7 +299,8 @@ const part001 = startSketchOn('-XZ')
         console.log(`snapshot failed for ${modelPath}`)
       }
     }
-})
+  }
+)
 
 const extrudeDefaultPlane = async (context: any, page: any, plane: string) => {
   await context.addInitScript(async () => {
@@ -357,7 +361,10 @@ const extrudeDefaultPlane = async (context: any, page: any, plane: string) => {
   await u.openKclCodePanel()
 }
 
-test.describe('extrude on default planes should be stable', () => {
+test.describe(
+  'extrude on default planes should be stable',
+  { tag: '@snapshot' },
+  () => {
     test('XY', async ({ page, context }) => {
       await extrudeDefaultPlane(context, page, 'XY')
     })
@@ -381,9 +388,13 @@ test.describe('extrude on default planes should be stable', () => {
     test('-YZ', async ({ page, context }) => {
       await extrudeDefaultPlane(context, page, '-YZ')
     })
-})
+  }
+)
 
-test('Draft segments should look right', async ({ page, context }) => {
+test(
+  'Draft segments should look right',
+  { tag: '@snapshot' },
+  async ({ page, context }) => {
     const u = await getUtils(page)
     await page.setViewportSize({ width: 1200, height: 500 })
     const PUR = 400 / 37.5 //pixeltoUnitRatio
@@ -394,7 +405,9 @@ test('Draft segments should look right', async ({ page, context }) => {
     await expect(
       page.getByRole('button', { name: 'Start Sketch' })
     ).not.toBeDisabled()
-  await expect(page.getByRole('button', { name: 'Start Sketch' })).toBeVisible()
+    await expect(
+      page.getByRole('button', { name: 'Start Sketch' })
+    ).toBeVisible()
 
     // click on "Start Sketch" button
     await u.clearCommandLogs()
@@ -442,9 +455,13 @@ test('Draft segments should look right', async ({ page, context }) => {
     await expect(page).toHaveScreenshot({
       maxDiffPixels: 100,
     })
-})
+  }
+)
 
-test('Draft rectangles should look right', async ({ page, context }) => {
+test(
+  'Draft rectangles should look right',
+  { tag: '@snapshot' },
+  async ({ page, context }) => {
     const u = await getUtils(page)
     await page.setViewportSize({ width: 1200, height: 500 })
     const PUR = 400 / 37.5 //pixeltoUnitRatio
@@ -455,7 +472,9 @@ test('Draft rectangles should look right', async ({ page, context }) => {
     await expect(
       page.getByRole('button', { name: 'Start Sketch' })
     ).not.toBeDisabled()
-  await expect(page.getByRole('button', { name: 'Start Sketch' })).toBeVisible()
+    await expect(
+      page.getByRole('button', { name: 'Start Sketch' })
+    ).toBeVisible()
 
     // click on "Start Sketch" button
     await u.clearCommandLogs()
@@ -490,9 +509,13 @@ test('Draft rectangles should look right', async ({ page, context }) => {
     await expect(page).toHaveScreenshot({
       maxDiffPixels: 100,
     })
-})
+  }
+)
 
-test.describe('Client side scene scale should match engine scale', () => {
+test.describe(
+  'Client side scene scale should match engine scale',
+  { tag: '@snapshot' },
+  () => {
     test('Inch scale', async ({ page }) => {
       const u = await getUtils(page)
       await page.setViewportSize({ width: 1200, height: 500 })
@@ -680,9 +703,13 @@ test.describe('Client side scene scale should match engine scale', () => {
         maxDiffPixels: 100,
       })
     })
-})
+  }
+)
 
-test('Sketch on face with none z-up', async ({ page, context }) => {
+test(
+  'Sketch on face with none z-up',
+  { tag: '@snapshot' },
+  async ({ page, context }) => {
     const u = await getUtils(page)
     await context.addInitScript(async (KCL_DEFAULT_LENGTH) => {
       localStorage.setItem(
@@ -728,7 +755,9 @@ const part002 = startSketchOn(part001, seg01)
 
     // click at 641, 135
     await page.mouse.click(641, 135)
-  await expect(page.locator('.cm-content')).not.toHaveText(previousCodeContent)
+    await expect(page.locator('.cm-content')).not.toHaveText(
+      previousCodeContent
+    )
     previousCodeContent = await page.locator('.cm-content').innerText()
 
     await page.waitForTimeout(300)
@@ -736,9 +765,13 @@ const part002 = startSketchOn(part001, seg01)
     await expect(page).toHaveScreenshot({
       maxDiffPixels: 100,
     })
-})
+  }
+)
 
-test('Zoom to fit on load - solid 2d', async ({ page, context }) => {
+test(
+  'Zoom to fit on load - solid 2d',
+  { tag: '@snapshot' },
+  async ({ page, context }) => {
     const u = await getUtils(page)
     await context.addInitScript(async () => {
       localStorage.setItem(
@@ -772,9 +805,13 @@ test('Zoom to fit on load - solid 2d', async ({ page, context }) => {
     await expect(page).toHaveScreenshot({
       maxDiffPixels: 100,
     })
-})
+  }
+)
 
-test('Zoom to fit on load - solid 3d', async ({ page, context }) => {
+test(
+  'Zoom to fit on load - solid 3d',
+  { tag: '@snapshot' },
+  async ({ page, context }) => {
     const u = await getUtils(page)
     await context.addInitScript(async () => {
       localStorage.setItem(
@@ -809,9 +846,10 @@ test('Zoom to fit on load - solid 3d', async ({ page, context }) => {
     await expect(page).toHaveScreenshot({
       maxDiffPixels: 100,
     })
-})
+  }
+)
 
-test.describe('Grid visibility', () => {
+test.describe('Grid visibility', { tag: '@snapshot' }, () => {
   test('Grid turned off', async ({ page }) => {
     const u = await getUtils(page)
     const stream = page.getByTestId('stream')

e2e/playwright/test-utils.ts

@@ -15,6 +15,23 @@ export const TEST_COLORS = {
   BLUE: [0, 0, 255] as TestColor,
 } as const
 
+async function waitForPageLoadWithRetry(page: Page) {
+  await expect(async () => {
+    await page.goto('/')
+    const errorMessage = 'App failed to load - 🔃 Retrying ...'
+    await expect(page.getByTestId('loading'), errorMessage).not.toBeAttached({
+      timeout: 20_000,
+    })
+
+    await expect(
+      page.getByRole('button', { name: 'Start Sketch' }),
+      errorMessage
+    ).toBeEnabled({
+      timeout: 20_000,
+    })
+  }).toPass({ timeout: 70_000, intervals: [1_000] })
+}
+
 async function waitForPageLoad(page: Page) {
   // wait for all spinners to be gone
   await expect(page.getByTestId('loading')).not.toBeAttached({
@@ -218,9 +235,12 @@ async function waitForAuthAndLsp(page: Page) {
     }
     return false
   })
-
+  if (process.env.CI) {
+    await waitForPageLoadWithRetry(page)
+  } else {
     await page.goto('/')
     await waitForPageLoad(page)
+  }
 
   return waitForLspPromise
 }
@@ -234,6 +254,7 @@ export async function getUtils(page: Page) {
   return {
     waitForAuthSkipAppStart: () => waitForAuthAndLsp(page),
     waitForPageLoad: () => waitForPageLoad(page),
+    waitForPageLoadWithRetry: () => waitForPageLoadWithRetry(page),
     removeCurrentCode: () => removeCurrentCode(page),
     sendCustomCmd: (cmd: EngineCommand) => sendCustomCmd(page, cmd),
     updateCamPosition: async (xyz: [number, number, number]) => {

playwright.config.ts

@@ -18,7 +18,7 @@ export default defineConfig({
   /* Do not retry */
   retries: process.env.CI ? 0 : 0,
   /* Different amount of parallelism on CI and local. */
-  workers: process.env.CI ? 4 : 4,
+  workers: process.env.CI ? 1 : 4,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: [
     [process.env.CI ? 'dot' : 'list'],