reposition timeout before mouseclick

.github/workflows/playwright.yml

@@ -105,108 +105,79 @@ jobs:
       run: yarn build:wasm
     - name: build web
       run: yarn build:local
-    - name: Run ubuntu/chrome snapshots
-      continue-on-error: true
-      run: |
-        yarn playwright test --project="Google Chrome" --update-snapshots e2e/playwright/snapshot-tests.spec.ts
-      env:
-        CI: true
-        token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
-        snapshottoken: ${{ secrets.KITTYCAD_API_TOKEN }}
-    - name: Clean up test-results
-      if: always()
-      continue-on-error: true
-      run: rm -r test-results
-    - name: check for changes
-      id: git-check
-      run: |
-          git add .
-          if git status | grep -q "Changes to be committed"
-          then echo "modified=true" >> $GITHUB_OUTPUT
-          else echo "modified=false" >> $GITHUB_OUTPUT
-          fi
-    - name: Commit changes, if any
-      if: steps.git-check.outputs.modified == 'true'
-      run: |
-        git add .
-        git config --local user.email "github-actions[bot]@users.noreply.github.com"
-        git config --local user.name "github-actions[bot]"
-        git remote set-url origin https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git
-        git fetch origin
-        echo ${{ github.head_ref }}
-        git checkout ${{ github.head_ref }}
-        # TODO when webkit works on ubuntu remove the os part of the commit message
-        git commit -am "A snapshot a day keeps the bugs away! 📷🐛 (OS: ubuntu)" || true
-        git push
-        git push origin ${{ github.head_ref }}
-    # only upload artifacts if there's actually changes
-    - uses: actions/upload-artifact@v4
-      if: steps.git-check.outputs.modified == 'true'
-      with:
-        name: playwright-report-ubuntu-${{ github.sha }}
-        path: playwright-report/
-        retention-days: 30
-    # if have previous run results, use them
-    - uses: actions/download-artifact@v4
-      if: always()
-      continue-on-error: true
-      with:
-        name: test-results-ubuntu-${{ github.sha }}
-        path: test-results/
+    # - name: Run ubuntu/chrome snapshots
+    #   continue-on-error: true
+    #   run: |
+    #     yarn playwright test --project="Google Chrome" --update-snapshots e2e/playwright/snapshot-tests.spec.ts
+    #   env:
+    #     CI: true
+    #     token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
+    #     snapshottoken: ${{ secrets.KITTYCAD_API_TOKEN }}
+    # - name: Clean up test-results
+    #   if: always()
+    #   continue-on-error: true
+    #   run: rm -r test-results
+    # - name: check for changes
+    #   id: git-check
+    #   run: |
+    #       git add .
+    #       if git status | grep -q "Changes to be committed"
+    #       then echo "modified=true" >> $GITHUB_OUTPUT
+    #       else echo "modified=false" >> $GITHUB_OUTPUT
+    #       fi
+    # - name: Commit changes, if any
+    #   if: steps.git-check.outputs.modified == 'true'
+    #   run: |
+    #     git add .
+    #     git config --local user.email "github-actions[bot]@users.noreply.github.com"
+    #     git config --local user.name "github-actions[bot]"
+    #     git remote set-url origin https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git
+    #     git fetch origin
+    #     echo ${{ github.head_ref }}
+    #     git checkout ${{ github.head_ref }}
+    #     # TODO when webkit works on ubuntu remove the os part of the commit message
+    #     git commit -am "A snapshot a day keeps the bugs away! 📷🐛 (OS: ubuntu)" || true
+    #     git push
+    #     git push origin ${{ github.head_ref }}
+    # # only upload artifacts if there's actually changes
+    # - uses: actions/upload-artifact@v4
+    #   if: steps.git-check.outputs.modified == 'true'
+    #   with:
+    #     name: playwright-report-ubuntu-${{ github.sha }}
+    #     path: playwright-report/
+    #     retention-days: 30
+    # # if have previous run results, use them
+    # - uses: actions/download-artifact@v4
+    #   if: always()
+    #   continue-on-error: true
+    #   with:
+    #     name: test-results-ubuntu-${{ github.sha }}
+    #     path: test-results/
     - name: Run ubuntu/chrome flow (with retries)
       id: retry
       if: always()
       run: |
-        if [[ ! -f "test-results/.last-run.json" ]]; then
-            # if no last run artifact, than run plawright normally
-            echo "run playwright normally"
-            yarn playwright test --project="Google Chrome" e2e/playwright/flow-tests.spec.ts || true
-            # # send to axiom
-            node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
-        fi
-        
-        retry=1
-        max_retrys=4
-        
-        # retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues
-        while [[ $retry -le $max_retrys ]]; do
-            if [[ -f "test-results/.last-run.json" ]]; then
-                failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
-                if [[ $failed_tests -gt 0 ]]; then
-                    echo "retried=true" >>$GITHUB_OUTPUT
-                    echo "run playwright with last failed tests and retry $retry"
-                    yarn playwright test --project="Google Chrome" --last-failed e2e/playwright/flow-tests.spec.ts || true
-                    # send to axiom
-                    node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
-                    retry=$((retry + 1))
-                else
-                    echo "retried=false" >>$GITHUB_OUTPUT
-                    exit 0
-                fi
-            else
-                echo "retried=false" >>$GITHUB_OUTPUT
-                exit 0
-            fi
-        done
-        
-        echo "retried=false" >>$GITHUB_OUTPUT
-        
-        if [[ -f "test-results/.last-run.json" ]]; then
-            failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
-            if [[ $failed_tests -gt 0 ]]; then
-                # if it still fails after 3 retrys, then fail the job
-                exit 1
-            fi
-        fi
-        exit 0
+        yarn playwright test --project="Google Chrome"  --workers=1  --grep=@focus --repeat-each=100
+        # if [[ ! -f "test-results/.last-run.json" ]]; then
+        #     # if no last run artifact, than run plawright normally
+        #     echo "run playwright normally"
+        #     yarn playwright test --project="Google Chrome" e2e/playwright/flow-tests.spec.ts
+        #     # # send to axiom
+        #     node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
+        # else 
+        #     echo "run playwright with last failed tests"
+        #     yarn playwright test --project="Google Chrome" --last-failed e2e/playwright/flow-tests.spec.ts
+        #     # send to axiom
+        #     node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
+        # fi
       env:
         CI: true
         token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
-    - name: send to axiom
-      if: always()
-      shell: bash
-      run: |
-        node playwrightProcess.mjs | tee /tmp/github-actions.log
+    # - name: send to axiom
+    #   if: always()
+    #   shell: bash
+    #   run: |
+    #     node playwrightProcess.mjs | tee /tmp/github-actions.log
     - uses: actions/upload-artifact@v4
       if: always()
       with:
@@ -306,48 +277,19 @@ jobs:
       id: retry
       if: always()
       run: |
-        if [[ ! -f "test-results/.last-run.json" ]]; then
-            # if no last run artifact, than run plawright normally
-            echo "run playwright normally"
-            yarn playwright test --project="webkit" e2e/playwright/flow-tests.spec.ts || true
-            # # send to axiom
-            node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
-        fi
-        
-        retry=1
-        max_retrys=4
-        
-        # retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues
-        while [[ $retry -le $max_retrys ]]; do
-            if [[ -f "test-results/.last-run.json" ]]; then
-                failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
-                if [[ $failed_tests -gt 0 ]]; then
-                    echo "retried=true" >>$GITHUB_OUTPUT
-                    echo "run playwright with last failed tests and retry $retry"
-                    yarn playwright test --project="webkit" --last-failed e2e/playwright/flow-tests.spec.ts || true
-                    # send to axiom
-                    node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
-                    retry=$((retry + 1))
-                else
-                    echo "retried=false" >>$GITHUB_OUTPUT
-                    exit 0
-                fi
-            else
-                echo "retried=false" >>$GITHUB_OUTPUT
-                exit 0
-            fi
-        done
-        
-        echo "retried=false" >>$GITHUB_OUTPUT
-        
-        if [[ -f "test-results/.last-run.json" ]]; then
-            failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
-            if [[ $failed_tests -gt 0 ]]; then
-                # if it still fails after 3 retrys, then fail the job
-                exit 1
-            fi
-        fi
-        exit 0
+        yarn playwright test --project="webkit"  --workers=1  --grep=@focus --repeat-each=1
+        # if [[ ! -f "test-results/.last-run.json" ]]; then
+        #     # if no last run artifact, than run plawright normally
+        #     echo "run playwright normally"
+        #     yarn playwright test --project="webkit" e2e/playwright/flow-tests.spec.ts
+        #     # # send to axiom
+        #     node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
+        # else
+        #     echo "run playwright with last failed tests"
+        #     yarn playwright test --project="webkit" --last-failed e2e/playwright/flow-tests.spec.ts
+        #     # send to axiom
+        #     node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
+        # fi
       env:
         CI: true
         token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}

e2e/playwright/flow-tests.spec.ts

@@ -1,4 +1,4 @@
-import { test, expect, Page } from '@playwright/test'
+import { test, expect, Page, TestInfo } from '@playwright/test'
 import {
   makeTemplate,
   getUtils,
@@ -139,22 +139,17 @@ async function doBasicSketch(page: Page, openPanes: string[]) {
     await expect(u.codeLocator)
       .toHaveText(`const sketch001 = startSketchOn('XZ')
   |> startProfileAt(${commonPoints.startAt}, %)`)
-  } else {
-    await page.waitForTimeout(500)
   }
-
-  await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 10)
   await page.waitForTimeout(500)
 
+  await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 10)
   if (openPanes.includes('code')) {
     await expect(u.codeLocator)
       .toHaveText(`const sketch001 = startSketchOn('XZ')
   |> startProfileAt(${commonPoints.startAt}, %)
   |> line([${commonPoints.num1}, 0], %)`)
-  } else {
-    await page.waitForTimeout(500)
   }
-
+  await page.waitForTimeout(500)
   await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 20)
   if (openPanes.includes('code')) {
     await expect(u.codeLocator)
@@ -162,9 +157,8 @@ async function doBasicSketch(page: Page, openPanes: string[]) {
   |> startProfileAt(${commonPoints.startAt}, %)
   |> line([${commonPoints.num1}, 0], %)
   |> line([0, ${commonPoints.num1 + 0.01}], %)`)
-  } else {
-    await page.waitForTimeout(500)
   }
+  await page.waitForTimeout(500)
   await page.mouse.click(startXPx, 500 - PUR * 20)
   if (openPanes.includes('code')) {
     await expect(u.codeLocator)
@@ -178,7 +172,6 @@ async function doBasicSketch(page: Page, openPanes: string[]) {
   // deselect line tool
   await page.getByRole('button', { name: 'Line', exact: true }).click()
   await page.waitForTimeout(500)
-
   const line1 = await u.getSegmentBodyCoords(`[data-overlay-index="${0}"]`, 0)
   if (openPanes.includes('code')) {
     expect(await u.getGreatestPixDiff(line1, TEST_COLORS.WHITE)).toBeLessThan(3)
@@ -217,7 +210,7 @@ async function doBasicSketch(page: Page, openPanes: string[]) {
 }
 
 test.describe('Basic sketch', () => {
-  test('code pane open at start', async ({ page }) => {
+  test('code pane open at start', { tag: '@focus' }, async ({ page }) => {
     await doBasicSketch(page, ['code'])
   })
 
@@ -3975,16 +3968,19 @@ test.describe('Regression tests', () => {
       )
     })
 
-    await page.goto('/')
-    await u.waitForPageLoad()
-
-    // error in guter
-    await expect(page.locator('.cm-lint-marker-error')).toBeVisible()
-    await page.waitForTimeout(200)
-    // expect it still to be there (sometimes it just clears for a bit?)
-    await expect(page.locator('.cm-lint-marker-error')).toBeVisible({
-      timeout: 10_000,
-    })
+    await expect(async () => {
+      await page.goto('/')
+      await u.waitForPageLoad()
+      // error in guter
+      await expect(page.locator('.cm-lint-marker-error')).toBeVisible({
+        timeout: 1_000,
+      })
+      await page.waitForTimeout(200)
+      // expect it still to be there (sometimes it just clears for a bit?)
+      await expect(page.locator('.cm-lint-marker-error')).toBeVisible({
+        timeout: 1_000,
+      })
+    }).toPass({ timeout: 40_000, intervals: [1_000] })
 
     // error text on hover
     await page.hover('.cm-lint-marker-error')
@@ -4148,9 +4144,9 @@ test.describe('Sketch tests', () => {
     await page.keyboard.press('Home')
     await page.keyboard.up('Shift')
     await page.keyboard.press('Backspace')
+    await u.expectCmdLog('[data-message-type="execution-done"]', 10_000)
     await u.openAndClearDebugPanel()
 
-    await u.expectCmdLog('[data-message-type="execution-done"]', 10_000)
     await page.waitForTimeout(100)
 
     await page.getByRole('button', { name: 'Line', exact: true }).click()
@@ -4854,7 +4850,7 @@ const sketch002 = startSketchOn(extrude001, 'END')
     )
   })
 })
-
+// flaky suite
 test.describe('Testing constraints', () => {
   test('Can constrain line length', async ({ page }) => {
     await page.addInitScript(async () => {
@@ -5793,7 +5789,7 @@ test.describe('Testing segment overlays', () => {
      * @param {number} options.steps - The number of steps to perform
      */
     const _clickConstrained =
-      (page: Page) =>
+      (page: Page, testInfo?: TestInfo) =>
       async ({
         hoverPos,
         constraintType,
@@ -5826,7 +5822,7 @@ test.describe('Testing segment overlays', () => {
         x = hoverPos.x + Math.cos(ang * deg) * 32
         y = hoverPos.y - Math.sin(ang * deg) * 32
         await page.mouse.move(x, y)
-        await wiggleMove(page, x, y, 20, 30, ang, 10, 5, locator)
+        await wiggleMove(page, x, y, 20, 30, ang, 10, 5, locator, testInfo)
 
         await expect(page.locator('.cm-content')).toContainText(
           expectBeforeUnconstrained
@@ -5947,7 +5943,7 @@ test.describe('Testing segment overlays', () => {
     test.setTimeout(120000)
     test('for segments [line, angledLine, lineTo, xLineTo]', async ({
       page,
-    }) => {
+    }, testInfo) => {
       await page.addInitScript(async () => {
         localStorage.setItem(
           'persistCode',
@@ -5991,7 +5987,7 @@ test.describe('Testing segment overlays', () => {
       await expect(page.getByTestId('segment-overlay')).toHaveCount(13)
 
       const clickUnconstrained = _clickUnconstrained(page)
-      const clickConstrained = _clickConstrained(page)
+      const clickConstrained = _clickConstrained(page, testInfo)
 
       await u.openAndClearDebugPanel()
       await u.sendCustomCmd({

e2e/playwright/test-utils.ts

@@ -1,4 +1,4 @@
-import { expect, Page, Download } from '@playwright/test'
+import { expect, Page, Download, TestInfo } from '@playwright/test'
 import { EngineCommand } from 'lang/std/artifactMap'
 import os from 'os'
 import fsp from 'fs/promises'
@@ -108,7 +108,7 @@ async function waitForCmdReceive(page: Page, commandType: string) {
 }
 
 export const wiggleMove = async (
-  page: any,
+  page: Page,
   x: number,
   y: number,
   steps: number,
@@ -116,11 +116,13 @@ export const wiggleMove = async (
   ang: number,
   amplitude: number,
   freq: number,
-  locator?: string
+  locator?: string,
+  testInfo?: TestInfo
 ) => {
   const tau = Math.PI * 2
   const deg = tau / 360
   const step = dist / steps
+  let mouseMovements: MouseMovement[] = []
   for (let i = 0, j = 0; i < dist; i += step, j += 1) {
     if (locator) {
       const isElVis = await page.locator(locator).isVisible()
@@ -133,7 +135,71 @@ export const wiggleMove = async (
     ]
     const [xr, yr] = [x2, y2]
     await page.mouse.move(x + xr, y + yr, { steps: 5 })
+    mouseMovements.push({ x: x + xr, y: y + yr, action: 'move' })
   }
+  // element was not visible, show the mouse movements in the screenshot
+  if (testInfo) {
+    testInfo?.annotations.push({
+      type: '⚠️',
+      description:
+        'Wiggling the mouse did not make the element visible - refer to the screenshot below to view mouse movements',
+    })
+    await showMovementsInScreenshot(page, mouseMovements, testInfo)
+  }
+}
+
+export type MouseMovement = {
+  x: number
+  y: number
+  action: 'click' | 'move'
+}
+
+const showMovementsInScreenshot = async (
+  page: Page,
+  mouseMovements: MouseMovement[],
+  testInfo: TestInfo
+) => {
+  await page.evaluate((mouseMovements) => {
+    for (let mouseMovement of mouseMovements) {
+      const { x, y, action } = mouseMovement
+      const elementWidth = action === 'move' ? 3 : 5
+      let circle = document.createElement('div')
+      circle.id = `pw-indicator-x${x}-y${y}`
+      circle.style.position = 'absolute'
+      circle.style.width = `${elementWidth}px`
+      circle.style.height = `${elementWidth}px`
+      circle.style.pointerEvents = 'none'
+
+      if (action === 'click') {
+        circle.style.backgroundColor = 'green'
+        circle.style.transform = 'rotate(45deg)'
+        // in case click and move are at the same position, ensure click is shown behind
+        circle.style.zIndex = '999'
+      } else {
+        circle.style.backgroundColor = 'red'
+        circle.style.borderRadius = '50%'
+        circle.style.zIndex = '1000'
+      }
+
+      circle.style.left = `${x - elementWidth / 2}px`
+      circle.style.top = `${y - elementWidth / 2}px`
+      document.body.appendChild(circle)
+    }
+  }, mouseMovements)
+
+  const screenshot = await page.screenshot()
+  await testInfo.attach('screenshot', {
+    body: screenshot,
+    contentType: 'image/png',
+  })
+  await removeIndicators(page)
+}
+
+const removeIndicators = async (page: Page) => {
+  await page.evaluate(() => {
+    let indicators = document.querySelectorAll('[id*="pw-indicator-"]')
+    indicators.forEach((e) => e.remove())
+  })
 }
 
 export const circleMove = async (

playwright.config.ts

@@ -16,13 +16,14 @@ export default defineConfig({
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Do not retry */
-  retries: process.env.CI ? 0 : 0,
+  retries: process.env.CI ? 0 : 2,
   /* Different amount of parallelism on CI and local. */
-  workers: process.env.CI ? 4 : 4,
+  workers: process.env.CI ? 1 : 4,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: [
     [process.env.CI ? 'dot' : 'list'],
     ['json', { outputFile: './test-results/report.json' }],
+    ['html'],
   ],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
@@ -31,6 +32,7 @@ export default defineConfig({
 
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
     trace: 'retain-on-failure',
+    screenshot: 'only-on-failure',
   },
 
   /* Configure projects for major browsers */