Nadro/4012/ci cd update (#4062)

* chore: saving off package.json progress unit tests fail in main

* fix: implementing a one liner for unit tests

* fix: renaming test:unit:local

* chore: adding playwright tests

* fix: making package.json not destructive to keep same pipeline commands for now

* fix: reordering

* fix: added tags for OS tests, moved kill-port to dev depen

* fix: OS skipping at tag level

* fix: lint, fmt, tsc, etc...

* Look at this (photo)Graph *in the voice of Nickelback*

* fix: new formatting

* fix: removing the ci copy, do not like it

* Look at this (photo)Graph *in the voice of Nickelback*

* chore: updating readme with explanation on the commands for CI CD simulation locally

* fix: package.json command for unit test, removing cached breaking cache in unit tests

* fix: fixing copy and typos in README.md for CI CD section

* fix: adding a duplicate command for a better name. CI CD will use it in a future PR

* chore: trying to clean up the copy and commands for CI CD tests

* chore: porting the bash code in the YAML to a bash script then using matrix permutations to control the runtime

* fix: typos

* fix: another typo, missed these went porting to the bash script logic

* fix: I think I need the checkout action since it has the repo code?

* fix: wrote absolute path not the relative hidden path, ope

* fix: does this cache give me the yarn install of playwright?

* fix: yarn cannot find the binary, use the yarn command

* fix: remove all uses...?

* chore: adding bash script for electron runtimes

* fix: copy cleanup

* fix: typo when copy and pasting the exclude logic, ope

* fix: this is wrong

* fix: build:wasm is a requirement for yarn tsc

* fix: reorder?

* fix: renaming integrations to e2e

* fix: windows is complaining about a pipe issue in the bash script?

* fix: escaping double quotes in windows?

* chore: consolidating commands into 1 file and easier YAML configuation for electron

* chore: mapped multiple OS playwright browser into a single bash script

* fix: removing old bash scripts, renaming matrix jobs

* fix: missed deleting this when I added the if statements.

* chore: removing unused job, xstate typegen more more since v5

* fix: trying to get these two tests to pass on first try

* fix: auto fixes

* fix: removing old unit test command

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/ci-cd-scripts/playwright-browser-chrome.sh

@@ -0,0 +1,59 @@
+# bash strict mode
+set -euo pipefail
+
+if [[ ! -f "test-results/.last-run.json" ]]; then
+    # if no last run artifact, than run plawright normally
+    echo "run playwright normally"
+    if [[ "$3" == "ubuntu-latest" ]]; then
+        yarn test:playwright:browser:chrome:ubuntu -- --shard=$1/$2 || true
+    elif [[ "$3" == "windows-latest" ]]; then
+        yarn test:playwright:browser:chrome:windows -- --shard=$1/$2 || true
+    else
+        echo "Do not run playwright. Unable to detect os runtime."
+        exit 1
+    fi
+    # # send to axiom
+    node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
+fi
+
+retry=1
+max_retrys=4
+
+# retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues
+while [[ $retry -le $max_retrys ]]; do
+    if [[ -f "test-results/.last-run.json" ]]; then
+        failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
+        if [[ $failed_tests -gt 0 ]]; then
+            echo "retried=true" >>$GITHUB_OUTPUT
+            echo "run playwright with last failed tests and retry $retry"
+            if [[ "$3" == "ubuntu-latest" ]]; then
+                yarn test:playwright:browser:chrome:ubuntu -- --last-failed || true
+            elif [[ "$3" == "windows-latest" ]]; then
+                yarn test:playwright:browser:chrome:windows -- --last-failed || true
+            else
+                echo "Do not run playwright. Unable to detect os runtime."
+                exit 1
+            fi
+            # send to axiom
+            node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
+            retry=$((retry + 1))
+        else
+            echo "retried=false" >>$GITHUB_OUTPUT
+            exit 0
+        fi
+    else
+        echo "retried=false" >>$GITHUB_OUTPUT
+        exit 0
+    fi
+done
+
+echo "retried=false" >>$GITHUB_OUTPUT
+
+if [[ -f "test-results/.last-run.json" ]]; then
+    failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
+    if [[ $failed_tests -gt 0 ]]; then
+        # if it still fails after 3 retrys, then fail the job
+        exit 1
+    fi
+fi
+exit 0

.github/ci-cd-scripts/playwright-electron.sh

@@ -0,0 +1,63 @@
+# bash strict mode
+set -euo pipefail
+
+if [[ ! -f "test-results/.last-run.json" ]]; then
+    # if no last run artifact, than run plawright normally
+    echo "run playwright normally"
+        if [[ "$1" == "ubuntu-latest" ]]; then
+            xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn test:playwright:electron:ubuntu || true
+        elif [[ "$1" == "windows-latest" ]]; then
+            yarn test:playwright:electron:windows || true
+        elif [[ "$1" == "macos-14" ]]; then
+            yarn test:playwright:electron:macos || true
+        else
+            echo "Do not run playwright. Unable to detect os runtime."
+            exit 1
+        fi
+        # # send to axiom
+        node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
+fi
+
+retry=1
+max_retrys=2
+
+# retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues
+while [[ $retry -le $max_retrys ]]; do
+    if [[ -f "test-results/.last-run.json" ]]; then
+        failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
+        if [[ $failed_tests -gt 0 ]]; then
+            echo "retried=true" >>$GITHUB_OUTPUT
+            echo "run playwright with last failed tests and retry $retry"
+            if [[ "$1" == "ubuntu-latest" ]]; then
+                xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn test:playwright:electron:ubuntu -- --last-failed || true
+            elif [[ "$1" == "windows-latest" ]]; then
+                yarn test:playwright:electron:windows -- --last-failed || true
+            elif [[ "$1" == "macos-14" ]]; then
+                yarn test:playwright:electron:macos -- --last-failed || true
+            else
+                echo "Do not run playwright. Unable to detect os runtime."
+                exit 1
+            fi
+            # send to axiom
+            node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
+            retry=$((retry + 1))
+        else
+            echo "retried=false" >>$GITHUB_OUTPUT
+            exit 0
+        fi
+    else
+        echo "retried=false" >>$GITHUB_OUTPUT
+        exit 0
+    fi
+done
+
+echo "retried=false" >>$GITHUB_OUTPUT
+
+if [[ -f "test-results/.last-run.json" ]]; then
+    failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
+    if [[ $failed_tests -gt 0 ]]; then
+        # if it still fails after 3 retrys, then fail the job
+        exit 1
+    fi
+fi
+exit 0

.github/workflows/e2e-tests.yml

@@ -1,4 +1,4 @@
-name: Playwright Tests
+name: E2E Tests
 on:
   push:
     branches: [ main ]
@@ -33,8 +33,9 @@ jobs:
             rust:
               - 'src/wasm-lib/**'
 
-  playwright-chrome:
+  browser:
     timeout-minutes: ${{ matrix.os == 'macos-14' && 60 || 50 }}
+    name: playwright:browser:${{ matrix.os }} ${{ matrix.shardIndex }} ${{ matrix.shardTotal }}
     strategy:
       fail-fast: false
       matrix:
@@ -188,48 +189,7 @@ jobs:
       if: ${{ !cancelled() && (success() || failure()) }}
       shell: bash
       run: |
-        if [[ ! -f "test-results/.last-run.json" ]]; then
+        .github/ci-cd-scripts/playwright-browser-chrome.sh ${{matrix.shardIndex}} ${{matrix.shardTotal}} ${{matrix.os}}
-            # if no last run artifact, than run plawright normally
-            echo "run playwright normally"
-            yarn playwright test --project="Google Chrome" --config=playwright.ci.config.ts --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --grep-invert="@snapshot|@electron" || true
-            # # send to axiom
-            node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
-        fi
-
-        retry=1
-        max_retrys=4
-
-        # retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues
-        while [[ $retry -le $max_retrys ]]; do
-            if [[ -f "test-results/.last-run.json" ]]; then
-                failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
-                if [[ $failed_tests -gt 0 ]]; then
-                    echo "retried=true" >>$GITHUB_OUTPUT
-                    echo "run playwright with last failed tests and retry $retry"
-                    yarn playwright test --project="Google Chrome" --config=playwright.ci.config.ts --last-failed --grep-invert="@snapshot|@electron" || true
-                    # send to axiom
-                    node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
-                    retry=$((retry + 1))
-                else
-                    echo "retried=false" >>$GITHUB_OUTPUT
-                    exit 0
-                fi
-            else
-                echo "retried=false" >>$GITHUB_OUTPUT
-                exit 0
-            fi
-        done
-
-        echo "retried=false" >>$GITHUB_OUTPUT
-
-        if [[ -f "test-results/.last-run.json" ]]; then
-            failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
-            if [[ $failed_tests -gt 0 ]]; then
-                # if it still fails after 3 retrys, then fail the job
-                exit 1
-            fi
-        fi
-        exit 0
       env:
         CI: true
         FAIL_ON_CONSOLE_ERRORS: true
@@ -258,7 +218,8 @@ jobs:
         overwrite: true
 
 
-  playwright-electron:
+  electron:
+    name: playwright:electron:${{matrix.os}}
     strategy:
       fail-fast: false
       matrix:
@@ -359,56 +320,7 @@ jobs:
       if: ${{ !cancelled() && (success() || failure()) }}
       shell: bash
       run: |
-        if [[ ! -f "test-results/.last-run.json" ]]; then
+        .github/ci-cd-scripts/playwright-electron.sh ${{ matrix.os }}
-            # if no last run artifact, than run plawright normally
-            echo "run playwright normally"
-            if [[ "$IS_UBUNTU" == "true" ]]; then
-              xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn playwright test --config=playwright.electron.config.ts --grep=@electron || true
-            else
-              yarn playwright test --config=playwright.electron.config.ts --grep=@electron || true
-            fi
-            # # send to axiom
-            node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
-        fi
-
-        retry=1
-        max_retrys=2
-
-        # retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues
-        while [[ $retry -le $max_retrys ]]; do
-            if [[ -f "test-results/.last-run.json" ]]; then
-                failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
-                if [[ $failed_tests -gt 0 ]]; then
-                    echo "retried=true" >>$GITHUB_OUTPUT
-                    echo "run playwright with last failed tests and retry $retry"
-                    if [[ "$IS_UBUNTU" == "true" ]]; then
-                      xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn playwright test --config=playwright.electron.config.ts --last-failed --grep=@electron || true
-                    else
-                      yarn playwright test --config=playwright.electron.config.ts --grep=@electron || true
-                    fi
-                    # send to axiom
-                    node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
-                    retry=$((retry + 1))
-                else
-                    echo "retried=false" >>$GITHUB_OUTPUT
-                    exit 0
-                fi
-            else
-                echo "retried=false" >>$GITHUB_OUTPUT
-                exit 0
-            fi
-        done
-
-        echo "retried=false" >>$GITHUB_OUTPUT
-
-        if [[ -f "test-results/.last-run.json" ]]; then
-            failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
-            if [[ $failed_tests -gt 0 ]]; then
-                # if it still fails after 3 retrys, then fail the job
-                exit 1
-            fi
-        fi
-        exit 0
       env:
         CI: true
         FAIL_ON_CONSOLE_ERRORS: true

.github/workflows/static-analysis.yml

@@ -1,4 +1,4 @@
-name: build-test-web
+name: Static Analysis
 
 on:
   pull_request:
@@ -16,7 +16,7 @@ permissions:
   actions: read
 
 jobs:
-  check-format:
+  yarn-fmt-check:
     runs-on: 'ubuntu-22.04'
     steps:
       - uses: actions/checkout@v4
@@ -27,7 +27,23 @@ jobs:
       - run: yarn install
       - run: yarn fmt-check
 
-  check-types:
+  yarn-build-wasm:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'yarn'
+      - run: yarn install
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: './src/wasm-lib'
+
+      - run: yarn build:wasm
+
+  yarn-tsc:
     runs-on: ubuntu-22.04
 
     steps:
@@ -42,13 +58,25 @@ jobs:
           workspaces: './src/wasm-lib'
 
       - run: yarn build:wasm
-      - run: yarn xstate:typegen
       - run: yarn tsc
-      - name: Lint
-        run: yarn eslint --max-warnings 0 src e2e packages/codemirror-lsp-client
 
+  yarn-lint:
+    runs-on: ubuntu-22.04
 
-  check-typos:
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'yarn'
+      - run: yarn install
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: './src/wasm-lib'
+
+      - run: yarn lint
+
+  python-codespell:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
@@ -62,7 +90,7 @@ jobs:
         run: codespell --config .codespellrc # Edit this file to tweak the typo list and other configuration.
 
 
-  build-test-web:
+  yarn-unit-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -80,7 +108,7 @@ jobs:
 
       - run: yarn build:wasm
 
-      - run: yarn simpleserver:ci
+      - run: yarn simpleserver:bg
         if: ${{ github.event_name != 'release' && github.event_name != 'schedule' }}
 
       - name: Install Chromium Browser
@@ -89,7 +117,7 @@ jobs:
 
       - name: run unit tests
         if: ${{ github.event_name != 'release' && github.event_name != 'schedule' }}
-        run: yarn test:nowatch
+        run: yarn test:unit
         env:
           VITE_KC_DEV_TOKEN: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
 

README.md

@@ -304,7 +304,7 @@ yarn start
 and finally:
 
 ```
-yarn test:nowatch
+yarn test:unit
 ```
 
 For individual testing:
@@ -348,7 +348,6 @@ yarn test-setup
 yarn tsc
 yarn fmt-check
 yarn lint
-yarn xstate:typegen
 yarn test:unit:local
 ```
 

e2e/playwright/sketch-tests.spec.ts

@@ -850,9 +850,17 @@ test.describe('Sketch tests', () => {
 
     await u.waitForAuthSkipAppStart()
 
+    // wait for execution done
+    await u.openDebugPanel()
+    await u.expectCmdLog('[data-message-type="execution-done"]')
+    await u.closeDebugPanel()
+
     // click "line([1.32, 0.38], %)"
     await page.getByText(`line([1.32, 0.38], %)`).click()
     await page.waitForTimeout(100)
+    await expect(page.getByRole('button', { name: 'Edit Sketch' })).toBeEnabled(
+      { timeout: 10_000 }
+    )
     // click edit sketch
     await page.getByRole('button', { name: 'Edit Sketch' }).click()
     await page.waitForTimeout(600) // wait for animation
@@ -872,7 +880,9 @@ test.describe('Sketch tests', () => {
     // otherwise the cmdbar would be waiting for a selection.
     await expect(
       page.getByRole('button', { name: 'selection : 1 face', exact: false })
-    ).toBeVisible()
+    ).toBeVisible({
+      timeout: 10_000,
+    })
   })
   test("Existing sketch with bad code delete user's code", async ({ page }) => {
     // this was a regression https://github.com/KittyCAD/modeling-app/issues/2832

e2e/playwright/various.spec.ts

@@ -407,8 +407,9 @@ test('Basic default modeling and sketch hotkeys work', async ({ page }) => {
     await page.waitForTimeout(500)
     await page.mouse.move(800, 200, { steps: 5 })
     await page.mouse.click(800, 200)
-    await page.waitForTimeout(500)
+    await expect(page.getByRole('button', { name: 'Continue' })).toBeVisible({
-    await expect(page.getByRole('button', { name: 'Continue' })).toBeVisible()
+      timeout: 20_000,
+    })
     await page.getByRole('button', { name: 'Continue' }).click()
     await expect(
       page.getByRole('button', { name: 'Submit command' })

package.json

@@ -104,11 +104,10 @@
     "tronb:package": "electron-builder --config electron-builder.yml",
     "test-setup": "yarn install && yarn build:wasm",
     "test": "vitest --mode development",
-    "test:nowatch": "vitest run --mode development",
     "test:unit": "vitest run --mode development",
-    "test:playwright:browser:chrome": "playwright test '--project=Google Chrome' --config=playwright.ci.config.ts '--grep-invert=@snapshot|@electron'",
+    "test:playwright:browser:chrome": "playwright test --project='Google Chrome' --config=playwright.ci.config.ts --grep-invert='@snapshot|@electron'",
-    "test:playwright:browser:chrome:windows": "playwright test '--project=Google Chrome' --config=playwright.ci.config.ts '--grep-invert=@snapshot|@electron|@skipWin'",
+    "test:playwright:browser:chrome:windows": "playwright test --project=\"Google Chrome\" --config=playwright.ci.config.ts --grep-invert=\"@snapshot|@electron|@skipWin\"",
-    "test:playwright:browser:chrome:ubuntu": "playwright test '--project=Google Chrome' --config=playwright.ci.config.ts '--grep-invert=@snapshot|@electron|@skipLinux'",
+    "test:playwright:browser:chrome:ubuntu": "playwright test --project='Google Chrome' --config=playwright.ci.config.ts --grep-invert='@snapshot|@electron|@skipLinux'",
     "test:playwright:electron": "playwright test --config=playwright.electron.config.ts --grep=@electron",
     "test:playwright:electron:windows": "playwright test --config=playwright.electron.config.ts --grep=@electron --grep-invert=@skipWin",
     "test:playwright:electron:macos": "playwright test --config=playwright.electron.config.ts --grep=@electron --grep-invert=@skipMacos",