Run end-to-end tests against the web app (#7171)

* Run end-to-end tests against the web app * Capture logs for web tests
2025-05-27 14:11:31 -04:00
parent f502e445cc
commit 8bae76000c
11 changed files with 108 additions and 23 deletions
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@ -100,9 +100,11 @@ jobs:
            rust/kcl-wasm-lib/pkg/kcl_wasm_lib*

  snapshots:
-    name: playwright:snapshots:ubuntu
-    runs-on: runs-on=${{ github.run_id }}/family=i7ie.2xlarge/image=ubuntu22-full-x64
    needs: [prepare-wasm]
+
+    runs-on: runs-on=${{ github.run_id }}/family=i7ie.2xlarge/image=ubuntu22-full-x64
+    name: playwright:snapshots:ubuntu
+
    steps:
      - uses: actions/create-github-app-token@v1
        id: app-token
@ -131,7 +133,6 @@ jobs:
          cache: 'npm'

      - name: Install dependencies
-        id: deps-install
        run: npm install

      - name: Cache browsers
@ -207,12 +208,92 @@ jobs:
          git push
          git push origin ${{ github.head_ref }}

-  electron:
+  web:
    needs: [prepare-wasm]
-    timeout-minutes: 60
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
    env:
      OS_NAME: ${{ contains(matrix.os, 'ubuntu') && 'ubuntu' || (contains(matrix.os, 'windows') && 'windows' || 'macos') }}
-    name: playwright:electron:${{ contains(matrix.os, 'ubuntu') && 'ubuntu' || (contains(matrix.os, 'windows') && 'windows' || 'macos') }} (shard ${{ matrix.shardIndex }})
+
+    steps:
+
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ secrets.MODELING_APP_GH_APP_ID }}
+          private-key: ${{ secrets.MODELING_APP_GH_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+
+      - uses: actions/download-artifact@v4
+        name: prepared-wasm
+
+      - name: Copy prepared Wasm
+        run: |
+          ls -R prepared-wasm
+          cp prepared-wasm/kcl_wasm_lib_bg.wasm public
+          mkdir rust/kcl-wasm-lib/pkg
+          cp prepared-wasm/kcl_wasm_lib* rust/kcl-wasm-lib/pkg
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Cache browsers
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/ms-playwright/
+          key: ${{ runner.os }}-playwright-${{ hashFiles('package-lock.json') }}
+
+      - name: Install browsers
+        run: npm run playwright install --with-deps
+
+      - name: Start Vector
+        if: ${{ !contains(matrix.os, 'windows') }}
+        run: .github/ci-cd-scripts/start-vector-${{ env.OS_NAME }}.sh
+        env:
+          GH_ACTIONS_AXIOM_TOKEN: ${{ secrets.GH_ACTIONS_AXIOM_TOKEN }}
+          OS_NAME: ${{ env.OS_NAME }}
+
+      - name: Test web
+        uses: nick-fields/retry@v3.0.2
+        with:
+          shell: bash
+          command: npm run test:e2e:web
+          timeout_minutes: 5
+          max_attempts: 5
+        env:
+          token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
+          TAB_API_URL: ${{ secrets.TAB_API_URL }}
+          TAB_API_KEY: ${{ secrets.TAB_API_KEY }}
+          CI_COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+          CI_PR_NUMBER: ${{ github.event.pull_request.number }}
+          CI_SUITE: e2e:web
+          TARGET: web
+
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() && (success() || failure()) }}
+        with:
+          path: playwright-report/
+          include-hidden-files: true
+          retention-days: 30
+          overwrite: true
+
+  desktop:
+    needs: [prepare-wasm]
+
    strategy:
      fail-fast: false
      matrix:
@ -255,6 +336,10 @@ jobs:
            shardIndex: 2
            shardTotal: 2
    runs-on: ${{ matrix.os }}
+    name: playwright:electron:${{ contains(matrix.os, 'ubuntu') && 'ubuntu' || (contains(matrix.os, 'windows') && 'windows' || 'macos') }} (shard ${{ matrix.shardIndex }})
+    env:
+      OS_NAME: ${{ contains(matrix.os, 'ubuntu') && 'ubuntu' || (contains(matrix.os, 'windows') && 'windows' || 'macos') }}
+
    steps:
      - uses: actions/checkout@v4

@ -277,14 +362,14 @@ jobs:
        id: deps-install
        run: npm install

-      - name: Cache Playwright Browsers
+      - name: Cache browsers
        uses: actions/cache@v4
        with:
          path: |
            ~/.cache/ms-playwright/
          key: ${{ runner.os }}-playwright-${{ hashFiles('package-lock.json') }}

-      - name: Install Playwright Browsers
+      - name: Install browsers
        run: npm run playwright install --with-deps

      - name: Build web
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -205,7 +205,7 @@ Prepare these system dependencies:

 #### Snapshot tests (Google Chrome on Ubuntu only)

-Only Ubunu and Google Chrome is supported for the set of tests evaluating screenshot snapshots.
+Only Ubuntu and Google Chrome is supported for the set of tests evaluating screenshot snapshots.
 If you don't run Ubuntu locally or in a VM, you may use a GitHub Codespace.
 ```
 npm run playwright -- install chrome
--- a/5
+++ b/5
@ -120,11 +120,10 @@ test-e2e: test-e2e-$(TARGET)

 .PHONY: test-e2e-web
 test-e2e-web: install build ## Run the web e2e tests
-	@ curl -fs localhost:3000 >/dev/null || ( echo "Error: localhost:3000 not available, 'make run-web' first" && exit 1 )
 ifdef E2E_GREP
-	npm run chrome:test -- --headed --grep="$(E2E_GREP)" --max-failures=$(E2E_FAILURES)
+	npm run test:e2e:web -- --headed --grep="$(E2E_GREP)" --max-failures=$(E2E_FAILURES)
 else
-	npm run chrome:test -- --headed --workers='100%'
+	npm run test:e2e:web -- --headed --workers='100%'
 endif

 .PHONY: test-e2e-desktop
--- a/e2e/playwright/command-bar-tests.spec.ts
+++ b/e2e/playwright/command-bar-tests.spec.ts
@ -519,7 +519,7 @@ test.describe('Command bar tests', () => {
    `Zoom to fit to shared model on web`,
    { tag: ['@web'] },
    async ({ page, scene }) => {
-      if (process.env.PLATFORM !== 'web') {
+      if (process.env.TARGET !== 'web') {
        // This test is web-only
        // TODO: re-enable on CI as part of a new @web test suite
        return
--- a/e2e/playwright/fixtures/fixtureSetup.ts
+++ b/e2e/playwright/fixtures/fixtureSetup.ts
@ -394,7 +394,7 @@ const fixturesBasedOnProcessEnvPlatform = {
  },
 }

-if (process.env.PLATFORM === 'web') {
+if (process.env.TARGET === 'web') {
  Object.assign(fixturesBasedOnProcessEnvPlatform, fixturesForWeb)
 } else {
  Object.assign(fixturesBasedOnProcessEnvPlatform, fixturesForElectron)
--- a/e2e/playwright/fixtures/homePageFixture.ts
+++ b/e2e/playwright/fixtures/homePageFixture.ts
@ -126,7 +126,7 @@ export class HomePageFixture {
  /** Returns the project name in case caller has used the default and needs it */
  goToModelingScene = async (name = 'testDefault') => {
    // On web this is a no-op. There is no project view.
-    if (process.env.PLATFORM === 'web') return ''
+    if (process.env.TARGET === 'web') return ''

    await this.createAndGoToProject(name)
    return name
--- a/e2e/playwright/share-link.spec.ts
+++ b/e2e/playwright/share-link.spec.ts
@ -17,7 +17,7 @@ test.describe('Share link tests', () => {
      `Open in desktop app with ${codeLength}-long code ${isWindows && showsErrorOnWindows ? 'shows error' : "doesn't show error"}`,
      { tag: ['@web'] },
      async ({ page }) => {
-        if (process.env.PLATFORM !== 'web') {
+        if (process.env.TARGET !== 'web') {
          // This test is web-only
          // TODO: re-enable on CI as part of a new @web test suite
          return
--- a/e2e/playwright/zoo-test.ts
+++ b/e2e/playwright/zoo-test.ts
@ -41,7 +41,7 @@ const playwrightTestFnWithFixtures_ = playwrightTestFn.extend<{
 }>({
  tronApp: [
    async ({}, use, testInfo) => {
-      if (process.env.PLATFORM === 'web') {
+      if (process.env.TARGET === 'web') {
        await use(undefined)
        return
      }
--- a/package-lock.json
+++ b/package-lock.json
@ -113,6 +113,7 @@
        "@xstate/cli": "^0.5.17",
        "autoprefixer": "^10.4.21",
        "babel-preset-vite": "^1.1.3",
+        "cross-env": "^7.0.3",
        "dpdm": "^3.14.0",
        "electron": "^34.1.1",
        "electron-builder": "^26.0.12",
@ -2492,7 +2493,6 @@
    },
    "node_modules/@clack/prompts/node_modules/is-unicode-supported": {
      "version": "1.3.0",
-      "extraneous": true,
      "inBundle": true,
      "license": "MIT",
      "engines": {
--- a/package.json
+++ b/package.json
@ -122,7 +122,6 @@
    "postinstall": "electron-rebuild",
    "generate:machine-api": "npx openapi-typescript ./openapi/machine-api.json -o src/lib/machine-api.d.ts",
    "tron:start": "electron-forge start",
-    "chrome:test": "PLATFORM=web NODE_ENV=development playwright test --config=playwright.config.ts --project='Google Chrome' --grep-invert=@snapshot",
    "tronb:vite:dev": "vite build -c vite.main.config.ts -m development && vite build -c vite.preload.config.ts -m development && vite build -c vite.renderer.config.ts -m development",
    "tronb:vite:prod": "vite build -c vite.main.config.ts && vite build -c vite.preload.config.ts && vite build -c vite.renderer.config.ts",
    "tronb:package:dev": "npm run tronb:vite:dev && electron-builder --config electron-builder.yml",
@ -130,9 +129,10 @@
    "test-setup": "npm install && npm run build:wasm",
    "test": "vitest --mode development",
    "test:rust": "(cd rust && just test && just lint)",
-    "test:snapshots": "PLATFORM=web NODE_ENV=development playwright test --config=playwright.config.ts --grep=@snapshot --trace=on --shard=1/1",
+    "test:snapshots": "cross-env TARGET=web NODE_ENV=development playwright test --config=playwright.config.ts --grep=@snapshot --trace=on",
    "test:unit": "vitest run --mode development --exclude **/jest-component-unit-tests/*",
    "test:unit:components": "jest -c jest-component-unit-tests/jest.config.ts --rootDir jest-component-unit-tests/",
+    "test:e2e:web": "cross-env TARGET=web NODE_ENV=development playwright test --config=playwright.config.ts --project=\"Google Chrome\" --grep=@web",
    "test:playwright:electron": "playwright test --config=playwright.electron.config.ts --grep-invert=@snapshot",
    "test:playwright:electron:local": "npm run tronb:vite:dev && playwright test --config=playwright.electron.config.ts --grep-invert=@snapshot --grep-invert=\"$(curl --silent https://test-analysis-bot.hawk-dinosaur.ts.net/projects/KittyCAD/modeling-app/tests/disabled/regex)\"",
    "test:playwright:electron:local-engine": "npm run tronb:vite:dev && playwright test --config=playwright.electron.config.ts --grep-invert='@snapshot|@skipLocalEngine' --grep-invert=\"$(curl --silent https://test-analysis-bot.hawk-dinosaur.ts.net/projects/KittyCAD/modeling-app/tests/disabled/regex)\"",
@ -188,6 +188,7 @@
    "@xstate/cli": "^0.5.17",
    "autoprefixer": "^10.4.21",
    "babel-preset-vite": "^1.1.3",
+    "cross-env": "^7.0.3",
    "dpdm": "^3.14.0",
    "electron": "^34.1.1",
    "electron-builder": "^26.0.12",
--- a/src/lib/commandTypes.ts
+++ b/src/lib/commandTypes.ts
@ -15,8 +15,8 @@ import type {
 } from '@src/machines/commandBarMachine'

 type Icon = CustomIconName
-const _PLATFORMS = ['both', 'web', 'desktop'] as const
-type PLATFORM = typeof _PLATFORMS
+const _TARGETS = ['both', 'web', 'desktop'] as const
+type TARGET = typeof _TARGETS
 const _INPUT_TYPES = [
  'options',
  'string',
@ -101,7 +101,7 @@ export type Command<
  displayName?: string
  description?: string
  icon?: Icon
-  hide?: PLATFORM[number]
+  hide?: TARGET[number]
  hideFromSearch?: boolean
  disabled?: boolean
  status?: CommandStatus