Run end-to-end tests against the web app (#7171)

* Run end-to-end tests against the web app

* Capture logs for web tests

.github/workflows/e2e-tests.yml

@@ -100,9 +100,11 @@ jobs:
             rust/kcl-wasm-lib/pkg/kcl_wasm_lib*
 
   snapshots:
-    name: playwright:snapshots:ubuntu
-    runs-on: runs-on=${{ github.run_id }}/family=i7ie.2xlarge/image=ubuntu22-full-x64
     needs: [prepare-wasm]
+
+    runs-on: runs-on=${{ github.run_id }}/family=i7ie.2xlarge/image=ubuntu22-full-x64
+    name: playwright:snapshots:ubuntu
+
     steps:
       - uses: actions/create-github-app-token@v1
         id: app-token
@@ -131,7 +133,6 @@ jobs:
           cache: 'npm'
 
       - name: Install dependencies
-        id: deps-install
         run: npm install
 
       - name: Cache browsers
@@ -207,12 +208,92 @@ jobs:
           git push
           git push origin ${{ github.head_ref }}
 
-  electron:
+  web:
     needs: [prepare-wasm]
-    timeout-minutes: 60
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
     env:
       OS_NAME: ${{ contains(matrix.os, 'ubuntu') && 'ubuntu' || (contains(matrix.os, 'windows') && 'windows' || 'macos') }}
-    name: playwright:electron:${{ contains(matrix.os, 'ubuntu') && 'ubuntu' || (contains(matrix.os, 'windows') && 'windows' || 'macos') }} (shard ${{ matrix.shardIndex }})
+
+    steps:
+
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ secrets.MODELING_APP_GH_APP_ID }}
+          private-key: ${{ secrets.MODELING_APP_GH_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+
+      - uses: actions/download-artifact@v4
+        name: prepared-wasm
+
+      - name: Copy prepared Wasm
+        run: |
+          ls -R prepared-wasm
+          cp prepared-wasm/kcl_wasm_lib_bg.wasm public
+          mkdir rust/kcl-wasm-lib/pkg
+          cp prepared-wasm/kcl_wasm_lib* rust/kcl-wasm-lib/pkg
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Cache browsers
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/ms-playwright/
+          key: ${{ runner.os }}-playwright-${{ hashFiles('package-lock.json') }}
+
+      - name: Install browsers
+        run: npm run playwright install --with-deps
+
+      - name: Start Vector
+        if: ${{ !contains(matrix.os, 'windows') }}
+        run: .github/ci-cd-scripts/start-vector-${{ env.OS_NAME }}.sh
+        env:
+          GH_ACTIONS_AXIOM_TOKEN: ${{ secrets.GH_ACTIONS_AXIOM_TOKEN }}
+          OS_NAME: ${{ env.OS_NAME }}
+
+      - name: Test web
+        uses: nick-fields/retry@v3.0.2
+        with:
+          shell: bash
+          command: npm run test:e2e:web
+          timeout_minutes: 5
+          max_attempts: 5
+        env:
+          token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
+          TAB_API_URL: ${{ secrets.TAB_API_URL }}
+          TAB_API_KEY: ${{ secrets.TAB_API_KEY }}
+          CI_COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+          CI_PR_NUMBER: ${{ github.event.pull_request.number }}
+          CI_SUITE: e2e:web
+          TARGET: web
+
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() && (success() || failure()) }}
+        with:
+          path: playwright-report/
+          include-hidden-files: true
+          retention-days: 30
+          overwrite: true
+
+  desktop:
+    needs: [prepare-wasm]
+
     strategy:
       fail-fast: false
       matrix:
@@ -255,6 +336,10 @@ jobs:
             shardIndex: 2
             shardTotal: 2
     runs-on: ${{ matrix.os }}
+    name: playwright:electron:${{ contains(matrix.os, 'ubuntu') && 'ubuntu' || (contains(matrix.os, 'windows') && 'windows' || 'macos') }} (shard ${{ matrix.shardIndex }})
+    env:
+      OS_NAME: ${{ contains(matrix.os, 'ubuntu') && 'ubuntu' || (contains(matrix.os, 'windows') && 'windows' || 'macos') }}
+
     steps:
       - uses: actions/checkout@v4
 
@@ -277,14 +362,14 @@ jobs:
         id: deps-install
         run: npm install
 
-      - name: Cache Playwright Browsers
+      - name: Cache browsers
         uses: actions/cache@v4
         with:
           path: |
             ~/.cache/ms-playwright/
           key: ${{ runner.os }}-playwright-${{ hashFiles('package-lock.json') }}
 
-      - name: Install Playwright Browsers
+      - name: Install browsers
         run: npm run playwright install --with-deps
 
       - name: Build web

CONTRIBUTING.md

@@ -205,7 +205,7 @@ Prepare these system dependencies:
 
 #### Snapshot tests (Google Chrome on Ubuntu only)
 
-Only Ubunu and Google Chrome is supported for the set of tests evaluating screenshot snapshots.
+Only Ubuntu and Google Chrome is supported for the set of tests evaluating screenshot snapshots.
 If you don't run Ubuntu locally or in a VM, you may use a GitHub Codespace.
 ```
 npm run playwright -- install chrome

Makefile

@@ -120,11 +120,10 @@ test-e2e: test-e2e-$(TARGET)
 
 .PHONY: test-e2e-web
 test-e2e-web: install build ## Run the web e2e tests
-	@ curl -fs localhost:3000 >/dev/null || ( echo "Error: localhost:3000 not available, 'make run-web' first" && exit 1 )
 ifdef E2E_GREP
-	npm run chrome:test -- --headed --grep="$(E2E_GREP)" --max-failures=$(E2E_FAILURES)
+	npm run test:e2e:web -- --headed --grep="$(E2E_GREP)" --max-failures=$(E2E_FAILURES)
 else
-	npm run chrome:test -- --headed --workers='100%'
+	npm run test:e2e:web -- --headed --workers='100%'
 endif
 
 .PHONY: test-e2e-desktop

e2e/playwright/command-bar-tests.spec.ts

@@ -519,7 +519,7 @@ test.describe('Command bar tests', () => {
     `Zoom to fit to shared model on web`,
     { tag: ['@web'] },
     async ({ page, scene }) => {
-      if (process.env.PLATFORM !== 'web') {
+      if (process.env.TARGET !== 'web') {
         // This test is web-only
         // TODO: re-enable on CI as part of a new @web test suite
         return

e2e/playwright/fixtures/fixtureSetup.ts

@@ -394,7 +394,7 @@ const fixturesBasedOnProcessEnvPlatform = {
   },
 }
 
-if (process.env.PLATFORM === 'web') {
+if (process.env.TARGET === 'web') {
   Object.assign(fixturesBasedOnProcessEnvPlatform, fixturesForWeb)
 } else {
   Object.assign(fixturesBasedOnProcessEnvPlatform, fixturesForElectron)

e2e/playwright/fixtures/homePageFixture.ts

@@ -126,7 +126,7 @@ export class HomePageFixture {
   /** Returns the project name in case caller has used the default and needs it */
   goToModelingScene = async (name = 'testDefault') => {
     // On web this is a no-op. There is no project view.
-    if (process.env.PLATFORM === 'web') return ''
+    if (process.env.TARGET === 'web') return ''
 
     await this.createAndGoToProject(name)
     return name

e2e/playwright/share-link.spec.ts

@@ -17,7 +17,7 @@ test.describe('Share link tests', () => {
       `Open in desktop app with ${codeLength}-long code ${isWindows && showsErrorOnWindows ? 'shows error' : "doesn't show error"}`,
       { tag: ['@web'] },
       async ({ page }) => {
-        if (process.env.PLATFORM !== 'web') {
+        if (process.env.TARGET !== 'web') {
           // This test is web-only
           // TODO: re-enable on CI as part of a new @web test suite
           return

e2e/playwright/zoo-test.ts

@@ -41,7 +41,7 @@ const playwrightTestFnWithFixtures_ = playwrightTestFn.extend<{
 }>({
   tronApp: [
     async ({}, use, testInfo) => {
-      if (process.env.PLATFORM === 'web') {
+      if (process.env.TARGET === 'web') {
         await use(undefined)
         return
       }

package-lock.json

@@ -113,6 +113,7 @@
         "@xstate/cli": "^0.5.17",
         "autoprefixer": "^10.4.21",
         "babel-preset-vite": "^1.1.3",
+        "cross-env": "^7.0.3",
         "dpdm": "^3.14.0",
         "electron": "^34.1.1",
         "electron-builder": "^26.0.12",
@@ -2492,7 +2493,6 @@
     },
     "node_modules/@clack/prompts/node_modules/is-unicode-supported": {
       "version": "1.3.0",
-      "extraneous": true,
       "inBundle": true,
       "license": "MIT",
       "engines": {

package.json

@@ -122,7 +122,6 @@
     "postinstall": "electron-rebuild",
     "generate:machine-api": "npx openapi-typescript ./openapi/machine-api.json -o src/lib/machine-api.d.ts",
     "tron:start": "electron-forge start",
-    "chrome:test": "PLATFORM=web NODE_ENV=development playwright test --config=playwright.config.ts --project='Google Chrome' --grep-invert=@snapshot",
     "tronb:vite:dev": "vite build -c vite.main.config.ts -m development && vite build -c vite.preload.config.ts -m development && vite build -c vite.renderer.config.ts -m development",
     "tronb:vite:prod": "vite build -c vite.main.config.ts && vite build -c vite.preload.config.ts && vite build -c vite.renderer.config.ts",
     "tronb:package:dev": "npm run tronb:vite:dev && electron-builder --config electron-builder.yml",
@@ -130,9 +129,10 @@
     "test-setup": "npm install && npm run build:wasm",
     "test": "vitest --mode development",
     "test:rust": "(cd rust && just test && just lint)",
-    "test:snapshots": "PLATFORM=web NODE_ENV=development playwright test --config=playwright.config.ts --grep=@snapshot --trace=on --shard=1/1",
+    "test:snapshots": "cross-env TARGET=web NODE_ENV=development playwright test --config=playwright.config.ts --grep=@snapshot --trace=on",
     "test:unit": "vitest run --mode development --exclude **/jest-component-unit-tests/*",
     "test:unit:components": "jest -c jest-component-unit-tests/jest.config.ts --rootDir jest-component-unit-tests/",
+    "test:e2e:web": "cross-env TARGET=web NODE_ENV=development playwright test --config=playwright.config.ts --project=\"Google Chrome\" --grep=@web",
     "test:playwright:electron": "playwright test --config=playwright.electron.config.ts --grep-invert=@snapshot",
     "test:playwright:electron:local": "npm run tronb:vite:dev && playwright test --config=playwright.electron.config.ts --grep-invert=@snapshot --grep-invert=\"$(curl --silent https://test-analysis-bot.hawk-dinosaur.ts.net/projects/KittyCAD/modeling-app/tests/disabled/regex)\"",
     "test:playwright:electron:local-engine": "npm run tronb:vite:dev && playwright test --config=playwright.electron.config.ts --grep-invert='@snapshot|@skipLocalEngine' --grep-invert=\"$(curl --silent https://test-analysis-bot.hawk-dinosaur.ts.net/projects/KittyCAD/modeling-app/tests/disabled/regex)\"",
@@ -188,6 +188,7 @@
     "@xstate/cli": "^0.5.17",
     "autoprefixer": "^10.4.21",
     "babel-preset-vite": "^1.1.3",
+    "cross-env": "^7.0.3",
     "dpdm": "^3.14.0",
     "electron": "^34.1.1",
     "electron-builder": "^26.0.12",

src/lib/commandTypes.ts

@@ -15,8 +15,8 @@ import type {
 } from '@src/machines/commandBarMachine'
 
 type Icon = CustomIconName
-const _PLATFORMS = ['both', 'web', 'desktop'] as const
+const _TARGETS = ['both', 'web', 'desktop'] as const
-type PLATFORM = typeof _PLATFORMS
+type TARGET = typeof _TARGETS
 const _INPUT_TYPES = [
   'options',
   'string',
@@ -101,7 +101,7 @@ export type Command<
   displayName?: string
   description?: string
   icon?: Icon
-  hide?: PLATFORM[number]
+  hide?: TARGET[number]
   hideFromSearch?: boolean
   disabled?: boolean
   status?: CommandStatus