Nadro/adhoc/e2e improvements (#4013)

* chore: saving off package.json progress unit tests fail in main * fix: implementing a one liner for unit tests * fix: renaming test:unit:local * chore: adding playwright tests * fix: making package.json not destructive to keep same pipeline commands for now * fix: reordering * fix: added tags for OS tests, moved kill-port to dev depen * fix: OS skipping at tag level * fix: lint, fmt, tsc, etc... * Look at this (photo)Graph *in the voice of Nickelback* * fix: new formatting * fix: removing the ci copy, do not like it * Look at this (photo)Graph *in the voice of Nickelback* * chore: updating readme with explanation on the commands for CI CD simulation locally * fix: package.json command for unit test, removing cached breaking cache in unit tests * fix: fixing copy and typos in README.md for CI CD section * fix: adding a duplicate command for a better name. CI CD will use it in a future PR * fix: this is wrong... removing it * A snapshot a day keeps the bugs away! 📷🐛 (OS: windows-latest) * Revert "A snapshot a day keeps the bugs away! 📷🐛 (OS: windows-latest)" This reverts commit f767dd46d4. * fix: typos in README.md --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Kurt Hutten Irev-Dev <k.hutten@protonmail.ch>
2024-10-02 11:58:17 -04:00
parent 74faf0461c
commit 3035ad16fc
11 changed files with 680 additions and 586 deletions
--- a/README.md
+++ b/README.md
@ -322,6 +322,70 @@ cd src/wasm-lib
 cargo test
 ```

+### Mapping CI CD jobs to local commands
+
+When you see the CI CD fail on jobs you may wonder three things
+- Do I have a bug in my code?
+- Is the test flaky?
+- Is there a bug in `main`?
+
+To answer these questions the following commands will give you confidence to locate the issue.
+
+#### Static Analysis
+
+Part of the CI CD pipeline performs static analysis on the code. Use the following commands to mimic the CI CD jobs.
+
+The following set of commands should get us closer to one and done commands to instantly retest issues.
+
+```
+yarn test-setup
+```
+
+> Gotcha, are packages up to date and is the wasm built?
+
+
+```
+yarn tsc
+yarn fmt-check
+yarn lint
+yarn xstate:typegen
+yarn test:unit:local
+```
+
+> Gotcha: Our unit tests have integration tests in them. You need to run a localhost server to run the unit tests.
+
+#### E2E Tests
+
+**Playwright Browser**
+
+These E2E tests run in a browser (without electron).
+There are tests that are skipped if they are ran in a windows OS or Linux OS. We can use playwright tags to implement test skipping.
+
+Breaking down the command `yarn test:playwright:browser:chrome:windows`
+- The application is `playwright`
+- The runtime is a `browser`
+- The specific `browser` is `chrome`
+- The test should run in a `windows` environment. It will skip tests that are broken or flaky in the windows OS.
+
+```
+yarn test:playwright:browser:chrome
+yarn test:playwright:browser:chrome:windows
+yarn test:playwright:browser:chrome:ubuntu
+```
+
+**Playwright Electron**
+
+These E2E tests run in electron. There are tests that are skipped if they are ran in a windows, linux, or macos environment. We can use playwright tags to implement test skipping.
+
+```
+yarn test:playwright:electron:local
+yarn test:playwright:electron:windows:local
+yarn test:playwright:electron:macos:local
+yarn test:playwright:electron:ubuntu:local
+```
+
+> Why does it say local? The CI CD commands that run in the pipeline cannot be ran locally. A single command will not properly setup the testing environment locally.
+
 #### Some notes on CI

 The tests are broken into snapshot tests and non-snapshot tests, and they run in that order, they automatically commit new snap shots, so if you see an image commit check it was an intended change. If we have non-determinism in the snapshots such that they are always committing new images, hopefully this annoyance makes us fix them asap, if you notice this happening let Kurt know. But for the odd occasion `git reset --hard HEAD~ && git push -f` is your friend.
--- a/e2e/playwright/basic-sketch.spec.ts
+++ b/e2e/playwright/basic-sketch.spec.ts
@ -154,7 +154,7 @@ async function doBasicSketch(page: Page, openPanes: string[]) {
 }

 test.describe('Basic sketch', () => {
-  test('code pane open at start', async ({ page }) => {
+  test('code pane open at start', { tag: ['@skipWin'] }, async ({ page }) => {
    // Skip on windows it is being weird.
    test.skip(process.platform === 'win32', 'Skip on windows')
    await doBasicSketch(page, ['code'])
--- a/e2e/playwright/point-click.spec.ts
+++ b/e2e/playwright/point-click.spec.ts
@ -5,13 +5,10 @@ import { ToolbarFixture } from './fixtures/toolbarFixture'

 // test file is for testing point an click code gen functionality that's not sketch mode related

-test('verify extruding circle works', async ({
-  app,
-  cmdBar,
-  editor,
-  toolbar,
-  scene,
-}) => {
+test(
+  'verify extruding circle works',
+  { tag: ['@skipWin'] },
+  async ({ app, cmdBar, editor, toolbar, scene }) => {
    test.skip(
      process.platform === 'win32',
      'Fails on windows in CI, can not be replicated locally on windows.'
@ -69,7 +66,8 @@ test('verify extruding circle works', async ({

      await editor.expectEditor.toContain(expectString)
    })
-})
+  }
+)

 test.describe('verify sketch on chamfer works', () => {
  const _sketchOnAChamfer =
@ -152,12 +150,10 @@ test.describe('verify sketch on chamfer works', () => {
        })
      })
    }
-  test('works on all edge selections and can break up multi edges in a chamfer array', async ({
-    app,
-    editor,
-    toolbar,
-    scene,
-  }) => {
+  test(
+    'works on all edge selections and can break up multi edges in a chamfer array',
+    { tag: ['@skipWin'] },
+    async ({ app, editor, toolbar, scene }) => {
      test.skip(
        process.platform === 'win32',
        'Fails on windows in CI, can not be replicated locally on windows.'
@ -360,14 +356,13 @@ test.describe('verify sketch on chamfer works', () => {
          { shouldNormalise: true }
        )
      })
-  })
+    }
+  )

-  test('Works on chamfers that are non in a pipeExpression can break up multi edges in a chamfer array', async ({
-    app,
-    editor,
-    toolbar,
-    scene,
-  }) => {
+  test(
+    'Works on chamfers that are non in a pipeExpression can break up multi edges in a chamfer array',
+    { tag: ['@skipWin'] },
+    async ({ app, editor, toolbar, scene }) => {
      test.skip(
        process.platform === 'win32',
        'Fails on windows in CI, can not be replicated locally on windows.'
@ -449,5 +444,6 @@ const sketch002 = startSketchOn(extrude001, seg03)
 `,
        { shouldNormalise: true }
      )
-  })
+    }
+  )
 })
--- a/e2e/playwright/regression-tests.spec.ts
+++ b/e2e/playwright/regression-tests.spec.ts
@ -221,7 +221,10 @@ const sketch001 = startSketchAt([-0, -0])
    // Make sure it's not a link
    await expect(zooLogo).not.toHaveAttribute('href')
  })
-  test('Position _ Is Out Of Range... regression test', async ({ page }) => {
+  test(
+    'Position _ Is Out Of Range... regression test',
+    { tag: ['@skipWin'] },
+    async ({ page }) => {
      // SKip on windows, its being weird.
      test.skip(
        process.platform === 'win32',
@ -298,7 +301,8 @@ const sketch001 = startSketchAt([-0, -0])
    thing: "blah"`)

      await expect(page.locator('.cm-lint-marker-error')).toBeVisible()
-  })
+    }
+  )

  test('when engine fails export we handle the failure and alert the user', async ({
    page,
@ -401,9 +405,10 @@ const sketch001 = startSketchAt([-0, -0])
    const successToastMessage = page.getByText(`Exported successfully`)
    await expect(successToastMessage).toBeVisible()
  })
-  test('ensure you can not export while an export is already going', async ({
-    page,
-  }) => {
+  test(
+    'ensure you can not export while an export is already going',
+    { tag: ['@skipLinux', '@skipWin'] },
+    async ({ page }) => {
      // This is being weird on ubuntu and windows.
      test.skip(
        // eslint-disable-next-line jest/valid-title
@ -489,7 +494,8 @@ const sketch001 = startSketchAt([-0, -0])

        await expect(successToastMessage).toBeVisible()
      })
-  })
+    }
+  )

  test(
    `Network health indicator only appears in modeling view`,
--- a/e2e/playwright/snapshot-tests.spec.ts
+++ b/e2e/playwright/snapshot-tests.spec.ts
@ -49,7 +49,7 @@ test.setTimeout(60_000)

 test(
  'exports of each format should work',
-  { tag: '@snapshot' },
+  { tag: ['@snapshot', '@skipWin', '@skipMacos'] },
  async ({ page, context }) => {
    // skip on macos and windows.
    test.skip(
--- a/e2e/playwright/testing-selections.spec.ts
+++ b/e2e/playwright/testing-selections.spec.ts
@ -15,7 +15,10 @@ test.afterEach(async ({ page }, testInfo) => {

 test.describe('Testing selections', () => {
  test.setTimeout(90_000)
-  test('Selections work on fresh and edited sketch', async ({ page }) => {
+  test(
+    'Selections work on fresh and edited sketch',
+    { tag: ['@skipWin'] },
+    async ({ page }) => {
      // Skip on windows its being weird.
      test.skip(process.platform === 'win32', 'Skip on windows')

@ -181,7 +184,9 @@ test.describe('Testing selections', () => {

        await expect(page.locator('.cm-cursor')).toHaveCount(2)
        await page.waitForTimeout(500)
-      await page.keyboard.up(process.platform === 'linux' ? 'Control' : 'Meta')
+        await page.keyboard.up(
+          process.platform === 'linux' ? 'Control' : 'Meta'
+        )

        // clear selection by clicking on nothing
        await emptySpaceClick()
@ -242,7 +247,8 @@ test.describe('Testing selections', () => {
      await test.step(`Test hovering and selecting on edited sketch`, async () => {
        await selectionSequence()
      })
-  })
+    }
+  )

  test('Solids should be select and deletable', async ({ page }) => {
    test.setTimeout(90_000)
--- a/e2e/playwright/testing-settings.spec.ts
+++ b/e2e/playwright/testing-settings.spec.ts
@ -258,7 +258,7 @@ test.describe('Testing settings', () => {

  test(
    `Project settings override user settings on desktop`,
-    { tag: '@electron' },
+    { tag: ['@electron', '@skipWin'] },
    async ({ browser: _ }, testInfo) => {
      test.skip(
        process.platform === 'win32',
--- a/e2e/playwright/text-to-cad-tests.spec.ts
+++ b/e2e/playwright/text-to-cad-tests.spec.ts
@ -447,9 +447,10 @@ test.describe('Text-to-CAD tests', () => {
    await expect(page.getByText(promptWithNewline)).toBeVisible()
  })

-  test('can do many at once and get many prompts back, and interact with many', async ({
-    page,
-  }) => {
+  test(
+    'can do many at once and get many prompts back, and interact with many',
+    { tag: ['@skipWin'] },
+    async ({ page }) => {
      // Let this test run longer since we've seen it timeout.
      test.setTimeout(180_000)
      // skip on windows
@ -565,7 +566,8 @@ test.describe('Text-to-CAD tests', () => {
      await expect(closeButton).toBeVisible()
      await closeButton.click()
      await expect(successToastMessage).not.toBeVisible()
-  })
+    }
+  )

  test('can do many at once with errors, clicking dismiss error does not dismiss all', async ({
    page,
--- a/package.json
+++ b/package.json
@ -75,11 +75,10 @@
    "build:both": "vite build",
    "build:both:local": "yarn build:wasm && vite build",
    "pretest": "yarn remove-importmeta",
-    "test": "vitest --mode development",
-    "test:nowatch": "vitest run --mode development",
    "test:rust": "(cd src/wasm-lib && cargo test --all && cargo clippy --all --tests --benches)",
    "simpleserver": "yarn pretest && http-server ./public --cors -p 3000",
    "simpleserver:ci": "yarn pretest && http-server ./public --cors -p 3000 &",
+    "simpleserver:bg": "yarn pretest && http-server ./public --cors -p 3000 &",
    "fmt": "prettier --write ./src *.ts *.json *.js ./e2e ./packages",
    "fmt-check": "prettier --check ./src *.ts *.json *.js ./e2e ./packages",
    "fetch:wasm": "./get-latest-wasm-bundle.sh",
@ -89,7 +88,8 @@
    "build:wasm": "yarn wasm-prep && cd src/wasm-lib && wasm-pack build --release --target web --out-dir pkg && cargo test -p kcl-lib export_bindings && cd ../.. && yarn isomorphic-copy-wasm && yarn fmt",
    "remove-importmeta": "sed -i 's/import.meta.url/window.location.origin/g' \"./src/wasm-lib/pkg/wasm_lib.js\"; sed -i '' 's/import.meta.url/window.location.origin/g' \"./src/wasm-lib/pkg/wasm_lib.js\" || echo \"sed for both mac and linux\"",
    "wasm-prep": "rimraf src/wasm-lib/pkg && mkdirp src/wasm-lib/pkg && rimraf src/wasm-lib/kcl/bindings",
-    "lint": "eslint --fix src e2e packages/codemirror-lsp-client",
+    "lint-fix": "eslint --fix src e2e packages/codemirror-lsp-client",
+    "lint": "eslint --max-warnings 0 src e2e packages/codemirror-lsp-client",
    "bump-jsons": "echo \"$(jq --arg v \"$VERSION\" '.version=$v' package.json --indent 2)\" > package.json",
    "postinstall": "yarn fetch:samples && yarn xstate:typegen && ./node_modules/.bin/electron-rebuild",
    "xstate:typegen": "yarn xstate typegen \"src/**/*.ts?(x)\"",
@ -101,7 +101,23 @@
    "tron:publish": "electron-forge publish",
    "tron:test": "NODE_ENV=development yarn playwright test --config=playwright.electron.config.ts --grep=@electron",
    "tronb:vite": "vite build -c vite.main.config.ts && vite build -c vite.preload.config.ts && vite build -c vite.renderer.config.ts",
-    "tronb:package": "electron-builder --config electron-builder.yml"
+    "tronb:package": "electron-builder --config electron-builder.yml",
+    "test-setup": "yarn install && yarn build:wasm",
+    "test": "vitest --mode development",
+    "test:nowatch": "vitest run --mode development",
+    "test:unit": "vitest run --mode development",
+    "test:playwright:browser:chrome": "playwright test '--project=Google Chrome' --config=playwright.ci.config.ts '--grep-invert=@snapshot|@electron'",
+    "test:playwright:browser:chrome:windows": "playwright test '--project=Google Chrome' --config=playwright.ci.config.ts '--grep-invert=@snapshot|@electron|@skipWin'",
+    "test:playwright:browser:chrome:ubuntu": "playwright test '--project=Google Chrome' --config=playwright.ci.config.ts '--grep-invert=@snapshot|@electron|@skipLinux'",
+    "test:playwright:electron": "playwright test --config=playwright.electron.config.ts --grep=@electron",
+    "test:playwright:electron:windows": "playwright test --config=playwright.electron.config.ts --grep=@electron --grep-invert=@skipWin",
+    "test:playwright:electron:macos": "playwright test --config=playwright.electron.config.ts --grep=@electron --grep-invert=@skipMacos",
+    "test:playwright:electron:ubuntu": "playwright test --config=playwright.electron.config.ts --grep=@electron --grep-invert=@skipLinux",
+    "test:playwright:electron:local": "yarn tron:package && NODE_ENV=development playwright test --config=playwright.electron.config.ts --grep=@electron",
+    "test:playwright:electron:windows:local": "yarn tron:package && NODE_ENV=development playwright test --config=playwright.electron.config.ts --grep=@electron --grep-invert=@skipWin",
+    "test:playwright:electron:macos:local": "yarn tron:package && NODE_ENV=development playwright test --config=playwright.electron.config.ts --grep=@electron --grep-invert=@skipMacos",
+    "test:playwright:electron:ubuntu:local": "yarn tron:package && NODE_ENV=development playwright test --config=playwright.electron.config.ts --grep=@electron --grep-invert=@skipLinux",
+    "test:unit:local": "yarn simpleserver:bg && yarn test:unit; kill-port 3000"
  },
  "prettier": {
    "trailingComma": "es5",
@ -174,6 +190,7 @@
    "happy-dom": "^14.3.10",
    "http-server": "^14.1.1",
    "husky": "^9.1.5",
+    "kill-port": "^2.0.1",
    "node-fetch": "^3.3.2",
    "pixelmatch": "^5.3.0",
    "pngjs": "^7.0.0",
--- a/src/lang/std/artifactGraph.test.ts
+++ b/src/lang/std/artifactGraph.test.ts
@ -14,7 +14,7 @@ import {
 } from './artifactGraph'
 import { err } from 'lib/trap'
 import { engineCommandManager, kclManager } from 'lib/singletons'
-import { CI, VITE_KC_DEV_TOKEN } from 'env'
+import { VITE_KC_DEV_TOKEN } from 'env'
 import fsp from 'fs/promises'
 import fs from 'fs'
 import { chromium } from 'playwright'
@ -97,21 +97,6 @@ type CacheShape = {
 beforeAll(async () => {
  await initPromise

-  let parsed
-  try {
-    const file = await fsp.readFile(fullPath, 'utf-8')
-    parsed = JSON.parse(file)
-  } catch (e) {
-    parsed = false
-  }
-
-  if (!CI && parsed) {
-    // caching the results of the websocket commands makes testing this locally much faster
-    // real calls to the engine are needed to test the artifact map
-    // bust the cache with: `rm -rf src/lang/std/artifactGraphCache`
-    return
-  }
-
  // THESE TEST WILL FAIL without VITE_KC_DEV_TOKEN set in .env.development.local
  await new Promise((resolve) => {
    engineCommandManager.start({
--- a/yarn.lock
+++ b/yarn.lock
@ -5695,6 +5695,11 @@ get-symbol-description@^1.0.2:
    es-errors "^1.3.0"
    get-intrinsic "^1.2.4"

+get-them-args@1.3.2:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/get-them-args/-/get-them-args-1.3.2.tgz#74a20ba8a4abece5ae199ad03f2bcc68fdfc9ba5"
+  integrity sha512-LRn8Jlk+DwZE4GTlDbT3Hikd1wSHgLMme/+7ddlqKd7ldwR6LjJgTVWzBnR01wnYGe4KgrXjg287RaI22UHmAw==
+
 glob-parent@^5.1.2, glob-parent@~5.1.2:
  version "5.1.2"
  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
@ -6645,6 +6650,14 @@ keyv@^4.0.0, keyv@^4.5.3:
  dependencies:
    json-buffer "3.0.1"

+kill-port@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/kill-port/-/kill-port-2.0.1.tgz#e5e18e2706b13d54320938be42cb7d40609b15cf"
+  integrity sha512-e0SVOV5jFo0mx8r7bS29maVWp17qGqLBZ5ricNSajON6//kmb7qqqNnml4twNE8Dtj97UQD+gNFOaipS/q1zzQ==
+  dependencies:
+    get-them-args "1.3.2"
+    shell-exec "1.0.2"
+
 klaw@^4.1.0:
  version "4.1.0"
  resolved "https://registry.yarnpkg.com/klaw/-/klaw-4.1.0.tgz#5df608067d8cb62bbfb24374f8e5d956323338f3"
@ -8579,6 +8592,11 @@ shebang-regex@^3.0.0:
  resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172"
  integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==

+shell-exec@1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/shell-exec/-/shell-exec-1.0.2.tgz#2e9361b0fde1d73f476c4b6671fa17785f696756"
+  integrity sha512-jyVd+kU2X+mWKMmGhx4fpWbPsjvD53k9ivqetutVW/BQ+WIZoDoP4d8vUMGezV6saZsiNoW2f9GIhg9Dondohg==
+
 side-channel@^1.0.4, side-channel@^1.0.6:
  version "1.0.6"
  resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.6.tgz#abd25fb7cd24baf45466406b1096b7831c9215f2"