From e1406012b4000d6fbdcc5f5676b87011b5401b1b Mon Sep 17 00:00:00 2001 From: Kevin Nadro Date: Thu, 3 Oct 2024 10:40:12 -0400 Subject: [PATCH] Nadro/4012/ci cd update (#4062) * chore: saving off package.json progress unit tests fail in main * fix: implementing a one liner for unit tests * fix: renaming test:unit:local * chore: adding playwright tests * fix: making package.json not destructive to keep same pipeline commands for now * fix: reordering * fix: added tags for OS tests, moved kill-port to dev depen * fix: OS skipping at tag level * fix: lint, fmt, tsc, etc... * Look at this (photo)Graph *in the voice of Nickelback* * fix: new formatting * fix: removing the ci copy, do not like it * Look at this (photo)Graph *in the voice of Nickelback* * chore: updating readme with explanation on the commands for CI CD simulation locally * fix: package.json command for unit test, removing cached breaking cache in unit tests * fix: fixing copy and typos in README.md for CI CD section * fix: adding a duplicate command for a better name. CI CD will use it in a future PR * chore: trying to clean up the copy and commands for CI CD tests * chore: porting the bash code in the YAML to a bash script then using matrix permutations to control the runtime * fix: typos * fix: another typo, missed these went porting to the bash script logic * fix: I think I need the checkout action since it has the repo code? * fix: wrote absolute path not the relative hidden path, ope * fix: does this cache give me the yarn install of playwright? * fix: yarn cannot find the binary, use the yarn command * fix: remove all uses...? * chore: adding bash script for electron runtimes * fix: copy cleanup * fix: typo when copy and pasting the exclude logic, ope * fix: this is wrong * fix: build:wasm is a requirement for yarn tsc * fix: reorder? * fix: renaming integrations to e2e * fix: windows is complaining about a pipe issue in the bash script? * fix: escaping double quotes in windows? * chore: consolidating commands into 1 file and easier YAML configuation for electron * chore: mapped multiple OS playwright browser into a single bash script * fix: removing old bash scripts, renaming matrix jobs * fix: missed deleting this when I added the if statements. * chore: removing unused job, xstate typegen more more since v5 * fix: trying to get these two tests to pass on first try * fix: auto fixes * fix: removing old unit test command --------- Co-authored-by: github-actions[bot] --- .../playwright-browser-chrome.sh | 59 ++++++++++ .github/ci-cd-scripts/playwright-electron.sh | 63 +++++++++++ .../{playwright.yml => e2e-tests.yml} | 102 ++---------------- ...build-test-web.yml => static-analysis.yml} | 48 +++++++-- README.md | 3 +- e2e/playwright/sketch-tests.spec.ts | 14 ++- e2e/playwright/various.spec.ts | 5 +- package.json | 7 +- 8 files changed, 186 insertions(+), 115 deletions(-) create mode 100755 .github/ci-cd-scripts/playwright-browser-chrome.sh create mode 100755 .github/ci-cd-scripts/playwright-electron.sh rename .github/workflows/{playwright.yml => e2e-tests.yml} (73%) rename .github/workflows/{build-test-web.yml => static-analysis.yml} (79%) diff --git a/.github/ci-cd-scripts/playwright-browser-chrome.sh b/.github/ci-cd-scripts/playwright-browser-chrome.sh new file mode 100755 index 000000000..676529d20 --- /dev/null +++ b/.github/ci-cd-scripts/playwright-browser-chrome.sh @@ -0,0 +1,59 @@ +# bash strict mode +set -euo pipefail + +if [[ ! -f "test-results/.last-run.json" ]]; then + # if no last run artifact, than run plawright normally + echo "run playwright normally" + if [[ "$3" == "ubuntu-latest" ]]; then + yarn test:playwright:browser:chrome:ubuntu -- --shard=$1/$2 || true + elif [[ "$3" == "windows-latest" ]]; then + yarn test:playwright:browser:chrome:windows -- --shard=$1/$2 || true + else + echo "Do not run playwright. Unable to detect os runtime." + exit 1 + fi + # # send to axiom + node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1 +fi + +retry=1 +max_retrys=4 + +# retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues +while [[ $retry -le $max_retrys ]]; do + if [[ -f "test-results/.last-run.json" ]]; then + failed_tests=$(jq '.failedTests | length' test-results/.last-run.json) + if [[ $failed_tests -gt 0 ]]; then + echo "retried=true" >>$GITHUB_OUTPUT + echo "run playwright with last failed tests and retry $retry" + if [[ "$3" == "ubuntu-latest" ]]; then + yarn test:playwright:browser:chrome:ubuntu -- --last-failed || true + elif [[ "$3" == "windows-latest" ]]; then + yarn test:playwright:browser:chrome:windows -- --last-failed || true + else + echo "Do not run playwright. Unable to detect os runtime." + exit 1 + fi + # send to axiom + node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1 + retry=$((retry + 1)) + else + echo "retried=false" >>$GITHUB_OUTPUT + exit 0 + fi + else + echo "retried=false" >>$GITHUB_OUTPUT + exit 0 + fi +done + +echo "retried=false" >>$GITHUB_OUTPUT + +if [[ -f "test-results/.last-run.json" ]]; then + failed_tests=$(jq '.failedTests | length' test-results/.last-run.json) + if [[ $failed_tests -gt 0 ]]; then + # if it still fails after 3 retrys, then fail the job + exit 1 + fi +fi +exit 0 diff --git a/.github/ci-cd-scripts/playwright-electron.sh b/.github/ci-cd-scripts/playwright-electron.sh new file mode 100755 index 000000000..c7d8777c8 --- /dev/null +++ b/.github/ci-cd-scripts/playwright-electron.sh @@ -0,0 +1,63 @@ +# bash strict mode +set -euo pipefail + +if [[ ! -f "test-results/.last-run.json" ]]; then + # if no last run artifact, than run plawright normally + echo "run playwright normally" + if [[ "$1" == "ubuntu-latest" ]]; then + xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn test:playwright:electron:ubuntu || true + elif [[ "$1" == "windows-latest" ]]; then + yarn test:playwright:electron:windows || true + elif [[ "$1" == "macos-14" ]]; then + yarn test:playwright:electron:macos || true + else + echo "Do not run playwright. Unable to detect os runtime." + exit 1 + fi + # # send to axiom + node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1 +fi + +retry=1 +max_retrys=2 + +# retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues +while [[ $retry -le $max_retrys ]]; do + if [[ -f "test-results/.last-run.json" ]]; then + failed_tests=$(jq '.failedTests | length' test-results/.last-run.json) + if [[ $failed_tests -gt 0 ]]; then + echo "retried=true" >>$GITHUB_OUTPUT + echo "run playwright with last failed tests and retry $retry" + if [[ "$1" == "ubuntu-latest" ]]; then + xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn test:playwright:electron:ubuntu -- --last-failed || true + elif [[ "$1" == "windows-latest" ]]; then + yarn test:playwright:electron:windows -- --last-failed || true + elif [[ "$1" == "macos-14" ]]; then + yarn test:playwright:electron:macos -- --last-failed || true + else + echo "Do not run playwright. Unable to detect os runtime." + exit 1 + fi + # send to axiom + node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1 + retry=$((retry + 1)) + else + echo "retried=false" >>$GITHUB_OUTPUT + exit 0 + fi + else + echo "retried=false" >>$GITHUB_OUTPUT + exit 0 + fi +done + +echo "retried=false" >>$GITHUB_OUTPUT + +if [[ -f "test-results/.last-run.json" ]]; then + failed_tests=$(jq '.failedTests | length' test-results/.last-run.json) + if [[ $failed_tests -gt 0 ]]; then + # if it still fails after 3 retrys, then fail the job + exit 1 + fi +fi +exit 0 diff --git a/.github/workflows/playwright.yml b/.github/workflows/e2e-tests.yml similarity index 73% rename from .github/workflows/playwright.yml rename to .github/workflows/e2e-tests.yml index 9036c415f..be5c33434 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/e2e-tests.yml @@ -1,4 +1,4 @@ -name: Playwright Tests +name: E2E Tests on: push: branches: [ main ] @@ -33,8 +33,9 @@ jobs: rust: - 'src/wasm-lib/**' - playwright-chrome: + browser: timeout-minutes: ${{ matrix.os == 'macos-14' && 60 || 50 }} + name: playwright:browser:${{ matrix.os }} ${{ matrix.shardIndex }} ${{ matrix.shardTotal }} strategy: fail-fast: false matrix: @@ -188,48 +189,7 @@ jobs: if: ${{ !cancelled() && (success() || failure()) }} shell: bash run: | - if [[ ! -f "test-results/.last-run.json" ]]; then - # if no last run artifact, than run plawright normally - echo "run playwright normally" - yarn playwright test --project="Google Chrome" --config=playwright.ci.config.ts --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --grep-invert="@snapshot|@electron" || true - # # send to axiom - node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1 - fi - - retry=1 - max_retrys=4 - - # retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues - while [[ $retry -le $max_retrys ]]; do - if [[ -f "test-results/.last-run.json" ]]; then - failed_tests=$(jq '.failedTests | length' test-results/.last-run.json) - if [[ $failed_tests -gt 0 ]]; then - echo "retried=true" >>$GITHUB_OUTPUT - echo "run playwright with last failed tests and retry $retry" - yarn playwright test --project="Google Chrome" --config=playwright.ci.config.ts --last-failed --grep-invert="@snapshot|@electron" || true - # send to axiom - node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1 - retry=$((retry + 1)) - else - echo "retried=false" >>$GITHUB_OUTPUT - exit 0 - fi - else - echo "retried=false" >>$GITHUB_OUTPUT - exit 0 - fi - done - - echo "retried=false" >>$GITHUB_OUTPUT - - if [[ -f "test-results/.last-run.json" ]]; then - failed_tests=$(jq '.failedTests | length' test-results/.last-run.json) - if [[ $failed_tests -gt 0 ]]; then - # if it still fails after 3 retrys, then fail the job - exit 1 - fi - fi - exit 0 + .github/ci-cd-scripts/playwright-browser-chrome.sh ${{matrix.shardIndex}} ${{matrix.shardTotal}} ${{matrix.os}} env: CI: true FAIL_ON_CONSOLE_ERRORS: true @@ -258,7 +218,8 @@ jobs: overwrite: true - playwright-electron: + electron: + name: playwright:electron:${{matrix.os}} strategy: fail-fast: false matrix: @@ -359,56 +320,7 @@ jobs: if: ${{ !cancelled() && (success() || failure()) }} shell: bash run: | - if [[ ! -f "test-results/.last-run.json" ]]; then - # if no last run artifact, than run plawright normally - echo "run playwright normally" - if [[ "$IS_UBUNTU" == "true" ]]; then - xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn playwright test --config=playwright.electron.config.ts --grep=@electron || true - else - yarn playwright test --config=playwright.electron.config.ts --grep=@electron || true - fi - # # send to axiom - node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1 - fi - - retry=1 - max_retrys=2 - - # retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues - while [[ $retry -le $max_retrys ]]; do - if [[ -f "test-results/.last-run.json" ]]; then - failed_tests=$(jq '.failedTests | length' test-results/.last-run.json) - if [[ $failed_tests -gt 0 ]]; then - echo "retried=true" >>$GITHUB_OUTPUT - echo "run playwright with last failed tests and retry $retry" - if [[ "$IS_UBUNTU" == "true" ]]; then - xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn playwright test --config=playwright.electron.config.ts --last-failed --grep=@electron || true - else - yarn playwright test --config=playwright.electron.config.ts --grep=@electron || true - fi - # send to axiom - node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1 - retry=$((retry + 1)) - else - echo "retried=false" >>$GITHUB_OUTPUT - exit 0 - fi - else - echo "retried=false" >>$GITHUB_OUTPUT - exit 0 - fi - done - - echo "retried=false" >>$GITHUB_OUTPUT - - if [[ -f "test-results/.last-run.json" ]]; then - failed_tests=$(jq '.failedTests | length' test-results/.last-run.json) - if [[ $failed_tests -gt 0 ]]; then - # if it still fails after 3 retrys, then fail the job - exit 1 - fi - fi - exit 0 + .github/ci-cd-scripts/playwright-electron.sh ${{ matrix.os }} env: CI: true FAIL_ON_CONSOLE_ERRORS: true diff --git a/.github/workflows/build-test-web.yml b/.github/workflows/static-analysis.yml similarity index 79% rename from .github/workflows/build-test-web.yml rename to .github/workflows/static-analysis.yml index 3c752a4e2..b5a8a4d3b 100644 --- a/.github/workflows/build-test-web.yml +++ b/.github/workflows/static-analysis.yml @@ -1,4 +1,4 @@ -name: build-test-web +name: Static Analysis on: pull_request: @@ -16,7 +16,7 @@ permissions: actions: read jobs: - check-format: + yarn-fmt-check: runs-on: 'ubuntu-22.04' steps: - uses: actions/checkout@v4 @@ -27,7 +27,23 @@ jobs: - run: yarn install - run: yarn fmt-check - check-types: + yarn-build-wasm: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'yarn' + - run: yarn install + - uses: Swatinem/rust-cache@v2 + with: + workspaces: './src/wasm-lib' + + - run: yarn build:wasm + + yarn-tsc: runs-on: ubuntu-22.04 steps: @@ -42,13 +58,25 @@ jobs: workspaces: './src/wasm-lib' - run: yarn build:wasm - - run: yarn xstate:typegen - run: yarn tsc - - name: Lint - run: yarn eslint --max-warnings 0 src e2e packages/codemirror-lsp-client + yarn-lint: + runs-on: ubuntu-22.04 - check-typos: + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'yarn' + - run: yarn install + - uses: Swatinem/rust-cache@v2 + with: + workspaces: './src/wasm-lib' + + - run: yarn lint + + python-codespell: runs-on: ubuntu-22.04 steps: - name: Checkout @@ -62,7 +90,7 @@ jobs: run: codespell --config .codespellrc # Edit this file to tweak the typo list and other configuration. - build-test-web: + yarn-unit-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -80,7 +108,7 @@ jobs: - run: yarn build:wasm - - run: yarn simpleserver:ci + - run: yarn simpleserver:bg if: ${{ github.event_name != 'release' && github.event_name != 'schedule' }} - name: Install Chromium Browser @@ -89,7 +117,7 @@ jobs: - name: run unit tests if: ${{ github.event_name != 'release' && github.event_name != 'schedule' }} - run: yarn test:nowatch + run: yarn test:unit env: VITE_KC_DEV_TOKEN: ${{ secrets.KITTYCAD_API_TOKEN_DEV }} diff --git a/README.md b/README.md index 1d6eda76c..c576b74c9 100644 --- a/README.md +++ b/README.md @@ -304,7 +304,7 @@ yarn start and finally: ``` -yarn test:nowatch +yarn test:unit ``` For individual testing: @@ -348,7 +348,6 @@ yarn test-setup yarn tsc yarn fmt-check yarn lint -yarn xstate:typegen yarn test:unit:local ``` diff --git a/e2e/playwright/sketch-tests.spec.ts b/e2e/playwright/sketch-tests.spec.ts index a9adcfffd..3064423bc 100644 --- a/e2e/playwright/sketch-tests.spec.ts +++ b/e2e/playwright/sketch-tests.spec.ts @@ -840,7 +840,7 @@ test.describe('Sketch tests', () => { |> line([1.02, -1.32], %, $seg01) |> line([-1.01, -0.77], %) |> lineTo([profileStartX(%), profileStartY(%)], %) - |> close(%) +|> close(%) ` ) }) @@ -850,9 +850,17 @@ test.describe('Sketch tests', () => { await u.waitForAuthSkipAppStart() + // wait for execution done + await u.openDebugPanel() + await u.expectCmdLog('[data-message-type="execution-done"]') + await u.closeDebugPanel() + // click "line([1.32, 0.38], %)" await page.getByText(`line([1.32, 0.38], %)`).click() await page.waitForTimeout(100) + await expect(page.getByRole('button', { name: 'Edit Sketch' })).toBeEnabled( + { timeout: 10_000 } + ) // click edit sketch await page.getByRole('button', { name: 'Edit Sketch' }).click() await page.waitForTimeout(600) // wait for animation @@ -872,7 +880,9 @@ test.describe('Sketch tests', () => { // otherwise the cmdbar would be waiting for a selection. await expect( page.getByRole('button', { name: 'selection : 1 face', exact: false }) - ).toBeVisible() + ).toBeVisible({ + timeout: 10_000, + }) }) test("Existing sketch with bad code delete user's code", async ({ page }) => { // this was a regression https://github.com/KittyCAD/modeling-app/issues/2832 diff --git a/e2e/playwright/various.spec.ts b/e2e/playwright/various.spec.ts index 3e73ae7ac..f606055f5 100644 --- a/e2e/playwright/various.spec.ts +++ b/e2e/playwright/various.spec.ts @@ -407,8 +407,9 @@ test('Basic default modeling and sketch hotkeys work', async ({ page }) => { await page.waitForTimeout(500) await page.mouse.move(800, 200, { steps: 5 }) await page.mouse.click(800, 200) - await page.waitForTimeout(500) - await expect(page.getByRole('button', { name: 'Continue' })).toBeVisible() + await expect(page.getByRole('button', { name: 'Continue' })).toBeVisible({ + timeout: 20_000, + }) await page.getByRole('button', { name: 'Continue' }).click() await expect( page.getByRole('button', { name: 'Submit command' }) diff --git a/package.json b/package.json index 6efa2b75f..f519c9424 100644 --- a/package.json +++ b/package.json @@ -104,11 +104,10 @@ "tronb:package": "electron-builder --config electron-builder.yml", "test-setup": "yarn install && yarn build:wasm", "test": "vitest --mode development", - "test:nowatch": "vitest run --mode development", "test:unit": "vitest run --mode development", - "test:playwright:browser:chrome": "playwright test '--project=Google Chrome' --config=playwright.ci.config.ts '--grep-invert=@snapshot|@electron'", - "test:playwright:browser:chrome:windows": "playwright test '--project=Google Chrome' --config=playwright.ci.config.ts '--grep-invert=@snapshot|@electron|@skipWin'", - "test:playwright:browser:chrome:ubuntu": "playwright test '--project=Google Chrome' --config=playwright.ci.config.ts '--grep-invert=@snapshot|@electron|@skipLinux'", + "test:playwright:browser:chrome": "playwright test --project='Google Chrome' --config=playwright.ci.config.ts --grep-invert='@snapshot|@electron'", + "test:playwright:browser:chrome:windows": "playwright test --project=\"Google Chrome\" --config=playwright.ci.config.ts --grep-invert=\"@snapshot|@electron|@skipWin\"", + "test:playwright:browser:chrome:ubuntu": "playwright test --project='Google Chrome' --config=playwright.ci.config.ts --grep-invert='@snapshot|@electron|@skipLinux'", "test:playwright:electron": "playwright test --config=playwright.electron.config.ts --grep=@electron", "test:playwright:electron:windows": "playwright test --config=playwright.electron.config.ts --grep=@electron --grep-invert=@skipWin", "test:playwright:electron:macos": "playwright test --config=playwright.electron.config.ts --grep=@electron --grep-invert=@skipMacos",