Dedicated job for snapshot tests (#5752)

* WIP: break out e2e snapshots * Separate prepare-wasm job * Force e2e to run on pierremtb/adhoc/break-out-snapshots * Quick fix * Quick fix2 * Quick fix3 * Quick fix4 * Quick fix5 * Remove ifs for flow tests * Clean up for review * should-run == 'true' madness
2025-03-11 17:18:42 -04:00
parent 724e65ac97
commit b8a0ad7144
1 changed files with 302 additions and 213 deletions
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@ -26,9 +26,11 @@ jobs:

    steps:
      - uses: actions/checkout@v4
+
      - name: Fetch the base branch
        if: ${{ github.event_name == 'pull_request' }}
        run: git fetch origin ${{ github.base_ref }} --depth=1
+
      - name: Check for path changes
        id: path-changes
        shell: bash
@ -48,6 +50,7 @@ jobs:
          else
            echo "significant=false" >> $GITHUB_OUTPUT
          fi
+
      - name: Should run
        id: should-run
        shell: bash
@ -60,6 +63,7 @@ jobs:
          else
            echo "should-run=false" >> $GITHUB_OUTPUT
          fi
+
      - name: Display conditions
        shell: bash
        run: |
@ -68,57 +72,34 @@ jobs:
          echo "significant: ${{ steps.path-changes.outputs.significant }}"
          echo "should-run: ${{ steps.should-run.outputs.should-run }}"

-  electron:
-    timeout-minutes: 60
-    name: playwright:electron:${{ matrix.os }} ${{ matrix.shardIndex }} ${{ matrix.shardTotal }}
-    strategy:
-      fail-fast: false
-      matrix:
-        # TODO: enable self-hosted-windows-8-cores once available
-        os: [namespace-profile-ubuntu-8-cores, namespace-profile-macos-8-cores, windows-16-cores]
-        shardIndex: [1, 2, 3, 4]
-        shardTotal: [4]
-        # TODO: add ref here for main and latest release tag
-    runs-on: ${{ matrix.os }}
+
+  prepare-wasm:
+    # seperate job on Ubuntu to build or fetch the wasm blob once on the fastest runner
+    runs-on: namespace-profile-ubuntu-8-cores
    needs: conditions
    steps:
-    - uses: actions/create-github-app-token@v1
-      id: app-token
-      with:
-        app-id: ${{ secrets.MODELING_APP_GH_APP_ID }}
-        private-key: ${{ secrets.MODELING_APP_GH_APP_PRIVATE_KEY }}
-        owner: ${{ github.repository_owner }}
      - uses: actions/checkout@v4
-      with:
-        token: ${{ steps.app-token.outputs.token }}
+        if: needs.conditions.outputs.should-run == 'true'
+
      - id: filter
+        if: needs.conditions.outputs.should-run == 'true'
        name: Check for Rust changes
        uses: dorny/paths-filter@v3
        with:
          filters: |
            rust:
              - 'rust/**'
+
      - uses: actions/setup-node@v4
        if: needs.conditions.outputs.should-run == 'true'
        with:
          node-version-file: '.nvmrc'
          cache: 'yarn'
+
      - name: Install dependencies
-      id: deps-install
        if: needs.conditions.outputs.should-run == 'true'
-      shell: bash
        run: yarn
-    - name: Cache Playwright Browsers
-      if: needs.conditions.outputs.should-run == 'true'
-      uses: actions/cache@v4
-      with:
-        path: |
-          ~/.cache/ms-playwright/
-        key: ${{ runner.os }}-playwright-${{ hashFiles('yarn.lock') }}
-    - name: Install Playwright Browsers
-      if: needs.conditions.outputs.should-run == 'true'
-      shell: bash
-      run: yarn playwright install --with-deps
+
      - name: Download Wasm Cache
        id: download-wasm
        if: ${{ needs.conditions.outputs.should-run == 'true' && github.event_name != 'schedule' && steps.filter.outputs.rust == 'false' }}
@ -130,15 +111,10 @@ jobs:
          workflow: build-and-store-wasm.yml
          branch: main
          path: rust/kcl-wasm-lib/pkg
-    - name: copy wasm blob
-      if: ${{ needs.conditions.outputs.should-run == 'true' && github.event_name != 'schedule' && steps.filter.outputs.rust == 'false' }}
-      shell: bash
-      run: cp rust/kcl-wasm-lib/pkg/kcl_wasm_lib_bg.wasm public
-      continue-on-error: true
+
      - name: Build WASM condition
        id: wasm
        if: needs.conditions.outputs.should-run == 'true'
-      shell: bash
        run: |
          set -euox pipefail
          # Build wasm if this is a scheduled run, there are Rust changes, or
@ -148,61 +124,102 @@ jobs:
          else
            echo "should-build-wasm=false" >> $GITHUB_OUTPUT
          fi
+
      - name: Use correct Rust toolchain
        if: ${{ needs.conditions.outputs.should-run == 'true' && steps.wasm.outputs.should-build-wasm == 'true' }}
        shell: bash
        run: |
          [ -e rust-toolchain.toml ] || cp rust/rust-toolchain.toml ./
+
      - name: Install rust
        if: ${{ needs.conditions.outputs.should-run == 'true' && steps.wasm.outputs.should-build-wasm == 'true' }}
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache: false # Configured below.
+
      - uses: taiki-e/install-action@955a6ff1416eae278c9f833008a9beb4b7f9afe3
        if: ${{ needs.conditions.outputs.should-run == 'true' && steps.wasm.outputs.should-build-wasm == 'true' }}
        with:
          tool: wasm-pack
+
      - name: Rust Cache
        if: ${{ needs.conditions.outputs.should-run == 'true' && steps.wasm.outputs.should-build-wasm == 'true' }}
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: './rust'
-    - name: install good sed
-      if: ${{ needs.conditions.outputs.should-run == 'true' && startsWith(matrix.os, 'macos') }}
-      shell: bash
-      run: |
-        brew install gnu-sed
-        echo "/opt/homebrew/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
-    - name: Install vector
-      shell: bash
-      # TODO: figure out what to do with this, it's failing
-      if: false
-      run: |
-        curl --proto '=https' --tlsv1.2 -sSfL https://sh.vector.dev > /tmp/vector.sh
-        chmod +x /tmp/vector.sh
-        /tmp/vector.sh -y -no-modify-path
-        mkdir -p /tmp/vector
-        cp .github/workflows/vector.toml /tmp/vector.toml
-        sed -i "s#GITHUB_WORKFLOW#${GITHUB_WORKFLOW}#g" /tmp/vector.toml
-        sed -i "s#GITHUB_REPOSITORY#${GITHUB_REPOSITORY}#g" /tmp/vector.toml
-        sed -i "s#GITHUB_SHA#${GITHUB_SHA}#g" /tmp/vector.toml
-        sed -i "s#GITHUB_REF_NAME#${GITHUB_REF_NAME}#g" /tmp/vector.toml
-        sed -i "s#GH_ACTIONS_AXIOM_TOKEN#${{secrets.GH_ACTIONS_AXIOM_TOKEN}}#g" /tmp/vector.toml
-        cat /tmp/vector.toml
-        ${HOME}/.vector/bin/vector --config /tmp/vector.toml &
+
      - name: Build Wasm
        if: ${{ needs.conditions.outputs.should-run == 'true' && steps.wasm.outputs.should-build-wasm == 'true' }}
        shell: bash
        run: yarn build:wasm
+
+      - uses: actions/upload-artifact@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        with:
+          name: prepared-wasm
+          path: |
+            rust/kcl-wasm-lib/pkg/kcl_wasm_lib*
+  
+
+  snapshots:
+    name: playwright:snapshots:ubuntu
+    runs-on: namespace-profile-ubuntu-8-cores
+    needs: [conditions, prepare-wasm]
+    steps:
+      - uses: actions/create-github-app-token@v1
+        if: needs.conditions.outputs.should-run == 'true'
+        id: app-token
+        with:
+          app-id: ${{ secrets.MODELING_APP_GH_APP_ID }}
+          private-key: ${{ secrets.MODELING_APP_GH_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - uses: actions/checkout@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+
+      - uses: actions/download-artifact@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        name: prepared-wasm
+
+      - name: Copy prepared wasm
+        if: needs.conditions.outputs.should-run == 'true'
+        run: |
+          ls -R prepared-wasm
+          cp prepared-wasm/kcl_wasm_lib_bg.wasm public
+          mkdir rust/kcl-wasm-lib/pkg
+          cp prepared-wasm/kcl_wasm_lib* rust/kcl-wasm-lib/pkg
+
+      - uses: actions/setup-node@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'yarn'
+
+      - name: Install dependencies
+        id: deps-install
+        if: needs.conditions.outputs.should-run == 'true'
+        run: yarn
+
+      - name: Cache Playwright Browsers
+        if: needs.conditions.outputs.should-run == 'true'
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/ms-playwright/
+          key: ${{ runner.os }}-playwright-${{ hashFiles('yarn.lock') }}
+
+      - name: Install Playwright Browsers
+        if: needs.conditions.outputs.should-run == 'true'
+        run: yarn playwright install --with-deps
+
      - name: build web
        if: needs.conditions.outputs.should-run == 'true'
-      shell: bash
        run: yarn tronb:vite:dev
+
      - name: Run ubuntu/chrome snapshots
-      if: ${{ needs.conditions.outputs.should-run == 'true' && matrix.os == 'namespace-profile-ubuntu-8-cores' && matrix.shardIndex == 1 }}
-      shell: bash
-      # TODO: break this in its own job, for now it's not slowing down the overall execution as ubuntu is the quickest,
-      # but we could do better. This forces a large 1/1 shard of all 20 snapshot tests that runs in about 3 minutes.
+        if: needs.conditions.outputs.should-run == 'true'
        run: |
          yarn test:snapshots
        env:
@ -212,6 +229,7 @@ jobs:
          VITE_KC_SKIP_AUTH: true
          token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
          snapshottoken: ${{ secrets.KITTYCAD_API_TOKEN }}
+
      - uses: actions/upload-artifact@v4
        if: ${{ needs.conditions.outputs.should-run == 'true' && !cancelled() && (success() || failure()) }}
        with:
@ -220,10 +238,12 @@ jobs:
          include-hidden-files: true
          retention-days: 30
          overwrite: true
+
      - name: Clean up test-results
        if: ${{ needs.conditions.outputs.should-run == 'true' && !cancelled() && (success() || failure()) }}
        continue-on-error: true
        run: rm -r test-results
+
      - name: check for changes
        if: ${{ needs.conditions.outputs.should-run == 'true' && matrix.os == 'namespace-profile-ubuntu-8-cores' && matrix.shardIndex == 1 && github.ref != 'refs/heads/main' }}
        shell: bash
@ -234,6 +254,7 @@ jobs:
            then echo "modified=true" >> $GITHUB_OUTPUT
            else echo "modified=false" >> $GITHUB_OUTPUT
            fi
+
      - name: Commit changes, if any
        if: ${{ needs.conditions.outputs.should-run == 'true' && steps.git-check.outputs.modified == 'true' }}
        shell: bash
@ -248,20 +269,87 @@ jobs:
          git commit -m "A snapshot a day keeps the bugs away! 📷🐛 (OS: ${{matrix.os}})" || true
          git push
          git push origin ${{ github.head_ref }}
+
      # only upload artifacts if there's actually changes
      - uses: actions/upload-artifact@v4
        if: ${{ needs.conditions.outputs.should-run == 'true' && steps.git-check.outputs.modified == 'true' }}
        with:
-        name: playwright-report-${{ matrix.os }}-${{ matrix.shardIndex }}-${{ github.sha }}
+          name: playwright-report-ubuntu-${{ github.sha }}
          path: playwright-report/
          include-hidden-files: true
          retention-days: 30
+
+
+  electron:
+    needs: [conditions, prepare-wasm]
+    timeout-minutes: 60
+    name: playwright:electron:${{ matrix.os }} ${{ matrix.shardIndex }} ${{ matrix.shardTotal }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO: enable self-hosted-windows-8-cores once available
+        os: [namespace-profile-ubuntu-8-cores, namespace-profile-macos-8-cores, windows-16-cores]
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
+        # TODO: add ref here for main and latest release tag
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        if: needs.conditions.outputs.should-run == 'true'
+
+      - uses: actions/download-artifact@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        name: prepared-wasm
+
+      - name: Copy prepared wasm
+        if: needs.conditions.outputs.should-run == 'true'
+        run: |
+          ls -R prepared-wasm
+          cp prepared-wasm/kcl_wasm_lib_bg.wasm public
+          mkdir rust/kcl-wasm-lib/pkg
+          cp prepared-wasm/kcl_wasm_lib* rust/kcl-wasm-lib/pkg
+
+      - uses: actions/setup-node@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'yarn'
+
+      - name: Install dependencies
+        id: deps-install
+        if: needs.conditions.outputs.should-run == 'true'
+        run: yarn
+
+      - name: Cache Playwright Browsers
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/ms-playwright/
+          key: ${{ runner.os }}-playwright-${{ hashFiles('yarn.lock') }}
+
+      - name: Install Playwright Browsers
+        run: yarn playwright install --with-deps
+
+      - name: Build web
+        if: needs.conditions.outputs.should-run == 'true'
+        run: yarn tronb:vite:dev
+
+      - name: Install good sed
+        if: startsWith(matrix.os, 'macos')
+        shell: bash
+        run: |
+          brew install gnu-sed
+          echo "/opt/homebrew/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
+
+        # TODO: Add back axiom logs
+
      - uses: actions/download-artifact@v4
        if: ${{ needs.conditions.outputs.should-run == 'true' && !cancelled() && (success() || failure()) }}
        continue-on-error: true
        with:
          name: test-results-${{ matrix.os }}-${{ matrix.shardIndex }}-${{ github.sha }}
          path: test-results/
+
      - name: Run playwright/electron flow (with retries)
        id: retry
        if: ${{ needs.conditions.outputs.should-run == 'true' && !cancelled() && steps.deps-install.outcome == 'success' }}
@ -278,6 +366,7 @@ jobs:
          VITE_KC_DEV_TOKEN: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
          VITE_KC_SKIP_AUTH: true
          token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
+
      - uses: actions/upload-artifact@v4
        if: ${{ needs.conditions.outputs.should-run == 'true' && always() }}
        with:
@ -286,6 +375,7 @@ jobs:
          include-hidden-files: true
          retention-days: 30
          overwrite: true
+
      - uses: actions/upload-artifact@v4
        if: ${{ needs.conditions.outputs.should-run == 'true' && always() }}
        with:
@ -294,4 +384,3 @@ jobs:
          include-hidden-files: true
          retention-days: 30
          overwrite: true
-