Dedicated job for snapshot tests (#5752)

* WIP: break out e2e snapshots

* Separate prepare-wasm job

* Force e2e to run on pierremtb/adhoc/break-out-snapshots

* Quick fix

* Quick fix2

* Quick fix3

* Quick fix4

* Quick fix5

* Remove ifs for flow tests

* Clean up for review

* should-run == 'true' madness

.github/workflows/e2e-tests.yml

@@ -26,9 +26,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+
       - name: Fetch the base branch
         if: ${{ github.event_name == 'pull_request' }}
         run: git fetch origin ${{ github.base_ref }} --depth=1
+
       - name: Check for path changes
         id: path-changes
         shell: bash
@@ -48,6 +50,7 @@ jobs:
           else
             echo "significant=false" >> $GITHUB_OUTPUT
           fi
+
       - name: Should run
         id: should-run
         shell: bash
@@ -60,6 +63,7 @@ jobs:
           else
             echo "should-run=false" >> $GITHUB_OUTPUT
           fi
+
       - name: Display conditions
         shell: bash
         run: |
@@ -68,57 +72,34 @@ jobs:
           echo "significant: ${{ steps.path-changes.outputs.significant }}"
           echo "should-run: ${{ steps.should-run.outputs.should-run }}"
 
-  electron:
+
-    timeout-minutes: 60
+  prepare-wasm:
-    name: playwright:electron:${{ matrix.os }} ${{ matrix.shardIndex }} ${{ matrix.shardTotal }}
+    # seperate job on Ubuntu to build or fetch the wasm blob once on the fastest runner
-    strategy:
+    runs-on: namespace-profile-ubuntu-8-cores
-      fail-fast: false
-      matrix:
-        # TODO: enable self-hosted-windows-8-cores once available
-        os: [namespace-profile-ubuntu-8-cores, namespace-profile-macos-8-cores, windows-16-cores]
-        shardIndex: [1, 2, 3, 4]
-        shardTotal: [4]
-        # TODO: add ref here for main and latest release tag
-    runs-on: ${{ matrix.os }}
     needs: conditions
     steps:
-    - uses: actions/create-github-app-token@v1
-      id: app-token
-      with:
-        app-id: ${{ secrets.MODELING_APP_GH_APP_ID }}
-        private-key: ${{ secrets.MODELING_APP_GH_APP_PRIVATE_KEY }}
-        owner: ${{ github.repository_owner }}
       - uses: actions/checkout@v4
-      with:
+        if: needs.conditions.outputs.should-run == 'true'
-        token: ${{ steps.app-token.outputs.token }}
+
       - id: filter
+        if: needs.conditions.outputs.should-run == 'true'
         name: Check for Rust changes
         uses: dorny/paths-filter@v3
         with:
           filters: |
             rust:
               - 'rust/**'
+
       - uses: actions/setup-node@v4
         if: needs.conditions.outputs.should-run == 'true'
         with:
           node-version-file: '.nvmrc'
           cache: 'yarn'
+
       - name: Install dependencies
-      id: deps-install
         if: needs.conditions.outputs.should-run == 'true'
-      shell: bash
         run: yarn
-    - name: Cache Playwright Browsers
+
-      if: needs.conditions.outputs.should-run == 'true'
-      uses: actions/cache@v4
-      with:
-        path: |
-          ~/.cache/ms-playwright/
-        key: ${{ runner.os }}-playwright-${{ hashFiles('yarn.lock') }}
-    - name: Install Playwright Browsers
-      if: needs.conditions.outputs.should-run == 'true'
-      shell: bash
-      run: yarn playwright install --with-deps
       - name: Download Wasm Cache
         id: download-wasm
         if: ${{ needs.conditions.outputs.should-run == 'true' && github.event_name != 'schedule' && steps.filter.outputs.rust == 'false' }}
@@ -130,15 +111,10 @@ jobs:
           workflow: build-and-store-wasm.yml
           branch: main
           path: rust/kcl-wasm-lib/pkg
-    - name: copy wasm blob
+
-      if: ${{ needs.conditions.outputs.should-run == 'true' && github.event_name != 'schedule' && steps.filter.outputs.rust == 'false' }}
-      shell: bash
-      run: cp rust/kcl-wasm-lib/pkg/kcl_wasm_lib_bg.wasm public
-      continue-on-error: true
       - name: Build WASM condition
         id: wasm
         if: needs.conditions.outputs.should-run == 'true'
-      shell: bash
         run: |
           set -euox pipefail
           # Build wasm if this is a scheduled run, there are Rust changes, or
@@ -148,61 +124,102 @@ jobs:
           else
             echo "should-build-wasm=false" >> $GITHUB_OUTPUT
           fi
+
       - name: Use correct Rust toolchain
         if: ${{ needs.conditions.outputs.should-run == 'true' && steps.wasm.outputs.should-build-wasm == 'true' }}
         shell: bash
         run: |
           [ -e rust-toolchain.toml ] || cp rust/rust-toolchain.toml ./
+
       - name: Install rust
         if: ${{ needs.conditions.outputs.should-run == 'true' && steps.wasm.outputs.should-build-wasm == 'true' }}
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           cache: false # Configured below.
+
       - uses: taiki-e/install-action@955a6ff1416eae278c9f833008a9beb4b7f9afe3
         if: ${{ needs.conditions.outputs.should-run == 'true' && steps.wasm.outputs.should-build-wasm == 'true' }}
         with:
           tool: wasm-pack
+
       - name: Rust Cache
         if: ${{ needs.conditions.outputs.should-run == 'true' && steps.wasm.outputs.should-build-wasm == 'true' }}
         uses: Swatinem/rust-cache@v2
         with:
           workspaces: './rust'
-    - name: install good sed
+
-      if: ${{ needs.conditions.outputs.should-run == 'true' && startsWith(matrix.os, 'macos') }}
-      shell: bash
-      run: |
-        brew install gnu-sed
-        echo "/opt/homebrew/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
-    - name: Install vector
-      shell: bash
-      # TODO: figure out what to do with this, it's failing
-      if: false
-      run: |
-        curl --proto '=https' --tlsv1.2 -sSfL https://sh.vector.dev > /tmp/vector.sh
-        chmod +x /tmp/vector.sh
-        /tmp/vector.sh -y -no-modify-path
-        mkdir -p /tmp/vector
-        cp .github/workflows/vector.toml /tmp/vector.toml
-        sed -i "s#GITHUB_WORKFLOW#${GITHUB_WORKFLOW}#g" /tmp/vector.toml
-        sed -i "s#GITHUB_REPOSITORY#${GITHUB_REPOSITORY}#g" /tmp/vector.toml
-        sed -i "s#GITHUB_SHA#${GITHUB_SHA}#g" /tmp/vector.toml
-        sed -i "s#GITHUB_REF_NAME#${GITHUB_REF_NAME}#g" /tmp/vector.toml
-        sed -i "s#GH_ACTIONS_AXIOM_TOKEN#${{secrets.GH_ACTIONS_AXIOM_TOKEN}}#g" /tmp/vector.toml
-        cat /tmp/vector.toml
-        ${HOME}/.vector/bin/vector --config /tmp/vector.toml &
       - name: Build Wasm
         if: ${{ needs.conditions.outputs.should-run == 'true' && steps.wasm.outputs.should-build-wasm == 'true' }}
         shell: bash
         run: yarn build:wasm
+
+      - uses: actions/upload-artifact@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        with:
+          name: prepared-wasm
+          path: |
+            rust/kcl-wasm-lib/pkg/kcl_wasm_lib*
+  
+
+  snapshots:
+    name: playwright:snapshots:ubuntu
+    runs-on: namespace-profile-ubuntu-8-cores
+    needs: [conditions, prepare-wasm]
+    steps:
+      - uses: actions/create-github-app-token@v1
+        if: needs.conditions.outputs.should-run == 'true'
+        id: app-token
+        with:
+          app-id: ${{ secrets.MODELING_APP_GH_APP_ID }}
+          private-key: ${{ secrets.MODELING_APP_GH_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - uses: actions/checkout@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+
+      - uses: actions/download-artifact@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        name: prepared-wasm
+
+      - name: Copy prepared wasm
+        if: needs.conditions.outputs.should-run == 'true'
+        run: |
+          ls -R prepared-wasm
+          cp prepared-wasm/kcl_wasm_lib_bg.wasm public
+          mkdir rust/kcl-wasm-lib/pkg
+          cp prepared-wasm/kcl_wasm_lib* rust/kcl-wasm-lib/pkg
+
+      - uses: actions/setup-node@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'yarn'
+
+      - name: Install dependencies
+        id: deps-install
+        if: needs.conditions.outputs.should-run == 'true'
+        run: yarn
+
+      - name: Cache Playwright Browsers
+        if: needs.conditions.outputs.should-run == 'true'
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/ms-playwright/
+          key: ${{ runner.os }}-playwright-${{ hashFiles('yarn.lock') }}
+
+      - name: Install Playwright Browsers
+        if: needs.conditions.outputs.should-run == 'true'
+        run: yarn playwright install --with-deps
+
       - name: build web
         if: needs.conditions.outputs.should-run == 'true'
-      shell: bash
         run: yarn tronb:vite:dev
+
       - name: Run ubuntu/chrome snapshots
-      if: ${{ needs.conditions.outputs.should-run == 'true' && matrix.os == 'namespace-profile-ubuntu-8-cores' && matrix.shardIndex == 1 }}
+        if: needs.conditions.outputs.should-run == 'true'
-      shell: bash
-      # TODO: break this in its own job, for now it's not slowing down the overall execution as ubuntu is the quickest,
-      # but we could do better. This forces a large 1/1 shard of all 20 snapshot tests that runs in about 3 minutes.
         run: |
           yarn test:snapshots
         env:
@@ -212,6 +229,7 @@ jobs:
           VITE_KC_SKIP_AUTH: true
           token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
           snapshottoken: ${{ secrets.KITTYCAD_API_TOKEN }}
+
       - uses: actions/upload-artifact@v4
         if: ${{ needs.conditions.outputs.should-run == 'true' && !cancelled() && (success() || failure()) }}
         with:
@@ -220,10 +238,12 @@ jobs:
           include-hidden-files: true
           retention-days: 30
           overwrite: true
+
       - name: Clean up test-results
         if: ${{ needs.conditions.outputs.should-run == 'true' && !cancelled() && (success() || failure()) }}
         continue-on-error: true
         run: rm -r test-results
+
       - name: check for changes
         if: ${{ needs.conditions.outputs.should-run == 'true' && matrix.os == 'namespace-profile-ubuntu-8-cores' && matrix.shardIndex == 1 && github.ref != 'refs/heads/main' }}
         shell: bash
@@ -234,6 +254,7 @@ jobs:
             then echo "modified=true" >> $GITHUB_OUTPUT
             else echo "modified=false" >> $GITHUB_OUTPUT
             fi
+
       - name: Commit changes, if any
         if: ${{ needs.conditions.outputs.should-run == 'true' && steps.git-check.outputs.modified == 'true' }}
         shell: bash
@@ -248,20 +269,87 @@ jobs:
           git commit -m "A snapshot a day keeps the bugs away! 📷🐛 (OS: ${{matrix.os}})" || true
           git push
           git push origin ${{ github.head_ref }}
+
       # only upload artifacts if there's actually changes
       - uses: actions/upload-artifact@v4
         if: ${{ needs.conditions.outputs.should-run == 'true' && steps.git-check.outputs.modified == 'true' }}
         with:
-        name: playwright-report-${{ matrix.os }}-${{ matrix.shardIndex }}-${{ github.sha }}
+          name: playwright-report-ubuntu-${{ github.sha }}
           path: playwright-report/
           include-hidden-files: true
           retention-days: 30
+
+
+  electron:
+    needs: [conditions, prepare-wasm]
+    timeout-minutes: 60
+    name: playwright:electron:${{ matrix.os }} ${{ matrix.shardIndex }} ${{ matrix.shardTotal }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO: enable self-hosted-windows-8-cores once available
+        os: [namespace-profile-ubuntu-8-cores, namespace-profile-macos-8-cores, windows-16-cores]
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
+        # TODO: add ref here for main and latest release tag
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        if: needs.conditions.outputs.should-run == 'true'
+
+      - uses: actions/download-artifact@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        name: prepared-wasm
+
+      - name: Copy prepared wasm
+        if: needs.conditions.outputs.should-run == 'true'
+        run: |
+          ls -R prepared-wasm
+          cp prepared-wasm/kcl_wasm_lib_bg.wasm public
+          mkdir rust/kcl-wasm-lib/pkg
+          cp prepared-wasm/kcl_wasm_lib* rust/kcl-wasm-lib/pkg
+
+      - uses: actions/setup-node@v4
+        if: needs.conditions.outputs.should-run == 'true'
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'yarn'
+
+      - name: Install dependencies
+        id: deps-install
+        if: needs.conditions.outputs.should-run == 'true'
+        run: yarn
+
+      - name: Cache Playwright Browsers
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/ms-playwright/
+          key: ${{ runner.os }}-playwright-${{ hashFiles('yarn.lock') }}
+
+      - name: Install Playwright Browsers
+        run: yarn playwright install --with-deps
+
+      - name: Build web
+        if: needs.conditions.outputs.should-run == 'true'
+        run: yarn tronb:vite:dev
+
+      - name: Install good sed
+        if: startsWith(matrix.os, 'macos')
+        shell: bash
+        run: |
+          brew install gnu-sed
+          echo "/opt/homebrew/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
+
+        # TODO: Add back axiom logs
+
       - uses: actions/download-artifact@v4
         if: ${{ needs.conditions.outputs.should-run == 'true' && !cancelled() && (success() || failure()) }}
         continue-on-error: true
         with:
           name: test-results-${{ matrix.os }}-${{ matrix.shardIndex }}-${{ github.sha }}
           path: test-results/
+
       - name: Run playwright/electron flow (with retries)
         id: retry
         if: ${{ needs.conditions.outputs.should-run == 'true' && !cancelled() && steps.deps-install.outcome == 'success' }}
@@ -278,6 +366,7 @@ jobs:
           VITE_KC_DEV_TOKEN: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
           VITE_KC_SKIP_AUTH: true
           token: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
+
       - uses: actions/upload-artifact@v4
         if: ${{ needs.conditions.outputs.should-run == 'true' && always() }}
         with:
@@ -286,6 +375,7 @@ jobs:
           include-hidden-files: true
           retention-days: 30
           overwrite: true
+
       - uses: actions/upload-artifact@v4
         if: ${{ needs.conditions.outputs.should-run == 'true' && always() }}
         with:
@@ -294,4 +384,3 @@ jobs:
           include-hidden-files: true
           retention-days: 30
           overwrite: true
-