Continue running broken tests for Axiom metrics (#5883)

* Install Vector on Ubuntu to log failed test to Axiom

* Allow flaky tests to run on main for Axiom metrics

* Enable problematic tests on a dedicated branch

.github/ci-cd-scripts/playwright-electron.sh

@@ -4,7 +4,7 @@
 set -euo pipefail
 
 if [[ ! -f "test-results/.last-run.json" ]]; then
-    # if no last run artifact, than run plawright normally
+    # If no last run artifact, than run Playwright normally
     echo "run playwright normally"
     if [[ "$3" == *ubuntu* ]]; then
         xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn test:playwright:electron:ubuntu -- --shard=$1/$2 || true
@@ -13,18 +13,18 @@ if [[ ! -f "test-results/.last-run.json" ]]; then
     elif [[ "$3" == *macos* ]]; then
         yarn test:playwright:electron:macos  -- --shard=$1/$2 || true
     else
-            echo "Do not run playwright. Unable to detect os runtime."
+        echo "Do not run Playwright. Unable to detect os runtime."
         exit 1
     fi
-        # # send to axiom
-        node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
+    # Log failures for Axiom to pick up
+    node playwrightProcess.mjs > /tmp/github-actions.log
 fi
 
 retry=1
-max_retrys=1
+max_retries=1
 
-# retry failed tests, doing our own retries because using inbuilt playwright retries causes connection issues
-while [[ $retry -le $max_retrys ]]; do
+# Retry failed tests, doing our own retries because using inbuilt Playwright retries causes connection issues
+while [[ $retry -le $max_retries ]]; do
     if [[ -f "test-results/.last-run.json" ]]; then
         failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
         if [[ $failed_tests -gt 0 ]]; then
@@ -40,8 +40,8 @@ while [[ $retry -le $max_retrys ]]; do
                 echo "Do not run playwright. Unable to detect os runtime."
                 exit 1
             fi
-            # send to axiom
-            node playwrightProcess.mjs | tee /tmp/github-actions.log > /dev/null 2>&1
+            # Log failures for Axiom to pick up
+            node playwrightProcess.mjs > /tmp/github-actions.log
             retry=$((retry + 1))
         else
             echo "retried=false" >>$GITHUB_OUTPUT
@@ -58,7 +58,7 @@ echo "retried=false" >>$GITHUB_OUTPUT
 if [[ -f "test-results/.last-run.json" ]]; then
     failed_tests=$(jq '.failedTests | length' test-results/.last-run.json)
     if [[ $failed_tests -gt 0 ]]; then
-        # if it still fails after 3 retrys, then fail the job
+        # If it still fails after 3 retries, then fail the job
         exit 1
     fi
 fi

.github/workflows/e2e-tests.yml

@@ -340,14 +340,22 @@ jobs:
         if: needs.conditions.outputs.should-run == 'true'
         run: yarn tronb:vite:dev
 
-      - name: Install good sed
-        if: startsWith(matrix.os, 'macos')
+      - name: Install vector
+        if: contains(matrix.os, 'ubuntu')
         shell: bash
         run: |
-          brew install gnu-sed
-          echo "/opt/homebrew/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
-
-        # TODO: Add back axiom logs
+          curl --proto '=https' --tlsv1.2 -sSfL https://sh.vector.dev > /tmp/vector.sh
+          chmod +x /tmp/vector.sh
+          /tmp/vector.sh -y -no-modify-path
+          mkdir -p /tmp/vector
+          cp .github/workflows/vector.toml /tmp/vector.toml
+          sed -i "s#GITHUB_WORKFLOW#${GITHUB_WORKFLOW}#g" /tmp/vector.toml
+          sed -i "s#GITHUB_REPOSITORY#${GITHUB_REPOSITORY}#g" /tmp/vector.toml
+          sed -i "s#GITHUB_SHA#${GITHUB_SHA}#g" /tmp/vector.toml
+          sed -i "s#GITHUB_REF_NAME#${GITHUB_REF_NAME}#g" /tmp/vector.toml
+          sed -i "s#GH_ACTIONS_AXIOM_TOKEN#${{secrets.GH_ACTIONS_AXIOM_TOKEN}}#g" /tmp/vector.toml
+          cat /tmp/vector.toml
+          ${HOME}/.vector/bin/vector --config /tmp/vector.toml &
 
       - uses: actions/download-artifact@v4
         if: ${{ needs.conditions.outputs.should-run == 'true' && !cancelled() && (success() || failure()) }}

e2e/playwright/basic-sketch.spec.ts

@@ -153,7 +153,8 @@ async function doBasicSketch(
 }
 
 test.describe('Basic sketch', { tag: ['@skipWin'] }, () => {
-  test.fixme('code pane open at start', async ({ page, homePage }) => {
+  test('code pane open at start', async ({ page, homePage }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     await doBasicSketch(page, homePage, ['code'])
   })
 

e2e/playwright/code-pane-and-errors.spec.ts

@@ -46,11 +46,12 @@ test.describe('Code pane and errors', { tag: ['@skipWin'] }, () => {
     await expect(codePaneButtonHolder).toContainText('notification')
   })
 
-  test.skip('Opening and closing the code pane will consistently show error diagnostics', async ({
+  test('Opening and closing the code pane will consistently show error diagnostics', async ({
     page,
     homePage,
     editor,
   }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     const u = await getUtils(page)
 
     // Load the app with the working starter code
@@ -119,9 +120,12 @@ test.describe('Code pane and errors', { tag: ['@skipWin'] }, () => {
     await expect(page.locator('.cm-tooltip').first()).toBeVisible()
   })
 
-  test.fixme(
-    'When error is not in view you can click the badge to scroll to it',
-    async ({ page, homePage, context }) => {
+  test('When error is not in view you can click the badge to scroll to it', async ({
+    page,
+    homePage,
+    context,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     // Load the app with the working starter code
     await context.addInitScript((code) => {
       localStorage.setItem('persistCode', code)
@@ -156,8 +160,7 @@ test.describe('Code pane and errors', { tag: ['@skipWin'] }, () => {
         )
         .first()
     ).toBeVisible()
-    }
-  )
+  })
 
   test('When error is not in view WITH LINTS you can click the badge to scroll to it', async ({
     context,

e2e/playwright/command-bar-tests.spec.ts

@@ -47,7 +47,8 @@ test.describe('Command bar tests', { tag: ['@skipWin'] }, () => {
   })
 
   // TODO: fix this test after the electron migration
-  test.fixme('Fillet from command bar', async ({ page, homePage }) => {
+  test('Fillet from command bar', async ({ page, homePage }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     await page.addInitScript(async () => {
       localStorage.setItem(
         'persistCode',

e2e/playwright/editor-tests.spec.ts

@@ -635,9 +635,11 @@ test.describe('Editor tests', { tag: ['@skipWin'] }, () => {
     await expect(page.locator('.cm-lint-marker-error')).not.toBeVisible()
   })
 
-  test.fixme(
-    'error with 2 source ranges gets 2 diagnostics',
-    async ({ page, homePage }) => {
+  test('error with 2 source ranges gets 2 diagnostics', async ({
+    page,
+    homePage,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     const u = await getUtils(page)
     await page.addInitScript(async () => {
       localStorage.setItem(
@@ -702,8 +704,7 @@ test.describe('Editor tests', { tag: ['@skipWin'] }, () => {
 
     // Make sure there are two diagnostics
     await expect(page.locator('.cm-lint-marker-error')).toHaveCount(2)
-    }
-  )
+  })
   test('if your kcl gets an error from the engine it is inlined', async ({
     context,
     page,
@@ -1121,10 +1122,11 @@ test.describe('Editor tests', { tag: ['@skipWin'] }, () => {
     }
   )
 
-  test.fixme(
+  test(
     `Can use the import stdlib function on a local OBJ file`,
     { tag: '@electron' },
     async ({ page, context }, testInfo) => {
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       await context.folderSetupFn(async (dir) => {
         const bracketDir = join(dir, 'cube')
         await fsp.mkdir(bracketDir, { recursive: true })

e2e/playwright/file-tree.spec.ts

@@ -266,12 +266,13 @@ test.describe('when using the file tree to', () => {
     }
   )
 
-  test.fixme(
+  test(
     'loading small file, then large, then back to small',
     {
       tag: '@electron',
     },
     async ({ page }, testInfo) => {
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       const {
         panesOpen,
         pasteCodeInEditor,

e2e/playwright/onboarding-tests.spec.ts

@@ -319,9 +319,13 @@ test.describe('Onboarding tests', () => {
   // (lee) The two avatar tests are weird because even on main, we don't have
   // anything to do with the avatar inside the onboarding test. Due to the
   // low impact of an avatar not showing I'm changing this to fixme.
-  test.fixme(
-    'Avatar text updates depending on image load success',
-    async ({ context, page, homePage, tronApp }) => {
+  test('Avatar text updates depending on image load success', async ({
+    context,
+    page,
+    homePage,
+    tronApp,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     if (!tronApp) {
       fail()
     }
@@ -349,9 +353,7 @@ test.describe('Onboarding tests', () => {
     await homePage.goToModelingScene()
 
     // Test that the text in this step is correct
-      const avatarLocator = page
-        .getByTestId('user-sidebar-toggle')
-        .locator('img')
+    const avatarLocator = page.getByTestId('user-sidebar-toggle').locator('img')
     const onboardingOverlayLocator = page
       .getByTestId('onboarding-content')
       .locator('div')
@@ -374,16 +376,13 @@ test.describe('Onboarding tests', () => {
     })
 
     // 404 the CI avatar image
-      await page.route(
-        'https://lh3.googleusercontent.com/**',
-        async (route) => {
+    await page.route('https://lh3.googleusercontent.com/**', async (route) => {
       await route.fulfill({
         status: 404,
         contentType: 'text/plain',
         body: 'Not Found!',
       })
-        }
-      )
+    })
 
     await page.reload({ waitUntil: 'domcontentloaded' })
 
@@ -391,12 +390,15 @@ test.describe('Onboarding tests', () => {
     await expect(avatarLocator).not.toBeVisible()
     await expect(onboardingOverlayLocator).toBeVisible()
     await expect(onboardingOverlayLocator).toContainText('the menu button')
-    }
-  )
+  })
 
-  test.fixme(
-    "Avatar text doesn't mention avatar when no avatar",
-    async ({ context, page, homePage, tronApp }) => {
+  test("Avatar text doesn't mention avatar when no avatar", async ({
+    context,
+    page,
+    homePage,
+    tronApp,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     if (!tronApp) {
       fail()
     }
@@ -447,13 +449,15 @@ test.describe('Onboarding tests', () => {
     for (const feature of userMenuFeatures) {
       await expect(onboardingOverlayLocator).toContainText(feature)
     }
-    }
-  )
+  })
 })
 
-test.fixme(
-  'Restarting onboarding on desktop takes one attempt',
-  async ({ context, page, tronApp }) => {
+test('Restarting onboarding on desktop takes one attempt', async ({
+  context,
+  page,
+  tronApp,
+}) => {
+  test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
   if (!tronApp) {
     fail()
   }
@@ -551,5 +555,4 @@ test.fixme(
     await expect(tutorialProjectIndicator).toBeVisible()
     await expect(tutorialModalText).toBeVisible()
   })
-  }
-)
+})

e2e/playwright/point-click.spec.ts

@@ -850,9 +850,13 @@ openSketch = startSketchOn('XY')
     })
   })
 
-  test.fixme(
-    `Shift-click to select and deselect sketch segments`,
-    async ({ page, homePage, scene, editor }) => {
+  test(`Shift-click to select and deselect sketch segments`, async ({
+    page,
+    homePage,
+    scene,
+    editor,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     // Locators
     const firstPointLocation = { x: 200, y: 100 }
     const secondPointLocation = { x: 800, y: 100 }
@@ -999,8 +1003,7 @@ openSketch = startSketchOn('XY')
         )
       })
     })
-    }
-  )
+  })
 
   test(`Offset plane point-and-click`, async ({
     context,

e2e/playwright/projects.spec.ts

@@ -1244,10 +1244,11 @@ test(
   }
 )
 
-test.fixme(
+test(
   'Deleting projects, can delete individual project, can still create projects after deleting all',
   { tag: '@electron' },
   async ({ context, page }, testInfo) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     const projectData = [
       ['router-template-slate', 'cylinder.kcl'],
       ['bracket', 'focusrite_scarlett_mounting_braket.kcl'],
@@ -1466,10 +1467,11 @@ test(
   }
 )
 
-test.fixme(
+test(
   'When the project folder is empty, user can create new project and open it.',
   { tag: '@electron' },
   async ({ page }, testInfo) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     const u = await getUtils(page)
     await page.setBodyDimensions({ width: 1200, height: 500 })
 
@@ -2050,10 +2052,11 @@ test(
 )
 
 // Flaky
-test.fixme(
+test(
   'Original project name persist after onboarding',
   { tag: '@electron' },
   async ({ page }, testInfo) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     await page.setBodyDimensions({ width: 1200, height: 500 })
 
     const getAllProjects = () => page.getByTestId('project-link').all()

e2e/playwright/prompt-to-edit.spec.ts

@@ -196,9 +196,15 @@ test.describe('Prompt-to-edit tests', { tag: '@skipWin' }, () => {
     })
   })
 
-  test.fixme(
-    `manual code selection rename`,
-    async ({ context, homePage, cmdBar, editor, page, scene }) => {
+  test(`manual code selection rename`, async ({
+    context,
+    homePage,
+    cmdBar,
+    editor,
+    page,
+    scene,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     const body1CapCoords = { x: 571, y: 311 }
 
     await context.addInitScript((file) => {
@@ -248,8 +254,7 @@ test.describe('Prompt-to-edit tests', { tag: '@skipWin' }, () => {
       await acceptBtn.click()
       await expect(successToast).not.toBeVisible()
     })
-    }
-  )
+  })
 
   test('multiple body selections', async ({
     context,

e2e/playwright/regression-tests.spec.ts

@@ -483,10 +483,11 @@ extrude001 = extrude(sketch001, length = 50)
     }
   )
 
-  test.fixme(
+  test(
     `Network health indicator only appears in modeling view`,
     { tag: '@electron' },
     async ({ context, page }, testInfo) => {
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       await context.folderSetupFn(async (dir) => {
         const bracketDir = path.join(dir, 'bracket')
         await fsp.mkdir(bracketDir, { recursive: true })

e2e/playwright/sketch-tests.spec.ts

@@ -188,7 +188,8 @@ sketch001 = startProfileAt([12.34, -12.34], sketch002)
       page.getByRole('button', { name: 'Start Sketch' })
     ).toBeVisible()
   })
-  test.fixme('Can edit segments by dragging their handles', () => {
+  test('Can edit segments by dragging their handles', () => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     const doEditSegmentsByDraggingHandle = async (
       page: Page,
       homePage: HomePageFixture,
@@ -1086,9 +1087,11 @@ profile001 = startProfileAt([${roundOff(scale * 69.6)}, ${roundOff(
     )
   })
   // TODO: fix after electron migration is merged
-  test.fixme(
-    'empty-scene default-planes act as expected',
-    async ({ page, homePage }) => {
+  test('empty-scene default-planes act as expected', async ({
+    page,
+    homePage,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     /**
      * Tests the following things
      * 1) The the planes are there on load because the scene is empty
@@ -1185,8 +1188,7 @@ profile001 = startProfileAt([${roundOff(scale * 69.6)}, ${roundOff(
     expect(
       await u.getGreatestPixDiff(XYPlanePoint, noPlanesColor)
     ).toBeLessThan(3)
-    }
-  )
+  })
 
   test('Can attempt to sketch on revolved face', async ({ page, homePage }) => {
     const u = await getUtils(page)
@@ -1460,10 +1462,11 @@ test.describe(`Sketching with offset planes`, () => {
 })
 
 test.describe('multi-profile sketching', () => {
-  test.fixme(
+  test(
     `test it removes half-finished expressions when changing tools in sketch mode`,
     { tag: ['@skipWin'] },
     async ({ context, page, scene, toolbar, editor, homePage, cmdBar }) => {
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       // We seed the scene with a single offset plane
       await context.addInitScript(() => {
         localStorage.setItem(
@@ -2852,9 +2855,14 @@ loft([profile001, profile002])
       )
     }
   )
-  test.fixme(
-    'Can enter sketch loft edges offsetPlane and continue sketch',
-    async ({ scene, toolbar, editor, page, homePage }) => {
+  test('Can enter sketch loft edges offsetPlane and continue sketch', async ({
+    scene,
+    toolbar,
+    editor,
+    page,
+    homePage,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     await page.addInitScript(async () => {
       localStorage.setItem(
         'persistCode',
@@ -2917,8 +2925,7 @@ loft([profile001, profile002])
       `angledLine([0, 106.42], %, $rectangleSegmentA001)`
     )
     await page.waitForTimeout(100)
-    }
-  )
+  })
 })
 
 // Regression test for https://github.com/KittyCAD/modeling-app/issues/4891

e2e/playwright/snapshot-tests.spec.ts

@@ -36,10 +36,11 @@ test.setTimeout(60_000)
 // a snapshot of it feels weird. I'd rather our regular tests fail.
 // The primary failure is doExport now relies on the filesystem. We can follow
 // up with another PR if we want this back.
-test.skip(
+test(
   'exports of each format should work',
   { tag: ['@snapshot', '@skipWin', '@skipMacos'] },
   async ({ page, context, scene, cmdBar, tronApp }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     if (!tronApp) {
       fail()
     }
@@ -406,9 +407,6 @@ test(
   'Draft segments should look right',
   { tag: '@snapshot' },
   async ({ page, scene, toolbar }) => {
-    // FIXME: Skip on macos its being weird.
-    // test.skip(process.platform === 'darwin', 'Skip on macos')
-
     const u = await getUtils(page)
     await page.setViewportSize({ width: 1200, height: 500 })
     const PUR = 400 / 37.5 //pixeltoUnitRatio
@@ -585,9 +583,6 @@ test(
   'Draft circle should look right',
   { tag: '@snapshot' },
   async ({ page, context, cmdBar, scene }) => {
-    // FIXME: Skip on macos its being weird.
-    // test.skip(process.platform === 'darwin', 'Skip on macos')
-
     const u = await getUtils(page)
     await page.setViewportSize({ width: 1200, height: 500 })
     const PUR = 400 / 37.5 //pixeltoUnitRatio
@@ -952,9 +947,6 @@ test(
 )
 
 test.describe('Grid visibility', { tag: '@snapshot' }, () => {
-  // FIXME: Skip on macos its being weird.
-  // test.skip(process.platform === 'darwin', 'Skip on macos')
-
   test('Grid turned off to on via command bar', async ({
     page,
     cmdBar,
@@ -1097,7 +1089,8 @@ test.describe('Grid visibility', { tag: '@snapshot' }, () => {
   })
 })
 
-test.fixme('theme persists', async ({ page, context }) => {
+test('theme persists', async ({ page, context }) => {
+  test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
   const u = await getUtils(page)
   await context.addInitScript(async () => {
     localStorage.setItem(

e2e/playwright/test-network-and-connection-issues.spec.ts

@@ -3,11 +3,12 @@ import { commonPoints, getUtils } from './test-utils'
 import { EngineCommand } from 'lang/std/artifactGraph'
 import { uuidv4 } from 'lib/utils'
 
-test.fixme('Test network and connection issues', () => {
+test.describe('Test network and connection issues', () => {
   test(
     'simulate network down and network little widget',
     { tag: '@skipLocalEngine' },
     async ({ page, homePage }) => {
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       const u = await getUtils(page)
       await page.setBodyDimensions({ width: 1200, height: 500 })
 
@@ -84,6 +85,7 @@ test.fixme('Test network and connection issues', () => {
     'Engine disconnect & reconnect in sketch mode',
     { tag: '@skipLocalEngine' },
     async ({ page, homePage, toolbar }) => {
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       const networkToggle = page.getByTestId('network-toggle')
 
       const u = await getUtils(page)

e2e/playwright/testing-camera-movement.spec.ts

@@ -179,9 +179,11 @@ test.describe('Testing Camera Movement', { tag: ['@skipWin'] }, () => {
   })
 
   // TODO: fix after electron migration is merged
-  test.fixme(
-    'Zoom should be consistent when exiting or entering sketches',
-    async ({ page, homePage }) => {
+  test('Zoom should be consistent when exiting or entering sketches', async ({
+    page,
+    homePage,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     // start new sketch pan and zoom before exiting, when exiting the sketch should stay in the same place
     // than zoom and pan outside of sketch mode and enter again and it should not change from where it is
     // than again for sketching
@@ -340,8 +342,7 @@ test.describe('Testing Camera Movement', { tag: ['@skipWin'] }, () => {
     await expect(page.getByTestId('hover-highlight').first()).toBeVisible({
       timeout: 10_000,
     })
-    }
-  )
+  })
 
   test(`Zoom by scroll should not fire while orbiting`, async ({
     homePage,

e2e/playwright/testing-constraints.spec.ts

@@ -1005,9 +1005,11 @@ part002 = startSketchOn('XZ')
     }
   })
 
-  test.fixme(
-    'Horizontally constrained line remains selected after applying constraint',
-    async ({ page, homePage }) => {
+  test('Horizontally constrained line remains selected after applying constraint', async ({
+    page,
+    homePage,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     test.setTimeout(70_000)
     await page.addInitScript(async () => {
       localStorage.setItem(
@@ -1025,9 +1027,9 @@ part002 = startSketchOn('XZ')
     await u.waitForPageLoad()
 
     await page.getByText('line(end = [3.79, 2.68], tag = $seg01)').click()
-      await expect(
-        page.getByRole('button', { name: 'Edit Sketch' })
-      ).toBeEnabled({ timeout: 10_000 })
+    await expect(page.getByRole('button', { name: 'Edit Sketch' })).toBeEnabled(
+      { timeout: 10_000 }
+    )
     await page.getByRole('button', { name: 'Edit Sketch' }).click()
 
     // Wait for overlays to populate
@@ -1054,9 +1056,7 @@ part002 = startSketchOn('XZ')
       })
       .click()
     await page.waitForTimeout(500)
-      await page
-        .getByRole('button', { name: 'Horizontal', exact: true })
-        .click()
+    await page.getByRole('button', { name: 'Horizontal', exact: true }).click()
     await page.waitForTimeout(500)
 
     await pollEditorLinesSelectedLength(page, 1)
@@ -1081,9 +1081,7 @@ part002 = startSketchOn('XZ')
     await page.mouse.click(linebb.x, linebb.y)
 
     await expect
-        .poll(
-          async () => await u.getGreatestPixDiff(lineAfter, TEST_COLORS.BLUE)
-        )
+      .poll(async () => await u.getGreatestPixDiff(lineAfter, TEST_COLORS.BLUE))
       .toBeLessThan(3)
 
     await page.waitForTimeout(500)
@@ -1093,10 +1091,7 @@ part002 = startSketchOn('XZ')
     // await page.getByRole('button', { name: 'length', exact: true }).click()
     await page.getByTestId('constraint-length').click()
 
-      await page
-        .getByTestId('cmd-bar-arg-value')
-        .getByRole('textbox')
-        .fill('10')
+    await page.getByTestId('cmd-bar-arg-value').getByRole('textbox').fill('10')
     await page
       .getByRole('button', {
         name: 'arrow right Continue',
@@ -1111,8 +1106,7 @@ part002 = startSketchOn('XZ')
 
     // checking the count of the overlays is a good proxy check that the client sketch scene is in a good state
     await expect(page.getByTestId('segment-overlay')).toHaveCount(2)
-    }
-  )
+  })
 })
 test.describe('Electron constraint tests', () => {
   test(

e2e/playwright/testing-perspective-toggle.spec.ts

@@ -2,7 +2,8 @@ import { test, expect } from './zoo-test'
 import { getUtils } from './test-utils'
 
 test.describe('Test toggling perspective', () => {
-  test.fixme('via command palette and toggle', async ({ page, homePage }) => {
+  test('via command palette and toggle', async ({ page, homePage }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     const u = await getUtils(page)
 
     // Locators and constants

e2e/playwright/testing-segment-overlays.spec.ts

@@ -7,11 +7,9 @@ import { uuidv4 } from 'lib/utils'
 import { EditorFixture } from './fixtures/editorFixture'
 
 test.describe('Testing segment overlays', { tag: ['@skipWin'] }, () => {
-  test.fixme(
-    'Hover over a segment should show its overlay, hovering over the input overlays should show its popover, clicking the input overlay should constrain/unconstrain it:\nfor the following segments',
-    () => {
+  test('Hover over a segment should show its overlay, hovering over the input overlays should show its popover, clicking the input overlay should constrain/unconstrain it:\nfor the following segments', () => {
     // TODO: fix this test on mac after the electron migration
-      test.skip(process.platform === 'darwin', 'Skip on mac')
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     /**
      * Clicks on an constrained element
      * @param {Page} page - The page to perform the action on
@@ -368,9 +366,12 @@ test.describe('Testing segment overlays', { tag: ['@skipWin'] }, () => {
     })
 
     // Broken on main at time of writing!
-      test.fixme(
-        'for segments [yLineTo, xLine]',
-        async ({ page, editor, homePage }) => {
+    test('for segments [yLineTo, xLine]', async ({
+      page,
+      editor,
+      homePage,
+    }) => {
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       await page.addInitScript(async () => {
         localStorage.setItem(
           'persistCode',
@@ -444,8 +445,7 @@ test.describe('Testing segment overlays', { tag: ['@skipWin'] }, () => {
         ang: ang + 180,
         locator: '[data-overlay-toolbar-index="5"]',
       })
-        }
-      )
+    })
     test('for segments [yLine, angledLineOfXLength, angledLineOfYLength]', async ({
       page,
       editor,
@@ -557,8 +557,7 @@ test.describe('Testing segment overlays', { tag: ['@skipWin'] }, () => {
           'angledLineOfYLength({ angle = -91, length = 19 + 0 }, %)',
         expectAfterUnconstrained:
           'angledLineOfYLength({ angle = angle002, length = 19 + 0 }, %)',
-          expectFinal:
-            'angledLineOfYLength({ angle = -91, length = 19 + 0 }, %)',
+        expectFinal: 'angledLineOfYLength({ angle = -91, length = 19 + 0 }, %)',
         ang: ang + 180,
         steps: 6,
         locator: '[data-overlay-toolbar-index="8"]',
@@ -657,9 +656,7 @@ test.describe('Testing segment overlays', { tag: ['@skipWin'] }, () => {
         locator: '[data-overlay-toolbar-index="9"]',
       })
 
-        const angledLineToY = await u.getBoundingBox(
-          '[data-overlay-index="10"]'
-        )
+      const angledLineToY = await u.getBoundingBox('[data-overlay-index="10"]')
       ang = await u.getAngle('[data-overlay-index="10"]')
       console.log('angledLineToY')
       await clickUnconstrained({
@@ -680,8 +677,7 @@ test.describe('Testing segment overlays', { tag: ['@skipWin'] }, () => {
         constraintType: 'yAbsolute',
         expectBeforeUnconstrained:
           'angledLineToY({ angle = 89, to = 9.14 + 0 }, %)',
-          expectAfterUnconstrained:
-            'angledLineToY({ angle = 89, to = 9.14 }, %)',
+        expectAfterUnconstrained: 'angledLineToY({ angle = 89, to = 9.14 }, %)',
         expectFinal: 'angledLineToY({ angle = 89, to = yAbs001 }, %)',
         ang: ang + 180,
         locator: '[data-overlay-toolbar-index="10"]',
@@ -1000,8 +996,7 @@ profile001 = startProfileAt([56.37, 120.33], sketch001)
       await clickUnconstrained({
         hoverPos,
         constraintType: 'yAbsolute',
-          expectBeforeUnconstrained:
-            'circle(center = [xAbs001, 0], radius = 8)',
+        expectBeforeUnconstrained: 'circle(center = [xAbs001, 0], radius = 8)',
         expectAfterUnconstrained:
           'circle(center = [xAbs001, yAbs001], radius =  8)',
         expectFinal: 'circle(center = [xAbs001, 0], radius =  8)',
@@ -1013,8 +1008,7 @@ profile001 = startProfileAt([56.37, 120.33], sketch001)
       await clickUnconstrained({
         hoverPos,
         constraintType: 'radius',
-          expectBeforeUnconstrained:
-            'circle(center = [xAbs001, 0], radius = 8)',
+        expectBeforeUnconstrained: 'circle(center = [xAbs001, 0], radius = 8)',
         expectAfterUnconstrained:
           'circle(center = [xAbs001, 0], radius = radius001)',
         expectFinal: 'circle(center = [xAbs001, 0], radius = 8)',
@@ -1023,8 +1017,7 @@ profile001 = startProfileAt([56.37, 120.33], sketch001)
         locator: '[data-overlay-toolbar-index="0"]',
       })
     })
-    }
-  )
+  })
   test.describe('Testing deleting a segment', () => {
     const _deleteSegmentSequence =
       (page: Page, editor: EditorFixture) =>

e2e/playwright/testing-selections.spec.ts

@@ -448,9 +448,14 @@ profile003 = startProfileAt([40.16, -120.48], sketch006)
     await page.waitForTimeout(200)
     await expect(u.codeLocator).not.toContainText(codeToBeDeletedSnippet)
   })
-  test.fixme(
-    'parent Solid should be select and deletable and uses custom planes to position children',
-    async ({ page, homePage, scene, cmdBar, editor }) => {
+  test('parent Solid should be select and deletable and uses custom planes to position children', async ({
+    page,
+    homePage,
+    scene,
+    cmdBar,
+    editor,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     test.setTimeout(90_000)
     const u = await getUtils(page)
     await page.addInitScript(async () => {
@@ -514,8 +519,7 @@ profile001 = startProfileAt([7.49, 9.96], sketch001)
       shouldNormalise: true,
     })
     await editor.snapshot()
-    }
-  )
+  })
   test('Hovering over 3d features highlights code, clicking puts the cursor in the right place and sends selection id to engine', async ({
     page,
     homePage,

e2e/playwright/testing-settings.spec.ts

@@ -55,9 +55,11 @@ test.describe('Testing settings', () => {
   })
 
   // The behavior is actually broken. Parent always takes precedence
-  test.fixme(
-    'Project settings can be set and override user settings',
-    async ({ page, homePage }) => {
+  test('Project settings can be set and override user settings', async ({
+    page,
+    homePage,
+  }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     const u = await getUtils(page)
     await test.step(`Setup`, async () => {
       await page.setBodyDimensions({ width: 1200, height: 500 })
@@ -82,14 +84,11 @@ test.describe('Testing settings', () => {
       /** Test to close https://github.com/KittyCAD/modeling-app/issues/2713 */
       await test.step(`Confirm that this dialog has a solid background`, async () => {
         await expect
-            .poll(
-              () => u.getGreatestPixDiff({ x: 600, y: 250 }, [28, 28, 28]),
-              {
+          .poll(() => u.getGreatestPixDiff({ x: 600, y: 250 }, [28, 28, 28]), {
             timeout: 1000,
             message:
               'Checking for solid background, should not see default plane colors',
-              }
-            )
+          })
           .toBeLessThan(15)
       })
 
@@ -122,9 +121,7 @@ test.describe('Testing settings', () => {
 
     // Roll back to default of "off"
     await await page
-        .getByText(
-          'show debug panelRoll back show debug panelRoll back to match'
-        )
+      .getByText('show debug panelRoll back show debug panelRoll back to match')
       .hover()
     await page
       .getByRole('button', {
@@ -138,8 +135,7 @@ test.describe('Testing settings', () => {
     await expect(
       page.locator('input[name="app-showDebugPanel"]')
     ).not.toBeChecked()
-    }
-  )
+  })
 
   test('Keybindings display the correct hotkey for Command Palette', async ({
     page,
@@ -175,9 +171,8 @@ test.describe('Testing settings', () => {
     await expect(hotkey).toHaveText(text)
   })
 
-  test.fixme(
-    'Project and user settings can be reset',
-    async ({ page, homePage }) => {
+  test('Project and user settings can be reset', async ({ page, homePage }) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     const u = await getUtils(page)
     await test.step(`Setup`, async () => {
       await page.setBodyDimensions({ width: 1200, height: 500 })
@@ -261,17 +256,13 @@ test.describe('Testing settings', () => {
         await expect(themeColorSetting).toHaveValue(settingValues.project)
       })
     })
-    }
-  )
+  })
 
-  test.fixme(
+  test(
     `Project settings override user settings on desktop`,
     { tag: ['@electron', '@skipWin'] },
     async ({ context, page }, testInfo) => {
-      test.skip(
-        process.platform === 'win32',
-        'TODO: remove this skip https://github.com/KittyCAD/modeling-app/issues/3557'
-      )
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       const projectName = 'bracket'
       const { dir: projectDirName } = await context.folderSetupFn(
         async (dir) => {
@@ -407,12 +398,13 @@ test.describe('Testing settings', () => {
   )
 
   // It was much easier to test the logo color than the background stream color.
-  test.fixme(
+  test(
     'user settings reload on external change, on project and modeling view',
     {
       tag: '@electron',
     },
     async ({ context, page, tronApp }, testInfo) => {
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       if (!tronApp) {
         fail()
       }
@@ -466,10 +458,11 @@ test.describe('Testing settings', () => {
     }
   )
 
-  test.fixme(
+  test(
     'project settings reload on external change',
     { tag: '@electron' },
     async ({ context, page }, testInfo) => {
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       const { dir: projectDirName } = await context.folderSetupFn(
         async () => {}
       )
@@ -982,9 +975,15 @@ fn cube`
   /**
    * This test assumes that the default value of the "highlight edges" setting is "on".
    */
-  test.fixme(
-    `Toggle stream settings multiple times`,
-    async ({ page, scene, homePage, context, toolbar, cmdBar }, testInfo) => {
+  test(`Toggle stream settings multiple times`, async ({
+    page,
+    scene,
+    homePage,
+    context,
+    toolbar,
+    cmdBar,
+  }, testInfo) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     await context.folderSetupFn(async (dir) => {
       const projectDir = join(dir, 'project-000')
       await fsp.mkdir(projectDir, { recursive: true })
@@ -1039,6 +1038,5 @@ fn cube`
         mask: [page.getByTestId('model-state-indicator')],
       }
     )
-    }
-  )
+  })
 })

e2e/playwright/text-to-cad-tests.spec.ts

@@ -431,10 +431,11 @@ test.describe('Text-to-CAD tests', { tag: ['@skipWin'] }, () => {
   })
 
   // This will be fine once greg makes prompt at top of file deterministic
-  test.fixme(
+  test(
     'can do many at once and get many prompts back, and interact with many',
     { tag: ['@skipWin'] },
     async ({ page, homePage }) => {
+      test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
       // Let this test run longer since we've seen it timeout.
       test.setTimeout(180_000)
 
@@ -619,10 +620,11 @@ async function sendPromptFromCommandBar(page: Page, promptStr: string) {
   })
 }
 
-test.fixme(
+test(
   'Text-to-CAD functionality',
   { tag: '@electron' },
   async ({ context, page }, testInfo) => {
+    test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
     const projectName = 'project-000'
     const prompt = 'lego 2x4'
     const textToCadFileName = 'lego-2x4.kcl'

e2e/playwright/various.spec.ts

@@ -2,7 +2,8 @@ import { test, expect } from './zoo-test'
 
 import { doExport, getUtils, makeTemplate } from './test-utils'
 
-test.fixme('Units menu', async ({ page, homePage }) => {
+test('Units menu', async ({ page, homePage }) => {
+  test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
   await page.setBodyDimensions({ width: 1200, height: 500 })
   await homePage.goToModelingScene()
 
@@ -263,9 +264,11 @@ test('First escape in tool pops you out of tool, second exits sketch mode', asyn
   ).not.toBeVisible()
 })
 
-test.fixme(
-  'Basic default modeling and sketch hotkeys work',
-  async ({ page, homePage }) => {
+test('Basic default modeling and sketch hotkeys work', async ({
+  page,
+  homePage,
+}) => {
+  test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
   const u = await getUtils(page)
 
   // This test can run long if it takes a little too long to load
@@ -441,8 +444,7 @@ test.fixme(
     await page.keyboard.press('Escape')
     await page.waitForTimeout(100)
   }
-  }
-)
+})
 
 test('Delete key does not navigate back', async ({ page, homePage }) => {
   await page.setBodyDimensions({ width: 1200, height: 500 })