Refactor logic to skip unreliable tests (#5919)

This commit is contained in:
Jace Browning
2025-03-20 23:52:30 -04:00
committed by GitHub
parent 89d1f7f3d3
commit 06b35b76ff
23 changed files with 106 additions and 56 deletions

View File

@ -53,6 +53,7 @@ format: install ## Format the code
.PHONY: lint
lint: install ## Lint the code
yarn tsc
yarn lint
###############################################################################

View File

@ -5,6 +5,7 @@ import {
TEST_COLORS,
commonPoints,
PERSIST_MODELING_CONTEXT,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { HomePageFixture } from './fixtures/homePageFixture'
@ -154,7 +155,7 @@ async function doBasicSketch(
test.describe('Basic sketch', { tag: ['@skipWin'] }, () => {
test('code pane open at start', async ({ page, homePage }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
await doBasicSketch(page, homePage, ['code'])
})

View File

@ -1,6 +1,9 @@
import { test, expect } from './zoo-test'
import { getUtils, executorInputPath } from './test-utils'
import {
orRunWhenFullSuiteEnabled,
getUtils,
executorInputPath,
} from './test-utils'
import { join } from 'path'
import { bracket } from 'lib/exampleKcl'
import { TEST_CODE_LONG_WITH_ERROR_OUT_OF_VIEW } from './storageStates'
@ -51,7 +54,7 @@ test.describe('Code pane and errors', { tag: ['@skipWin'] }, () => {
homePage,
editor,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const u = await getUtils(page)
// Load the app with the working starter code
@ -125,7 +128,7 @@ test.describe('Code pane and errors', { tag: ['@skipWin'] }, () => {
homePage,
context,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
// Load the app with the working starter code
await context.addInitScript((code) => {
localStorage.setItem('persistCode', code)

View File

@ -1,6 +1,10 @@
import { test, expect } from './zoo-test'
import * as fsp from 'fs/promises'
import { executorInputPath, getUtils } from './test-utils'
import {
executorInputPath,
getUtils,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { KCL_DEFAULT_LENGTH } from 'lib/constants'
import path, { join } from 'path'
@ -48,7 +52,7 @@ test.describe('Command bar tests', { tag: ['@skipWin'] }, () => {
// TODO: fix this test after the electron migration
test('Fillet from command bar', async ({ page, homePage }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
await page.addInitScript(async () => {
localStorage.setItem(
'persistCode',

View File

@ -1,7 +1,12 @@
import { test, expect } from './zoo-test'
import fsp from 'fs/promises'
import { uuidv4 } from 'lib/utils'
import { executorInputPath, getUtils, TEST_COLORS } from './test-utils'
import {
executorInputPath,
getUtils,
orRunWhenFullSuiteEnabled,
TEST_COLORS,
} from './test-utils'
import { join } from 'path'
@ -634,7 +639,7 @@ test.describe('Editor tests', { tag: ['@skipWin'] }, () => {
page,
homePage,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const u = await getUtils(page)
await page.addInitScript(async () => {
localStorage.setItem(
@ -1121,7 +1126,7 @@ test.describe('Editor tests', { tag: ['@skipWin'] }, () => {
`Can use the import stdlib function on a local OBJ file`,
{ tag: '@electron' },
async ({ page, context }, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
await context.folderSetupFn(async (dir) => {
const bracketDir = join(dir, 'cube')
await fsp.mkdir(bracketDir, { recursive: true })

View File

@ -1,7 +1,12 @@
import { test, expect } from './zoo-test'
import * as fsp from 'fs/promises'
import * as fs from 'fs'
import { createProject, executorInputPath, getUtils } from './test-utils'
import {
createProject,
executorInputPath,
getUtils,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { join } from 'path'
import { FILE_EXT } from 'lib/constants'
@ -272,7 +277,7 @@ test.describe('when using the file tree to', () => {
tag: '@electron',
},
async ({ page }, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const {
panesOpen,
pasteCodeInEditor,

View File

@ -6,6 +6,7 @@ import {
executorInputPath,
createProject,
settingsToToml,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { bracket } from 'lib/exampleKcl'
import { onboardingPaths } from 'routes/Onboarding/paths'
@ -325,7 +326,7 @@ test.describe('Onboarding tests', () => {
homePage,
tronApp,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
if (!tronApp) {
fail()
}
@ -398,7 +399,7 @@ test.describe('Onboarding tests', () => {
homePage,
tronApp,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
if (!tronApp) {
fail()
}
@ -457,7 +458,7 @@ test('Restarting onboarding on desktop takes one attempt', async ({
page,
tronApp,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
if (!tronApp) {
fail()
}

View File

@ -5,7 +5,7 @@ import { SceneFixture } from './fixtures/sceneFixture'
import { ToolbarFixture } from './fixtures/toolbarFixture'
import fs from 'node:fs/promises'
import path from 'node:path'
import { getUtils } from './test-utils'
import { getUtils, orRunWhenFullSuiteEnabled } from './test-utils'
import { Locator } from '@playwright/test'
// test file is for testing point an click code gen functionality that's not sketch mode related
@ -856,7 +856,7 @@ openSketch = startSketchOn('XY')
scene,
editor,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
// Locators
const firstPointLocation = { x: 200, y: 100 }
const secondPointLocation = { x: 800, y: 100 }

View File

@ -7,6 +7,7 @@ import {
Paths,
createProject,
getPlaywrightDownloadDir,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import fsp from 'fs/promises'
import fs from 'fs'
@ -1248,7 +1249,7 @@ test(
'Deleting projects, can delete individual project, can still create projects after deleting all',
{ tag: '@electron' },
async ({ context, page }, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const projectData = [
['router-template-slate', 'cylinder.kcl'],
['bracket', 'focusrite_scarlett_mounting_braket.kcl'],
@ -1471,7 +1472,7 @@ test(
'When the project folder is empty, user can create new project and open it.',
{ tag: '@electron' },
async ({ page }, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const u = await getUtils(page)
await page.setBodyDimensions({ width: 1200, height: 500 })
@ -2056,7 +2057,7 @@ test(
'Original project name persist after onboarding',
{ tag: '@electron' },
async ({ page }, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
await page.setBodyDimensions({ width: 1200, height: 500 })
const getAllProjects = () => page.getByTestId('project-link').all()

View File

@ -1,4 +1,5 @@
import { test, expect } from './zoo-test'
import { orRunWhenFullSuiteEnabled } from './test-utils'
/* eslint-disable jest/no-conditional-expect */
@ -204,7 +205,7 @@ test.describe('Prompt-to-edit tests', { tag: '@skipWin' }, () => {
page,
scene,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const body1CapCoords = { x: 571, y: 311 }
await context.addInitScript((file) => {

View File

@ -7,6 +7,7 @@ import {
executorInputPath,
TEST_COLORS,
TestColor,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { TEST_CODE_TRIGGER_ENGINE_EXPORT_ERROR } from './storageStates'
import { bracket } from 'lib/exampleKcl'
@ -576,7 +577,7 @@ extrude002 = extrude(profile002, length = 150)
`Network health indicator only appears in modeling view`,
{ tag: '@electron' },
async ({ context, page }, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
await context.folderSetupFn(async (dir) => {
const bracketDir = path.join(dir, 'bracket')
await fsp.mkdir(bracketDir, { recursive: true })

View File

@ -9,6 +9,7 @@ import {
getUtils,
PERSIST_MODELING_CONTEXT,
TEST_COLORS,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { uuidv4, roundOff } from 'lib/utils'
import { SceneFixture } from './fixtures/sceneFixture'
@ -189,7 +190,7 @@ sketch001 = startProfileAt([12.34, -12.34], sketch002)
).toBeVisible()
})
test('Can edit segments by dragging their handles', () => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const doEditSegmentsByDraggingHandle = async (
page: Page,
homePage: HomePageFixture,
@ -1091,7 +1092,7 @@ profile001 = startProfileAt([${roundOff(scale * 69.6)}, ${roundOff(
page,
homePage,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
/**
* Tests the following things
* 1) The the planes are there on load because the scene is empty
@ -1466,7 +1467,7 @@ test.describe('multi-profile sketching', () => {
`test it removes half-finished expressions when changing tools in sketch mode`,
{ tag: ['@skipWin'] },
async ({ context, page, scene, toolbar, editor, homePage, cmdBar }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
// We seed the scene with a single offset plane
await context.addInitScript(() => {
localStorage.setItem(
@ -2862,7 +2863,7 @@ loft([profile001, profile002])
page,
homePage,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
await page.addInitScript(async () => {
localStorage.setItem(
'persistCode',

View File

@ -1,6 +1,12 @@
import { test, expect } from './zoo-test'
import { secrets } from './secrets'
import { Paths, doExport, getUtils, settingsToToml } from './test-utils'
import {
Paths,
doExport,
getUtils,
settingsToToml,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { Models } from '@kittycad/lib'
import fsp from 'fs/promises'
import { spawn } from 'child_process'
@ -40,7 +46,7 @@ test(
'exports of each format should work',
{ tag: ['@snapshot', '@skipWin', '@skipMacos'] },
async ({ page, context, scene, cmdBar, tronApp }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
if (!tronApp) {
fail()
}
@ -1093,7 +1099,7 @@ test.describe('Grid visibility', { tag: '@snapshot' }, () => {
})
test('theme persists', async ({ page, context }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const u = await getUtils(page)
await context.addInitScript(async () => {
localStorage.setItem(

View File

@ -1,5 +1,5 @@
import { test, expect } from './zoo-test'
import { commonPoints, getUtils } from './test-utils'
import { commonPoints, getUtils, orRunWhenFullSuiteEnabled } from './test-utils'
import { EngineCommand } from 'lang/std/artifactGraph'
import { uuidv4 } from 'lib/utils'
@ -8,7 +8,7 @@ test.describe('Test network and connection issues', () => {
'simulate network down and network little widget',
{ tag: '@skipLocalEngine' },
async ({ page, homePage }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const u = await getUtils(page)
await page.setBodyDimensions({ width: 1200, height: 500 })
@ -85,7 +85,7 @@ test.describe('Test network and connection issues', () => {
'Engine disconnect & reconnect in sketch mode',
{ tag: '@skipLocalEngine' },
async ({ page, homePage, toolbar }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const networkToggle = page.getByTestId('network-toggle')
const u = await getUtils(page)

View File

@ -55,6 +55,10 @@ export const commonPoints = {
export const editorSelector = '[role="textbox"][data-language="kcl"]'
type PaneId = 'variables' | 'code' | 'files' | 'logs'
export function orRunWhenFullSuiteEnabled() {
return process.env.GITHUB_HEAD_REF !== 'all-e2e'
}
async function waitForPageLoadWithRetry(page: Page) {
await expect(async () => {
await page.goto('/')

View File

@ -1,7 +1,7 @@
import { test, expect } from './zoo-test'
import { EngineCommand } from 'lang/std/artifactGraph'
import { uuidv4 } from 'lib/utils'
import { getUtils } from './test-utils'
import { getUtils, orRunWhenFullSuiteEnabled } from './test-utils'
test.describe('Testing Camera Movement', { tag: ['@skipWin'] }, () => {
test('Can move camera reliably', async ({ page, context, homePage }) => {
@ -183,7 +183,7 @@ test.describe('Testing Camera Movement', { tag: ['@skipWin'] }, () => {
page,
homePage,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
// start new sketch pan and zoom before exiting, when exiting the sketch should stay in the same place
// than zoom and pan outside of sketch mode and enter again and it should not change from where it is
// than again for sketching

View File

@ -5,6 +5,7 @@ import {
TEST_COLORS,
pollEditorLinesSelectedLength,
executorInputPath,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { XOR } from 'lib/utils'
import path from 'node:path'
@ -1009,7 +1010,7 @@ part002 = startSketchOn('XZ')
page,
homePage,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
test.setTimeout(70_000)
await page.addInitScript(async () => {
localStorage.setItem(

View File

@ -1,9 +1,9 @@
import { test, expect } from './zoo-test'
import { getUtils } from './test-utils'
import { getUtils, orRunWhenFullSuiteEnabled } from './test-utils'
test.describe('Test toggling perspective', () => {
test('via command palette and toggle', async ({ page, homePage }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const u = await getUtils(page)
// Locators and constants

View File

@ -1,7 +1,12 @@
import { Page } from '@playwright/test'
import { test, expect } from './zoo-test'
import { deg, getUtils, wiggleMove } from './test-utils'
import {
deg,
getUtils,
wiggleMove,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { LineInputsType } from 'lang/std/sketchcombos'
import { uuidv4 } from 'lib/utils'
import { EditorFixture } from './fixtures/editorFixture'
@ -9,7 +14,7 @@ import { EditorFixture } from './fixtures/editorFixture'
test.describe('Testing segment overlays', { tag: ['@skipWin'] }, () => {
test('Hover over a segment should show its overlay, hovering over the input overlays should show its popover, clicking the input overlay should constrain/unconstrain it:\nfor the following segments', () => {
// TODO: fix this test on mac after the electron migration
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
/**
* Clicks on an constrained element
* @param {Page} page - The page to perform the action on
@ -371,7 +376,7 @@ test.describe('Testing segment overlays', { tag: ['@skipWin'] }, () => {
editor,
homePage,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
await page.addInitScript(async () => {
localStorage.setItem(
'persistCode',

View File

@ -1,6 +1,6 @@
import { test, expect } from './zoo-test'
import { commonPoints, getUtils } from './test-utils'
import { commonPoints, getUtils, orRunWhenFullSuiteEnabled } from './test-utils'
import { Coords2d } from 'lang/std/sketch'
import { KCL_DEFAULT_LENGTH } from 'lib/constants'
import { uuidv4 } from 'lib/utils'
@ -455,7 +455,7 @@ profile003 = startProfileAt([40.16, -120.48], sketch006)
cmdBar,
editor,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
test.setTimeout(90_000)
const u = await getUtils(page)
await page.addInitScript(async () => {

View File

@ -7,6 +7,7 @@ import {
createProject,
tomlToSettings,
TEST_COLORS,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { SettingsLevel } from 'lib/settings/settingsTypes'
import { SETTINGS_FILE_NAME, PROJECT_SETTINGS_FILE_NAME } from 'lib/constants'
@ -60,7 +61,7 @@ test.describe('Testing settings', () => {
page,
homePage,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const u = await getUtils(page)
await test.step(`Setup`, async () => {
await page.setBodyDimensions({ width: 1200, height: 500 })
@ -173,7 +174,7 @@ test.describe('Testing settings', () => {
})
test('Project and user settings can be reset', async ({ page, homePage }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const u = await getUtils(page)
await test.step(`Setup`, async () => {
await page.setBodyDimensions({ width: 1200, height: 500 })
@ -263,7 +264,7 @@ test.describe('Testing settings', () => {
`Project settings override user settings on desktop`,
{ tag: ['@electron', '@skipWin'] },
async ({ context, page }, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const projectName = 'bracket'
const { dir: projectDirName } = await context.folderSetupFn(
async (dir) => {
@ -405,7 +406,7 @@ test.describe('Testing settings', () => {
tag: '@electron',
},
async ({ context, page, tronApp }, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
if (!tronApp) {
fail()
}
@ -463,7 +464,7 @@ test.describe('Testing settings', () => {
'project settings reload on external change',
{ tag: '@electron' },
async ({ context, page }, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const { dir: projectDirName } = await context.folderSetupFn(
async () => {}
)
@ -984,7 +985,7 @@ fn cube`
toolbar,
cmdBar,
}, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
await context.folderSetupFn(async (dir) => {
const projectDir = join(dir, 'project-000')
await fsp.mkdir(projectDir, { recursive: true })

View File

@ -1,6 +1,10 @@
import { Page } from '@playwright/test'
import { test, expect } from './zoo-test'
import { getUtils, createProject } from './test-utils'
import {
getUtils,
createProject,
orRunWhenFullSuiteEnabled,
} from './test-utils'
import { join } from 'path'
import fs from 'fs'
@ -435,7 +439,7 @@ test.describe('Text-to-CAD tests', { tag: ['@skipWin'] }, () => {
'can do many at once and get many prompts back, and interact with many',
{ tag: ['@skipWin'] },
async ({ page, homePage }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
// Let this test run longer since we've seen it timeout.
test.setTimeout(180_000)
@ -624,7 +628,7 @@ test(
'Text-to-CAD functionality',
{ tag: '@electron' },
async ({ context, page }, testInfo) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const projectName = 'project-000'
const prompt = 'lego 2x4'
const textToCadFileName = 'lego-2x4.kcl'

View File

@ -1,9 +1,14 @@
import { test, expect } from './zoo-test'
import { doExport, getUtils, makeTemplate } from './test-utils'
import {
doExport,
getUtils,
makeTemplate,
orRunWhenFullSuiteEnabled,
} from './test-utils'
test('Units menu', async ({ page, homePage }) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
await page.setBodyDimensions({ width: 1200, height: 500 })
await homePage.goToModelingScene()
@ -268,7 +273,7 @@ test('Basic default modeling and sketch hotkeys work', async ({
page,
homePage,
}) => {
test.fixme(process.env.GITHUB_HEAD_REF !== 'all-e2e')
test.fixme(orRunWhenFullSuiteEnabled())
const u = await getUtils(page)
// This test can run long if it takes a little too long to load