Revert Playwright tests to use addInitScript to adjust storage state (#2077)

* Revert Playwright tests to use addInitScript to adjust storage state

* A snapshot a day keeps the bugs away! 📷🐛 (OS: ubuntu)

* Fix tsc

* Rerun CI

* Rerun CI

* Only use page.addInitScript within tests
because technically adding multiple init scripts to the context has an indeterminate run order, per the [Playwright docs](https://playwright.dev/docs/api/class-page#page-add-init-script)

* Rerun CI

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Frank Noirot
2024-04-11 13:37:49 -04:00
committed by GitHub
parent d28555a070
commit 15b9f43f2c
10 changed files with 277 additions and 249 deletions

View File

@ -2,10 +2,15 @@ import { test, expect } from '@playwright/test'
import { getUtils } from './test-utils'
import waitOn from 'wait-on'
import { roundOff } from 'lib/utils'
import { basicStorageState } from './storageStates'
import * as TOML from '@iarna/toml'
import { SaveSettingsPayload } from 'lib/settings/settingsTypes'
import { Themes } from 'lib/theme'
import { secrets } from './secrets'
import {
TEST_SETTINGS,
TEST_SETTINGS_KEY,
TEST_SETTINGS_CORRUPTED,
TEST_SETTINGS_ONBOARDING,
} from './storageStates'
/*
debug helper: unfortunately we do rely on exact coord mouse clicks in a few places
@ -32,13 +37,25 @@ test.beforeEach(async ({ context, page }) => {
timeout: 5000,
})
await context.addInitScript(
async ({ token, settingsKey, settings }) => {
localStorage.setItem('TOKEN_PERSIST_KEY', token)
localStorage.setItem('persistCode', ``)
localStorage.setItem(settingsKey, settings)
},
{
token: secrets.token,
settingsKey: TEST_SETTINGS_KEY,
settings: TOML.stringify({ settings: TEST_SETTINGS }),
}
)
// kill animations, speeds up tests and reduced flakiness
await page.emulateMedia({ reducedMotion: 'reduce' })
})
test.setTimeout(60000)
test('Basic sketch', async ({ page, context }) => {
test('Basic sketch', async ({ page }) => {
const u = getUtils(page)
await page.setViewportSize({ width: 1200, height: 500 })
const PUR = 400 / 37.5 //pixeltoUnitRatio
@ -308,9 +325,9 @@ test('if you write invalid kcl you get inlined errors', async ({ page }) => {
await expect(page.locator('.cm-lint-marker-error')).not.toBeVisible()
})
test('executes on load', async ({ page, context }) => {
test('executes on load', async ({ page }) => {
const u = getUtils(page)
await context.addInitScript(async () => {
await page.addInitScript(async () => {
localStorage.setItem(
'persistCode',
`const part001 = startSketchOn('-XZ')
@ -340,9 +357,9 @@ test('executes on load', async ({ page, context }) => {
).toBeVisible()
})
test('re-executes', async ({ page, context }) => {
test('re-executes', async ({ page }) => {
const u = getUtils(page)
await context.addInitScript(async (token) => {
await page.addInitScript(async () => {
localStorage.setItem('persistCode', `const myVar = 5`)
})
await page.setViewportSize({ width: 1000, height: 500 })
@ -512,134 +529,131 @@ test('Auto complete works', async ({ page }) => {
|> xLine(5, %) // lin`)
})
// Stored settings validation test
test.describe('Settings persistence and validation tests', () => {
// Override test setup
test('Stored settings are validated and fall back to defaults', async ({
page,
}) => {
// Override beforeEach test setup
// with corrupted settings
const storageState = structuredClone(basicStorageState)
const s = TOML.parse(storageState.origins[0].localStorage[2].value) as {
settings: SaveSettingsPayload
}
s.settings.app.theme = Themes.Dark
s.settings.app.projectDirectory = 123 as any
s.settings.modeling.defaultUnit = 'invalid' as any
s.settings.modeling.mouseControls = `() => alert('hack the planet')` as any
s.settings.projects.defaultProjectName = false as any
storageState.origins[0].localStorage[2].value = TOML.stringify(s)
await page.addInitScript(
async ({ settingsKey, settings }) => {
localStorage.setItem(settingsKey, settings)
},
{
settingsKey: TEST_SETTINGS_KEY,
settings: TOML.stringify({ settings: TEST_SETTINGS_CORRUPTED }),
}
)
test.use({ storageState })
await page.setViewportSize({ width: 1200, height: 500 })
await page.goto('/')
test('Stored settings are validated and fall back to defaults', async ({
page,
}) => {
const u = getUtils(page)
await page.setViewportSize({ width: 1200, height: 500 })
await page.goto('/')
await u.waitForAuthSkipAppStart()
// Check the settings were reset
const storedSettings = TOML.parse(
await page.evaluate(
({ settingsKey }) => localStorage.getItem(settingsKey) || '{}',
{ settingsKey: TEST_SETTINGS_KEY }
)
) as { settings: SaveSettingsPayload }
// Check the settings were reset
const storedSettings = TOML.parse(
await page.evaluate(() => localStorage.getItem('/user.toml') || '{}')
) as { settings: SaveSettingsPayload }
expect(storedSettings.settings.app?.theme).toBe('dark')
expect(storedSettings.settings.app?.theme).toBe('dark')
// Check that the invalid settings were removed
expect(storedSettings.settings.modeling?.defaultUnit).toBe(undefined)
expect(storedSettings.settings.modeling?.mouseControls).toBe(undefined)
expect(storedSettings.settings.app?.projectDirectory).toBe(undefined)
expect(storedSettings.settings.projects?.defaultProjectName).toBe(undefined)
})
test('Project settings can be set and override user settings', async ({
page,
}) => {
const u = getUtils(page)
await page.setViewportSize({ width: 1200, height: 500 })
await page.goto('/')
await u.waitForAuthSkipAppStart()
// Open the settings modal with the browser keyboard shortcut
await page.keyboard.press('Meta+Shift+,')
await expect(
page.getByRole('heading', { name: 'Settings', exact: true })
).toBeVisible()
await page
.locator('select[name="app-theme"]')
.selectOption({ value: 'light' })
// Verify the toast appeared
await expect(
page.getByText(`Set theme to "light" for this project`)
).toBeVisible()
// Check that the theme changed
await expect(page.locator('body')).not.toHaveClass(`body-bg dark`)
// Check that the user setting was not changed
await page.getByRole('radio', { name: 'User' }).click()
await expect(page.locator('select[name="app-theme"]')).toHaveValue('dark')
// Roll back to default "system" theme
await page
.getByText(
'themeRoll back themeRoll back to match defaultThe overall appearance of the appl'
)
.hover()
await page
.getByRole('button', {
name: 'Roll back theme ; Has tooltip: Roll back to match default',
})
.click()
await expect(page.locator('select[name="app-theme"]')).toHaveValue('system')
// Check that the project setting did not change
await page.getByRole('radio', { name: 'Project' }).click()
await expect(page.locator('select[name="app-theme"]')).toHaveValue('light')
})
// Check that the invalid settings were removed
expect(storedSettings.settings.modeling?.defaultUnit).toBe(undefined)
expect(storedSettings.settings.modeling?.mouseControls).toBe(undefined)
expect(storedSettings.settings.app?.projectDirectory).toBe(undefined)
expect(storedSettings.settings.projects?.defaultProjectName).toBe(undefined)
})
// Onboarding tests
test.describe('Onboarding tests', () => {
// Override test setup
const storageState = structuredClone(basicStorageState)
const s = TOML.parse(storageState.origins[0].localStorage[2].value) as {
settings: SaveSettingsPayload
}
s.settings.app.onboardingStatus = '/export'
storageState.origins[0].localStorage[2].value = TOML.stringify(s)
test.use({ storageState })
test('Project settings can be set and override user settings', async ({
page,
}) => {
await page.setViewportSize({ width: 1200, height: 500 })
await page.goto('/', { waitUntil: 'domcontentloaded' })
await page
.getByRole('button', { name: 'Start Sketch' })
.waitFor({ state: 'visible' })
test('Onboarding redirects and code updating', async ({ page, context }) => {
const u = getUtils(page)
// Open the settings modal with the browser keyboard shortcut
await page.keyboard.press('Meta+Shift+,')
await page.setViewportSize({ width: 1200, height: 500 })
await page.goto('/')
await u.waitForAuthSkipAppStart()
await expect(
page.getByRole('heading', { name: 'Settings', exact: true })
).toBeVisible()
await page
.locator('select[name="app-theme"]')
.selectOption({ value: 'light' })
// Test that the redirect happened
await expect(page.url().split(':3000').slice(-1)[0]).toBe(
`/file/%2Fbrowser%2Fmain.kcl/onboarding/export`
// Verify the toast appeared
await expect(
page.getByText(`Set theme to "light" for this project`)
).toBeVisible()
// Check that the theme changed
await expect(page.locator('body')).not.toHaveClass(`body-bg dark`)
// Check that the user setting was not changed
await page.getByRole('radio', { name: 'User' }).click()
await expect(page.locator('select[name="app-theme"]')).toHaveValue('dark')
// Roll back to default "system" theme
await page
.getByText(
'themeRoll back themeRoll back to match defaultThe overall appearance of the appl'
)
.hover()
await page
.getByRole('button', {
name: 'Roll back theme ; Has tooltip: Roll back to match default',
})
.click()
await expect(page.locator('select[name="app-theme"]')).toHaveValue('system')
// Test that you come back to this page when you refresh
await page.reload()
await expect(page.url().split(':3000').slice(-1)[0]).toBe(
`/file/%2Fbrowser%2Fmain.kcl/onboarding/export`
)
// Check that the project setting did not change
await page.getByRole('radio', { name: 'Project' }).click()
await expect(page.locator('select[name="app-theme"]')).toHaveValue('light')
})
// Test that the onboarding pane loaded
const title = page.locator('[data-testid="onboarding-content"]')
await expect(title).toBeAttached()
test('Onboarding redirects and code updating', async ({ page }) => {
const u = getUtils(page)
// Test that the code changes when you advance to the next step
await page.locator('[data-testid="onboarding-next"]').click()
await expect(page.locator('.cm-content')).toHaveText('')
// Override beforeEach test setup
await page.addInitScript(
async ({ settingsKey, settings }) => {
// Give some initial code, so we can test that it's cleared
localStorage.setItem('persistCode', 'const sigmaAllow = 15000')
localStorage.setItem(settingsKey, settings)
},
{
settingsKey: TEST_SETTINGS_KEY,
settings: TOML.stringify({ settings: TEST_SETTINGS_ONBOARDING }),
}
)
// Test that the code is not empty when you click on the next step
await page.locator('[data-testid="onboarding-next"]').click()
await expect(page.locator('.cm-content')).toHaveText(/.+/)
})
await page.setViewportSize({ width: 1200, height: 500 })
await page.goto('/')
await u.waitForAuthSkipAppStart()
// Test that the redirect happened
await expect(page.url().split(':3000').slice(-1)[0]).toBe(
`/file/%2Fbrowser%2Fmain.kcl/onboarding/export`
)
// Test that you come back to this page when you refresh
await page.reload()
await expect(page.url().split(':3000').slice(-1)[0]).toBe(
`/file/%2Fbrowser%2Fmain.kcl/onboarding/export`
)
// Test that the onboarding pane loaded
const title = page.locator('[data-testid="onboarding-content"]')
await expect(title).toBeAttached()
// Test that the code changes when you advance to the next step
await page.locator('[data-testid="onboarding-next"]').click()
await expect(page.locator('.cm-content')).toHaveText('')
// Test that the code is not empty when you click on the next step
await page.locator('[data-testid="onboarding-next"]').click()
await expect(page.locator('.cm-content')).toHaveText(/.+/)
})
test('Selections work on fresh and edited sketch', async ({ page }) => {
@ -851,19 +865,22 @@ test.describe('Command bar tests', () => {
await expect(page.locator('body')).not.toHaveClass(`body-bg dark`)
})
// Override test setup code
const storageState = structuredClone(basicStorageState)
storageState.origins[0].localStorage[1].value = `const distance = sqrt(20)
const part001 = startSketchOn('-XZ')
|> startProfileAt([-6.95, 4.98], %)
|> line([25.1, 0.41], %)
|> line([0.73, -14.93], %)
|> line([-23.44, 0.52], %)
|> close(%)
`
test.use({ storageState })
test('Can extrude from the command bar', async ({ page }) => {
await page.addInitScript(async () => {
localStorage.setItem(
'persistCode',
`
const distance = sqrt(20)
const part001 = startSketchOn('-XZ')
|> startProfileAt([-6.95, 4.98], %)
|> line([25.1, 0.41], %)
|> line([0.73, -14.93], %)
|> line([-23.44, 0.52], %)
|> close(%)
`
)
})
test('Can extrude from the command bar', async ({ page, context }) => {
const u = getUtils(page)
await page.setViewportSize({ width: 1200, height: 500 })
await page.goto('/')
@ -1055,9 +1072,9 @@ const part002 = startSketchOn('XY')
)
})
test('ProgramMemory can be serialised', async ({ page, context }) => {
test('ProgramMemory can be serialised', async ({ page }) => {
const u = getUtils(page)
await context.addInitScript(async () => {
await page.addInitScript(async () => {
localStorage.setItem(
'persistCode',
`const part = startSketchOn('XY')
@ -1096,7 +1113,6 @@ test('ProgramMemory can be serialised', async ({ page, context }) => {
test("Various pipe expressions should and shouldn't allow edit and or extrude", async ({
page,
context,
}) => {
const u = getUtils(page)
const selectionsSnippets = {
@ -1105,7 +1121,7 @@ test("Various pipe expressions should and shouldn't allow edit and or extrude",
extrudeAndEditAllowed: '|> startProfileAt([15.72, 4.7], %)',
editOnly: '|> startProfileAt([15.79, -14.6], %)',
}
await context.addInitScript(
await page.addInitScript(
async ({
extrudeAndEditBlocked,
extrudeAndEditBlockedInFunction,
@ -1265,12 +1281,9 @@ test('Deselecting line tool should mean nothing happens on click', async ({
previousCodeContent = await page.locator('.cm-content').innerText()
})
test('Can edit segments by dragging their handles', async ({
page,
context,
}) => {
test('Can edit segments by dragging their handles', async ({ page }) => {
const u = getUtils(page)
await context.addInitScript(async () => {
await page.addInitScript(async () => {
localStorage.setItem(
'persistCode',
`const part001 = startSketchOn('-XZ')
@ -1422,9 +1435,9 @@ test('Snap to close works (at any scale)', async ({ page }) => {
await doSnapAtDifferentScales([0, 10000, 10000], codeTemplate())
})
test('Sketch on face', async ({ page, context }) => {
test('Sketch on face', async ({ page }) => {
const u = getUtils(page)
await context.addInitScript(async () => {
await page.addInitScript(async () => {
localStorage.setItem(
'persistCode',
`const part001 = startSketchOn('-XZ')

View File

@ -7,16 +7,26 @@ import { spawn } from 'child_process'
import { APP_NAME } from 'lib/constants'
import JSZip from 'jszip'
import path from 'path'
import { basicSettings, basicStorageState } from './storageStates'
import { TEST_SETTINGS, TEST_SETTINGS_KEY } from './storageStates'
import * as TOML from '@iarna/toml'
test.beforeEach(async ({ page }) => {
// reducedMotion kills animations, which speeds up tests and reduces flakiness
await page.emulateMedia({ reducedMotion: 'reduce' })
})
test.use({
storageState: structuredClone(basicStorageState),
// set the default settings
await page.addInitScript(
async ({ token, settingsKey, settings }) => {
localStorage.setItem('TOKEN_PERSIST_KEY', token)
localStorage.setItem('persistCode', ``)
localStorage.setItem(settingsKey, settings)
},
{
token: secrets.token,
settingsKey: TEST_SETTINGS_KEY,
settings: TOML.stringify({ settings: TEST_SETTINGS }),
}
)
})
test.setTimeout(60_000)
@ -447,105 +457,108 @@ test('Draft segments should look right', async ({ page, context }) => {
})
})
test('Client side scene scale should match engine scale - Inch', async ({
page,
}) => {
const u = getUtils(page)
await page.setViewportSize({ width: 1200, height: 500 })
const PUR = 400 / 37.5 //pixeltoUnitRatio
await page.goto('/')
await u.waitForAuthSkipAppStart()
await u.openDebugPanel()
test.describe('Client side scene scale should match engine scale', () => {
test('Inch', async ({ page }) => {
const u = getUtils(page)
await page.setViewportSize({ width: 1200, height: 500 })
const PUR = 400 / 37.5 //pixeltoUnitRatio
await page.goto('/')
await u.waitForAuthSkipAppStart()
await u.openDebugPanel()
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).not.toBeDisabled()
await expect(page.getByRole('button', { name: 'Start Sketch' })).toBeVisible()
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).not.toBeDisabled()
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).toBeVisible()
// click on "Start Sketch" button
await u.clearCommandLogs()
await u.doAndWaitForImageDiff(
() => page.getByRole('button', { name: 'Start Sketch' }).click(),
200
)
// click on "Start Sketch" button
await u.clearCommandLogs()
await u.doAndWaitForImageDiff(
() => page.getByRole('button', { name: 'Start Sketch' }).click(),
200
)
// select a plane
await page.mouse.click(700, 200)
// select a plane
await page.mouse.click(700, 200)
await expect(page.locator('.cm-content')).toHaveText(
`const part001 = startSketchOn('-XZ')`
)
await expect(page.locator('.cm-content')).toHaveText(
`const part001 = startSketchOn('-XZ')`
)
await page.waitForTimeout(300) // TODO detect animation ending, or disable animation
await page.waitForTimeout(300) // TODO detect animation ending, or disable animation
const startXPx = 600
await page.mouse.click(startXPx + PUR * 10, 500 - PUR * 10)
await expect(page.locator('.cm-content'))
.toHaveText(`const part001 = startSketchOn('-XZ')
|> startProfileAt([9.06, -12.22], %)`)
await page.waitForTimeout(100)
const startXPx = 600
await page.mouse.click(startXPx + PUR * 10, 500 - PUR * 10)
await expect(page.locator('.cm-content'))
.toHaveText(`const part001 = startSketchOn('-XZ')
|> startProfileAt([9.06, -12.22], %)`)
await page.waitForTimeout(100)
await u.closeDebugPanel()
await u.closeDebugPanel()
await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 10)
await page.waitForTimeout(100)
await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 10)
await page.waitForTimeout(100)
await expect(page.locator('.cm-content'))
.toHaveText(`const part001 = startSketchOn('-XZ')
|> startProfileAt([9.06, -12.22], %)
|> line([9.14, 0], %)`)
await expect(page.locator('.cm-content'))
.toHaveText(`const part001 = startSketchOn('-XZ')
|> startProfileAt([9.06, -12.22], %)
|> line([9.14, 0], %)`)
await page.getByRole('button', { name: 'Tangential Arc' }).click()
await page.waitForTimeout(100)
await page.getByRole('button', { name: 'Tangential Arc' }).click()
await page.waitForTimeout(100)
await page.mouse.click(startXPx + PUR * 30, 500 - PUR * 20)
await page.mouse.click(startXPx + PUR * 30, 500 - PUR * 20)
await expect(page.locator('.cm-content'))
.toHaveText(`const part001 = startSketchOn('-XZ')
|> startProfileAt([9.06, -12.22], %)
|> line([9.14, 0], %)
|> tangentialArcTo([27.34, -3.08], %)`)
await expect(page.locator('.cm-content'))
.toHaveText(`const part001 = startSketchOn('-XZ')
|> startProfileAt([9.06, -12.22], %)
|> line([9.14, 0], %)
|> tangentialArcTo([27.34, -3.08], %)`)
// click tangential arc tool again to unequip it
await page.getByRole('button', { name: 'Tangential Arc' }).click()
await page.waitForTimeout(100)
// click tangential arc tool again to unequip it
await page.getByRole('button', { name: 'Tangential Arc' }).click()
await page.waitForTimeout(100)
// screen shot should show the sketch
await expect(page).toHaveScreenshot({
maxDiffPixels: 100,
})
// screen shot should show the sketch
await expect(page).toHaveScreenshot({
maxDiffPixels: 100,
})
// exit sketch
await u.openAndClearDebugPanel()
await page.getByRole('button', { name: 'Exit Sketch' }).click()
// exit sketch
await u.openAndClearDebugPanel()
await page.getByRole('button', { name: 'Exit Sketch' }).click()
// wait for execution done
await u.expectCmdLog('[data-message-type="execution-done"]')
await u.clearAndCloseDebugPanel()
await page.waitForTimeout(200)
// wait for execution done
await u.expectCmdLog('[data-message-type="execution-done"]')
await u.clearAndCloseDebugPanel()
await page.waitForTimeout(200)
// second screen shot should look almost identical, i.e. scale should be the same.
await expect(page).toHaveScreenshot({
maxDiffPixels: 100,
})
})
test.describe('Client side scene scale should match engine scale - Millimeters', () => {
const storageState = structuredClone(basicStorageState)
storageState.origins[0].localStorage[2].value = TOML.stringify({
settings: {
...basicSettings,
modeling: {
...basicSettings.modeling,
defaultUnit: 'mm',
},
},
})
test.use({
storageState,
// second screen shot should look almost identical, i.e. scale should be the same.
await expect(page).toHaveScreenshot({
maxDiffPixels: 100,
})
})
test('Millimeters', async ({ page }) => {
await page.addInitScript(
async ({ settingsKey, settings }) => {
localStorage.setItem(settingsKey, settings)
},
{
settingsKey: TEST_SETTINGS_KEY,
settings: TOML.stringify({
settings: {
...TEST_SETTINGS,
modeling: {
...TEST_SETTINGS.modeling,
defaultUnit: 'mm',
},
},
}),
}
)
const u = getUtils(page)
await page.setViewportSize({ width: 1200, height: 500 })
const PUR = 400 / 37.5 //pixeltoUnitRatio

Binary file not shown.

Before

Width:  |  Height:  |  Size: 44 KiB

After

Width:  |  Height:  |  Size: 43 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 48 KiB

After

Width:  |  Height:  |  Size: 47 KiB

View File

@ -1,9 +1,8 @@
import { SaveSettingsPayload } from 'lib/settings/settingsTypes'
import { secrets } from './secrets'
import * as TOML from '@iarna/toml'
import { Themes } from 'lib/theme'
export const basicSettings = {
export const TEST_SETTINGS_KEY = '/user.toml'
export const TEST_SETTINGS = {
app: {
theme: Themes.Dark,
onboardingStatus: 'dismissed',
@ -22,19 +21,26 @@ export const basicSettings = {
},
} satisfies Partial<SaveSettingsPayload>
export const basicStorageState = {
cookies: [],
origins: [
{
origin: 'http://localhost:3000',
localStorage: [
{ name: 'TOKEN_PERSIST_KEY', value: secrets.token },
{ name: 'persistCode', value: '' },
{
name: '/user.toml',
value: TOML.stringify({ settings: basicSettings }),
},
],
},
],
}
export const TEST_SETTINGS_ONBOARDING = {
...TEST_SETTINGS,
app: { ...TEST_SETTINGS.app, onboardingStatus: '/export ' },
} satisfies Partial<SaveSettingsPayload>
export const TEST_SETTINGS_CORRUPTED = {
app: {
theme: Themes.Dark,
onboardingStatus: 'dismissed',
projectDirectory: 123 as any,
},
modeling: {
defaultUnit: 'invalid' as any,
mouseControls: `() => alert('hack the planet')` as any,
showDebugPanel: true,
},
projects: {
defaultProjectName: false as any,
},
textEditor: {
textWrapping: true,
},
} satisfies Partial<SaveSettingsPayload>

View File

@ -1,5 +1,4 @@
import { defineConfig, devices } from '@playwright/test'
import { basicStorageState } from './e2e/playwright/storageStates'
/**
* Read environment variables from file.
@ -29,9 +28,6 @@ export default defineConfig({
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
trace: 'on-first-retry',
/* Use a common shared localStorage */
storageState: basicStorageState,
},
/* Configure projects for major browsers */