diff --git a/e2e/playwright/fixtures/fixtureSetup.ts b/e2e/playwright/fixtures/fixtureSetup.ts index 7e5897fde..5b19806c8 100644 --- a/e2e/playwright/fixtures/fixtureSetup.ts +++ b/e2e/playwright/fixtures/fixtureSetup.ts @@ -90,8 +90,9 @@ export class ElectronZoo { constructor() {} + // Help remote end by signaling we're done with the connection. + // If it takes longer than 10s to stop, just resolve. async makeAvailableAgain() { - // Help remote end by signaling we're done with the connection. await this.page.evaluate(async () => { return new Promise((resolve) => { if (!window.engineCommandManager.engineConnection?.state?.type) { @@ -99,7 +100,9 @@ export class ElectronZoo { } window.engineCommandManager.tearDown() + // Keep polling (per js event tick) until state is Disconnected. + const timeA = Date.now() const checkDisconnected = () => { // It's possible we never even created an engineConnection // e.g. never left Projects view. @@ -109,6 +112,11 @@ export class ElectronZoo { ) { return resolve(undefined) } + + if (Date.now() - timeA > 10000) { + return resolve(undefined) + } + setTimeout(checkDisconnected, 0) } checkDisconnected() @@ -130,6 +138,7 @@ export class ElectronZoo { const that = this const options = { + timeout: 120000, args: ['.', '--no-sandbox'], env: { ...process.env, @@ -155,8 +164,28 @@ export class ElectronZoo { // Do this once and then reuse window on subsequent calls. if (!this.electron) { this.electron = await electron.launch(options) + + // Mac takes quite a long time to create the first window in CI. + // Turns out we can't trust firstWindow() either. So loop. + let timeoutId: ReturnType + const tryToGetWindowPage = () => + new Promise((resolve) => { + const fn = () => { + this.page = this.electron.windows()[0] + timeoutId = setTimeout(() => { + if (this.page) { + clearTimeout(timeoutId) + return resolve(undefined) + } + fn() + }, 0) + } + fn() + }) + + await tryToGetWindowPage() + this.context = this.electron.context() - this.page = await this.electron.firstWindow() await this.context.tracing.start({ screenshots: true, snapshots: true }) } @@ -304,16 +333,13 @@ const fixturesForElectron = { use: FnUse, testInfo: TestInfo ) => { - await tronApp.createInstanceIfMissing(testInfo) await use(tronApp.page) - await tronApp?.makeAvailableAgain() }, context: async ( { tronApp }: { tronApp: ElectronZoo }, use: FnUse, testInfo: TestInfo ) => { - await tronApp.createInstanceIfMissing(testInfo) await use(tronApp.context) }, } diff --git a/e2e/playwright/prompt-to-edit.spec.ts b/e2e/playwright/prompt-to-edit.spec.ts index 7f4976b53..15014b74b 100644 --- a/e2e/playwright/prompt-to-edit.spec.ts +++ b/e2e/playwright/prompt-to-edit.spec.ts @@ -233,7 +233,7 @@ test.describe('Prompt-to-edit tests', { tag: '@skipWin' }, () => { await cmdBar.openCmdBar('promptToEdit') await page .getByTestId('cmd-bar-arg-value') - .fill('Please rename to mySketch') + .fill('Please rename to mySketch001') await page.waitForTimeout(100) await cmdBar.progressCmdBar() await expect(submittingToast).toBeVisible() @@ -244,10 +244,10 @@ test.describe('Prompt-to-edit tests', { tag: '@skipWin' }, () => { }) await test.step('verify rename change and accept it', async () => { - await editor.expectEditor.toContain('mySketch = startSketchOn') + await editor.expectEditor.toContain('mySketch001 = startSketchOn') await editor.expectEditor.not.toContain('sketch002 = startSketchOn') await editor.expectEditor.toContain( - 'extrude002 = extrude(mySketch, length = 50)' + 'extrude002 = extrude(mySketch001, length = 50)' ) await acceptBtn.click() diff --git a/e2e/playwright/test-network-and-connection-issues.spec.ts b/e2e/playwright/test-network-and-connection-issues.spec.ts index 9d861cf16..410741334 100644 --- a/e2e/playwright/test-network-and-connection-issues.spec.ts +++ b/e2e/playwright/test-network-and-connection-issues.spec.ts @@ -84,7 +84,6 @@ test.describe('Test network and connection issues', () => { 'Engine disconnect & reconnect in sketch mode', { tag: '@skipLocalEngine' }, async ({ page, homePage }) => { - // TODO: Don't skip Mac for these. After `window.engineCommandManager.tearDown` is working in Safari, these should work on webkit const networkToggle = page.getByTestId('network-toggle') const u = await getUtils(page) diff --git a/e2e/playwright/text-to-cad-tests.spec.ts b/e2e/playwright/text-to-cad-tests.spec.ts index 38ccfc347..6f19f18f3 100644 --- a/e2e/playwright/text-to-cad-tests.spec.ts +++ b/e2e/playwright/text-to-cad-tests.spec.ts @@ -492,8 +492,15 @@ test.describe('Text-to-CAD tests', { tag: ['@skipWin'] }, () => { // Click the button. await copyToClipboardButton.first().click() - // Expect the code to be pasted. - await expect(page.locator('.cm-content')).toContainText(`2x8`) + // Do NOT do AI tests like this: "Expect the code to be pasted." + // Reason: AI tests are NONDETERMINISTIC. Thus we need to be as most + // general as we can for the assertion. + // We can use Kolmogorov complexity as a measurement of the + // "probably most minimal version of this program" to have a lower + // bound to work with. It is completely by feel because there are + // no proofs that any program is its smallest self. + const code2x8 = await page.locator('.cm-content').innerText() + await expect(code2x8.length).toBeGreaterThan(249) // Ensure the final toast remains. await expect(page.getByText(`a 2x10 lego`)).not.toBeVisible() @@ -506,7 +513,8 @@ test.describe('Text-to-CAD tests', { tag: ['@skipWin'] }, () => { await copyToClipboardButton.click() // Expect the code to be pasted. - await expect(page.locator('.cm-content')).toContainText(`2x4`) + const code2x4 = await page.locator('.cm-content').innerText() + await expect(code2x4.length).toBeGreaterThan(249) } ) diff --git a/e2e/playwright/zoo-test.ts b/e2e/playwright/zoo-test.ts index 3e3c433da..3d73c652a 100644 --- a/e2e/playwright/zoo-test.ts +++ b/e2e/playwright/zoo-test.ts @@ -45,7 +45,9 @@ const playwrightTestFnWithFixtures_ = playwrightTestFn.extend<{ return } + await electronZooInstance.createInstanceIfMissing(testInfo) await use(electronZooInstance) + await electronZooInstance.makeAvailableAgain() }, })