diff --git a/e2e/playwright/flow-tests.spec.ts b/e2e/playwright/flow-tests.spec.ts index b9db33e86..a0f52ed83 100644 --- a/e2e/playwright/flow-tests.spec.ts +++ b/e2e/playwright/flow-tests.spec.ts @@ -52,7 +52,24 @@ const commonPoints = { // num2: 19.19, } -// Utilities for writing tests that depend on test values +test.afterEach(async ({ context, page }, testInfo) => { + if (testInfo.status === 'skipped') return + if (testInfo.status === 'failed') return + + const u = await getUtils(page) + // Kill the network so shutdown happens properly + await u.emulateNetworkConditions({ + offline: true, + // values of 0 remove any active throttling. crbug.com/456324#c9 + latency: 0, + downloadThroughput: -1, + uploadThroughput: -1, + }) + + // It seems it's best to give the browser about 3s to close things + // It's not super reliable but we have no real other choice for now + await page.waitForTimeout(3000) +}) test.beforeEach(async ({ context, page }) => { // wait for Vite preview server to be up @@ -78,7 +95,7 @@ test.beforeEach(async ({ context, page }) => { await page.emulateMedia({ reducedMotion: 'reduce' }) }) -test.setTimeout(60000) +test.setTimeout(120000) async function doBasicSketch(page: Page, openPanes: string[]) { const u = await getUtils(page) diff --git a/e2e/playwright/test-utils.ts b/e2e/playwright/test-utils.ts index daaea2adf..b0735538e 100644 --- a/e2e/playwright/test-utils.ts +++ b/e2e/playwright/test-utils.ts @@ -312,9 +312,9 @@ export async function getUtils(page: Page) { fullPage: true, }) const screenshot = await PNG.sync.read(buffer) - // most likely related to pixel density but the screenshots for webkit are 2x the size - // there might be a more robust way of doing this. - const pixMultiplier = browserType === 'webkit' ? 2 : 1 + const pixMultiplier: number = await page.evaluate( + 'window.devicePixelRatio' + ) const index = (screenshot.width * coords.y * pixMultiplier + coords.x * pixMultiplier) * @@ -377,11 +377,10 @@ export async function getUtils(page: Page) { emulateNetworkConditions: async ( networkOptions: Protocol.Network.emulateNetworkConditionsParameters ) => { - // Skip on non-Chromium browsers, since we need to use the CDP. - test.skip( - cdpSession === null, - 'Network emulation is only supported in Chromium' - ) + if (cdpSession === null) { + // Use a fail safe if we can't simulate disconnect (on Safari) + return page.evaluate('window.tearDown()') + } cdpSession?.send('Network.emulateNetworkConditions', networkOptions) }, diff --git a/src/components/ModelingMachineProvider.tsx b/src/components/ModelingMachineProvider.tsx index 833255be8..045429518 100644 --- a/src/components/ModelingMachineProvider.tsx +++ b/src/components/ModelingMachineProvider.tsx @@ -163,6 +163,8 @@ export const ModelingMachineProvider = ({ store.videoElement?.pause() kclManager.executeCode(true).then(() => { + if (engineCommandManager.engineConnection?.freezeFrame) return + store.videoElement?.play() }) })() diff --git a/src/components/Stream.tsx b/src/components/Stream.tsx index 31a55e831..d16a64caf 100644 --- a/src/components/Stream.tsx +++ b/src/components/Stream.tsx @@ -8,7 +8,7 @@ import { NetworkHealthState } from 'hooks/useNetworkStatus' import { ClientSideScene } from 'clientSideScene/ClientSideSceneComp' import { butName } from 'lib/cameraControls' import { sendSelectEventToEngine } from 'lib/selections' -import { kclManager } from 'lib/singletons' +import { kclManager, engineCommandManager } from 'lib/singletons' export const Stream = () => { const [isLoading, setIsLoading] = useState(true) @@ -18,6 +18,7 @@ export const Stream = () => { const { settings } = useSettingsAuthContext() const { state, send, context } = useModelingContext() const { overallState } = useNetworkContext() + const [isFreezeFrame, setIsFreezeFrame] = useState(false) const isNetworkOkay = overallState === NetworkHealthState.Ok || @@ -49,14 +50,69 @@ export const Stream = () => { globalThis?.window?.document?.addEventListener('paste', handlePaste, { capture: true, }) - return () => + + // Teardown everything if we go hidden or reconnect + if (globalThis?.window?.document) { + globalThis.window.document.onvisibilitychange = () => { + if (globalThis.window.document.visibilityState === 'hidden') { + videoRef.current?.pause() + setIsFreezeFrame(true) + window.requestAnimationFrame(() => { + engineCommandManager.engineConnection?.tearDown({ freeze: true }) + }) + } else { + engineCommandManager.engineConnection?.connect(true) + } + } + } + + const IDLE_TIME_MS = 1000 * 20 + let timeoutIdIdle: ReturnType | undefined = undefined + + const onIdle = () => { + videoRef.current?.pause() + setIsFreezeFrame(true) + kclManager.isFirstRender = true + setIsFirstRender(true) + // Give video time to pause + window.requestAnimationFrame(() => { + engineCommandManager.engineConnection?.tearDown({ freeze: true }) + }) + } + const onAnyInput = () => { + if (!engineCommandManager.engineConnection?.isReady()) { + engineCommandManager.engineConnection?.connect(true) + } + clearTimeout(timeoutIdIdle) + timeoutIdIdle = setTimeout(onIdle, IDLE_TIME_MS) + } + + globalThis?.window?.document?.addEventListener('keydown', onAnyInput) + globalThis?.window?.document?.addEventListener('mousemove', onAnyInput) + globalThis?.window?.document?.addEventListener('mousedown', onAnyInput) + globalThis?.window?.document?.addEventListener('scroll', onAnyInput) + globalThis?.window?.document?.addEventListener('touchstart', onAnyInput) + + timeoutIdIdle = setTimeout(onIdle, IDLE_TIME_MS) + + return () => { globalThis?.window?.document?.removeEventListener('paste', handlePaste, { capture: true, }) + globalThis?.window?.document?.removeEventListener('keydown', onAnyInput) + globalThis?.window?.document?.removeEventListener('mousemove', onAnyInput) + globalThis?.window?.document?.removeEventListener('mousedown', onAnyInput) + globalThis?.window?.document?.removeEventListener('scroll', onAnyInput) + globalThis?.window?.document?.removeEventListener( + 'touchstart', + onAnyInput + ) + } }, []) useEffect(() => { setIsFirstRender(kclManager.isFirstRender) + if (!kclManager.isFirstRender) videoRef.current?.play() }, [kclManager.isFirstRender]) useEffect(() => { @@ -67,7 +123,10 @@ export const Stream = () => { return if (!videoRef.current) return if (!context.store?.mediaStream) return + + // Do not immediately play the stream! videoRef.current.srcObject = context.store.mediaStream + videoRef.current.pause() send({ type: 'Set context', @@ -172,17 +231,12 @@ export const Stream = () => { - {!isNetworkOkay && !isLoading && ( + {(!isNetworkOkay || isLoading || isFirstRender) && !isFreezeFrame && (
- Stream disconnected... - -
- )} - {(isLoading || isFirstRender) && ( -
- - {!isLoading && isFirstRender ? ( + {!isNetworkOkay && !isLoading ? ( + Stream disconnected... + ) : !isLoading && isFirstRender ? ( Building scene... ) : ( Loading stream... diff --git a/src/lang/std/engineConnection.ts b/src/lang/std/engineConnection.ts index 7dc1b6540..d03f3e6ec 100644 --- a/src/lang/std/engineConnection.ts +++ b/src/lang/std/engineConnection.ts @@ -300,6 +300,7 @@ class EngineConnection extends EventTarget { pc?: RTCPeerConnection unreliableDataChannel?: RTCDataChannel mediaStream?: MediaStream + freezeFrame: boolean = false private _state: EngineConnectionState = { type: EngineConnectionStateType.Fresh, @@ -365,7 +366,11 @@ class EngineConnection extends EventTarget { this.pingPongSpan = { ping: undefined, pong: undefined } // Without an interval ping, our connection will timeout. + // If this.freezeFrame is true we skip this logic so only reconnect + // happens on mouse move setInterval(() => { + if (this.freezeFrame) return + switch (this.state.type as EngineConnectionStateType) { case EngineConnectionStateType.ConnectionEstablished: // If there was no reply to the last ping, report a timeout. @@ -426,7 +431,8 @@ class EngineConnection extends EventTarget { return this.state.type === EngineConnectionStateType.ConnectionEstablished } - tearDown() { + tearDown(opts?: { freeze: boolean }) { + this.freezeFrame = opts?.freeze ?? false this.disconnectAll() this.state = { type: EngineConnectionStateType.Disconnecting, @@ -996,6 +1002,9 @@ class EngineConnection extends EventTarget { this.pc?.connectionState === 'closed' && this.unreliableDataChannel?.readyState === 'closed' if (allClosed) { + // Do not notify the rest of the program that we have cut off anything. + if (this.freezeFrame) return + this.state = { type: EngineConnectionStateType.Disconnected } } } @@ -1619,7 +1628,15 @@ export class EngineCommandManager extends EventTarget { } } tearDown() { - this.engineConnection?.tearDown() + if (this.engineConnection) { + this.engineConnection?.tearDown() + // Our window.tearDown assignment causes this case to happen which is + // only really for tests. + // @ts-ignore + } else if (this.engineCommandManager?.engineConnection) { + // @ts-ignore + this.engineCommandManager?.engineConnection?.tearDown() + } } async startNewSession() { this.lastArtifactMap = this.artifactMap diff --git a/src/lib/singletons.ts b/src/lib/singletons.ts index a29e0dda4..01f4e6838 100644 --- a/src/lib/singletons.ts +++ b/src/lib/singletons.ts @@ -9,6 +9,10 @@ export const codeManager = new CodeManager() export const engineCommandManager = new EngineCommandManager() +// Accessible for tests mostly +// @ts-ignore +window.tearDown = engineCommandManager.tearDown + // This needs to be after codeManager is created. export const kclManager = new KclManager(engineCommandManager) kclManager.isFirstRender = true diff --git a/src/machines/modelingMachine.ts b/src/machines/modelingMachine.ts index 0a0f52834..f097610c0 100644 --- a/src/machines/modelingMachine.ts +++ b/src/machines/modelingMachine.ts @@ -1051,7 +1051,9 @@ export const modelingMachine = createMachine( type: 'start_path', }, }) - store.videoElement?.play() + if (!engineCommandManager.engineConnection?.freezeFrame) { + store.videoElement?.play() + } if (updatedAst?.selections) { editorManager.selectRange(updatedAst?.selections) }