skip network tests mac

* Trigger shutdown operations after each test

* Idle mode states

* Don't show the reconnect when coming back from tab

e2e/playwright/flow-tests.spec.ts

@@ -52,7 +52,24 @@ const commonPoints = {
   // num2: 19.19,
 }
 
-// Utilities for writing tests that depend on test values
+test.afterEach(async ({ context, page }, testInfo) => {
+  if (testInfo.status === 'skipped') return
+  if (testInfo.status === 'failed') return
+
+  const u = await getUtils(page)
+  // Kill the network so shutdown happens properly
+  await u.emulateNetworkConditions({
+    offline: true,
+    // values of 0 remove any active throttling. crbug.com/456324#c9
+    latency: 0,
+    downloadThroughput: -1,
+    uploadThroughput: -1,
+  })
+
+  // It seems it's best to give the browser about 3s to close things
+  // It's not super reliable but we have no real other choice for now
+  await page.waitForTimeout(3000)
+})
 
 test.beforeEach(async ({ context, page }) => {
   // wait for Vite preview server to be up
@@ -78,7 +95,7 @@ test.beforeEach(async ({ context, page }) => {
   await page.emulateMedia({ reducedMotion: 'reduce' })
 })
 
-test.setTimeout(60000)
+test.setTimeout(120000)
 
 async function doBasicSketch(page: Page, openPanes: string[]) {
   const u = await getUtils(page)
@@ -6705,6 +6722,11 @@ ${extraLine ? 'const myVar = segLen(seg01, part001)' : ''}`
 
 test.describe('Test network and connection issues', () => {
   test('simulate network down and network little widget', async ({ page }) => {
+    const browserType = page.context().browser()?.browserType().name()
+    test.skip(
+      browserType !== 'chromium',
+      'emulateNetworkConditions only works in chromium'
+    )
     const u = await getUtils(page)
     await page.setViewportSize({ width: 1200, height: 500 })
 
@@ -6775,6 +6797,11 @@ test.describe('Test network and connection issues', () => {
   })
 
   test('Engine disconnect & reconnect in sketch mode', async ({ page }) => {
+    const browserType = page.context().browser()?.browserType().name()
+    test.skip(
+      browserType !== 'chromium',
+      'emulateNetworkConditions only works in chromium'
+    )
     const u = await getUtils(page)
     await page.setViewportSize({ width: 1200, height: 500 })
     const PUR = 400 / 37.5 //pixeltoUnitRatio

e2e/playwright/test-utils.ts

@@ -312,9 +312,9 @@ export async function getUtils(page: Page) {
         fullPage: true,
       })
       const screenshot = await PNG.sync.read(buffer)
-      // most likely related to pixel density but the screenshots for webkit are 2x the size
-      // there might be a more robust way of doing this.
-      const pixMultiplier = browserType === 'webkit' ? 2 : 1
+      const pixMultiplier: number = await page.evaluate(
+        'window.devicePixelRatio'
+      )
       const index =
         (screenshot.width * coords.y * pixMultiplier +
           coords.x * pixMultiplier) *
@@ -377,11 +377,13 @@ export async function getUtils(page: Page) {
     emulateNetworkConditions: async (
       networkOptions: Protocol.Network.emulateNetworkConditionsParameters
     ) => {
-      // Skip on non-Chromium browsers, since we need to use the CDP.
-      test.skip(
-        cdpSession === null,
-        'Network emulation is only supported in Chromium'
-      )
+      if (browserType !== 'chromium') {
+        console.warn('emulateNetworkConditions will not work on this browser')
+      }
+      if (cdpSession === null) {
+        // Use a fail safe if we can't simulate disconnect (on Safari)
+        return page.evaluate('window.tearDown()')
+      }
 
       cdpSession?.send('Network.emulateNetworkConditions', networkOptions)
     },

src/components/ModelingMachineProvider.tsx

@@ -163,6 +163,8 @@ export const ModelingMachineProvider = ({
 
             store.videoElement?.pause()
             kclManager.executeCode(true).then(() => {
+              if (engineCommandManager.engineConnection?.freezeFrame) return
+
               store.videoElement?.play()
             })
           })()

src/components/Stream.tsx

@@ -8,7 +8,7 @@ import { NetworkHealthState } from 'hooks/useNetworkStatus'
 import { ClientSideScene } from 'clientSideScene/ClientSideSceneComp'
 import { butName } from 'lib/cameraControls'
 import { sendSelectEventToEngine } from 'lib/selections'
-import { kclManager } from 'lib/singletons'
+import { kclManager, engineCommandManager } from 'lib/singletons'
 
 export const Stream = () => {
   const [isLoading, setIsLoading] = useState(true)
@@ -18,6 +18,7 @@ export const Stream = () => {
   const { settings } = useSettingsAuthContext()
   const { state, send, context } = useModelingContext()
   const { overallState } = useNetworkContext()
+  const [isFreezeFrame, setIsFreezeFrame] = useState(false)
 
   const isNetworkOkay =
     overallState === NetworkHealthState.Ok ||
@@ -49,14 +50,69 @@ export const Stream = () => {
     globalThis?.window?.document?.addEventListener('paste', handlePaste, {
       capture: true,
     })
-    return () =>
+
+    // Teardown everything if we go hidden or reconnect
+    if (globalThis?.window?.document) {
+      globalThis.window.document.onvisibilitychange = () => {
+        if (globalThis.window.document.visibilityState === 'hidden') {
+          videoRef.current?.pause()
+          setIsFreezeFrame(true)
+          window.requestAnimationFrame(() => {
+            engineCommandManager.engineConnection?.tearDown({ freeze: true })
+          })
+        } else {
+          engineCommandManager.engineConnection?.connect(true)
+        }
+      }
+    }
+
+    const IDLE_TIME_MS = 1000 * 20
+    let timeoutIdIdle: ReturnType<typeof setTimeout> | undefined = undefined
+
+    const onIdle = () => {
+      videoRef.current?.pause()
+      setIsFreezeFrame(true)
+      kclManager.isFirstRender = true
+      setIsFirstRender(true)
+      // Give video time to pause
+      window.requestAnimationFrame(() => {
+        engineCommandManager.engineConnection?.tearDown({ freeze: true })
+      })
+    }
+    const onAnyInput = () => {
+      if (!engineCommandManager.engineConnection?.isReady()) {
+        engineCommandManager.engineConnection?.connect(true)
+      }
+      clearTimeout(timeoutIdIdle)
+      timeoutIdIdle = setTimeout(onIdle, IDLE_TIME_MS)
+    }
+
+    globalThis?.window?.document?.addEventListener('keydown', onAnyInput)
+    globalThis?.window?.document?.addEventListener('mousemove', onAnyInput)
+    globalThis?.window?.document?.addEventListener('mousedown', onAnyInput)
+    globalThis?.window?.document?.addEventListener('scroll', onAnyInput)
+    globalThis?.window?.document?.addEventListener('touchstart', onAnyInput)
+
+    timeoutIdIdle = setTimeout(onIdle, IDLE_TIME_MS)
+
+    return () => {
       globalThis?.window?.document?.removeEventListener('paste', handlePaste, {
         capture: true,
       })
+      globalThis?.window?.document?.removeEventListener('keydown', onAnyInput)
+      globalThis?.window?.document?.removeEventListener('mousemove', onAnyInput)
+      globalThis?.window?.document?.removeEventListener('mousedown', onAnyInput)
+      globalThis?.window?.document?.removeEventListener('scroll', onAnyInput)
+      globalThis?.window?.document?.removeEventListener(
+        'touchstart',
+        onAnyInput
+      )
+    }
   }, [])
 
   useEffect(() => {
     setIsFirstRender(kclManager.isFirstRender)
+    if (!kclManager.isFirstRender) videoRef.current?.play()
   }, [kclManager.isFirstRender])
 
   useEffect(() => {
@@ -67,7 +123,10 @@ export const Stream = () => {
       return
     if (!videoRef.current) return
     if (!context.store?.mediaStream) return
+
+    // Do not immediately play the stream!
     videoRef.current.srcObject = context.store.mediaStream
+    videoRef.current.pause()
 
     send({
       type: 'Set context',
@@ -172,17 +231,12 @@ export const Stream = () => {
       <ClientSideScene
         cameraControls={settings.context.modeling.mouseControls.current}
       />
-      {!isNetworkOkay && !isLoading && (
+      {(!isNetworkOkay || isLoading || isFirstRender) && !isFreezeFrame && (
         <div className="text-center absolute inset-0">
           <Loading>
-            <span data-testid="loading-stream">Stream disconnected...</span>
-          </Loading>
-        </div>
-      )}
-      {(isLoading || isFirstRender) && (
-        <div className="text-center absolute inset-0">
-          <Loading>
-            {!isLoading && isFirstRender ? (
+            {!isNetworkOkay && !isLoading ? (
+              <span data-testid="loading-stream">Stream disconnected...</span>
+            ) : !isLoading && isFirstRender ? (
               <span data-testid="loading-stream">Building scene...</span>
             ) : (
               <span data-testid="loading-stream">Loading stream...</span>

src/lang/std/engineConnection.ts

@@ -300,6 +300,7 @@ class EngineConnection extends EventTarget {
   pc?: RTCPeerConnection
   unreliableDataChannel?: RTCDataChannel
   mediaStream?: MediaStream
+  freezeFrame: boolean = false
 
   private _state: EngineConnectionState = {
     type: EngineConnectionStateType.Fresh,
@@ -365,7 +366,11 @@ class EngineConnection extends EventTarget {
     this.pingPongSpan = { ping: undefined, pong: undefined }
 
     // Without an interval ping, our connection will timeout.
+    // If this.freezeFrame is true we skip this logic so only reconnect
+    // happens on mouse move
     setInterval(() => {
+      if (this.freezeFrame) return
+
       switch (this.state.type as EngineConnectionStateType) {
         case EngineConnectionStateType.ConnectionEstablished:
           // If there was no reply to the last ping, report a timeout.
@@ -426,7 +431,8 @@ class EngineConnection extends EventTarget {
     return this.state.type === EngineConnectionStateType.ConnectionEstablished
   }
 
-  tearDown() {
+  tearDown(opts?: { freeze: boolean }) {
+    this.freezeFrame = opts?.freeze ?? false
     this.disconnectAll()
     this.state = {
       type: EngineConnectionStateType.Disconnecting,
@@ -996,6 +1002,9 @@ class EngineConnection extends EventTarget {
       this.pc?.connectionState === 'closed' &&
       this.unreliableDataChannel?.readyState === 'closed'
     if (allClosed) {
+      // Do not notify the rest of the program that we have cut off anything.
+      if (this.freezeFrame) return
+
       this.state = { type: EngineConnectionStateType.Disconnected }
     }
   }
@@ -1619,7 +1628,15 @@ export class EngineCommandManager extends EventTarget {
     }
   }
   tearDown() {
-    this.engineConnection?.tearDown()
+    if (this.engineConnection) {
+      this.engineConnection?.tearDown()
+      // Our window.tearDown assignment causes this case to happen which is
+      // only really for tests.
+      // @ts-ignore
+    } else if (this.engineCommandManager?.engineConnection) {
+      // @ts-ignore
+      this.engineCommandManager?.engineConnection?.tearDown()
+    }
   }
   async startNewSession() {
     this.lastArtifactMap = this.artifactMap

src/lib/singletons.ts

@@ -9,6 +9,10 @@ export const codeManager = new CodeManager()
 
 export const engineCommandManager = new EngineCommandManager()
 
+// Accessible for tests mostly
+// @ts-ignore
+window.tearDown = engineCommandManager.tearDown
+
 // This needs to be after codeManager is created.
 export const kclManager = new KclManager(engineCommandManager)
 kclManager.isFirstRender = true

src/machines/modelingMachine.ts

@@ -1051,7 +1051,9 @@ export const modelingMachine = createMachine(
             type: 'start_path',
           },
         })
-        store.videoElement?.play()
+        if (!engineCommandManager.engineConnection?.freezeFrame) {
+          store.videoElement?.play()
+        }
         if (updatedAst?.selections) {
           editorManager.selectRange(updatedAst?.selections)
         }