Try to avoid the black screen again & improve error messages (#7327)

* Fix the black screen of death

* fmt

* make check

* Clean up

* Fix up zoom to fit

* Change how emulateNetworkConditions work

* Do NOT use browser's offline/online mechanisms

* Fix test

* Improve network error messages

* Signal offline when failed event comes in

* Don't use logic on components that only want a loader

* Remove unnecessary pause state transition

---------

Co-authored-by: jacebrowning <jacebrowning@gmail.com>

e2e/playwright/test-network-and-connection-issues.spec.ts

@@ -63,7 +63,7 @@ test.describe('Test network related behaviors', () => {
       })
 
       // Expect the network to be down
-      await expect(networkToggle).toContainText('Problem')
+      await expect(networkToggle).toContainText('Network health (Offline)')
 
       // Click the network widget
       await networkWidget.click()
@@ -160,7 +160,8 @@ test.describe('Test network related behaviors', () => {
 
       // Expect the network to be down
       await networkToggle.hover()
-      await expect(networkToggle).toContainText('Problem')
+
+      await expect(networkToggle).toContainText('Network health (Offline)')
 
       // Ensure we are not in sketch mode
       await expect(

e2e/playwright/test-utils.ts

@@ -364,11 +364,6 @@ export async function getUtils(page: Page, test_?: typeof test) {
     )
   }
 
-  // Chrome devtools protocol session only works in Chromium
-  const browserType = page.context().browser()?.browserType().name()
-  const cdpSession =
-    browserType !== 'chromium' ? null : await page.context().newCDPSession(page)
-
   const util = {
     waitForAuthSkipAppStart: () => waitForAuthAndLsp(page),
     waitForPageLoad: () => waitForPageLoad(page),
@@ -489,15 +484,9 @@ export async function getUtils(page: Page, test_?: typeof test) {
     emulateNetworkConditions: async (
       networkOptions: Protocol.Network.emulateNetworkConditionsParameters
     ) => {
-      if (cdpSession === null) {
+      return networkOptions.offline
-        // Use a fail safe if we can't simulate disconnect (on Safari)
+        ? page.evaluate('window.engineCommandManager.offline()')
-        return page.evaluate('window.engineCommandManager.tearDown()')
+        : page.evaluate('window.engineCommandManager.online()')
-      }
-
-      return cdpSession?.send(
-        'Network.emulateNetworkConditions',
-        networkOptions
-      )
     },
 
     toNormalizedCode(text: string) {

src/App.tsx

@@ -114,6 +114,9 @@ export function App() {
     // by the Projects view.
     billingActor.send({ type: BillingTransition.Update, apiToken: authToken })
 
+    // Tell engineStream to wait for dependencies to start streaming.
+    engineStreamActor.send({ type: EngineStreamTransition.WaitForDependencies })
+
     // When leaving the modeling scene, cut the engine stream.
     return () => {
       // When leaving the modeling scene, cut the engine stream.

src/clientSideScene/sceneEntities.ts

@@ -3588,7 +3588,8 @@ export class SceneEntities {
     })
 
     if (!resp) {
-      return Promise.reject('no response')
+      console.warn('No response')
+      return {} as Models['GetSketchModePlane_type']
     }
 
     if (isArray(resp)) {

src/components/EngineStream.tsx

@@ -1,4 +1,3 @@
-import { isPlaywright } from '@src/lib/isPlaywright'
 import { useAppState } from '@src/AppState'
 import { ClientSideScene } from '@src/clientSideScene/ClientSideSceneComp'
 import { ViewControlContextMenu } from '@src/components/ViewControlMenu'
@@ -52,8 +51,10 @@ export const EngineStream = (props: {
   const last = useRef<number>(Date.now())
 
   const [firstPlay, setFirstPlay] = useState(true)
-  const [isRestartRequestStarting, setIsRestartRequestStarting] =
+  const [goRestart, setGoRestart] = useState(false)
-    useState(false)
+  const [timeoutId, setTimeoutId] = useState<
+    ReturnType<typeof setTimeout> | undefined
+  >(undefined)
   const [attemptTimes, setAttemptTimes] = useState<[number, number]>([
     0,
     TIME_1_SECOND,
@@ -85,18 +86,21 @@ export const EngineStream = (props: {
   const streamIdleMode = settings.app.streamIdleMode.current
 
   useEffect(() => {
+    // Will cause a useEffect loop if not checked for.
+    if (engineStreamState.context.videoRef.current !== null) return
     engineStreamActor.send({
       type: EngineStreamTransition.SetVideoRef,
       videoRef: { current: videoRef.current },
     })
-  }, [videoRef.current])
+  }, [videoRef.current, engineStreamState])
 
   useEffect(() => {
+    if (engineStreamState.context.canvasRef.current !== null) return
     engineStreamActor.send({
       type: EngineStreamTransition.SetCanvasRef,
       canvasRef: { current: canvasRef.current },
     })
-  }, [canvasRef.current])
+  }, [canvasRef.current, engineStreamState])
 
   useEffect(() => {
     engineStreamActor.send({
@@ -131,24 +135,6 @@ export const EngineStream = (props: {
     })
   }
 
-  useEffect(() => {
-    // Only try to start the stream if we're stopped or think we're done
-    // waiting for dependencies.
-    if (
-      !(
-        engineStreamState.value === EngineStreamState.WaitingForDependencies ||
-        engineStreamState.value === EngineStreamState.Stopped
-      )
-    )
-      return
-
-    // Don't bother trying to connect if the auth token is empty.
-    // We have the checks in the machine but this can cause a hot loop.
-    if (!engineStreamState.context.authToken) return
-
-    startOrReconfigureEngine()
-  }, [engineStreamState, setAppState])
-
   // I would inline this but it needs to be a function for removeEventListener.
   const play = () => {
     engineStreamActor.send({
@@ -174,12 +160,13 @@ export const EngineStream = (props: {
     console.log('scene is ready, execute kcl')
     const kmp = kclManager.executeCode().catch(trap)
 
-    if (!firstPlay) return
-
-    setFirstPlay(false)
     // Reset the restart timeouts
     setAttemptTimes([0, TIME_1_SECOND])
 
+    console.log(firstPlay)
+    if (!firstPlay) return
+
+    setFirstPlay(false)
     console.log('firstPlay true, zoom to fit')
     kmp
       .then(async () => {
@@ -211,51 +198,76 @@ export const EngineStream = (props: {
     // We do a back-off restart, using a fibonacci sequence, since it
     // has a nice retry time curve (somewhat quick then exponential)
     const attemptRestartIfNecessary = () => {
-      if (isRestartRequestStarting) return
+      // Timeout already set.
-      setIsRestartRequestStarting(true)
+      if (timeoutId) return
-      setTimeout(() => {
+
-        engineStreamState.context.videoRef.current?.pause()
+      setTimeoutId(
-        engineCommandManager.tearDown()
+        setTimeout(() => {
-        startOrReconfigureEngine()
+          engineStreamState.context.videoRef.current?.pause()
-        setFirstPlay(false)
+          engineCommandManager.tearDown()
-        setIsRestartRequestStarting(false)
+          startOrReconfigureEngine()
-      }, attemptTimes[0] + attemptTimes[1])
+          setFirstPlay(true)
+
+          setTimeoutId(undefined)
+          setGoRestart(false)
+        }, attemptTimes[0] + attemptTimes[1])
+      )
       setAttemptTimes([attemptTimes[1], attemptTimes[0] + attemptTimes[1]])
     }
 
-    // Poll that we're connected. If not, send a reset signal.
+    const onOffline = () => {
-    // Do not restart if we're in idle mode.
+      if (
-    const connectionCheckIntervalId = setInterval(() => {
+        !(
-      // SKIP DURING TESTS BECAUSE IT WILL MESS WITH REUSING THE
+          EngineConnectionStateType.Disconnected ===
-      // ELECTRON INSTANCE.
+            engineCommandManager.engineConnection?.state.type ||
-      if (isPlaywright()) {
+          EngineConnectionStateType.Disconnecting ===
+            engineCommandManager.engineConnection?.state.type
+        )
+      ) {
         return
       }
-
+      engineStreamActor.send({ type: EngineStreamTransition.Stop })
-      // Don't try try to restart if we're already connected!
-      const hasEngineConnectionInst = engineCommandManager.engineConnection
-      const isDisconnected =
-        engineCommandManager.engineConnection?.state.type ===
-        EngineConnectionStateType.Disconnected
-      const inIdleMode = engineStreamState.value === EngineStreamState.Paused
-      if ((hasEngineConnectionInst && !isDisconnected) || inIdleMode) return
-
       attemptRestartIfNecessary()
-    }, TIME_1_SECOND)
+    }
+
+    if (
+      !goRestart &&
+      engineStreamState.value === EngineStreamState.WaitingForDependencies
+    ) {
+      setGoRestart(true)
+    }
+
+    if (goRestart && !timeoutId) {
+      attemptRestartIfNecessary()
+    }
 
     engineCommandManager.addEventListener(
       EngineCommandManagerEvents.EngineRestartRequest,
       attemptRestartIfNecessary
     )
-    return () => {
-      clearInterval(connectionCheckIntervalId)
 
+    engineCommandManager.addEventListener(
+      EngineCommandManagerEvents.Offline,
+      onOffline
+    )
+
+    return () => {
       engineCommandManager.removeEventListener(
         EngineCommandManagerEvents.EngineRestartRequest,
         attemptRestartIfNecessary
       )
+      engineCommandManager.removeEventListener(
+        EngineCommandManagerEvents.Offline,
+        onOffline
+      )
     }
-  }, [engineStreamState, attemptTimes, isRestartRequestStarting])
+  }, [
+    engineStreamState,
+    attemptTimes,
+    goRestart,
+    timeoutId,
+    engineCommandManager.engineConnection?.state.type,
+  ])
 
   useEffect(() => {
     // If engineStreamMachine is already reconfiguring, bail.
@@ -269,7 +281,7 @@ export const EngineStream = (props: {
     const canvas = engineStreamState.context.canvasRef?.current
     if (!canvas) return
 
-    new ResizeObserver(() => {
+    const observer = new ResizeObserver(() => {
       // Prevents:
       // `Uncaught ResizeObserver loop completed with undelivered notifications`
       window.requestAnimationFrame(() => {
@@ -280,13 +292,19 @@ export const EngineStream = (props: {
         if (
           (Math.abs(video.width - window.innerWidth) > 4 ||
             Math.abs(video.height - window.innerHeight) > 4) &&
-          !engineStreamState.matches(EngineStreamState.WaitingToPlay)
+          engineStreamState.matches(EngineStreamState.Playing)
         ) {
           timeoutStart.current = Date.now()
           startOrReconfigureEngine()
         }
       })
-    }).observe(document.body)
+    })
+
+    observer.observe(document.body)
+
+    return () => {
+      observer.disconnect()
+    }
   }, [engineStreamState.value])
 
   /**
@@ -345,8 +363,21 @@ export const EngineStream = (props: {
         timeoutStart.current = null
       } else if (timeoutStart.current) {
         const elapsed = Date.now() - timeoutStart.current
-        if (elapsed >= IDLE_TIME_MS) {
+        // Don't pause if we're already disconnected.
+        if (
+          // It's unnecessary to once again setup an event listener for
+          // offline/online to capture this state, when this state already
+          // exists on the window.navigator object. In hindsight it makes
+          // me (lee) regret we set React state variables such as
+          // isInternetConnected in other files when we could check this
+          // object instead.
+          engineCommandManager.engineConnection?.state.type ===
+            EngineConnectionStateType.ConnectionEstablished &&
+          elapsed >= IDLE_TIME_MS &&
+          engineStreamState.value === EngineStreamState.Playing
+        ) {
           timeoutStart.current = null
+          console.log('PAUSING')
           engineStreamActor.send({ type: EngineStreamTransition.Pause })
         }
       }
@@ -357,7 +388,7 @@ export const EngineStream = (props: {
     return () => {
       window.cancelAnimationFrame(frameId)
     }
-  }, [modelingMachineState])
+  }, [modelingMachineState, engineStreamState.value])
 
   useEffect(() => {
     if (!streamIdleMode) return
@@ -370,9 +401,18 @@ export const EngineStream = (props: {
         return
       }
 
-      if (engineStreamState.value === EngineStreamState.Paused) {
+      engineStreamActor.send({
-        startOrReconfigureEngine()
+        type: EngineStreamTransition.Resume,
-      }
+        modelingMachineActorSend,
+        settings: settingsEngine,
+        setAppState,
+        onMediaStream(mediaStream: MediaStream) {
+          engineStreamActor.send({
+            type: EngineStreamTransition.SetMediaStream,
+            mediaStream,
+          })
+        },
+      })
 
       timeoutStart.current = Date.now()
     }
@@ -471,7 +511,11 @@ export const EngineStream = (props: {
     }
 
     sendSelectEventToEngine(e)
-      .then(({ entity_id }) => {
+      .then((result) => {
+        if (!result) {
+          return
+        }
+        const { entity_id } = result
         if (!entity_id) {
           // No entity selected. This is benign
           return
@@ -535,7 +579,7 @@ export const EngineStream = (props: {
         EngineStreamState.Resuming,
       ].some((s) => s === engineStreamState.value) && (
         <Loading dataTestId="loading-engine" className="fixed inset-0 h-screen">
-          Connecting to engine
+          Connecting to engine...
         </Loading>
       )}
     </div>

src/components/Loading.tsx

@@ -1,6 +1,7 @@
 import type { MarkedOptions } from '@ts-stack/markdown'
 import { Marked, escape, unescape } from '@ts-stack/markdown'
 import { useEffect, useState } from 'react'
+import type { ReactNode } from 'react'
 
 import { CustomIcon } from '@src/components/CustomIcon'
 import { Spinner } from '@src/components/Spinner'
@@ -17,6 +18,7 @@ import { SafeRenderer } from '@src/lib/markdown'
 import { engineCommandManager } from '@src/lib/singletons'
 
 interface LoadingProps extends React.PropsWithChildren {
+  isDummy?: boolean
   className?: string
   dataTestId?: string
 }
@@ -29,7 +31,49 @@ const markedOptions: MarkedOptions = {
   escape,
 }
 
-const Loading = ({ children, className, dataTestId }: LoadingProps) => {
+// This exists here and not in engineConnection because we want some styling
+// available to us.
+export const CONNECTION_ERROR_CALL_TO_ACTION_TEXT: Record<
+  ConnectionError,
+  ReactNode
+> = {
+  [ConnectionError.Unset]: '',
+  [ConnectionError.LongLoadingTime]:
+    'Loading is taking longer than expected, check your network connection.',
+  [ConnectionError.VeryLongLoadingTime]:
+    'Check the connection is being blocked by a firewall, or if your internet is disconnected.',
+  [ConnectionError.ICENegotiate]:
+    'The modeling session was created, but there is an issue connecting to the stream.',
+  [ConnectionError.DataChannelError]:
+    'A modeling session was created, but there was an issue creating a modeling commands channel.',
+  [ConnectionError.WebSocketError]:
+    "An unexpected issue regarding the connection to Zoo's KittyCAD API happened. We suggest re-opening Zoo Design Studio to try again.",
+  [ConnectionError.LocalDescriptionInvalid]:
+    'The modeling session was created, but there is an issue connecting to the stream.',
+  [ConnectionError.MissingAuthToken]:
+    'Your authorization token is missing; please login again.',
+  [ConnectionError.BadAuthToken]:
+    'Your authorization token is invalid; please login again.',
+  [ConnectionError.TooManyConnections]:
+    'There are too many open engine connections associated with your account. Please close web browser windows and tabs with app.zoo.dev open, and close multiple Zoo Design Studio windows.',
+  [ConnectionError.Outage]: (
+    <>
+      We seem to be experiencing an outage. Please visit{' '}
+      <a href="https://status.zoo.dev">status.zoo.dev</a> for updates.
+    </>
+  ),
+  [ConnectionError.PeerConnectionRemoteDisconnected]:
+    'The remote end has disconnected. Zoo Design Studio will reconnect you.',
+  [ConnectionError.Unknown]:
+    'An unexpected error occurred. Please report this to us.',
+}
+
+const Loading = ({
+  isDummy,
+  children,
+  className,
+  dataTestId,
+}: LoadingProps) => {
   const [error, setError] = useState<ErrorType>({
     error: ConnectionError.Unset,
   })
@@ -109,6 +153,20 @@ const Loading = ({ children, className, dataTestId }: LoadingProps) => {
     }
   }, [error, setError, isUnrecoverableError])
 
+  // Useful for particular cases where we want a loading spinner but no other
+  // logic, such as when the feature tree is being built.
+  if (isDummy) {
+    return (
+      <div
+        className={`body-bg flex flex-col items-center justify-center ${colorClass} ${className}`}
+        data-testid={dataTestId ? dataTestId : 'loading'}
+      >
+        <Spinner />
+        <p className={`text-base mt-4`}>{children}</p>
+      </div>
+    )
+  }
+
   return (
     <div
       className={`body-bg flex flex-col items-center justify-center ${colorClass} ${className}`}
@@ -116,39 +174,52 @@ const Loading = ({ children, className, dataTestId }: LoadingProps) => {
     >
       {isUnrecoverableError ? (
         <CustomIcon
-          name="close"
+          name="exclamationMark"
           className="w-8 h-8 !text-chalkboard-10 bg-destroy-60 rounded-full"
         />
       ) : (
         <Spinner />
       )}
       <p className={`text-base mt-4`}>
-        {isUnrecoverableError ? 'An error occurred' : children || 'Loading'}
+        {isUnrecoverableError ? '' : children || 'Loading'}
       </p>
       {CONNECTION_ERROR_TEXT[error.error] && (
-        <div
+        <div>
-          className={
+          <div className="max-w-3xl text-base flex flex-col gap-2 px-2 pt-2 mt-2 pb-6 mb-6 border-b border-chalkboard-30">
-            'text-center text-sm mt-4 text-opacity-70 transition-opacity duration-500' +
+            {CONNECTION_ERROR_CALL_TO_ACTION_TEXT[error.error]}
-            (error.error !== ConnectionError.Unset
+            <div className="text-sm">
-              ? ' opacity-100'
+              If the issue persists, please visit the community support thread
-              : ' opacity-0')
+              on{' '}
-          }
+              <a href="https://community.zoo.dev/t/diagnosing-network-connection-issues/156">
-          dangerouslySetInnerHTML={{
+                diagnosing network connection issues
-            __html: Marked.parse(
+              </a>
-              CONNECTION_ERROR_TEXT[error.error] +
+              .
-                (error.context
+            </div>
-                  ? '\n\nThe error details are: ' +
+          </div>
-                    (error.context instanceof Object
+          <div
-                      ? JSON.stringify(error.context)
+            className={
-                      : error.context)
+              'font-mono text-xs px-2 text-opacity-70 transition-opacity duration-500' +
-                  : ''),
+              (error.error !== ConnectionError.Unset
-              {
+                ? ' opacity-100'
-                renderer: new SafeRenderer(markedOptions),
+                : ' opacity-0')
-                ...markedOptions,
+            }
-              }
+            dangerouslySetInnerHTML={{
-            ),
+              __html: Marked.parse(
-          }}
+                CONNECTION_ERROR_TEXT[error.error] +
-        ></div>
+                  (error.context
+                    ? '\n\nThe error details are: ' +
+                      (error.context instanceof Object
+                        ? JSON.stringify(error.context)
+                        : error.context)
+                    : ''),
+                {
+                  renderer: new SafeRenderer(markedOptions),
+                  ...markedOptions,
+                }
+              ),
+            }}
+          ></div>
+        </div>
       )}
     </div>
   )

src/components/ModelingSidebar/ModelingPanes/FeatureTreePane.tsx

@@ -155,7 +155,9 @@ export const FeatureTreePane = () => {
         className="absolute inset-0 p-1 box-border overflow-auto"
       >
         {kclManager.isExecuting ? (
-          <Loading className="h-full">Building feature tree...</Loading>
+          <Loading className="h-full" isDummy={true}>
+            Building feature tree...
+          </Loading>
         ) : (
           <>
             {!modelingState.matches('Sketch') && <DefaultPlanes />}

src/hooks/useNetworkStatus.tsx

@@ -117,18 +117,19 @@ export function useNetworkStatus() {
   }, [hasIssues, internetConnected, pingEMA, overallState])
 
   useEffect(() => {
-    const onlineCallback = () => {
-      setInternetConnected(true)
-    }
     const offlineCallback = () => {
       setInternetConnected(false)
       setSteps(structuredClone(initialConnectingTypeGroupState))
     }
-    window.addEventListener('online', onlineCallback)
+    engineCommandManager.addEventListener(
-    window.addEventListener('offline', offlineCallback)
+      EngineCommandManagerEvents.Offline,
+      offlineCallback
+    )
     return () => {
-      window.removeEventListener('online', onlineCallback)
+      engineCommandManager.removeEventListener(
-      window.removeEventListener('offline', offlineCallback)
+        EngineCommandManagerEvents.Offline,
+        offlineCallback
+      )
     }
   }, [])
 
@@ -178,6 +179,8 @@ export function useNetworkStatus() {
         if (
           engineConnectionState.type === EngineConnectionStateType.Connecting
         ) {
+          setInternetConnected(true)
+
           const groups = Object.values(nextSteps)
           for (let group of groups) {
             for (let step of group) {
@@ -207,6 +210,10 @@ export function useNetworkStatus() {
 
             if (engineConnectionState.value.type === DisconnectingType.Error) {
               setError(engineConnectionState.value.value)
+            } else if (
+              engineConnectionState.value.type === DisconnectingType.Quit
+            ) {
+              return structuredClone(initialConnectingTypeGroupState)
             }
           }
         }

src/lang/std/engineConnection.ts

@@ -1,4 +1,3 @@
-import { TEST } from '@src/env'
 import type { Models } from '@kittycad/lib'
 import { VITE_KC_API_WS_MODELING_URL, VITE_KC_DEV_TOKEN } from '@src/env'
 import { jsAppSettings } from '@src/lib/settings/settingsUtils'
@@ -83,6 +82,9 @@ export enum ConnectionError {
   TooManyConnections,
   Outage,
 
+  // Observed to happen on a local network outage.
+  PeerConnectionRemoteDisconnected,
+
   // An unknown error is the most severe because it has not been classified
   // or encountered before.
   Unknown,
@@ -93,22 +95,18 @@ export const CONNECTION_ERROR_TEXT: Record<ConnectionError, string> = {
   [ConnectionError.LongLoadingTime]:
     'Loading is taking longer than expected...',
   [ConnectionError.VeryLongLoadingTime]:
-    'Loading seems stuck. Do you have a firewall turned on?',
+    "It's possible there's a connection issue.",
   [ConnectionError.ICENegotiate]: 'ICE negotiation failed.',
-  [ConnectionError.DataChannelError]: 'The data channel signaled an error.',
+  [ConnectionError.DataChannelError]: 'Data channel error.',
-  [ConnectionError.WebSocketError]: 'The websocket signaled an error.',
+  [ConnectionError.WebSocketError]: 'Websocket error.',
-  [ConnectionError.LocalDescriptionInvalid]:
+  [ConnectionError.LocalDescriptionInvalid]: 'Local description invalid',
-    'The local description is invalid.',
+  [ConnectionError.MissingAuthToken]: 'Missing authorization token',
-  [ConnectionError.MissingAuthToken]:
+  [ConnectionError.BadAuthToken]: 'Bad authorization token',
-    'Your authorization token is missing; please login again.',
+  [ConnectionError.TooManyConnections]: 'Too many connections',
-  [ConnectionError.BadAuthToken]:
+  [ConnectionError.Outage]: 'Outage',
-    'Your authorization token is invalid; please login again.',
+  [ConnectionError.PeerConnectionRemoteDisconnected]:
-  [ConnectionError.TooManyConnections]:
+    'Peer connection disconnected',
-    'There are too many open engine connections associated with your account.',
+  [ConnectionError.Unknown]: 'Unknown',
-  [ConnectionError.Outage]:
-    'We seem to be experiencing an outage. Please visit [status.zoo.dev](https://status.zoo.dev) for updates.',
-  [ConnectionError.Unknown]:
-    'An unexpected error occurred. Please report this to us.',
 }
 
 export const WEBSOCKET_READYSTATE_TEXT: Record<number, string> = {
@@ -226,6 +224,9 @@ export enum EngineConnectionEvents {
   Opened = 'opened', // (engineConnection: EngineConnection) => void
   Closed = 'closed', // (engineConnection: EngineConnection) => void
   NewTrack = 'new-track', // (track: NewTrackArgs) => void
+
+  // A general offline state.
+  Offline = 'offline',
 }
 
 function toRTCSessionDescriptionInit(
@@ -669,14 +670,28 @@ class EngineConnection extends EventTarget {
                   },
                 },
               }
+              this.dispatchEvent(
+                new CustomEvent(EngineConnectionEvents.Offline, {})
+              )
               this.disconnectAll()
               break
 
             // The remote end broke up with us! :(
             case 'disconnected':
+              this.state = {
+                type: EngineConnectionStateType.Disconnecting,
+                value: {
+                  type: DisconnectingType.Error,
+                  value: {
+                    error: ConnectionError.PeerConnectionRemoteDisconnected,
+                    context: event,
+                  },
+                },
+              }
               this.dispatchEvent(
-                new CustomEvent(EngineConnectionEvents.RestartRequest, {})
+                new CustomEvent(EngineConnectionEvents.Offline, {})
               )
+              this.disconnectAll()
               break
             case 'closed':
               this.pc?.removeEventListener('icecandidate', this.onIceCandidate)
@@ -847,7 +862,6 @@ class EngineConnection extends EventTarget {
               'message',
               this.onDataChannelMessage
             )
-            this.disconnectAll()
           }
 
           this.unreliableDataChannel?.addEventListener(
@@ -866,7 +880,6 @@ class EngineConnection extends EventTarget {
                 },
               },
             }
-            this.disconnectAll()
           }
           this.unreliableDataChannel?.addEventListener(
             'error',
@@ -956,6 +969,9 @@ class EngineConnection extends EventTarget {
             this.onNetworkStatusReady
           )
 
+          this.dispatchEvent(
+            new CustomEvent(EngineConnectionEvents.Offline, {})
+          )
           this.disconnectAll()
         }
         this.websocket.addEventListener('close', this.onWebSocketClose)
@@ -974,8 +990,6 @@ class EngineConnection extends EventTarget {
               },
             }
           }
-
-          this.disconnectAll()
         }
         this.websocket.addEventListener('error', this.onWebSocketError)
 
@@ -1331,6 +1345,9 @@ export enum EngineCommandManagerEvents {
 
   // the whole scene is ready (settings loaded)
   SceneReady = 'scene-ready',
+
+  // we're offline
+  Offline = 'offline',
 }
 
 /**
@@ -1380,6 +1397,7 @@ export class EngineCommandManager extends EventTarget {
    * This is compared to the {@link outSequence} number to determine if we should ignore
    * any out-of-order late responses in the unreliable channel.
    */
+  keepForcefulOffline = false
   inSequence = 1
   engineConnection?: EngineConnection
   commandLogs: CommandLog[] = []
@@ -1453,13 +1471,8 @@ export class EngineCommandManager extends EventTarget {
     )
   }
 
-  private onOffline = () => {
+  private onEngineOffline = () => {
-    console.log('Browser reported network is offline')
+    this.dispatchEvent(new CustomEvent(EngineCommandManagerEvents.Offline, {}))
-    if (TEST) {
-      console.warn('DURING TESTS ENGINECONNECTION.ONOFFLINE WILL DO NOTHING.')
-      return
-    }
-    this.onEngineConnectionRestartRequest()
   }
 
   idleMode: boolean = false
@@ -1494,6 +1507,11 @@ export class EngineCommandManager extends EventTarget {
     if (settings) {
       this.settings = settings
     }
+
+    if (this.keepForcefulOffline) {
+      return
+    }
+
     if (width === 0 || height === 0) {
       return
     }
@@ -1509,8 +1527,6 @@ export class EngineCommandManager extends EventTarget {
       return
     }
 
-    window.addEventListener('offline', this.onOffline)
-
     let additionalSettings = this.settings.enableSSAO ? '&post_effect=ssao' : ''
     additionalSettings +=
       '&show_grid=' + (this.settings.showScaleGrid ? 'true' : 'false')
@@ -1537,6 +1553,11 @@ export class EngineCommandManager extends EventTarget {
       this.onEngineConnectionRestartRequest as EventListener
     )
 
+    this.engineConnection.addEventListener(
+      EngineConnectionEvents.Offline,
+      this.onEngineOffline as EventListener
+    )
+
     // eslint-disable-next-line @typescript-eslint/no-misused-promises
     this.onEngineConnectionOpened = async () => {
       console.log('onEngineConnectionOpened')
@@ -1548,13 +1569,8 @@ export class EngineCommandManager extends EventTarget {
           this.codeManager?.currentFilePath || undefined
         )
       } catch (e) {
-        // If this happens shit's actually gone south aka the websocket closed.
+        // If this happens, the websocket may have closed and we need to restart
-        // Let's restart.
+        console.warn('unknown error:', e)
-        console.warn("shit's gone south")
-        console.warn(e)
-        this.engineConnection?.dispatchEvent(
-          new CustomEvent(EngineConnectionEvents.RestartRequest, {})
-        )
         return
       }
 
@@ -1597,23 +1613,7 @@ export class EngineCommandManager extends EventTarget {
       console.log('camControlsCameraChange')
       this._camControlsCameraChange()
 
-      // We should eventually only have 1 restoral call.
+      await this.sceneInfra?.camControls.restoreRemoteCameraStateAndTriggerSync()
-      if (this.idleMode) {
-        await this.sceneInfra?.camControls.restoreRemoteCameraStateAndTriggerSync()
-      } else {
-        // NOTE: This code is old. It uses the old hack to restore camera.
-        console.log('call default_camera_get_settings')
-        // eslint-disable-next-line @typescript-eslint/no-floating-promises
-        await this.sendSceneCommand({
-          // CameraControls subscribes to default_camera_get_settings response events
-          // firing this at connection ensure the camera's are synced initially
-          type: 'modeling_cmd_req',
-          cmd_id: uuidv4(),
-          cmd: {
-            type: 'default_camera_get_settings',
-          },
-        })
-      }
 
       setIsStreamReady(true)
 
@@ -1877,8 +1877,6 @@ export class EngineCommandManager extends EventTarget {
   tearDown(opts?: { idleMode: boolean }) {
     this.idleMode = opts?.idleMode ?? false
 
-    window.removeEventListener('offline', this.onOffline)
-
     if (this.engineConnection) {
       for (const [cmdId, pending] of Object.entries(this.pendingCommands)) {
         pending.reject([
@@ -1928,7 +1926,26 @@ export class EngineCommandManager extends EventTarget {
       this.engineCommandManager.engineConnection = null
     }
     this.engineConnection = undefined
+
+    // It is possible all connections never even started, but we still want
+    // to signal to the whole application we are "offline".
+    this.dispatchEvent(new CustomEvent(EngineCommandManagerEvents.Offline, {}))
   }
+
+  offline() {
+    this.keepForcefulOffline = true
+    this.tearDown()
+    console.log('offline')
+  }
+
+  online() {
+    this.keepForcefulOffline = false
+    this.dispatchEvent(
+      new CustomEvent(EngineCommandManagerEvents.EngineRestartRequest, {})
+    )
+    console.log('online')
+  }
+
   async startNewSession() {
     this.responseMap = {}
   }

src/lib/selections.ts

@@ -703,7 +703,8 @@ export async function sendSelectEventToEngine(
     cmd_id: uuidv4(),
   })
   if (!res) {
-    return Promise.reject('no response')
+    console.warn('No response')
+    return undefined
   }
 
   if (isArray(res)) {

src/machines/engineStreamMachine.ts

@@ -70,7 +70,6 @@ export async function holdOntoVideoFrameInCanvas(
   video: HTMLVideoElement,
   canvas: HTMLCanvasElement
 ) {
-  video.pause()
   canvas.width = video.videoWidth
   canvas.height = video.videoHeight
   canvas.style.width = video.videoWidth + 'px'
@@ -220,11 +219,14 @@ export const engineStreamMachine = setup({
         if (context.videoRef.current && context.canvasRef.current) {
           await context.videoRef.current.pause()
 
-          await holdOntoVideoFrameInCanvas(
+          // It's possible we've already frozen the frame due to a disconnect.
-            context.videoRef.current,
+          if (context.videoRef.current.style.display !== 'none') {
-            context.canvasRef.current
+            await holdOntoVideoFrameInCanvas(
-          )
+              context.videoRef.current,
-          context.videoRef.current.style.display = 'none'
+              context.canvasRef.current
+            )
+            context.videoRef.current.style.display = 'none'
+          }
         }
 
         await rootContext.sceneInfra.camControls.saveRemoteCameraState()
@@ -365,9 +367,12 @@ export const engineStreamMachine = setup({
         }),
       },
       on: {
-        [EngineStreamTransition.StartOrReconfigureEngine]: {
+        [EngineStreamTransition.Resume]: {
           target: EngineStreamState.Resuming,
         },
+        [EngineStreamTransition.Stop]: {
+          target: EngineStreamState.Stopped,
+        },
       },
     },
     [EngineStreamState.Stopped]: {
@@ -398,11 +403,17 @@ export const engineStreamMachine = setup({
           rootContext: args.self.system.get('root').getSnapshot().context,
           event: args.event,
         }),
+        // Usually only fails if there was a disconnection mid-way.
+        onError: [
+          {
+            target: EngineStreamState.WaitingForDependencies,
+            reenter: true,
+          },
+        ],
       },
       on: {
-        // The stream can be paused as it's resuming.
+        [EngineStreamTransition.Stop]: {
-        [EngineStreamTransition.Pause]: {
+          target: EngineStreamState.Stopped,
-          target: EngineStreamState.Paused,
         },
         [EngineStreamTransition.SetMediaStream]: {
           target: EngineStreamState.Playing,

src/routes/Home.tsx

@@ -521,7 +521,7 @@ function ProjectGrid({
   return (
     <section data-testid="home-section" {...rest}>
       {state.matches(SystemIOMachineStates.readingFolders) ? (
-        <Loading>Loading your Projects...</Loading>
+        <Loading isDummy={true}>Loading your Projects...</Loading>
       ) : (
         <>
           {searchResults.length > 0 ? (