Backoff Reconnect (#6358)

* Add a Stop command that does a bit of cleanup in the engineStateMachine

* Major network code startup refactor; backoff reconnect; many more logs for us

* Save camera state after every user interaction in case of disconnection and restoral

* Translate basic WebSocket error numbers into useful text

* Add a RestartRequest event to the engineCommandManager

* Adjust for a rebase

* Add E2E test for stream pause behavior

* Fix snapshot test

* fmt, lint

* small issue

* Fix tests

* Fix up idle test

* One last fix

* fixes

* Remove circ dep

* fix test

* use a const for time

* TEST -> isPlaywright instead

* whoops

e2e/playwright/snapshot-tests.spec.ts

@@ -766,7 +766,7 @@ test.describe('Grid visibility', { tag: '@snapshot' }, () => {
   })
 })
 
-test('theme persists', async ({ page, context }) => {
+test('theme persists', async ({ page, context, homePage }) => {
   const u = await getUtils(page)
   await context.addInitScript(async () => {
     localStorage.setItem(
@@ -784,7 +784,7 @@ test('theme persists', async ({ page, context }) => {
 
   await page.setViewportSize({ width: 1200, height: 500 })
 
-  await u.waitForAuthSkipAppStart()
+  await homePage.goToModelingScene()
   await page.waitForTimeout(500)
 
   // await page.getByRole('link', { name: 'Settings Settings (tooltip)' }).click()
@@ -812,7 +812,7 @@ test('theme persists', async ({ page, context }) => {
   // Disconnect and reconnect to check the theme persists through a reload
 
   // Expect the network to be down
-  await expect(networkToggle).toContainText('Offline')
+  await expect(networkToggle).toContainText('Problem')
 
   // simulate network up
   await u.emulateNetworkConditions({

e2e/playwright/test-network-and-connection-issues.spec.ts

@@ -1,236 +1,257 @@
 import type { EngineCommand } from '@src/lang/std/artifactGraph'
 import { uuidv4 } from '@src/lib/utils'
 
-import { commonPoints, getUtils } from '@e2e/playwright/test-utils'
+import {
+  commonPoints,
+  getUtils,
+  TEST_COLORS,
+  circleMove,
+} from '@e2e/playwright/test-utils'
 import { expect, test } from '@e2e/playwright/zoo-test'
 
-test.describe(
+test.describe('Test network related behaviors', () => {
-  'Test network and connection issues',
+  test(
-  {
+    'simulate network down and network little widget',
-    tag: ['@macos', '@windows'],
+    { tag: '@skipLocalEngine' },
-  },
+    async ({ page, homePage }) => {
-  () => {
+      const networkToggleConnectedText = page.getByText('Connected')
-    test(
+      const networkToggleWeakText = page.getByText('Network health (Weak)')
-      'simulate network down and network little widget',
-      { tag: '@skipLocalEngine' },
-      async ({ page, homePage }) => {
-        const u = await getUtils(page)
-        await page.setBodyDimensions({ width: 1200, height: 500 })
 
-        await homePage.goToModelingScene()
+      const u = await getUtils(page)
+      await page.setBodyDimensions({ width: 1200, height: 500 })
 
-        const networkToggle = page.getByTestId('network-toggle')
+      await homePage.goToModelingScene()
 
-        // This is how we wait until the stream is online
+      const networkToggle = page.getByTestId('network-toggle')
-        await expect(
-          page.getByRole('button', { name: 'Start Sketch' })
-        ).not.toBeDisabled({ timeout: 15000 })
 
-        const networkWidget = page.locator('[data-testid="network-toggle"]')
+      // This is how we wait until the stream is online
-        await expect(networkWidget).toBeVisible()
+      await expect(
-        await networkWidget.hover()
+        page.getByRole('button', { name: 'Start Sketch' })
+      ).not.toBeDisabled({ timeout: 15000 })
 
-        const networkPopover = page.locator('[data-testid="network-popover"]')
+      const networkWidget = page.locator('[data-testid="network-toggle"]')
-        await expect(networkPopover).not.toBeVisible()
+      await expect(networkWidget).toBeVisible()
+      await networkWidget.hover()
 
-        // (First check) Expect the network to be up
+      const networkPopover = page.locator('[data-testid="network-popover"]')
-        await expect(networkToggle).toContainText('Connected')
+      await expect(networkPopover).not.toBeVisible()
 
-        // Click the network widget
+      // (First check) Expect the network to be up
-        await networkWidget.click()
+      await expect(
+        networkToggleConnectedText.or(networkToggleWeakText)
+      ).toBeVisible()
 
-        // Check the modal opened.
+      // Click the network widget
-        await expect(networkPopover).toBeVisible()
+      await networkWidget.click()
 
-        // Click off the modal.
+      // Check the modal opened.
-        await page.mouse.click(100, 100)
+      await expect(networkPopover).toBeVisible()
-        await expect(networkPopover).not.toBeVisible()
 
-        // Turn off the network
+      // Click off the modal.
-        await u.emulateNetworkConditions({
+      await page.mouse.click(100, 100)
-          offline: true,
+      await expect(networkPopover).not.toBeVisible()
-          // values of 0 remove any active throttling. crbug.com/456324#c9
-          latency: 0,
-          downloadThroughput: -1,
-          uploadThroughput: -1,
-        })
 
-        // Expect the network to be down
+      // Turn off the network
-        await expect(networkToggle).toContainText('Problem')
+      await u.emulateNetworkConditions({
+        offline: true,
+        // values of 0 remove any active throttling. crbug.com/456324#c9
+        latency: 0,
+        downloadThroughput: -1,
+        uploadThroughput: -1,
+      })
 
-        // Click the network widget
+      // Expect the network to be down
-        await networkWidget.click()
+      await expect(networkToggle).toContainText('Problem')
 
-        // Check the modal opened.
+      // Click the network widget
-        await expect(networkPopover).toBeVisible()
+      await networkWidget.click()
 
-        // Click off the modal.
+      // Check the modal opened.
-        await page.mouse.click(0, 0)
+      await expect(networkPopover).toBeVisible()
-        await expect(networkPopover).not.toBeVisible()
 
-        // Turn back on the network
+      // Click off the modal.
-        await u.emulateNetworkConditions({
+      await page.mouse.click(0, 0)
-          offline: false,
+      await expect(networkPopover).not.toBeVisible()
-          // values of 0 remove any active throttling. crbug.com/456324#c9
-          latency: 0,
-          downloadThroughput: -1,
-          uploadThroughput: -1,
-        })
 
-        await expect(
+      // Turn back on the network
-          page.getByRole('button', { name: 'Start Sketch' })
+      await u.emulateNetworkConditions({
-        ).not.toBeDisabled({ timeout: 15000 })
+        offline: false,
+        // values of 0 remove any active throttling. crbug.com/456324#c9
+        latency: 0,
+        downloadThroughput: -1,
+        uploadThroughput: -1,
+      })
 
-        // (Second check) expect the network to be up
+      await expect(
-        await expect(networkToggle).toContainText('Connected')
+        page.getByRole('button', { name: 'Start Sketch' })
-      }
+      ).not.toBeDisabled({ timeout: 15000 })
-    )
 
-    test(
+      // (Second check) expect the network to be up
-      'Engine disconnect & reconnect in sketch mode',
+      await expect(
-      { tag: '@skipLocalEngine' },
+        networkToggleConnectedText.or(networkToggleWeakText)
-      async ({ page, homePage, toolbar, scene, cmdBar }) => {
+      ).toBeVisible()
-        const networkToggle = page.getByTestId('network-toggle')
+    }
+  )
 
-        const u = await getUtils(page)
+  test(
-        await page.setBodyDimensions({ width: 1200, height: 500 })
+    'Engine disconnect & reconnect in sketch mode',
-        const PUR = 400 / 37.5 //pixeltoUnitRatio
+    { tag: '@skipLocalEngine' },
+    async ({ page, homePage, toolbar, scene, cmdBar }) => {
+      const networkToggle = page.getByTestId('network-toggle')
+      const networkToggleConnectedText = page.getByText('Connected')
+      const networkToggleWeakText = page.getByText('Network health (Weak)')
 
-        await homePage.goToModelingScene()
+      const u = await getUtils(page)
-        await u.waitForPageLoad()
+      await page.setBodyDimensions({ width: 1200, height: 500 })
+      const PUR = 400 / 37.5 //pixeltoUnitRatio
 
-        await u.openDebugPanel()
+      await homePage.goToModelingScene()
-        // click on "Start Sketch" button
+      await u.waitForPageLoad()
-        await u.clearCommandLogs()
-        await page.getByRole('button', { name: 'Start Sketch' }).click()
-        await page.waitForTimeout(100)
 
-        // select a plane
+      await u.openDebugPanel()
-        await page.mouse.click(700, 200)
+      // click on "Start Sketch" button
+      await u.clearCommandLogs()
+      await page.getByRole('button', { name: 'Start Sketch' }).click()
+      await page.waitForTimeout(100)
 
-        await expect(page.locator('.cm-content')).toHaveText(
+      // select a plane
-          `sketch001 = startSketchOn(XZ)`
+      await page.mouse.click(700, 200)
-        )
-        await u.closeDebugPanel()
 
-        await page.waitForTimeout(500) // TODO detect animation ending, or disable animation
+      await expect(page.locator('.cm-content')).toHaveText(
+        `@settings(defaultLengthUnit = in)sketch001 = startSketchOn(XZ)`
+      )
+      await u.closeDebugPanel()
 
-        const startXPx = 600
+      await page.waitForTimeout(500) // TODO detect animation ending, or disable animation
-        await page.mouse.click(startXPx + PUR * 10, 500 - PUR * 10)
-        await expect(page.locator('.cm-content')).toHaveText(
-          `sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt})`
-        )
-        await page.waitForTimeout(100)
 
-        await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 10)
+      const startXPx = 600
-        await page.waitForTimeout(100)
+      await page.mouse.click(startXPx + PUR * 10, 500 - PUR * 10)
+      await expect(page.locator('.cm-content')).toHaveText(
+        `@settings(defaultLengthUnit = in)sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt})`
+      )
+      await page.waitForTimeout(100)
 
-        await expect(
+      await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 10)
-          page.locator('.cm-content')
+      await page.waitForTimeout(100)
-        ).toHaveText(`sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt})
+
+      await expect(
+        page.locator('.cm-content')
+      ).toHaveText(`@settings(defaultLengthUnit = in)sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt})
       |> xLine(length = ${commonPoints.num1})`)
 
-        // Expect the network to be up
+      // Expect the network to be up
-        await expect(networkToggle).toContainText('Connected')
+      await networkToggle.hover()
+      await expect(
+        networkToggleConnectedText.or(networkToggleWeakText)
+      ).toBeVisible()
 
-        // simulate network down
+      // simulate network down
-        await u.emulateNetworkConditions({
+      await u.emulateNetworkConditions({
-          offline: true,
+        offline: true,
-          // values of 0 remove any active throttling. crbug.com/456324#c9
+        // values of 0 remove any active throttling. crbug.com/456324#c9
-          latency: 0,
+        latency: 0,
-          downloadThroughput: -1,
+        downloadThroughput: -1,
-          uploadThroughput: -1,
+        uploadThroughput: -1,
-        })
+      })
 
-        // Expect the network to be down
+      // Expect the network to be down
-        await expect(networkToggle).toContainText('Problem')
+      await networkToggle.hover()
+      await expect(networkToggle).toContainText('Problem')
 
-        // Ensure we are not in sketch mode
+      // Ensure we are not in sketch mode
-        await expect(
+      await expect(
-          page.getByRole('button', { name: 'Exit Sketch' })
+        page.getByRole('button', { name: 'Exit Sketch' })
-        ).not.toBeVisible()
+      ).not.toBeVisible()
-        await expect(
+      await expect(
-          page.getByRole('button', { name: 'Start Sketch' })
+        page.getByRole('button', { name: 'Start Sketch' })
-        ).toBeVisible()
+      ).toBeVisible()
 
-        // simulate network up
+      // simulate network up
-        await u.emulateNetworkConditions({
+      await u.emulateNetworkConditions({
-          offline: false,
+        offline: false,
-          // values of 0 remove any active throttling. crbug.com/456324#c9
+        // values of 0 remove any active throttling. crbug.com/456324#c9
-          latency: 0,
+        latency: 0,
-          downloadThroughput: -1,
+        downloadThroughput: -1,
-          uploadThroughput: -1,
+        uploadThroughput: -1,
-        })
+      })
 
-        // Wait for the app to be ready for use
+      // Wait for the app to be ready for use
-        await expect(
+      await expect(
-          page.getByRole('button', { name: 'Start Sketch' })
+        page.getByRole('button', { name: 'Start Sketch' })
-        ).not.toBeDisabled({ timeout: 15000 })
+      ).not.toBeDisabled({ timeout: 15000 })
 
-        // Expect the network to be up
+      // Expect the network to be up
-        await expect(networkToggle).toContainText('Connected')
+      await networkToggle.hover()
-        await scene.settled(cmdBar)
+      await expect(
+        networkToggleConnectedText.or(networkToggleWeakText)
+      ).toBeVisible()
 
-        // Click off the code pane.
+      await scene.settled(cmdBar)
-        await page.mouse.click(100, 100)
 
-        // select a line
+      // Click off the code pane.
-        await page
+      await page.mouse.click(100, 100)
-          .getByText(`startProfile(sketch001, at = ${commonPoints.startAt})`)
-          .click()
 
-        // enter sketch again
+      // select a line
-        await toolbar.editSketch()
+      await page
+        .getByText(`startProfile(sketch001, at = ${commonPoints.startAt})`)
+        .click()
 
-        // Click the line tool
+      // enter sketch again
-        await page
+      await toolbar.editSketch()
-          .getByRole('button', { name: 'line Line', exact: true })
-          .click()
 
-        await page.waitForTimeout(150)
+      // Click the line tool
+      await page.getByRole('button', { name: 'line Line', exact: true }).click()
 
-        const camCommand: EngineCommand = {
+      await page.waitForTimeout(150)
-          type: 'modeling_cmd_req',
-          cmd_id: uuidv4(),
-          cmd: {
-            type: 'default_camera_look_at',
-            center: { x: 109, y: 0, z: -152 },
-            vantage: { x: 115, y: -505, z: -152 },
-            up: { x: 0, y: 0, z: 1 },
-          },
-        }
-        const updateCamCommand: EngineCommand = {
-          type: 'modeling_cmd_req',
-          cmd_id: uuidv4(),
-          cmd: {
-            type: 'default_camera_get_settings',
-          },
-        }
-        await toolbar.openPane('debug')
-        await u.sendCustomCmd(camCommand)
-        await page.waitForTimeout(100)
-        await u.sendCustomCmd(updateCamCommand)
-        await page.waitForTimeout(100)
 
-        // click to continue profile
+      const camCommand: EngineCommand = {
-        await page.mouse.click(1007, 400)
+        type: 'modeling_cmd_req',
-        await page.waitForTimeout(100)
+        cmd_id: uuidv4(),
-        // Ensure we can continue sketching
+        cmd: {
-        await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 20)
+          type: 'default_camera_look_at',
-        await expect
+          center: { x: 109, y: 0, z: -152 },
-          .poll(u.normalisedEditorCode)
+          vantage: { x: 115, y: -505, z: -152 },
-          .toBe(`sketch001 = startSketchOn(XZ)
+          up: { x: 0, y: 0, z: 1 },
+        },
+      }
+      const updateCamCommand: EngineCommand = {
+        type: 'modeling_cmd_req',
+        cmd_id: uuidv4(),
+        cmd: {
+          type: 'default_camera_get_settings',
+        },
+      }
+      await toolbar.openPane('debug')
+      await u.sendCustomCmd(camCommand)
+      await page.waitForTimeout(100)
+      await u.sendCustomCmd(updateCamCommand)
+      await page.waitForTimeout(100)
+
+      // click to continue profile
+      await page.mouse.click(1007, 400)
+      await page.waitForTimeout(100)
+      // Ensure we can continue sketching
+      await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 20)
+      await expect
+        .poll(u.normalisedEditorCode)
+        .toBe(`@settings(defaultLengthUnit = in)
+
+
+sketch001 = startSketchOn(XZ)
 profile001 = startProfile(sketch001, at = [12.34, -12.34])
   |> xLine(length = 12.34)
   |> line(end = [-12.34, 12.34])
 
 `)
-        await page.waitForTimeout(100)
+      await page.waitForTimeout(100)
-        await page.mouse.click(startXPx, 500 - PUR * 20)
+      await page.mouse.click(startXPx, 500 - PUR * 20)
 
-        await expect
+      await expect
-          .poll(u.normalisedEditorCode)
+        .poll(u.normalisedEditorCode)
-          .toBe(`sketch001 = startSketchOn(XZ)
+        .toBe(`@settings(defaultLengthUnit = in)
+
+
+sketch001 = startSketchOn(XZ)
 profile001 = startProfile(sketch001, at = [12.34, -12.34])
   |> xLine(length = 12.34)
   |> line(end = [-12.34, 12.34])
@@ -238,22 +259,105 @@ profile001 = startProfile(sketch001, at = [12.34, -12.34])
 
 `)
 
-        // Unequip line tool
+      // Unequip line tool
-        await page.keyboard.press('Escape')
+      await page.keyboard.press('Escape')
-        // Make sure we didn't pop out of sketch mode.
+      // Make sure we didn't pop out of sketch mode.
-        await expect(
+      await expect(
-          page.getByRole('button', { name: 'Exit Sketch' })
+        page.getByRole('button', { name: 'Exit Sketch' })
-        ).toBeVisible()
+      ).toBeVisible()
-        await expect(
+      await expect(
-          page.getByRole('button', { name: 'line Line', exact: true })
+        page.getByRole('button', { name: 'line Line', exact: true })
-        ).not.toHaveAttribute('aria-pressed', 'true')
+      ).not.toHaveAttribute('aria-pressed', 'true')
 
-        // Exit sketch
+      // Exit sketch
-        await page.keyboard.press('Escape')
+      await page.keyboard.press('Escape')
-        await expect(
+      await expect(
-          page.getByRole('button', { name: 'Exit Sketch' })
+        page.getByRole('button', { name: 'Exit Sketch' })
-        ).not.toBeVisible()
+      ).not.toBeVisible()
+    }
+  )
+
+  test(
+    'Paused stream freezes view frame, unpause reconnect is seamless to user',
+    { tag: ['@electron', '@skipLocalEngine'] },
+    async ({ page, homePage, scene, cmdBar, toolbar, tronApp }) => {
+      const networkToggle = page.getByTestId('network-toggle')
+      const networkToggleConnectedText = page.getByText('Connected')
+      const networkToggleWeakText = page.getByText('Network health (Weak)')
+
+      if (!tronApp) {
+        fail()
       }
-    )
+
-  }
+      await tronApp.cleanProjectDir({
-)
+        app: {
+          stream_idle_mode: 5000,
+        },
+      })
+
+      await page.addInitScript(async () => {
+        localStorage.setItem(
+          'persistCode',
+          `sketch001 = startSketchOn(XY)
+profile001 = startProfile(sketch001, at = [0.0, 0.0])
+  |> line(end = [10.0, 0])
+  |> line(end = [0, 10.0])
+  |> close()`
+        )
+      })
+
+      const dim = { width: 1200, height: 500 }
+      await page.setBodyDimensions(dim)
+
+      await test.step('Go to modeling scene', async () => {
+        await homePage.goToModelingScene()
+        await scene.settled(cmdBar)
+      })
+
+      await test.step('Verify pausing behavior', async () => {
+        // Wait 5s + 1s to pause.
+        await page.waitForTimeout(6000)
+
+        // We should now be paused. To the user, it should appear we're still
+        // connected.
+        await networkToggle.hover()
+        await expect(
+          networkToggleConnectedText.or(networkToggleWeakText)
+        ).toBeVisible()
+
+        const center = {
+          x: dim.width / 2,
+          y: dim.height / 2,
+        }
+
+        let probe = { x: 0, y: 0 }
+
+        // ... and the model's still visibly there
+        probe.x = center.x + dim.width / 100
+        probe.y = center.y
+        await scene.expectPixelColor(TEST_COLORS.GREY, probe, 15)
+        probe = { ...center }
+
+        // Now move the mouse around to unpause!
+        await circleMove(page, probe.x, probe.y, 20, 10)
+
+        // ONCE AGAIN! Check the view area hasn't changed at all.
+        // Check the pixel a couple times as it reconnects.
+        // NOTE: Remember, idle behavior is still on at this point -
+        // if this test takes longer than 5s shit WILL go south!
+        probe.x = center.x + dim.width / 100
+        probe.y = center.y
+        await scene.expectPixelColor(TEST_COLORS.GREY, probe, 15)
+        await page.waitForTimeout(1000)
+        await scene.expectPixelColor(TEST_COLORS.GREY, probe, 15)
+        probe = { ...center }
+
+        // Ensure we're still connected
+        await networkToggle.hover()
+        await expect(
+          networkToggleConnectedText.or(networkToggleWeakText)
+        ).toBeVisible()
+      })
+    }
+  )
+})

e2e/playwright/test-utils.ts

@@ -44,6 +44,8 @@ export const lowerRightMasks = (page: Page) => [
 export type TestColor = [number, number, number]
 export const TEST_COLORS: { [key: string]: TestColor } = {
   WHITE: [249, 249, 249],
+  OFFWHITE: [237, 237, 237],
+  GREY: [142, 142, 142],
   YELLOW: [255, 255, 0],
   BLUE: [0, 0, 255],
   DARK_MODE_BKGD: [27, 27, 27],

src/App.tsx

@@ -117,7 +117,9 @@ export function App() {
 
     // When leaving the modeling scene, cut the engine stream.
     return () => {
-      engineStreamActor.send({ type: EngineStreamTransition.Pause })
+      // When leaving the modeling scene, cut the engine stream.
+      // Stop is more serious than Pause
+      engineStreamActor.send({ type: EngineStreamTransition.Stop })
     }
   }, [])
 

src/clientSideScene/CameraControls.ts

@@ -975,7 +975,6 @@ export class CameraControls {
         },
       })
     }
-
     await this.engineCommandManager.sendSceneCommand({
       type: 'modeling_cmd_req',
       cmd_id: uuidv4(),

src/components/EngineStream.tsx

@@ -1,3 +1,4 @@
+import { isPlaywright } from '@src/lib/isPlaywright'
 import { useAppState } from '@src/AppState'
 import { ClientSideScene } from '@src/clientSideScene/ClientSideSceneComp'
 import { ViewControlContextMenu } from '@src/components/ViewControlMenu'
@@ -5,7 +6,10 @@ import { useModelingContext } from '@src/hooks/useModelingContext'
 import { useNetworkContext } from '@src/hooks/useNetworkContext'
 import { NetworkHealthState } from '@src/hooks/useNetworkStatus'
 import { getArtifactOfTypes } from '@src/lang/std/artifactGraph'
-import { EngineCommandManagerEvents } from '@src/lang/std/engineConnection'
+import {
+  EngineCommandManagerEvents,
+  EngineConnectionStateType,
+} from '@src/lang/std/engineConnection'
 import { btnName } from '@src/lib/cameraControls'
 import { PATHS } from '@src/lib/paths'
 import { sendSelectEventToEngine } from '@src/lib/selections'
@@ -33,22 +37,38 @@ import { createThumbnailPNGOnDesktop } from '@src/lib/screenshot'
 import type { SettingsViaQueryString } from '@src/lib/settings/settingsTypes'
 import { resetCameraPosition } from '@src/lib/resetCameraPosition'
 
+const TIME_1_SECOND = 1000
+
 export const EngineStream = (props: {
   pool: string | null
   authToken: string | undefined
 }) => {
   const { setAppState } = useAppState()
-  const [firstPlay, setFirstPlay] = useState(true)
-
-  const { overallState } = useNetworkContext()
   const settings = useSettings()
-
+  const { state: modelingMachineState, send: modelingMachineActorSend } =
-  const engineStreamState = useSelector(engineStreamActor, (state) => state)
+    useModelingContext()
 
   const { file, project } = useRouteLoaderData(PATHS.FILE) as IndexLoaderData
   const last = useRef<number>(Date.now())
+
+  const [firstPlay, setFirstPlay] = useState(true)
+  const [isRestartRequestStarting, setIsRestartRequestStarting] =
+    useState(false)
+  const [attemptTimes, setAttemptTimes] = useState<[number, number]>([
+    0,
+    TIME_1_SECOND,
+  ])
+
+  // These will be passed to the engineStreamActor to handle.
+  const videoRef = useRef<HTMLVideoElement>(null)
+  const canvasRef = useRef<HTMLCanvasElement>(null)
+
+  // For attaching right-click menu events
   const videoWrapperRef = useRef<HTMLDivElement>(null)
 
+  const { overallState } = useNetworkContext()
+  const engineStreamState = useSelector(engineStreamActor, (state) => state)
+
   /**
    * We omit `pool` here because `engineStreamMachine` will override it anyway
    * within the `EngineStreamTransition.StartOrReconfigureEngine` Promise actor.
@@ -62,19 +82,46 @@ export const EngineStream = (props: {
     cameraOrbit: settings.modeling.cameraOrbit.current,
   }
 
-  const { state: modelingMachineState, send: modelingMachineActorSend } =
-    useModelingContext()
-
   const streamIdleMode = settings.app.streamIdleMode.current
 
+  useEffect(() => {
+    engineStreamActor.send({
+      type: EngineStreamTransition.SetVideoRef,
+      videoRef: { current: videoRef.current },
+    })
+  }, [videoRef.current])
+
+  useEffect(() => {
+    engineStreamActor.send({
+      type: EngineStreamTransition.SetCanvasRef,
+      canvasRef: { current: canvasRef.current },
+    })
+  }, [canvasRef.current])
+
+  useEffect(() => {
+    engineStreamActor.send({
+      type: EngineStreamTransition.SetPool,
+      pool: props.pool,
+    })
+  }, [props.pool])
+
+  useEffect(() => {
+    engineStreamActor.send({
+      type: EngineStreamTransition.SetAuthToken,
+      authToken: props.authToken,
+    })
+  }, [props.authToken])
+
+  // We have to call this here because of the dependencies:
+  // modelingMachineActorSend, setAppState, settingsEngine
+  // It's possible to pass these in earlier but I (lee) don't want to
+  // restructure this further at the moment.
   const startOrReconfigureEngine = () => {
     engineStreamActor.send({
       type: EngineStreamTransition.StartOrReconfigureEngine,
       modelingMachineActorSend,
       settings: settingsEngine,
       setAppState,
-
-      // It's possible a reconnect happens as we drag the window :')
       onMediaStream(mediaStream: MediaStream) {
         engineStreamActor.send({
           type: EngineStreamTransition.SetMediaStream,
@@ -84,18 +131,47 @@ export const EngineStream = (props: {
     })
   }
 
-  // When the scene is ready play the stream and execute!
+  useEffect(() => {
+    if (
+      engineStreamState.value !== EngineStreamState.WaitingForDependencies &&
+      engineStreamState.value !== EngineStreamState.Stopped
+    )
+      return
+    startOrReconfigureEngine()
+  }, [engineStreamState, setAppState])
+
+  // I would inline this but it needs to be a function for removeEventListener.
   const play = () => {
     engineStreamActor.send({
       type: EngineStreamTransition.Play,
     })
+  }
 
+  useEffect(() => {
+    engineCommandManager.addEventListener(
+      EngineCommandManagerEvents.SceneReady,
+      play
+    )
+    return () => {
+      engineCommandManager.removeEventListener(
+        EngineCommandManagerEvents.SceneReady,
+        play
+      )
+    }
+  }, [])
+
+  // When the scene is ready, execute kcl!
+  const executeKcl = () => {
+    console.log('scene is ready, execute kcl')
     const kmp = kclManager.executeCode().catch(trap)
 
     if (!firstPlay) return
-    setFirstPlay(false)
-    console.log('scene is ready, fire!')
 
+    setFirstPlay(false)
+    // Reset the restart timeouts
+    setAttemptTimes([0, TIME_1_SECOND])
+
+    console.log('firstPlay true, zoom to fit')
     kmp
       .then(async () => {
         await resetCameraPosition()
@@ -112,51 +188,65 @@ export const EngineStream = (props: {
   useEffect(() => {
     engineCommandManager.addEventListener(
       EngineCommandManagerEvents.SceneReady,
-      play
+      executeKcl
     )
     return () => {
       engineCommandManager.removeEventListener(
         EngineCommandManagerEvents.SceneReady,
-        play
+        executeKcl
       )
     }
   }, [firstPlay])
 
   useEffect(() => {
+    // We do a back-off restart, using a fibonacci sequence, since it
+    // has a nice retry time curve (somewhat quick then exponential)
+    const attemptRestartIfNecessary = () => {
+      if (isRestartRequestStarting) return
+      setIsRestartRequestStarting(true)
+      setTimeout(() => {
+        engineStreamState.context.videoRef.current?.pause()
+        engineCommandManager.tearDown()
+        startOrReconfigureEngine()
+        setFirstPlay(false)
+        setIsRestartRequestStarting(false)
+      }, attemptTimes[0] + attemptTimes[1])
+      setAttemptTimes([attemptTimes[1], attemptTimes[0] + attemptTimes[1]])
+    }
+
+    // Poll that we're connected. If not, send a reset signal.
+    // Do not restart if we're in idle mode.
+    const connectionCheckIntervalId = setInterval(() => {
+      // SKIP DURING TESTS BECAUSE IT WILL MESS WITH REUSING THE
+      // ELECTRON INSTANCE.
+      if (isPlaywright()) {
+        return
+      }
+
+      // Don't try try to restart if we're already connected!
+      const hasEngineConnectionInst = engineCommandManager.engineConnection
+      const isDisconnected =
+        engineCommandManager.engineConnection?.state.type ===
+        EngineConnectionStateType.Disconnected
+      const inIdleMode = engineStreamState.value === EngineStreamState.Paused
+      if ((hasEngineConnectionInst && !isDisconnected) || inIdleMode) return
+
+      attemptRestartIfNecessary()
+    }, TIME_1_SECOND)
+
     engineCommandManager.addEventListener(
-      EngineCommandManagerEvents.SceneReady,
+      EngineCommandManagerEvents.EngineRestartRequest,
-      play
+      attemptRestartIfNecessary
     )
-
-    engineStreamActor.send({
-      type: EngineStreamTransition.SetPool,
-      data: { pool: props.pool },
-    })
-    engineStreamActor.send({
-      type: EngineStreamTransition.SetAuthToken,
-      data: { authToken: props.authToken },
-    })
-
     return () => {
-      engineCommandManager.tearDown()
+      clearInterval(connectionCheckIntervalId)
-    }
-  }, [])
 
-  // In the past we'd try to play immediately, but the proper thing is to way
+      engineCommandManager.removeEventListener(
-  // for the 'canplay' event to tell us data is ready.
+        EngineCommandManagerEvents.EngineRestartRequest,
-  useEffect(() => {
+        attemptRestartIfNecessary
-    const videoRef = engineStreamState.context.videoRef.current
+      )
-    if (!videoRef) {
-      return
     }
-    const play = () => {
+  }, [engineStreamState, attemptTimes, isRestartRequestStarting])
-      videoRef.play().catch(console.error)
-    }
-    videoRef.addEventListener('canplay', play)
-    return () => {
-      videoRef.removeEventListener('canplay', play)
-    }
-  }, [engineStreamState.context.videoRef.current])
 
   useEffect(() => {
     if (engineStreamState.value === EngineStreamState.Reconfiguring) return
@@ -184,25 +274,6 @@ export const EngineStream = (props: {
     }).observe(document.body)
   }, [engineStreamState.value])
 
-  // When the video and canvas element references are set, start the engine.
-  useEffect(() => {
-    if (
-      engineStreamState.context.canvasRef.current &&
-      engineStreamState.context.videoRef.current
-    ) {
-      startOrReconfigureEngine()
-    }
-  }, [
-    engineStreamState.context.canvasRef.current,
-    engineStreamState.context.videoRef.current,
-  ])
-
-  // On settings change, reconfigure the engine. When paused this gets really tricky,
-  // and also requires onMediaStream to be set!
-  useEffect(() => {
-    startOrReconfigureEngine()
-  }, Object.values(settingsEngine))
-
   /**
    * Subscribe to execute code when the file changes
    * but only if the scene is already ready.
@@ -285,18 +356,7 @@ export const EngineStream = (props: {
       }
 
       if (engineStreamState.value === EngineStreamState.Paused) {
-        engineStreamActor.send({
+        startOrReconfigureEngine()
-          type: EngineStreamTransition.StartOrReconfigureEngine,
-          modelingMachineActorSend,
-          settings: settingsEngine,
-          setAppState,
-          onMediaStream(mediaStream: MediaStream) {
-            engineStreamActor.send({
-              type: EngineStreamTransition.SetMediaStream,
-              mediaStream,
-            })
-          },
-        })
       }
 
       timeoutStart.current = Date.now()
@@ -314,7 +374,7 @@ export const EngineStream = (props: {
     window.document.addEventListener('mouseup', onAnyInput)
     window.document.addEventListener('scroll', onAnyInput)
     window.document.addEventListener('touchstart', onAnyInput)
-    window.document.addEventListener('touchstop', onAnyInput)
+    window.document.addEventListener('touchend', onAnyInput)
 
     return () => {
       timeoutStart.current = null
@@ -325,10 +385,34 @@ export const EngineStream = (props: {
       window.document.removeEventListener('mouseup', onAnyInput)
       window.document.removeEventListener('scroll', onAnyInput)
       window.document.removeEventListener('touchstart', onAnyInput)
-      window.document.removeEventListener('touchstop', onAnyInput)
+      window.document.removeEventListener('touchend', onAnyInput)
     }
   }, [streamIdleMode, engineStreamState.value])
 
+  // On various inputs save the camera state, in case we get disconnected.
+  useEffect(() => {
+    const onInput = () => {
+      // Save the remote camera state to restore on stream restore.
+      // Fire-and-forget because we don't know when a camera movement is
+      // completed on the engine side (there are no responses to data channel
+      // mouse movements.)
+      sceneInfra.camControls.saveRemoteCameraState().catch(trap)
+    }
+
+    // These usually signal a user is done some sort of operation.
+    window.document.addEventListener('keyup', onInput)
+    window.document.addEventListener('mouseup', onInput)
+    window.document.addEventListener('scroll', onInput)
+    window.document.addEventListener('touchend', onInput)
+
+    return () => {
+      window.document.removeEventListener('keyup', onInput)
+      window.document.removeEventListener('mouseup', onInput)
+      window.document.removeEventListener('scroll', onInput)
+      window.document.removeEventListener('touchend', onInput)
+    }
+  }, [])
+
   const isNetworkOkay =
     overallState === NetworkHealthState.Ok ||
     overallState === NetworkHealthState.Weak
@@ -399,7 +483,7 @@ export const EngineStream = (props: {
         autoPlay
         muted
         key={engineStreamActor.id + 'video'}
-        ref={engineStreamState.context.videoRef}
+        ref={videoRef}
         controls={false}
         className="w-full cursor-pointer h-full"
         disablePictureInPicture
@@ -407,7 +491,7 @@ export const EngineStream = (props: {
       />
       <canvas
         key={engineStreamActor.id + 'canvas'}
-        ref={engineStreamState.context.canvasRef}
+        ref={canvasRef}
         className="cursor-pointer"
         id="freeze-frame"
       >
@@ -424,9 +508,11 @@ export const EngineStream = (props: {
         }
         menuTargetElement={videoWrapperRef}
       />
-      {![EngineStreamState.Playing, EngineStreamState.Paused].some(
+      {![
-        (s) => s === engineStreamState.value
+        EngineStreamState.Playing,
-      ) && (
+        EngineStreamState.Paused,
+        EngineStreamState.Resuming,
+      ].some((s) => s === engineStreamState.value) && (
         <Loading dataTestId="loading-engine" className="fixed inset-0 h-screen">
           Connecting to engine
         </Loading>

src/components/Loading.tsx

@@ -138,7 +138,9 @@ const Loading = ({ children, className, dataTestId }: LoadingProps) => {
               CONNECTION_ERROR_TEXT[error.error] +
                 (error.context
                   ? '\n\nThe error details are: ' +
-                    JSON.stringify(error.context)
+                    (error.context instanceof Object
+                      ? JSON.stringify(error.context)
+                      : error.context)
                   : ''),
               {
                 renderer: new SafeRenderer(markedOptions),

src/components/ModelingMachineProvider.tsx

@@ -204,12 +204,13 @@ export const ModelingMachineProvider = ({
 
             sceneInfra.camControls.syncDirection = 'engineToClient'
 
+            // TODO: Re-evaluate if this pause/play logic is needed.
             store.videoElement?.pause()
 
             return kclManager
               .executeCode()
               .then(() => {
-                if (engineCommandManager.engineConnection?.idleMode) return
+                if (engineCommandManager.idleMode) return
 
                 store.videoElement?.play().catch((e) => {
                   console.warn('Video playing was prevented', e)

src/lang/std/engineConnection.ts

@@ -1,3 +1,4 @@
+import { TEST } from '@src/env'
 import type { Models } from '@kittycad/lib'
 import { VITE_KC_API_WS_MODELING_URL, VITE_KC_DEV_TOKEN } from '@src/env'
 import { jsAppSettings } from '@src/lib/settings/settingsUtils'
@@ -7,6 +8,7 @@ import type { MachineManager } from '@src/components/MachineManagerProvider'
 import type { useModelingContext } from '@src/hooks/useModelingContext'
 import type { KclManager } from '@src/lang/KclSingleton'
 import type CodeManager from '@src/lang/codeManager'
+import type { SceneInfra } from '@src/clientSideScene/sceneInfra'
 import type { EngineCommand, ResponseMap } from '@src/lang/std/artifactGraph'
 import type { CommandLog } from '@src/lang/std/commandLog'
 import { CommandLogType } from '@src/lang/std/commandLog'
@@ -109,6 +111,13 @@ export const CONNECTION_ERROR_TEXT: Record<ConnectionError, string> = {
     'An unexpected error occurred. Please report this to us.',
 }
 
+export const WEBSOCKET_READYSTATE_TEXT: Record<number, string> = {
+  [WebSocket.CONNECTING]: 'WebSocket.CONNECTING',
+  [WebSocket.OPEN]: 'WebSocket.OPEN',
+  [WebSocket.CLOSING]: 'WebSocket.CLOSING',
+  [WebSocket.CLOSED]: 'WebSocket.CLOSED',
+}
+
 export interface ErrorType {
   // The error we've encountered.
   error: ConnectionError
@@ -208,6 +217,9 @@ export enum EngineConnectionEvents {
   // We can eventually use it for more, but one step at a time.
   ConnectionStateChanged = 'connection-state-changed', // (state: EngineConnectionState) => void
 
+  // There are various failure scenarios where we want to try a restart.
+  RestartRequest = 'restart-request',
+
   // These are used for the EngineCommandManager and were created
   // before onConnectionStateChange existed.
   ConnectionStarted = 'connection-started', // (engineConnection: EngineConnection) => void
@@ -238,11 +250,23 @@ class EngineConnection extends EventTarget {
   pc?: RTCPeerConnection
   unreliableDataChannel?: RTCDataChannel
   mediaStream?: MediaStream
-  idleMode: boolean = false
   promise?: Promise<void>
   sdpAnswer?: RTCSessionDescriptionInit
   triggeredStart = false
 
+  onWebSocketOpen = function (event: Event) {}
+  onWebSocketClose = function (event: Event) {}
+  onWebSocketError = function (event: Event) {}
+  onWebSocketMessage = function (event: MessageEvent) {}
+  onIceGatheringStateChange = function (
+    this: RTCPeerConnection,
+    event: Event
+  ) {}
+  onIceConnectionStateChange = function (
+    this: RTCPeerConnection,
+    event: Event
+  ) {}
+  onNegotiationNeeded = function (this: RTCPeerConnection, event: Event) {}
   onIceCandidate = function (
     this: RTCPeerConnection,
     event: RTCPeerConnectionIceEvent
@@ -252,6 +276,9 @@ class EngineConnection extends EventTarget {
     event: RTCPeerConnectionIceErrorEvent
   ) {}
   onConnectionStateChange = function (this: RTCPeerConnection, event: Event) {}
+  onSignalingStateChange = function (this: RTCDataChannel, event: Event) {}
+
+  onTrack = function (this: RTCPeerConnection, event: RTCTrackEvent) {}
   onDataChannelOpen = function (this: RTCDataChannel, event: Event) {}
   onDataChannelClose = function (this: RTCDataChannel, event: Event) {}
   onDataChannelError = function (this: RTCDataChannel, event: Event) {}
@@ -260,11 +287,7 @@ class EngineConnection extends EventTarget {
     this: RTCPeerConnection,
     event: RTCDataChannelEvent
   ) {}
-  onTrack = function (this: RTCPeerConnection, event: RTCTrackEvent) {}
+
-  onWebSocketOpen = function (event: Event) {}
-  onWebSocketClose = function (event: Event) {}
-  onWebSocketError = function (event: Event) {}
-  onWebSocketMessage = function (event: MessageEvent) {}
   onNetworkStatusReady = () => {}
 
   private _state: EngineConnectionState = {
@@ -309,10 +332,10 @@ class EngineConnection extends EventTarget {
   private engineCommandManager: EngineCommandManager
 
   private pingPongSpan: { ping?: number; pong?: number }
-  private pingIntervalId: ReturnType<typeof setInterval> | null = null
+  private pingIntervalId: ReturnType<typeof setInterval> | undefined = undefined
   isUsingConnectionLite: boolean = false
 
-  timeoutToForceConnectId: ReturnType<typeof setTimeout> | null = null
+  timeoutToForceConnectId: ReturnType<typeof setTimeout> | undefined = undefined
 
   constructor({
     engineCommandManager,
@@ -416,18 +439,18 @@ class EngineConnection extends EventTarget {
     return this.state.type === EngineConnectionStateType.ConnectionEstablished
   }
 
-  tearDown(opts?: { idleMode: boolean }) {
+  tearDown() {
-    this.idleMode = opts?.idleMode ?? false
+    clearInterval(this.pingIntervalId)
-    if (this.pingIntervalId) {
+    clearTimeout(this.timeoutToForceConnectId)
-      clearInterval(this.pingIntervalId)
+
-    }
+    // As each network connection (websocket, webrtc, peer connection) is
-    if (this.timeoutToForceConnectId) {
+    // closed, they will handle removing their own event listeners.
-      clearTimeout(this.timeoutToForceConnectId)
+    // If they didn't then it'd be possible we stop listened to close events
-    }
+    // which is what we want to do in the first place :)
 
     this.disconnectAll()
 
-    if (this.idleMode) {
+    if (this.engineCommandManager.idleMode) {
       this.state = {
         type: EngineConnectionStateType.Disconnecting,
         value: {
@@ -435,6 +458,7 @@ class EngineConnection extends EventTarget {
         },
       }
     }
+
     // Pass the state along
     if (this.state.type === EngineConnectionStateType.Disconnecting) return
     if (this.state.type === EngineConnectionStateType.Disconnected) return
@@ -568,30 +592,41 @@ class EngineConnection extends EventTarget {
           }, 3000)
         }
         this.pc?.addEventListener?.('icecandidate', this.onIceCandidate)
+
+        // Watch out human! The names of the next couple events are really similar!
+        this.onIceGatheringStateChange = (event) => {
+          console.log('icegatheringstatechange', event)
+
+          that.initiateConnectionExclusive()
+        }
         this.pc?.addEventListener?.(
           'icegatheringstatechange',
-          function (_event) {
+          this.onIceGatheringStateChange
-            console.log('icegatheringstatechange', this.iceGatheringState)
-
-            if (this.iceGatheringState !== 'complete') return
-            that.initiateConnectionExclusive()
-          }
         )
 
+        this.onIceConnectionStateChange = (event: Event) => {
+          console.log('iceconnectionstatechange', event)
+        }
         this.pc?.addEventListener?.(
           'iceconnectionstatechange',
-          function (_event) {
+          this.onIceConnectionStateChange
-            console.log('iceconnectionstatechange', this.iceConnectionState)
+        )
-            console.log('iceconnectionstatechange', this.iceGatheringState)
+
-          }
+        this.onNegotiationNeeded = (event: Event) => {
+          console.log('negotiationneeded', event)
+        }
+        this.pc?.addEventListener?.(
+          'negotiationneeded',
+          this.onNegotiationNeeded
+        )
+
+        this.onSignalingStateChange = (event) => {
+          console.log('signalingstatechange', event)
+        }
+        this.pc?.addEventListener?.(
+          'signalingstatechange',
+          this.onSignalingStateChange
         )
-        this.pc?.addEventListener?.('negotiationneeded', function (_event) {
-          console.log('negotiationneeded', this.iceConnectionState)
-          console.log('negotiationneeded', this.iceGatheringState)
-        })
-        this.pc?.addEventListener?.('signalingstatechange', function (event) {
-          console.log('signalingstatechange', this.signalingState)
-        })
 
         this.onIceCandidateError = (_event: Event) => {
           const event = _event as RTCPeerConnectionIceErrorEvent
@@ -618,38 +653,12 @@ class EngineConnection extends EventTarget {
                   detail: { conn: this, mediaStream: this.mediaStream! },
                 })
               )
-
-              setTimeout(() => {
-                // Everything is now connected.
-                this.state = {
-                  type: EngineConnectionStateType.ConnectionEstablished,
-                }
-
-                this.engineCommandManager.inSequence = 1
-
-                this.dispatchEvent(
-                  new CustomEvent(EngineConnectionEvents.Opened, {
-                    detail: this,
-                  })
-                )
-                markOnce('code/endInitialEngineConnect')
-              }, 2000)
               break
+
             case 'connecting':
               break
-            case 'disconnected':
-            case 'failed':
-              this.pc?.removeEventListener('icecandidate', this.onIceCandidate)
-              this.pc?.removeEventListener(
-                'icecandidateerror',
-                this.onIceCandidateError
-              )
-              this.pc?.removeEventListener(
-                'connectionstatechange',
-                this.onConnectionStateChange
-              )
-              this.pc?.removeEventListener('track', this.onTrack)
 
+            case 'failed':
               this.state = {
                 type: EngineConnectionStateType.Disconnecting,
                 value: {
@@ -662,6 +671,43 @@ class EngineConnection extends EventTarget {
               }
               this.disconnectAll()
               break
+
+            // The remote end broke up with us! :(
+            case 'disconnected':
+              this.dispatchEvent(
+                new CustomEvent(EngineConnectionEvents.RestartRequest, {})
+              )
+              break
+            case 'closed':
+              this.pc?.removeEventListener('icecandidate', this.onIceCandidate)
+              this.pc?.removeEventListener(
+                'icegatheringstatechange',
+                this.onIceGatheringStateChange
+              )
+              this.pc?.removeEventListener(
+                'iceconnectionstatechange',
+                this.onIceConnectionStateChange
+              )
+              this.pc?.removeEventListener(
+                'negotiationneeded',
+                this.onNegotiationNeeded
+              )
+              this.pc?.removeEventListener(
+                'signalingstatechange',
+                this.onSignalingStateChange
+              )
+              this.pc?.removeEventListener(
+                'icecandidateerror',
+                this.onIceCandidateError
+              )
+              this.pc?.removeEventListener(
+                'connectionstatechange',
+                this.onConnectionStateChange
+              )
+              this.pc?.removeEventListener('track', this.onTrack)
+              this.pc?.removeEventListener('datachannel', this.onDataChannel)
+              break
+
             default:
               break
           }
@@ -735,9 +781,7 @@ class EngineConnection extends EventTarget {
           // The app is eager to use the MediaStream; as soon as onNewTrack is
           // called, the following sequence happens:
           // EngineConnection.onNewTrack -> StoreState.setMediaStream ->
-          // Stream.tsx reacts to mediaStream change, setting a video element.
+          // EngineStream.tsx reacts to mediaStream change, setting a video element.
-          // We wait until connectionstatechange changes to "connected"
-          // to pass it to the rest of the application.
 
           this.mediaStream = mediaStream
         }
@@ -761,6 +805,25 @@ class EngineConnection extends EventTarget {
                 type: ConnectingType.DataChannelEstablished,
               },
             }
+
+            // Start firing off engine commands at this point.
+            // They could be fired at an earlier time, onWebSocketOpen,
+            // but DataChannel can offer some benefits like speed,
+            // and it's nice to say everything's connected before interacting
+            // with the server.
+
+            this.state = {
+              type: EngineConnectionStateType.ConnectionEstablished,
+            }
+
+            this.engineCommandManager.inSequence = 1
+
+            this.dispatchEvent(
+              new CustomEvent(EngineConnectionEvents.Opened, {
+                detail: this,
+              })
+            )
+            markOnce('code/endInitialEngineConnect')
           }
           this.unreliableDataChannel?.addEventListener(
             'open',
@@ -784,7 +847,6 @@ class EngineConnection extends EventTarget {
               'message',
               this.onDataChannelMessage
             )
-            this.pc?.removeEventListener('datachannel', this.onDataChannel)
             this.disconnectAll()
           }
 
@@ -898,16 +960,19 @@ class EngineConnection extends EventTarget {
         }
         this.websocket.addEventListener('close', this.onWebSocketClose)
 
-        this.onWebSocketError = (event) => {
+        this.onWebSocketError = (event: Event) => {
-          this.state = {
+          if (event.target instanceof WebSocket) {
-            type: EngineConnectionStateType.Disconnecting,
+            this.state = {
-            value: {
+              type: EngineConnectionStateType.Disconnecting,
-              type: DisconnectingType.Error,
               value: {
-                error: ConnectionError.WebSocketError,
+                type: DisconnectingType.Error,
-                context: event,
+                value: {
+                  error: ConnectionError.WebSocketError,
+                  context:
+                    WEBSOCKET_READYSTATE_TEXT[event.target.readyState] ?? event,
+                },
               },
-            },
+            }
           }
 
           this.disconnectAll()
@@ -1213,20 +1278,27 @@ class EngineConnection extends EventTarget {
       !this.websocket ||
       this.websocket?.readyState === 3
 
-    if (closedPc && closedUDC && closedWS) {
+    if (!(closedPc && closedUDC && closedWS)) {
-      if (!this.idleMode) {
+      return
-        // Do not notify the rest of the program that we have cut off anything.
-        this.state = { type: EngineConnectionStateType.Disconnected }
-      } else {
-        this.state = {
-          type: EngineConnectionStateType.Disconnecting,
-          value: {
-            type: DisconnectingType.Pause,
-          },
-        }
-      }
-      this.triggeredStart = false
     }
+
+    // Clean up all the event listeners.
+
+    if (!this.engineCommandManager.idleMode) {
+      // Do not notify the rest of the program that we have cut off anything.
+      this.state = { type: EngineConnectionStateType.Disconnected }
+      this.dispatchEvent(
+        new CustomEvent(EngineConnectionEvents.RestartRequest, {})
+      )
+    } else {
+      this.state = {
+        type: EngineConnectionStateType.Disconnecting,
+        value: {
+          type: DisconnectingType.Pause,
+        },
+      }
+    }
+    this.triggeredStart = false
   }
 }
 
@@ -1254,6 +1326,9 @@ export enum EngineCommandManagerEvents {
   // engineConnection is available but scene setup may not have run
   EngineAvailable = 'engine-available',
 
+  // request a restart of engineConnection
+  EngineRestartRequest = 'engine-restart-request',
+
   // the whole scene is ready (settings loaded)
   SceneReady = 'scene-ready',
 }
@@ -1366,10 +1441,29 @@ export class EngineCommandManager extends EventTarget {
   kclManager: null | KclManager = null
   codeManager?: CodeManager
   rustContext?: RustContext
+  sceneInfra?: SceneInfra
 
   // The current "manufacturing machine" aka 3D printer, CNC, etc.
   public machineManager: MachineManager | null = null
 
+  // Dispatch to the application the engine needs a restart.
+  private onEngineConnectionRestartRequest = () => {
+    this.dispatchEvent(
+      new CustomEvent(EngineCommandManagerEvents.EngineRestartRequest, {})
+    )
+  }
+
+  private onOffline = () => {
+    console.log('Browser reported network is offline')
+    if (TEST) {
+      console.warn('DURING TESTS ENGINECONNECTION.ONOFFLINE WILL DO NOTHING.')
+      return
+    }
+    this.onEngineConnectionRestartRequest()
+  }
+
+  idleMode: boolean = false
+
   start({
     setMediaStream,
     setIsStreamReady,
@@ -1415,6 +1509,8 @@ export class EngineCommandManager extends EventTarget {
       return
     }
 
+    window.addEventListener('offline', this.onOffline)
+
     let additionalSettings = this.settings.enableSSAO ? '&post_effect=ssao' : ''
     additionalSettings +=
       '&show_grid=' + (this.settings.showScaleGrid ? 'true' : 'false')
@@ -1436,29 +1532,51 @@ export class EngineCommandManager extends EventTarget {
       })
     )
 
+    this.engineConnection.addEventListener(
+      EngineConnectionEvents.RestartRequest,
+      this.onEngineConnectionRestartRequest as EventListener
+    )
+
     // eslint-disable-next-line @typescript-eslint/no-misused-promises
     this.onEngineConnectionOpened = async () => {
-      await this.rustContext?.clearSceneAndBustCache(
+      console.log('onEngineConnectionOpened')
-        await jsAppSettings(),
+
-        this.codeManager?.currentFilePath || undefined
+      try {
-      )
+        console.log('clearing scene and busting cache')
+        await this.rustContext?.clearSceneAndBustCache(
+          await jsAppSettings(),
+          this.codeManager?.currentFilePath || undefined
+        )
+      } catch (e) {
+        // If this happens shit's actually gone south aka the websocket closed.
+        // Let's restart.
+        console.warn("shit's gone south")
+        console.warn(e)
+        this.engineConnection?.dispatchEvent(
+          new CustomEvent(EngineConnectionEvents.RestartRequest, {})
+        )
+        return
+      }
 
       // Set the stream's camera projection type
       // We don't send a command to the engine if in perspective mode because
       // for now it's the engine's default.
       if (settings.cameraProjection === 'orthographic') {
-        this.sendSceneCommand({
+        console.log('Setting camera to orthographic')
+        await this.sendSceneCommand({
           type: 'modeling_cmd_req',
           cmd_id: uuidv4(),
           cmd: {
             type: 'default_camera_set_orthographic',
           },
-        }).catch(reportRejection)
+        })
       }
 
       // Set the theme
-      this.setTheme(this.settings.theme).catch(reportRejection)
+      console.log('Setting theme', this.settings.theme)
+      await this.setTheme(this.settings.theme)
       // Set up a listener for the dark theme media query
+      console.log('Setup theme media query change')
       darkModeMatcher?.addEventListener(
         'change',
         this.onDarkThemeMediaQueryChange
@@ -1466,7 +1584,8 @@ export class EngineCommandManager extends EventTarget {
 
       // Set the edge lines visibility
       // eslint-disable-next-line @typescript-eslint/no-floating-promises
-      this.sendSceneCommand({
+      console.log('setting edge_lines_visible')
+      await this.sendSceneCommand({
         type: 'modeling_cmd_req',
         cmd_id: uuidv4(),
         cmd: {
@@ -1475,21 +1594,30 @@ export class EngineCommandManager extends EventTarget {
         },
       })
 
+      console.log('camControlsCameraChange')
       this._camControlsCameraChange()
 
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
+      // We should eventually only have 1 restoral call.
-      this.sendSceneCommand({
+      if (this.idleMode) {
-        // CameraControls subscribes to default_camera_get_settings response events
+        await this.sceneInfra?.camControls.restoreRemoteCameraStateAndTriggerSync()
-        // firing this at connection ensure the camera's are synced initially
+      } else {
-        type: 'modeling_cmd_req',
+        // NOTE: This code is old. It uses the old hack to restore camera.
-        cmd_id: uuidv4(),
+        console.log('call default_camera_get_settings')
-        cmd: {
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
-          type: 'default_camera_get_settings',
+        await this.sendSceneCommand({
-        },
+          // CameraControls subscribes to default_camera_get_settings response events
-      })
+          // firing this at connection ensure the camera's are synced initially
+          type: 'modeling_cmd_req',
+          cmd_id: uuidv4(),
+          cmd: {
+            type: 'default_camera_get_settings',
+          },
+        })
+      }
 
       setIsStreamReady(true)
 
+      console.log('Dispatching SceneReady')
       // Other parts of the application should use this to react on scene ready.
       this.dispatchEvent(
         new CustomEvent(EngineCommandManagerEvents.SceneReady, {
@@ -1555,7 +1683,7 @@ export class EngineCommandManager extends EventTarget {
       }) as EventListener)
 
       this.onVideoTrackMute = () => {
-        console.error('video track mute: check webrtc internals -> inbound rtp')
+        console.warn('video track mute - potentially lost stream for a moment')
       }
 
       this.onEngineConnectionNewTrack = ({
@@ -1746,9 +1874,11 @@ export class EngineCommandManager extends EventTarget {
     this.engineConnection?.send(resizeCmd)
   }
 
-  tearDown(opts?: {
+  tearDown(opts?: { idleMode: boolean }) {
-    idleMode: boolean
+    this.idleMode = opts?.idleMode ?? false
-  }) {
+
+    window.removeEventListener('offline', this.onOffline)
+
     if (this.engineConnection) {
       for (const [cmdId, pending] of Object.entries(this.pendingCommands)) {
         pending.reject([
@@ -1786,14 +1916,14 @@ export class EngineCommandManager extends EventTarget {
         this.onDarkThemeMediaQueryChange
       )
 
-      this.engineConnection?.tearDown(opts)
+      this.engineConnection?.tearDown()
 
       // Our window.engineCommandManager.tearDown assignment causes this case to happen which is
       // only really for tests.
       // @ts-ignore
     } else if (this.engineCommandManager?.engineConnection) {
       // @ts-ignore
-      this.engineCommandManager?.engineConnection?.tearDown(opts)
+      this.engineCommandManager?.engineConnection?.tearDown()
       // @ts-ignore
       this.engineCommandManager.engineConnection = null
     }
@@ -2112,25 +2242,25 @@ export class EngineCommandManager extends EventTarget {
     // Set the stream background color
     // This takes RGBA values from 0-1
     // So we convert from the conventional 0-255 found in Figma
-    this.sendSceneCommand({
+    await this.sendSceneCommand({
       cmd_id: uuidv4(),
       type: 'modeling_cmd_req',
       cmd: {
         type: 'set_background_color',
         color: getThemeColorForEngine(theme),
       },
-    }).catch(reportRejection)
+    })
 
     // Sets the default line colors
     const opposingTheme = getOppositeTheme(theme)
-    this.sendSceneCommand({
+    await this.sendSceneCommand({
       cmd_id: uuidv4(),
       type: 'modeling_cmd_req',
       cmd: {
         type: 'set_default_system_properties',
         color: getThemeColorForEngine(opposingTheme),
       },
-    }).catch(reportRejection)
+    })
   }
 }
 

src/lib/singletons.ts

@@ -74,6 +74,7 @@ editorManager.kclManager = kclManager
 // TODO: proper dependency injection.
 engineCommandManager.kclManager = kclManager
 engineCommandManager.codeManager = codeManager
+engineCommandManager.sceneInfra = sceneInfra
 engineCommandManager.rustContext = rustContext
 
 kclManager.sceneInfraBaseUnitMultiplierSetter = (unit: BaseUnit) => {

src/machines/engineStreamMachine.ts

@@ -4,41 +4,53 @@ import { assign, fromPromise, setup } from 'xstate'
 import type { AppMachineContext } from '@src/lib/types'
 
 export enum EngineStreamState {
-  Off = 'off',
+  WaitingForDependencies = 'waiting-for-dependencies',
-  On = 'on',
+  WaitingForMediaStream = 'waiting-for-media-stream',
-  WaitForMediaStream = 'wait-for-media-stream',
+  WaitingToPlay = 'waiting-to-play',
   Playing = 'playing',
   Reconfiguring = 'reconfiguring',
   Paused = 'paused',
+  Stopped = 'stopped',
   // The is the state in-between Paused and Playing *specifically that order*.
   Resuming = 'resuming',
 }
 
 export enum EngineStreamTransition {
-  SetMediaStream = 'set-media-stream',
+  // This brings us back to the configuration loop
+  WaitForDependencies = 'wait-for-dependencies',
+
+  // Our dependencies to set
   SetPool = 'set-pool',
   SetAuthToken = 'set-auth-token',
+  SetVideoRef = 'set-video-ref',
+  SetCanvasRef = 'set-canvas-ref',
+  SetMediaStream = 'set-media-stream',
+
+  // Stream operations
   Play = 'play',
   Resume = 'resume',
   Pause = 'pause',
+  Stop = 'stop',
+
+  // Used to reconfigure the stream during connection
   StartOrReconfigureEngine = 'start-or-reconfigure-engine',
 }
 
 export interface EngineStreamContext {
   pool: string | null
   authToken: string | undefined
-  mediaStream: MediaStream | null
   videoRef: MutableRefObject<HTMLVideoElement | null>
   canvasRef: MutableRefObject<HTMLCanvasElement | null>
+  mediaStream: MediaStream | null
   zoomToFit: boolean
 }
 
 export const engineStreamContextCreate = (): EngineStreamContext => ({
   pool: null,
   authToken: undefined,
-  mediaStream: null,
   videoRef: { current: null },
   canvasRef: { current: null },
+  mediaStream: null,
   zoomToFit: true,
 })
 
@@ -77,76 +89,6 @@ export const engineStreamMachine = setup({
     input: {} as EngineStreamContext,
   },
   actors: {
-    [EngineStreamTransition.Play]: fromPromise(
-      async ({
-        input: { context, params, rootContext },
-      }: {
-        input: {
-          context: EngineStreamContext
-          params: { zoomToFit: boolean }
-          rootContext: AppMachineContext
-        }
-      }) => {
-        const canvas = context.canvasRef.current
-        if (!canvas) return false
-
-        const video = context.videoRef.current
-        if (!video) return false
-
-        const mediaStream = context.mediaStream
-        if (!mediaStream) return false
-
-        // If the video is already playing it means we're doing a reconfigure.
-        // We don't want to re-run the KCL or touch the video element at all.
-        if (!video.paused) {
-          return
-        }
-
-        await rootContext.sceneInfra.camControls.restoreRemoteCameraStateAndTriggerSync()
-
-        video.style.display = 'block'
-        canvas.style.display = 'none'
-
-        video.srcObject = mediaStream
-      }
-    ),
-    [EngineStreamTransition.Pause]: fromPromise(
-      async ({
-        input: { context, rootContext },
-      }: {
-        input: { context: EngineStreamContext; rootContext: AppMachineContext }
-      }) => {
-        const video = context.videoRef.current
-        if (!video) return
-
-        video.pause()
-
-        const canvas = context.canvasRef.current
-        if (!canvas) return
-
-        await holdOntoVideoFrameInCanvas(video, canvas)
-        video.style.display = 'none'
-
-        await rootContext.sceneInfra.camControls.saveRemoteCameraState()
-
-        // Make sure we're on the next frame for no flickering between canvas
-        // and the video elements.
-        window.requestAnimationFrame(
-          () =>
-            void (async () => {
-              // Destroy the media stream. We will re-establish it. We could
-              // leave everything at pausing, preventing video decoders from running
-              // but we can do even better by significantly reducing network
-              // cards also.
-              context.mediaStream?.getVideoTracks()[0].stop()
-              context.mediaStream = null
-              video.srcObject = null
-
-              rootContext.engineCommandManager.tearDown({ idleMode: true })
-            })()
-        )
-      }
-    ),
     [EngineStreamTransition.StartOrReconfigureEngine]: fromPromise(
       async ({
         input: { context, event, rootContext },
@@ -157,21 +99,17 @@ export const engineStreamMachine = setup({
           rootContext: AppMachineContext
         }
       }) => {
-        if (!context.authToken) return
+        if (!context.authToken) return Promise.reject()
-
+        if (!context.videoRef.current) return Promise.reject()
-        const video = context.videoRef.current
+        if (!context.canvasRef.current) return Promise.reject()
-        if (!video) return
-
-        const canvas = context.canvasRef.current
-        if (!canvas) return
 
         const { width, height } = getDimensions(
           window.innerWidth,
           window.innerHeight
         )
 
-        video.width = width
+        context.videoRef.current.width = width
-        video.height = height
+        context.videoRef.current.height = height
 
         const settingsNext = {
           // override the pool param (?pool=) to request a specific engine instance
@@ -206,51 +144,183 @@ export const engineStreamMachine = setup({
         })
       }
     ),
+    [EngineStreamTransition.Play]: fromPromise(
+      async ({
+        input: { context, params },
+      }: {
+        input: { context: EngineStreamContext; params: { zoomToFit: boolean } }
+      }) => {
+        if (!context.canvasRef.current) return
+        if (!context.videoRef.current) return
+        if (!context.mediaStream) return
+
+        // If the video is already playing it means we're doing a reconfigure.
+        // We don't want to re-run the KCL or touch the video element at all.
+        if (!context.videoRef.current.paused) {
+          return
+        }
+
+        // In the past we'd try to play immediately, but the proper thing is to way
+        // for the 'canplay' event to tell us data is ready.
+        const onCanPlay = () => {
+          if (!context.videoRef.current) {
+            return
+          }
+
+          context.videoRef.current.play().catch(console.error)
+
+          // Yes, event listeners can remove themselves because of the
+          // lazy nature of interpreted languages :D
+          context.videoRef.current.removeEventListener('canplay', onCanPlay)
+        }
+
+        // We're receiving video frames, so show the video now.
+        const onPlay = () => {
+          // We have to give engine time to crunch all the scene setup we
+          // ask it to do. As far as I can tell it doesn't block until
+          // they are done, so we must wait.
+          setTimeout(() => {
+            if (!context.videoRef.current) {
+              return
+            }
+            if (!context.canvasRef.current) {
+              return
+            }
+
+            context.videoRef.current.style.display = 'block'
+            context.canvasRef.current.style.display = 'none'
+
+            context.videoRef.current.removeEventListener('play', onPlay)
+            // I've tried < 400ms and sometimes it's possible to see a flash
+            // and the camera snap.
+          }, 400)
+        }
+
+        context.videoRef.current.addEventListener('canplay', onCanPlay)
+        context.videoRef.current.addEventListener('play', onPlay)
+
+        // THIS ASSIGNMENT IS *EXTREMELY* EFFECTFUL! The amount of logic
+        // this triggers is quite far and wide. It drives the above events.
+        context.videoRef.current.srcObject = context.mediaStream
+      }
+    ),
+
+    // Pause is also called when leaving the modeling scene. It's possible
+    // then videoRef and canvasRef are now null due to their DOM elements
+    // being destroyed.
+    [EngineStreamTransition.Pause]: fromPromise(
+      async ({
+        input: { context, rootContext },
+      }: {
+        input: {
+          context: EngineStreamContext
+          rootContext: AppMachineContext
+        }
+      }) => {
+        if (context.videoRef.current && context.canvasRef.current) {
+          await context.videoRef.current.pause()
+
+          await holdOntoVideoFrameInCanvas(
+            context.videoRef.current,
+            context.canvasRef.current
+          )
+          context.videoRef.current.style.display = 'none'
+        }
+
+        await rootContext.sceneInfra.camControls.saveRemoteCameraState()
+
+        // Make sure we're on the next frame for no flickering between canvas
+        // and the video elements.
+        window.requestAnimationFrame(
+          () =>
+            void (async () => {
+              // Destroy the media stream. We will re-establish it. We could
+              // leave everything at pausing, preventing video decoders from running
+              // but we can do even better by significantly reducing network
+              // cards also.
+              context.mediaStream?.getVideoTracks()[0].stop()
+              context.mediaStream = null
+
+              if (context.videoRef.current) {
+                context.videoRef.current.srcObject = null
+              }
+
+              rootContext.engineCommandManager.tearDown({ idleMode: true })
+            })()
+        )
+      }
+    ),
   },
 }).createMachine({
-  initial: EngineStreamState.Off,
+  initial: EngineStreamState.WaitingForDependencies,
   context: (initial) => initial.input,
   states: {
-    [EngineStreamState.Off]: {
+    [EngineStreamState.WaitingForDependencies]: {
-      reenter: true,
       on: {
         [EngineStreamTransition.SetPool]: {
-          target: EngineStreamState.Off,
+          target: EngineStreamState.WaitingForDependencies,
-          actions: [assign({ pool: ({ context, event }) => event.data.pool })],
+          actions: [assign({ pool: ({ context, event }) => event.pool })],
         },
         [EngineStreamTransition.SetAuthToken]: {
-          target: EngineStreamState.Off,
+          target: EngineStreamState.WaitingForDependencies,
           actions: [
-            assign({ authToken: ({ context, event }) => event.data.authToken }),
+            assign({ authToken: ({ context, event }) => event.authToken }),
+          ],
+        },
+        [EngineStreamTransition.SetVideoRef]: {
+          target: EngineStreamState.WaitingForDependencies,
+          actions: [
+            assign({ videoRef: ({ context, event }) => event.videoRef }),
+          ],
+        },
+        [EngineStreamTransition.SetCanvasRef]: {
+          target: EngineStreamState.WaitingForDependencies,
+          actions: [
+            assign({ canvasRef: ({ context, event }) => event.canvasRef }),
           ],
         },
         [EngineStreamTransition.StartOrReconfigureEngine]: {
-          target: EngineStreamState.On,
+          target: EngineStreamState.WaitingForMediaStream,
         },
       },
     },
-    [EngineStreamState.On]: {
+    [EngineStreamState.WaitingForMediaStream]: {
-      reenter: true,
       invoke: {
         src: EngineStreamTransition.StartOrReconfigureEngine,
         input: (args) => ({
           context: args.context,
           rootContext: args.self.system.get('root').getSnapshot().context,
-          params: { zoomToFit: args.context.zoomToFit },
           event: args.event,
         }),
+        onError: [
+          {
+            target: EngineStreamState.WaitingForDependencies,
+            reenter: true,
+          },
+        ],
       },
       on: {
-        // Transition requested by engineConnection
+        [EngineStreamTransition.StartOrReconfigureEngine]: {
+          target: EngineStreamState.WaitingForMediaStream,
+          reenter: true,
+        },
         [EngineStreamTransition.SetMediaStream]: {
-          target: EngineStreamState.On,
+          target: EngineStreamState.WaitingToPlay,
           actions: [
             assign({ mediaStream: ({ context, event }) => event.mediaStream }),
           ],
         },
+      },
+    },
+    [EngineStreamState.WaitingToPlay]: {
+      on: {
         [EngineStreamTransition.Play]: {
           target: EngineStreamState.Playing,
-          actions: [assign({ zoomToFit: () => true })],
+        },
+        // We actually failed inbetween needing to play and sending commands.
+        [EngineStreamTransition.StartOrReconfigureEngine]: {
+          target: EngineStreamState.WaitingForMediaStream,
+          reenter: true,
         },
       },
     },
@@ -270,6 +340,9 @@ export const engineStreamMachine = setup({
         [EngineStreamTransition.Pause]: {
           target: EngineStreamState.Paused,
         },
+        [EngineStreamTransition.Stop]: {
+          target: EngineStreamState.Stopped,
+        },
       },
     },
     [EngineStreamState.Reconfiguring]: {
@@ -280,9 +353,7 @@ export const engineStreamMachine = setup({
           rootContext: args.self.system.get('root').getSnapshot().context,
           event: args.event,
         }),
-        onDone: {
+        onDone: [{ target: EngineStreamState.Playing }],
-          target: EngineStreamState.Playing,
-        },
       },
     },
     [EngineStreamState.Paused]: {
@@ -299,8 +370,27 @@ export const engineStreamMachine = setup({
         },
       },
     },
+    [EngineStreamState.Stopped]: {
+      invoke: {
+        src: EngineStreamTransition.Pause,
+        input: (args) => ({
+          context: args.context,
+          rootContext: args.self.system.get('root').getSnapshot().context,
+        }),
+        onDone: [
+          {
+            target: EngineStreamState.WaitingForDependencies,
+            actions: [
+              assign({
+                videoRef: { current: null },
+                canvasRef: { current: null },
+              }),
+            ],
+          },
+        ],
+      },
+    },
     [EngineStreamState.Resuming]: {
-      reenter: true,
       invoke: {
         src: EngineStreamTransition.StartOrReconfigureEngine,
         input: (args) => ({
@@ -315,14 +405,11 @@ export const engineStreamMachine = setup({
           target: EngineStreamState.Paused,
         },
         [EngineStreamTransition.SetMediaStream]: {
+          target: EngineStreamState.Playing,
           actions: [
             assign({ mediaStream: ({ context, event }) => event.mediaStream }),
           ],
         },
-        [EngineStreamTransition.Play]: {
-          target: EngineStreamState.Playing,
-          actions: [assign({ zoomToFit: () => false })],
-        },
       },
     },
   },