Add back stream idle mode

rust/kcl-lib/src/settings/types/mod.rs

@@ -5,7 +5,7 @@ pub mod project;
 use anyhow::Result;
 use parse_display::{Display, FromStr};
 use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
+use serde::{Deserializer, Deserialize, Serialize};
 use validator::{Validate, ValidateRange};
 
 const DEFAULT_THEME_COLOR: f64 = 264.5;
@@ -131,9 +131,9 @@ pub struct AppSettings {
     /// This setting only applies to the web app. And is temporary until we have Linux support.
     #[serde(default, alias = "dismissWebBanner", skip_serializing_if = "is_default")]
     pub dismiss_web_banner: bool,
-    /// When the user is idle, and this is true, the stream will be torn down.
+    /// When the user is idle, teardown the stream after some time.
-    #[serde(default, alias = "streamIdleMode", skip_serializing_if = "is_default")]
+    #[serde(default, deserialize_with = "deserialize_stream_idle_mode", alias = "streamIdleMode", skip_serializing_if = "is_default")]
-    pub stream_idle_mode: bool,
+    stream_idle_mode: Option<u32>,
     /// When the user is idle, and this is true, the stream will be torn down.
     #[serde(default, alias = "allowOrbitInSketchMode", skip_serializing_if = "is_default")]
     pub allow_orbit_in_sketch_mode: bool,
@@ -143,7 +143,29 @@ pub struct AppSettings {
     pub show_debug_panel: bool,
 }
 
-// TODO: When we remove backwards compatibility with the old settings file, we can remove this.
+fn deserialize_stream_idle_mode<'de, D>(deserializer: D) -> Result<Option<u32>, D::Error>
+where
+  D: Deserializer<'de>,
+{
+  #[derive(Deserialize)]
+  #[serde(untagged)]
+  enum StreamIdleModeValue {
+    String(String),
+    Boolean(bool),
+  }
+
+  const DEFAULT_TIMEOUT: u32 = 1000 * 60 * 5;
+
+  Ok(match StreamIdleModeValue::deserialize(deserializer) {
+    Ok(StreamIdleModeValue::String(value)) => Some(value.parse::<u32>().unwrap_or(DEFAULT_TIMEOUT)),
+    // The old type of this value. I'm willing to say no one used it but
+    // we can never guarantee it.
+    Ok(StreamIdleModeValue::Boolean(true)) => Some(DEFAULT_TIMEOUT),
+    Ok(StreamIdleModeValue::Boolean(false)) => None,
+    _ => None
+  })
+}
+
 #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, ts_rs::TS, PartialEq)]
 #[ts(export)]
 #[serde(untagged)]
@@ -626,7 +648,7 @@ textWrapping = true
                         theme_color: None,
                         dismiss_web_banner: false,
                         enable_ssao: None,
-                        stream_idle_mode: false,
+                        stream_idle_mode: None,
                         allow_orbit_in_sketch_mode: false,
                         show_debug_panel: true,
                     },
@@ -691,7 +713,7 @@ includeSettings = false
                         dismiss_web_banner: false,
                         enable_ssao: None,
                         show_debug_panel: true,
-                        stream_idle_mode: false,
+                        stream_idle_mode: None,
                         allow_orbit_in_sketch_mode: false,
                     },
                     modeling: ModelingSettings {
@@ -759,7 +781,7 @@ defaultProjectName = "projects-$nnn"
                         theme_color: None,
                         dismiss_web_banner: false,
                         enable_ssao: None,
-                        stream_idle_mode: false,
+                        stream_idle_mode: None,
                         allow_orbit_in_sketch_mode: false,
                         show_debug_panel: true,
                     },
@@ -841,7 +863,7 @@ projectDirectory = "/Users/macinatormax/Documents/kittycad-modeling-projects""#;
                         dismiss_web_banner: false,
                         enable_ssao: None,
                         show_debug_panel: false,
-                        stream_idle_mode: false,
+                        stream_idle_mode: None,
                         allow_orbit_in_sketch_mode: false,
                     },
                     modeling: ModelingSettings {

src/App.tsx

@@ -41,6 +41,8 @@ import { onboardingPaths } from '@src/routes/Onboarding/paths'
 maybeWriteToDisk()
   .then(() => {})
   .catch(() => {})
+import EngineStreamContext from 'hooks/useEngineStreamContext'
+import { EngineStream } from 'components/EngineStream'
 
 export function App() {
   const { project, file } = useLoaderData() as IndexLoaderData
@@ -64,6 +66,12 @@ export function App() {
   // the coredump.
   const ref = useRef<HTMLDivElement>(null)
 
+  // Stream related refs and data
+  const videoRef = useRef<HTMLVideoElement>(null)
+  const canvasRef = useRef<HTMLCanvasElement>(null)
+  let [searchParams] = useSearchParams()
+  const pool = searchParams.get('pool')
+
   const projectName = project?.name || null
   const projectPath = project?.path || null
 
@@ -149,13 +157,26 @@ export function App() {
       />
       <ModalContainer />
       <ModelingSidebar paneOpacity={paneOpacity} />
-      <Stream />
+      <EngineStreamContext.Provider
-      {/* <CamToggle /> */}
+        options={{
-      <LowerRightControls coreDumpManager={coreDumpManager}>
+          input: {
-        <UnitsMenu />
+            videoRef,
-        <Gizmo />
+            canvasRef,
-        <CameraProjectionToggle />
+            mediaStream: null,
-      </LowerRightControls>
+            authToken: token ?? null,
+            pool,
+            zoomToFit: true,
+          },
+        }}
+      >
+        <EngineStream />
+        {/* <CamToggle /> */}
+        <LowerRightControls coreDumpManager={coreDumpManager}>
+          <UnitsMenu />
+          <Gizmo />
+          <CameraProjectionToggle />
+        </LowerRightControls>
+      </EngineStreamContext.Provider>
     </div>
   )
 }

src/Toolbar.tsx

@@ -52,7 +52,7 @@ export function Toolbar({
   }, [kclManager.artifactGraph, context.selectionRanges])
 
   const toolbarButtonsRef = useRef<HTMLUListElement>(null)
-  const { overallState } = useNetworkContext()
+  const { overallState, immediateState } = useNetworkContext()
   const { isExecuting } = useKclContext()
   const { isStreamReady } = useAppState()
   const [showRichContent, setShowRichContent] = useState(false)
@@ -61,6 +61,7 @@ export function Toolbar({
     (overallState !== NetworkHealthState.Ok &&
       overallState !== NetworkHealthState.Weak) ||
     isExecuting ||
+    immediateState.type !== EngineConnectionStateType.ConnectionEstablished ||
     !isStreamReady
 
   const currentMode =

src/clientSideScene/CameraControls.ts

@@ -105,6 +105,12 @@ export class CameraControls {
   wasDragging: boolean
   mouseDownPosition: Vector2
   mouseNewPosition: Vector2
+  old:
+    | {
+        camera: PerspectiveCamera | OrthographicCamera
+        target: Vector3
+      }
+    | undefined
   rotationSpeed = 0.3
   enableRotate = true
   enablePan = true
@@ -956,6 +962,28 @@ export class CameraControls {
     })
   }
 
+  async restoreCameraPosition(): Promise<void> {
+    if (!this.old) return
+
+    this.camera = this.old.camera.clone()
+    this.target = this.old.target.clone()
+
+    void this.engineCommandManager.sendSceneCommand({
+      type: 'modeling_cmd_req',
+      cmd_id: uuidv4(),
+      cmd: {
+        type: 'default_camera_look_at',
+        ...convertThreeCamValuesToEngineCam({
+          isPerspective: true,
+          position: this.camera.position,
+          quaternion: this.camera.quaternion,
+          zoom: this.camera.zoom,
+          target: this.target,
+        }),
+      },
+    })
+  }
+
   async tweenCameraToQuaternion(
     targetQuaternion: Quaternion,
     targetPosition = new Vector3(),

src/components/CommandBar/CommandBar.tsx

@@ -18,6 +18,7 @@ export const COMMAND_PALETTE_HOTKEY = 'mod+k'
 export const CommandBar = () => {
   const { pathname } = useLocation()
   const commandBarState = useCommandBarState()
+  const { immediateState } = useNetworkContext()
   const {
     context: { selectedCommand, currentArgument, commands },
   } = commandBarState
@@ -32,6 +33,14 @@ export const CommandBar = () => {
     commandBarActor.send({ type: 'Close' })
   }, [pathname])
 
+  useEffect(() => {
+    if (
+      immediateState.type !== EngineConnectionStateType.ConnectionEstablished
+    ) {
+      commandBarActor.send({ type: 'Close' })
+    }
+  }, [immediateState])
+
   // Hook up keyboard shortcuts
   useHotkeyWrapper([COMMAND_PALETTE_HOTKEY], () => {
     if (commandBarState.context.commands.length === 0) return

src/components/CommandBarOpenButton.tsx

@@ -4,10 +4,16 @@ import { hotkeyDisplay } from '@src/lib/hotkeyWrapper'
 import { commandBarActor } from '@src/machines/commandBarMachine'
 
 export function CommandBarOpenButton() {
+  const { immediateState } = useNetworkContext()
+
   const platform = usePlatform()
 
+  const isDisabled =
+    immediateState.type !== EngineConnectionStateType.ConnectionEstablished
+
   return (
     <button
+      disabled={isDisabled}
       className="group rounded-full flex items-center justify-center gap-2 px-2 py-1 bg-primary/10 dark:bg-chalkboard-90 dark:backdrop-blur-sm border-primary hover:border-primary dark:border-chalkboard-50 dark:hover:border-inherit text-primary dark:text-inherit"
       onClick={() => commandBarActor.send({ type: 'Open' })}
       data-testid="command-bar-open-button"

src/components/EngineStream.tsx

@@ -0,0 +1,338 @@
+import { MouseEventHandler, useEffect, useRef } from 'react'
+import { useAppState } from 'AppState'
+import { useSettingsAuthContext } from 'hooks/useSettingsAuthContext'
+import { useModelingContext } from 'hooks/useModelingContext'
+import { useNetworkContext } from 'hooks/useNetworkContext'
+import { NetworkHealthState } from 'hooks/useNetworkStatus'
+import { ClientSideScene } from 'clientSideScene/ClientSideSceneComp'
+import { btnName } from 'lib/cameraControls'
+import { sendSelectEventToEngine } from 'lib/selections'
+import { kclManager, engineCommandManager, sceneInfra } from 'lib/singletons'
+import { EngineCommandManagerEvents } from 'lang/std/engineConnection'
+import { useRouteLoaderData } from 'react-router-dom'
+import { PATHS } from 'lib/paths'
+import { IndexLoaderData } from 'lib/types'
+import { err, reportRejection, trap } from 'lib/trap'
+import { getArtifactOfTypes } from 'lang/std/artifactGraph'
+import { ViewControlContextMenu } from './ViewControlMenu'
+import { commandBarActor, useCommandBarState } from 'machines/commandBarMachine'
+import { useSelector } from '@xstate/react'
+import useEngineStreamContext, {
+  EngineStreamState,
+  EngineStreamTransition,
+} from 'hooks/useEngineStreamContext'
+import { REASONABLE_TIME_TO_REFRESH_STREAM_SIZE } from 'lib/timings'
+
+export const EngineStream = () => {
+  const { setAppState } = useAppState()
+
+  const { overallState } = useNetworkContext()
+  const { settings } = useSettingsAuthContext()
+  const { file } = useRouteLoaderData(PATHS.FILE) as IndexLoaderData
+  const last = useRef<number>(Date.now())
+  const videoWrapperRef = useRef<HTMLDivElement>(null)
+
+  const settingsEngine = {
+    theme: settings.context.app.theme.current,
+    enableSSAO: settings.context.app.enableSSAO.current,
+    highlightEdges: settings.context.modeling.highlightEdges.current,
+    showScaleGrid: settings.context.modeling.showScaleGrid.current,
+    cameraProjection: settings.context.modeling.cameraProjection.current,
+  }
+
+  const { state: modelingMachineState, send: modelingMachineActorSend } =
+    useModelingContext()
+
+  const commandBarState = useCommandBarState()
+
+  const engineStreamActor = useEngineStreamContext.useActorRef()
+  const engineStreamState = engineStreamActor.getSnapshot()
+
+  const streamIdleMode = settings.context.app.streamIdleMode.current
+
+  const startOrReconfigureEngine = () => {
+    engineStreamActor.send({
+      type: EngineStreamTransition.StartOrReconfigureEngine,
+      modelingMachineActorSend,
+      settings: settingsEngine,
+      setAppState,
+
+      // It's possible a reconnect happens as we drag the window :')
+      onMediaStream(mediaStream: MediaStream) {
+        engineStreamActor.send({
+          type: EngineStreamTransition.SetMediaStream,
+          mediaStream,
+        })
+      },
+    })
+  }
+
+  useEffect(() => {
+    const play = () => {
+      engineStreamActor.send({
+        type: EngineStreamTransition.Play,
+      })
+    }
+    engineCommandManager.addEventListener(
+      EngineCommandManagerEvents.SceneReady,
+      play
+    )
+
+    return () => {
+      engineCommandManager.removeEventListener(
+        EngineCommandManagerEvents.SceneReady,
+        play
+      )
+    }
+  }, [])
+
+  useEffect(() => {
+    const video = engineStreamState.context.videoRef?.current
+    if (!video) return
+    const canvas = engineStreamState.context.canvasRef?.current
+    if (!canvas) return
+
+    new ResizeObserver(() => {
+      if (Date.now() - last.current < REASONABLE_TIME_TO_REFRESH_STREAM_SIZE)
+        return
+      last.current = Date.now()
+
+      if (
+        Math.abs(video.width - window.innerWidth) > 4 ||
+        Math.abs(video.height - window.innerHeight) > 4
+      ) {
+        timeoutStart.current = Date.now()
+        startOrReconfigureEngine()
+      }
+    }).observe(document.body)
+  }, [engineStreamState.value])
+
+  // When the video and canvas element references are set, start the engine.
+  useEffect(() => {
+    if (
+      engineStreamState.context.canvasRef.current &&
+      engineStreamState.context.videoRef.current
+    ) {
+      startOrReconfigureEngine()
+    }
+  }, [
+    engineStreamState.context.canvasRef.current,
+    engineStreamState.context.videoRef.current,
+  ])
+
+  // On settings change, reconfigure the engine. When paused this gets really tricky,
+  // and also requires onMediaStream to be set!
+  useEffect(() => {
+    if (engineStreamState.value === EngineStreamState.Playing) {
+      startOrReconfigureEngine()
+    }
+  }, [settings.context, engineStreamState.value])
+
+  /**
+   * Subscribe to execute code when the file changes
+   * but only if the scene is already ready.
+   * See onSceneReady for the initial scene setup.
+   */
+  useEffect(() => {
+    if (engineCommandManager.engineConnection?.isReady() && file?.path) {
+      console.log('execute on file change')
+      void kclManager.executeCode(true).catch(trap)
+    }
+  }, [file?.path, engineCommandManager.engineConnection])
+
+  const IDLE_TIME_MS = Number(streamIdleMode)
+
+  // When streamIdleMode is changed, setup or teardown the timeouts
+  const timeoutStart = useRef<number | null>(null)
+
+  useEffect(() => {
+    timeoutStart.current = streamIdleMode ? Date.now() : null
+  }, [streamIdleMode])
+
+  useEffect(() => {
+    let frameId: ReturnType<typeof window.requestAnimationFrame> = 0
+    const frameLoop = () => {
+      // Do not pause if the user is in the middle of an operation
+      if (!modelingMachineState.matches('idle')) {
+        // In fact, stop the timeout, because we don't want to trigger the
+        // pause when we exit the operation.
+        timeoutStart.current = null
+      } else if (timeoutStart.current) {
+        const elapsed = Date.now() - timeoutStart.current
+        if (elapsed >= IDLE_TIME_MS) {
+          timeoutStart.current = null
+          engineStreamActor.send({ type: EngineStreamTransition.Pause })
+        }
+      }
+      frameId = window.requestAnimationFrame(frameLoop)
+    }
+    frameId = window.requestAnimationFrame(frameLoop)
+
+    return () => {
+      window.cancelAnimationFrame(frameId)
+    }
+  }, [modelingMachineState])
+
+  useEffect(() => {
+    if (!streamIdleMode) return
+
+    const onAnyInput = () => {
+      // Just in case it happens in the middle of the user turning off
+      // idle mode.
+      if (!streamIdleMode) {
+        timeoutStart.current = null
+        return
+      }
+
+      if (engineStreamState.value === EngineStreamState.Paused) {
+        engineStreamActor.send({
+          type: EngineStreamTransition.StartOrReconfigureEngine,
+          modelingMachineActorSend,
+          settings: settingsEngine,
+          setAppState,
+          onMediaStream(mediaStream: MediaStream) {
+            engineStreamActor.send({
+              type: EngineStreamTransition.SetMediaStream,
+              mediaStream,
+            })
+          },
+        })
+      }
+
+      timeoutStart.current = Date.now()
+    }
+
+    // It's possible after a reconnect, the user doesn't move their mouse at
+    // all, meaning the timer is not reset to run. We need to set it every
+    // time our effect dependencies change then.
+    timeoutStart.current = Date.now()
+
+    window.document.addEventListener('keydown', onAnyInput)
+    window.document.addEventListener('keyup', onAnyInput)
+    window.document.addEventListener('mousemove', onAnyInput)
+    window.document.addEventListener('mousedown', onAnyInput)
+    window.document.addEventListener('mouseup', onAnyInput)
+    window.document.addEventListener('scroll', onAnyInput)
+    window.document.addEventListener('touchstart', onAnyInput)
+    window.document.addEventListener('touchstop', onAnyInput)
+
+    return () => {
+      timeoutStart.current = null
+      window.document.removeEventListener('keydown', onAnyInput)
+      window.document.removeEventListener('keyup', onAnyInput)
+      window.document.removeEventListener('mousemove', onAnyInput)
+      window.document.removeEventListener('mousedown', onAnyInput)
+      window.document.removeEventListener('mouseup', onAnyInput)
+      window.document.removeEventListener('scroll', onAnyInput)
+      window.document.removeEventListener('touchstart', onAnyInput)
+      window.document.removeEventListener('touchstop', onAnyInput)
+    }
+  }, [streamIdleMode, engineStreamState.value])
+
+  const isNetworkOkay =
+    overallState === NetworkHealthState.Ok ||
+    overallState === NetworkHealthState.Weak
+
+  const handleMouseUp: MouseEventHandler<HTMLDivElement> = (e) => {
+    if (!isNetworkOkay) return
+    if (!engineStreamState.context.videoRef.current) return
+    // If we're in sketch mode, don't send a engine-side select event
+    if (modelingMachineState.matches('Sketch')) return
+    // Only respect default plane selection if we're on a selection command argument
+    if (
+      modelingMachineState.matches({ idle: 'showPlanes' }) &&
+      !(
+        commandBarState.matches('Gathering arguments') &&
+        commandBarState.context.currentArgument?.inputType === 'selection'
+      )
+    )
+      return
+    // If we're mousing up from a camera drag, don't send a select event
+    if (sceneInfra.camControls.wasDragging === true) return
+
+    if (btnName(e.nativeEvent).left) {
+      // eslint-disable-next-line @typescript-eslint/no-floating-promises
+      sendSelectEventToEngine(e, engineStreamState.context.videoRef.current)
+    }
+  }
+
+  /**
+   * On double-click of sketch entities we automatically enter sketch mode with the selected sketch,
+   * allowing for quick editing of sketches. TODO: This should be moved to a more central place.
+   */
+  const enterSketchModeIfSelectingSketch: MouseEventHandler<HTMLDivElement> = (
+    e
+  ) => {
+    if (
+      !isNetworkOkay ||
+      !engineStreamState.context.videoRef.current ||
+      modelingMachineState.matches('Sketch') ||
+      modelingMachineState.matches({ idle: 'showPlanes' }) ||
+      sceneInfra.camControls.wasDragging === true ||
+      !btnName(e.nativeEvent).left
+    ) {
+      return
+    }
+
+    sendSelectEventToEngine(e, engineStreamState.context.videoRef.current)
+      .then(({ entity_id }) => {
+        if (!entity_id) {
+          // No entity selected. This is benign
+          return
+        }
+        const path = getArtifactOfTypes(
+          { key: entity_id, types: ['path', 'solid2d', 'segment', 'helix'] },
+          engineCommandManager.artifactGraph
+        )
+        if (err(path)) {
+          return path
+        }
+        sceneInfra.modelingSend({ type: 'Enter sketch' })
+      })
+      .catch(reportRejection)
+  }
+
+  return (
+    // eslint-disable-next-line jsx-a11y/no-static-element-interactions
+    <div
+      ref={videoWrapperRef}
+      className="absolute inset-0 z-0"
+      id="stream"
+      data-testid="stream"
+      onMouseUp={handleMouseUp}
+      onDoubleClick={enterSketchModeIfSelectingSketch}
+      onContextMenu={(e) => e.preventDefault()}
+      onContextMenuCapture={(e) => e.preventDefault()}
+    >
+      <video
+        autoPlay
+        muted
+        key={engineStreamActor.id + 'video'}
+        ref={engineStreamState.context.videoRef}
+        controls={false}
+        className="w-full cursor-pointer h-full"
+        disablePictureInPicture
+        id="video-stream"
+      />
+      <canvas
+        key={engineStreamActor.id + 'canvas'}
+        ref={engineStreamState.context.canvasRef}
+        className="cursor-pointer"
+        id="freeze-frame"
+      >
+        No canvas support
+      </canvas>
+      <ClientSideScene
+        cameraControls={settings.context.modeling.mouseControls.current}
+      />
+      <ViewControlContextMenu
+        event="mouseup"
+        guard={(e) =>
+          sceneInfra.camControls.wasDragging === false &&
+          btnName(e).right === true
+        }
+        menuTargetElement={videoWrapperRef}
+      />
+    </div>
+  )
+}

src/components/ModelStateIndicator.tsx

@@ -4,24 +4,34 @@ import { Spinner } from '@src/components/Spinner'
 
 export const ModelStateIndicator = () => {
   const [commands] = useEngineCommands()
+  const [isDone, setIsDone] = useState<boolean>(false)
+
+  const engineStreamActor = useEngineStreamContext.useActorRef()
+  const engineStreamState = engineStreamActor.getSnapshot()
+
   const lastCommandType = commands[commands.length - 1]?.type
 
+  useEffect(() => {
+    if (lastCommandType === CommandLogType.SetDefaultSystemProperties) {
+      setIsDone(false)
+    }
+    if (lastCommandType === CommandLogType.ExecutionDone) {
+      setIsDone(true)
+    }
+  }, [lastCommandType])
+
   let className = 'w-6 h-6 '
-  let icon = <Spinner className={className} />
+  let icon = <div className={className}></div>
   let dataTestId = 'model-state-indicator'
 
-  if (lastCommandType === 'receive-reliable') {
+  if (engineStreamState.value === EngineStreamState.Paused) {
-    className +=
+    className += 'text-secondary'
-      'bg-chalkboard-20 dark:bg-chalkboard-80 !group-disabled:bg-chalkboard-30 !dark:group-disabled:bg-chalkboard-80 rounded-sm bg-succeed-10/30 dark:bg-succeed'
+    icon = <CustomIcon data-testid={dataTestId + '-paused'} name="parallel" />
-    icon = (
+  } else if (engineStreamState.value === EngineStreamState.Resuming) {
-      <CustomIcon
+    className += 'text-secondary'
-        data-testid={dataTestId + '-receive-reliable'}
+    icon = <CustomIcon data-testid={dataTestId + '-resuming'} name="parallel" />
-        name="checkmark"
+  } else if (isDone) {
-      />
+    className += 'text-secondary'
-    )
-  } else if (lastCommandType === 'execution-done') {
-    className +=
-      'border-6 border border-solid border-chalkboard-60 dark:border-chalkboard-80 bg-chalkboard-20 dark:bg-chalkboard-80 !group-disabled:bg-chalkboard-30 !dark:group-disabled:bg-chalkboard-80 rounded-sm bg-succeed-10/30 dark:bg-succeed'
     icon = (
       <CustomIcon
         data-testid={dataTestId + '-execution-done'}

src/components/ModelingMachineProvider.tsx

@@ -147,6 +147,7 @@ export const ModelingMachineProvider = ({
       showScaleGrid,
       cameraOrbit,
       enableSSAO,
+      cameraProjection,
     },
   } = useSettings()
   const navigate = useNavigate()
@@ -163,6 +164,7 @@ export const ModelingMachineProvider = ({
     commandBarActor,
     commandBarIsClosedSelector
   )
+
   // Settings machine setup
   // const retrievedSettings = useRef(
   // localStorage?.getItem(MODELING_PERSIST_KEY) || '{}'
@@ -1913,22 +1915,6 @@ export const ModelingMachineProvider = ({
     }
   }, [modelingActor])
 
-  useSetupEngineManager(
-    streamRef,
-    modelingSend,
-    modelingState.context,
-    {
-      pool: pool,
-      theme: theme.current,
-      highlightEdges: highlightEdges.current,
-      enableSSAO: enableSSAO.current,
-      showScaleGrid: showScaleGrid.current,
-      cameraProjection: cameraProjection.current,
-      cameraOrbit: cameraOrbit.current,
-    },
-    token
-  )
-
   useEffect(() => {
     kclManager.registerExecuteCallback(() => {
       modelingSend({ type: 'Re-execute' })

src/hooks/useEngineStreamContext.ts

@@ -0,0 +1,254 @@
+import { makeDefaultPlanes, clearSceneAndBustCache } from 'lang/wasm'
+import { MutableRefObject } from 'react'
+import { setup, assign, fromPromise } from 'xstate'
+import { createActorContext } from '@xstate/react'
+import { kclManager, sceneInfra, engineCommandManager } from 'lib/singletons'
+import { trap } from 'lib/trap'
+
+export enum EngineStreamState {
+  Off = 'off',
+  On = 'on',
+  WaitForMediaStream = 'wait-for-media-stream',
+  Playing = 'playing',
+  Paused = 'paused',
+  // The is the state inbetween Paused and Playing *specifically that order*.
+  Resuming = 'resuming',
+}
+
+export enum EngineStreamTransition {
+  SetMediaStream = 'set-context',
+  Play = 'play',
+  Resume = 'resume',
+  Pause = 'pause',
+  StartOrReconfigureEngine = 'start-or-reconfigure-engine',
+}
+
+export interface EngineStreamContext {
+  pool: string | null
+  authToken: string | null
+  mediaStream: MediaStream | null
+  videoRef: MutableRefObject<HTMLVideoElement | null>
+  canvasRef: MutableRefObject<HTMLCanvasElement | null>
+  zoomToFit: boolean
+}
+
+export function getDimensions(streamWidth: number, streamHeight: number) {
+  const factorOf = 4
+  const maxResolution = 2160
+  const ratio = Math.min(
+    Math.min(maxResolution / streamWidth, maxResolution / streamHeight),
+    1.0
+  )
+  const quadWidth = Math.round((streamWidth * ratio) / factorOf) * factorOf
+  const quadHeight = Math.round((streamHeight * ratio) / factorOf) * factorOf
+  return { width: quadWidth, height: quadHeight }
+}
+
+const engineStreamMachine = setup({
+  types: {
+    context: {} as EngineStreamContext,
+    input: {} as EngineStreamContext,
+  },
+  actors: {
+    [EngineStreamTransition.Play]: fromPromise(
+      async ({
+        input: { context, params },
+      }: {
+        input: { context: EngineStreamContext; params: { zoomToFit: boolean } }
+      }) => {
+        const canvas = context.canvasRef.current
+        if (!canvas) return false
+
+        const video = context.videoRef.current
+        if (!video) return false
+
+        const mediaStream = context.mediaStream
+        if (!mediaStream) return false
+
+        video.style.display = 'block'
+        canvas.style.display = 'none'
+
+        await sceneInfra.camControls.restoreCameraPosition()
+        await clearSceneAndBustCache(kclManager.engineCommandManager)
+
+        video.srcObject = mediaStream
+        await video.play()
+
+        await kclManager.executeCode(params.zoomToFit)
+      }
+    ),
+    [EngineStreamTransition.Pause]: fromPromise(
+      async ({
+        input: { context },
+      }: {
+        input: { context: EngineStreamContext }
+      }) => {
+        const video = context.videoRef.current
+        if (!video) return
+
+        video.pause()
+
+        const canvas = context.canvasRef.current
+        if (!canvas) return
+
+        canvas.width = video.videoWidth
+        canvas.height = video.videoHeight
+        canvas.style.width = video.videoWidth + 'px'
+        canvas.style.height = video.videoHeight + 'px'
+        canvas.style.display = 'block'
+
+        const ctx = canvas.getContext('2d')
+        if (!ctx) return
+
+        ctx.drawImage(video, 0, 0, canvas.width, canvas.height)
+
+        // Make sure we're on the next frame for no flickering between canvas
+        // and the video elements.
+        window.requestAnimationFrame(() => {
+          video.style.display = 'none'
+
+          // Destroy the media stream only. We will re-establish it. We could
+          // leave everything at pausing, preventing video decoders from running
+          // but we can do even better by significantly reducing network
+          // cards also.
+          context.mediaStream?.getVideoTracks()[0].stop()
+          video.srcObject = null
+
+          sceneInfra.camControls.old = {
+            camera: sceneInfra.camControls.camera.clone(),
+            target: sceneInfra.camControls.target.clone(),
+          }
+
+          engineCommandManager.tearDown({ idleMode: true })
+        })
+      }
+    ),
+    [EngineStreamTransition.StartOrReconfigureEngine]: fromPromise(
+      async ({
+        input: { context, event },
+      }: {
+        input: { context: EngineStreamContext; event: any }
+      }) => {
+        if (!context.authToken) return
+
+        const video = context.videoRef.current
+        if (!video) return
+
+        const { width, height } = getDimensions(
+          window.innerWidth,
+          window.innerHeight
+        )
+
+        video.width = width
+        video.height = height
+
+        const settingsNext = {
+          // override the pool param (?pool=) to request a specific engine instance
+          // from a particular pool.
+          pool: context.pool,
+          ...event.settings,
+        }
+
+        engineCommandManager.settings = settingsNext
+
+        engineCommandManager.start({
+          setMediaStream: event.onMediaStream,
+          setIsStreamReady: (isStreamReady) =>
+            event.setAppState({ isStreamReady }),
+          width,
+          height,
+          token: context.authToken,
+          settings: settingsNext,
+          makeDefaultPlanes: () => {
+            return makeDefaultPlanes(kclManager.engineCommandManager)
+          },
+        })
+
+        event.modelingMachineActorSend({
+          type: 'Set context',
+          data: {
+            streamDimensions: {
+              streamWidth: width,
+              streamHeight: height,
+            },
+          },
+        })
+      }
+    ),
+  },
+}).createMachine({
+  context: (initial) => initial.input,
+  initial: EngineStreamState.Off,
+  states: {
+    [EngineStreamState.Off]: {
+      on: {
+        [EngineStreamTransition.StartOrReconfigureEngine]: {
+          target: EngineStreamState.On,
+        },
+      },
+    },
+    [EngineStreamState.On]: {
+      reenter: true,
+      invoke: {
+        src: EngineStreamTransition.StartOrReconfigureEngine,
+        input: (args) => args,
+      },
+      on: {
+        [EngineStreamTransition.SetMediaStream]: {
+          target: EngineStreamState.On,
+          actions: [
+            assign({ mediaStream: ({ context, event }) => event.mediaStream }),
+          ],
+        },
+        [EngineStreamTransition.Play]: {
+          target: EngineStreamState.Playing,
+          actions: [assign({ zoomToFit: () => true })],
+        },
+      },
+    },
+    [EngineStreamState.Playing]: {
+      invoke: {
+        src: EngineStreamTransition.Play,
+        input: (args) => ({
+          context: args.context,
+          params: { zoomToFit: args.context.zoomToFit },
+        }),
+      },
+      on: {
+        [EngineStreamTransition.Pause]: {
+          target: EngineStreamState.Paused,
+        },
+      },
+    },
+    [EngineStreamState.Paused]: {
+      invoke: {
+        src: EngineStreamTransition.Pause,
+        input: (args) => args,
+      },
+      on: {
+        [EngineStreamTransition.StartOrReconfigureEngine]: {
+          target: EngineStreamState.Resuming,
+        },
+      },
+    },
+    [EngineStreamState.Resuming]: {
+      invoke: {
+        src: EngineStreamTransition.StartOrReconfigureEngine,
+        input: (args) => args,
+      },
+      on: {
+        [EngineStreamTransition.SetMediaStream]: {
+          actions: [
+            assign({ mediaStream: ({ context, event }) => event.mediaStream }),
+          ],
+        },
+        [EngineStreamTransition.Play]: {
+          target: EngineStreamState.Playing,
+          actions: [assign({ zoomToFit: () => false })],
+        },
+      },
+    },
+  },
+})
+
+export default createActorContext(engineStreamMachine)

src/lang/KclSingleton.ts

@@ -473,7 +473,7 @@ export class KclManager {
       sceneInfra.modelingSend({ type: 'code edit during sketch' })
     }
     this.engineCommandManager.addCommandLog({
-      type: 'execution-done',
+      type: CommandLogType.ExecutionDone,
       data: null,
     })
 
@@ -492,7 +492,7 @@ export class KclManager {
     this.isExecuting = false
     this.executeIsStale = null
     this.engineCommandManager.addCommandLog({
-      type: 'execution-done',
+      type: CommandLogType.ExecutionDone,
       data: null,
     })
     markOnce('code/endExecuteAst')

src/lang/std/engineConnection.ts

@@ -394,13 +394,14 @@ class EngineConnection extends EventTarget {
         default:
           if (this.isConnecting()) break
           // Means we never could do an initial connection. Reconnect everything.
-          if (!this.pingPongSpan.ping) this.connect().catch(reportRejection)
+          if (!this.pingPongSpan.ping)
+            this.connect({ reconnect: false }).catch(reportRejection)
           break
       }
     }, pingIntervalMs)
 
     // eslint-disable-next-line @typescript-eslint/no-floating-promises
-    this.connect()
+    this.connect({ reconnect: false })
   }
 
   // SHOULD ONLY BE USED FOR VITESTS
@@ -511,7 +512,9 @@ class EngineConnection extends EventTarget {
     this.idleMode = opts?.idleMode ?? false
     clearInterval(this.pingIntervalId)
 
-    if (opts?.idleMode) {
+    this.disconnectAll()
+
+    if (this.idleMode) {
       this.state = {
         type: EngineConnectionStateType.Disconnecting,
         value: {
@@ -530,8 +533,6 @@ class EngineConnection extends EventTarget {
         type: DisconnectingType.Quit,
       },
     }
-
-    this.disconnectAll()
   }
 
   initiateConnectionExclusive(): boolean {
@@ -585,8 +586,7 @@ class EngineConnection extends EventTarget {
    * This will attempt the full handshake, and retry if the connection
    * did not establish.
    */
-  connect(reconnecting?: boolean): Promise<void> {
+  connect(args: { reconnect: boolean }): Promise<void> {
-    // eslint-disable-next-line @typescript-eslint/no-this-alias
     const that = this
     return new Promise((resolve) => {
       if (this.isConnecting() || this.isReady()) {
@@ -1197,7 +1197,7 @@ class EngineConnection extends EventTarget {
         this.websocket.addEventListener('message', this.onWebSocketMessage)
       }
 
-      if (reconnecting) {
+      if (args.reconnect) {
         createWebSocketConnection()
       } else {
         this.onNetworkStatusReady = () => {
@@ -1210,6 +1210,7 @@ class EngineConnection extends EventTarget {
       }
     })
   }
+
   // Do not change this back to an object or any, we should only be sending the
   // WebSocketRequest type!
   unreliableSend(message: Models['WebSocketRequest_type']) {
@@ -1261,8 +1262,17 @@ class EngineConnection extends EventTarget {
       this.websocket?.readyState === 3
 
     if (closedPc && closedUDC && closedWS) {
-      // Do not notify the rest of the program that we have cut off anything.
+      if (!this.idleMode) {
-      this.state = { type: EngineConnectionStateType.Disconnected }
+        // Do not notify the rest of the program that we have cut off anything.
+        this.state = { type: EngineConnectionStateType.Disconnected }
+      } else {
+        this.state = {
+          type: EngineConnectionStateType.Disconnecting,
+          value: {
+            type: DisconnectingType.Pause,
+          },
+        }
+      }
       this.triggeredStart = false
     }
   }
@@ -1288,23 +1298,32 @@ export interface Subscription<T extends ModelTypes> {
   ) => void
 }
 
+export enum CommandLogType {
+  SendModeling = 'send-modeling',
+  SendScene = 'send-scene',
+  ReceiveReliable = 'receive-reliable',
+  ExecutionDone = 'execution-done',
+  ExportDone = 'export-done',
+  SetDefaultSystemProperties = 'set_default_system_properties',
+}
+
 export type CommandLog =
   | {
-      type: 'send-modeling'
+      type: CommandLogType.SendModeling
       data: EngineCommand
     }
   | {
-      type: 'send-scene'
+      type: CommandLogType.SendScene
       data: EngineCommand
     }
   | {
-      type: 'receive-reliable'
+      type: CommandLogType.ReceiveReliable
       data: OkWebSocketResponseData
       id: string
       cmd_type?: string
     }
   | {
-      type: 'execution-done'
+      type: CommandLogType.ExecutionDone
       data: null
     }
 
@@ -1646,7 +1665,7 @@ export class EngineCommandManager extends EventTarget {
           message.request_id
         ) {
           this.addCommandLog({
-            type: 'receive-reliable',
+            type: CommandLogType.ReceiveReliable,
             data: message.resp,
             id: message?.request_id || '',
             cmd_type: pending?.command?.cmd?.type,
@@ -1680,7 +1699,7 @@ export class EngineCommandManager extends EventTarget {
               if (!command) return
               if (command.type === 'modeling_cmd_req')
                 this.addCommandLog({
-                  type: 'receive-reliable',
+                  type: CommandLogType.ReceiveReliable,
                   data: {
                     type: 'modeling',
                     data: {
@@ -1722,7 +1741,7 @@ export class EngineCommandManager extends EventTarget {
       )
 
       // eslint-disable-next-line @typescript-eslint/no-floating-promises
-      this.engineConnection?.connect()
+      this.engineConnection?.connect({ reconnect: false })
     }
     this.engineConnection.addEventListener(
       EngineConnectionEvents.ConnectionStarted,
@@ -1782,6 +1801,7 @@ export class EngineCommandManager extends EventTarget {
       )
 
       this.engineConnection?.tearDown(opts)
+      this.engineConnection = undefined
 
       // Our window.engineCommandManager.tearDown assignment causes this case to happen which is
       // only really for tests.
@@ -1789,6 +1809,8 @@ export class EngineCommandManager extends EventTarget {
     } else if (this.engineCommandManager?.engineConnection) {
       // @ts-ignore
       this.engineCommandManager?.engineConnection?.tearDown(opts)
+      // @ts-ignore
+      this.engineCommandManager.engineConnection = null
     }
   }
   async startNewSession() {
@@ -1864,7 +1886,7 @@ export class EngineCommandManager extends EventTarget {
     ) {
       // highlight_set_entity, mouse_move and camera_drag_move are sent over the unreliable channel and are too noisy
       this.addCommandLog({
-        type: 'send-scene',
+        type: CommandLogType.SendScene,
         data: command,
       })
     }

src/lib/settings/initialSettings.tsx

@@ -123,6 +123,8 @@ export class Setting<T = unknown> {
   }
 }
 
+const MS_IN_MINUTE = 1000 * 60
+
 export function createSettings() {
   return {
     /** Settings that affect the behavior of the entire app,
@@ -208,13 +210,58 @@ export function createSettings() {
       /**
        * Stream resource saving behavior toggle
        */
-      streamIdleMode: new Setting<boolean>({
+      streamIdleMode: new Setting<number | undefined>({
-        defaultValue: false,
+        defaultValue: undefined,
         description: 'Toggle stream idling, saving bandwidth and battery',
-        validate: (v) => typeof v === 'boolean',
+        validate: (v) =>
-        commandConfig: {
+          v === undefined ||
-          inputType: 'boolean',
+          (typeof v === 'number' &&
-        },
+            v >= 1 * MS_IN_MINUTE &&
+            v <= 60 * MS_IN_MINUTE),
+        Component: ({ value, updateValue }) => (
+          <div className="flex item-center gap-4 px-2 m-0 py-0">
+            <div className="flex flex-col">
+              <input
+                type="checkbox"
+                checked={value !== undefined}
+                onChange={(e) =>
+                  updateValue(
+                    !e.currentTarget.checked ? undefined : 5 * MS_IN_MINUTE
+                  )
+                }
+                className="block w-4 h-4"
+              />
+              <div></div>
+            </div>
+            <div className="flex flex-col grow">
+              <input
+                type="range"
+                onChange={(e) =>
+                  updateValue(Number(e.currentTarget.value) * MS_IN_MINUTE)
+                }
+                disabled={value === undefined}
+                value={
+                  value !== null && value !== undefined
+                    ? value / MS_IN_MINUTE
+                    : 5
+                }
+                min={1}
+                max={60}
+                step={1}
+                className="block flex-1"
+              />
+              {value !== undefined && value !== null && (
+                <div>
+                  {value / MS_IN_MINUTE === 60
+                    ? '1 hour'
+                    : value / MS_IN_MINUTE === 1
+                    ? '1 minute'
+                    : value / MS_IN_MINUTE + ' minutes'}
+                </div>
+              )}
+            </div>
+          </div>
+        ),
       }),
       allowOrbitInSketchMode: new Setting<boolean>({
         defaultValue: false,

src/lib/settings/settingsUtils.ts

@@ -33,6 +33,10 @@ import { appThemeToTheme } from '@src/lib/theme'
 import { err } from '@src/lib/trap'
 import type { DeepPartial } from '@src/lib/types'
 
+type OmitNull<T> = T extends null ? undefined : T
+const toUndefinedIfNull = (a: any): OmitNull<any> =>
+  a === null ? undefined : a
+
 /**
  * Convert from a rust settings struct into the JS settings struct.
  * We do this because the JS settings type has all the fancy shit
@@ -49,7 +53,9 @@ export function configurationToSettingsPayload(
         : undefined,
       onboardingStatus: configuration?.settings?.app?.onboarding_status,
       dismissWebBanner: configuration?.settings?.app?.dismiss_web_banner,
-      streamIdleMode: configuration?.settings?.app?.stream_idle_mode,
+      streamIdleMode: toUndefinedIfNull(
+        configuration?.settings?.app?.stream_idle_mode
+      ),
       allowOrbitInSketchMode:
         configuration?.settings?.app?.allow_orbit_in_sketch_mode,
       projectDirectory: configuration?.settings?.project?.directory,
@@ -128,7 +134,9 @@ export function projectConfigurationToSettingsPayload(
         : undefined,
       onboardingStatus: configuration?.settings?.app?.onboarding_status,
       dismissWebBanner: configuration?.settings?.app?.dismiss_web_banner,
-      streamIdleMode: configuration?.settings?.app?.stream_idle_mode,
+      streamIdleMode: toUndefinedIfNull(
+        configuration?.settings?.app?.stream_idle_mode
+      ),
       allowOrbitInSketchMode:
         configuration?.settings?.app?.allow_orbit_in_sketch_mode,
       namedViews: deepPartialNamedViewsToNamedViews(

src/lib/timings.ts

@@ -0,0 +1,3 @@
+// 0.25s is the average visual reaction time for humans so we'll go a bit less
+// so those exception people don't see.
+export const REASONABLE_TIME_TO_REFRESH_STREAM_SIZE = 100