ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add back stream idle mode

Browse Source
This commit is contained in:
49lf
2025-02-07 17:03:09 -05:00
committed by lee-at-zoo-corp
parent 1aa27bd91c
commit e642731529
15 changed files with 829 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
rust/kcl-lib/src/settings/types/mod.rs
View File

@ -5,7 +5,7 @@ pub mod project;
use anyhow::Result;
use parse_display::{Display, FromStr};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde::{Deserializer, Deserialize, Serialize};
use validator::{Validate, ValidateRange};
const DEFAULT_THEME_COLOR: f64 = 264.5;
@ -131,9 +131,9 @@ pub struct AppSettings {
/// This setting only applies to the web app. And is temporary until we have Linux support.
#[serde(default, alias = "dismissWebBanner", skip_serializing_if = "is_default")]
pub dismiss_web_banner: bool,
/// When the user is idle, and this is true, the stream will be torn down.
#[serde(default, alias = "streamIdleMode", skip_serializing_if = "is_default")]
pub stream_idle_mode: bool,
/// When the user is idle, teardown the stream after some time.
#[serde(default, deserialize_with = "deserialize_stream_idle_mode", alias = "streamIdleMode", skip_serializing_if = "is_default")]
stream_idle_mode: Option<u32>,
/// When the user is idle, and this is true, the stream will be torn down.
#[serde(default, alias = "allowOrbitInSketchMode", skip_serializing_if = "is_default")]
pub allow_orbit_in_sketch_mode: bool,
@ -143,7 +143,29 @@ pub struct AppSettings {
pub show_debug_panel: bool,
}
// TODO: When we remove backwards compatibility with the old settings file, we can remove this.
fn deserialize_stream_idle_mode<'de, D>(deserializer: D) -> Result<Option<u32>, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum StreamIdleModeValue {
String(String),
Boolean(bool),
}
const DEFAULT_TIMEOUT: u32 = 1000 * 60 * 5;
Ok(match StreamIdleModeValue::deserialize(deserializer) {
Ok(StreamIdleModeValue::String(value)) => Some(value.parse::<u32>().unwrap_or(DEFAULT_TIMEOUT)),
// The old type of this value. I'm willing to say no one used it but
// we can never guarantee it.
Ok(StreamIdleModeValue::Boolean(true)) => Some(DEFAULT_TIMEOUT),
Ok(StreamIdleModeValue::Boolean(false)) => None,
_ => None
})
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, ts_rs::TS, PartialEq)]
#[ts(export)]
#[serde(untagged)]
@ -626,7 +648,7 @@ textWrapping = true
theme_color: None,
dismiss_web_banner: false,
enable_ssao: None,
stream_idle_mode: false,
stream_idle_mode: None,
allow_orbit_in_sketch_mode: false,
show_debug_panel: true,
},
@ -691,7 +713,7 @@ includeSettings = false
dismiss_web_banner: false,
enable_ssao: None,
show_debug_panel: true,
stream_idle_mode: false,
stream_idle_mode: None,
allow_orbit_in_sketch_mode: false,
},
modeling: ModelingSettings {
@ -759,7 +781,7 @@ defaultProjectName = "projects-$nnn"
theme_color: None,
dismiss_web_banner: false,
enable_ssao: None,
stream_idle_mode: false,
stream_idle_mode: None,
allow_orbit_in_sketch_mode: false,
show_debug_panel: true,
},
@ -841,7 +863,7 @@ projectDirectory = "/Users/macinatormax/Documents/kittycad-modeling-projects""#;
dismiss_web_banner: false,
enable_ssao: None,
show_debug_panel: false,
stream_idle_mode: false,
stream_idle_mode: None,
allow_orbit_in_sketch_mode: false,
},
modeling: ModelingSettings {

35
src/App.tsx
View File

@ -41,6 +41,8 @@ import { onboardingPaths } from '@src/routes/Onboarding/paths'
maybeWriteToDisk()
.then(() => {})
.catch(() => {})
import EngineStreamContext from 'hooks/useEngineStreamContext'
import { EngineStream } from 'components/EngineStream'
export function App() {
const { project, file } = useLoaderData() as IndexLoaderData
@ -64,6 +66,12 @@ export function App() {
// the coredump.
const ref = useRef<HTMLDivElement>(null)
// Stream related refs and data
const videoRef = useRef<HTMLVideoElement>(null)
const canvasRef = useRef<HTMLCanvasElement>(null)
let [searchParams] = useSearchParams()
const pool = searchParams.get('pool')
const projectName = project?.name || null
const projectPath = project?.path || null
@ -149,13 +157,26 @@ export function App() {
/>
<ModalContainer />
<ModelingSidebar paneOpacity={paneOpacity} />
<Stream />
{/* <CamToggle /> */}
<LowerRightControls coreDumpManager={coreDumpManager}>
<UnitsMenu />
<Gizmo />
<CameraProjectionToggle />
</LowerRightControls>
<EngineStreamContext.Provider
options={{
input: {
videoRef,
canvasRef,
mediaStream: null,
authToken: token ?? null,
pool,
zoomToFit: true,
},
}}
>
<EngineStream />
{/* <CamToggle /> */}
<LowerRightControls coreDumpManager={coreDumpManager}>
<UnitsMenu />
<Gizmo />
<CameraProjectionToggle />
</LowerRightControls>
</EngineStreamContext.Provider>
</div>
)
}

3
src/Toolbar.tsx
View File

@ -52,7 +52,7 @@ export function Toolbar({
}, [kclManager.artifactGraph, context.selectionRanges])
const toolbarButtonsRef = useRef<HTMLUListElement>(null)
const { overallState } = useNetworkContext()
const { overallState, immediateState } = useNetworkContext()
const { isExecuting } = useKclContext()
const { isStreamReady } = useAppState()
const [showRichContent, setShowRichContent] = useState(false)
@ -61,6 +61,7 @@ export function Toolbar({
(overallState !== NetworkHealthState.Ok &&
overallState !== NetworkHealthState.Weak) ||
isExecuting ||
immediateState.type !== EngineConnectionStateType.ConnectionEstablished ||
!isStreamReady
const currentMode =

28
src/clientSideScene/CameraControls.ts
View File

@ -105,6 +105,12 @@ export class CameraControls {
wasDragging: boolean
mouseDownPosition: Vector2
mouseNewPosition: Vector2
old:
| {
camera: PerspectiveCamera | OrthographicCamera
target: Vector3
}
| undefined
rotationSpeed = 0.3
enableRotate = true
enablePan = true
@ -956,6 +962,28 @@ export class CameraControls {
})
}
async restoreCameraPosition(): Promise<void> {
if (!this.old) return
this.camera = this.old.camera.clone()
this.target = this.old.target.clone()
void this.engineCommandManager.sendSceneCommand({
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'default_camera_look_at',
...convertThreeCamValuesToEngineCam({
isPerspective: true,
position: this.camera.position,
quaternion: this.camera.quaternion,
zoom: this.camera.zoom,
target: this.target,
}),
},
})
}
async tweenCameraToQuaternion(
targetQuaternion: Quaternion,
targetPosition = new Vector3(),

9
src/components/CommandBar/CommandBar.tsx
View File

@ -18,6 +18,7 @@ export const COMMAND_PALETTE_HOTKEY = 'mod+k'
export const CommandBar = () => {
const { pathname } = useLocation()
const commandBarState = useCommandBarState()
const { immediateState } = useNetworkContext()
const {
context: { selectedCommand, currentArgument, commands },
} = commandBarState
@ -32,6 +33,14 @@ export const CommandBar = () => {
commandBarActor.send({ type: 'Close' })
}, [pathname])
useEffect(() => {
if (
immediateState.type !== EngineConnectionStateType.ConnectionEstablished
) {
commandBarActor.send({ type: 'Close' })
}
}, [immediateState])
// Hook up keyboard shortcuts
useHotkeyWrapper([COMMAND_PALETTE_HOTKEY], () => {
if (commandBarState.context.commands.length === 0) return

6
src/components/CommandBarOpenButton.tsx
View File

@ -4,10 +4,16 @@ import { hotkeyDisplay } from '@src/lib/hotkeyWrapper'
import { commandBarActor } from '@src/machines/commandBarMachine'
export function CommandBarOpenButton() {
const { immediateState } = useNetworkContext()
const platform = usePlatform()
const isDisabled =
immediateState.type !== EngineConnectionStateType.ConnectionEstablished
return (
<button
disabled={isDisabled}
className="group rounded-full flex items-center justify-center gap-2 px-2 py-1 bg-primary/10 dark:bg-chalkboard-90 dark:backdrop-blur-sm border-primary hover:border-primary dark:border-chalkboard-50 dark:hover:border-inherit text-primary dark:text-inherit"
onClick={() => commandBarActor.send({ type: 'Open' })}
data-testid="command-bar-open-button"

338
src/components/EngineStream.tsx Normal file
View File

@ -0,0 +1,338 @@
import { MouseEventHandler, useEffect, useRef } from 'react'
import { useAppState } from 'AppState'
import { useSettingsAuthContext } from 'hooks/useSettingsAuthContext'
import { useModelingContext } from 'hooks/useModelingContext'
import { useNetworkContext } from 'hooks/useNetworkContext'
import { NetworkHealthState } from 'hooks/useNetworkStatus'
import { ClientSideScene } from 'clientSideScene/ClientSideSceneComp'
import { btnName } from 'lib/cameraControls'
import { sendSelectEventToEngine } from 'lib/selections'
import { kclManager, engineCommandManager, sceneInfra } from 'lib/singletons'
import { EngineCommandManagerEvents } from 'lang/std/engineConnection'
import { useRouteLoaderData } from 'react-router-dom'
import { PATHS } from 'lib/paths'
import { IndexLoaderData } from 'lib/types'
import { err, reportRejection, trap } from 'lib/trap'
import { getArtifactOfTypes } from 'lang/std/artifactGraph'
import { ViewControlContextMenu } from './ViewControlMenu'
import { commandBarActor, useCommandBarState } from 'machines/commandBarMachine'
import { useSelector } from '@xstate/react'
import useEngineStreamContext, {
EngineStreamState,
EngineStreamTransition,
} from 'hooks/useEngineStreamContext'
import { REASONABLE_TIME_TO_REFRESH_STREAM_SIZE } from 'lib/timings'
export const EngineStream = () => {
const { setAppState } = useAppState()
const { overallState } = useNetworkContext()
const { settings } = useSettingsAuthContext()
const { file } = useRouteLoaderData(PATHS.FILE) as IndexLoaderData
const last = useRef<number>(Date.now())
const videoWrapperRef = useRef<HTMLDivElement>(null)
const settingsEngine = {
theme: settings.context.app.theme.current,
enableSSAO: settings.context.app.enableSSAO.current,
highlightEdges: settings.context.modeling.highlightEdges.current,
showScaleGrid: settings.context.modeling.showScaleGrid.current,
cameraProjection: settings.context.modeling.cameraProjection.current,
}
const { state: modelingMachineState, send: modelingMachineActorSend } =
useModelingContext()
const commandBarState = useCommandBarState()
const engineStreamActor = useEngineStreamContext.useActorRef()
const engineStreamState = engineStreamActor.getSnapshot()
const streamIdleMode = settings.context.app.streamIdleMode.current
const startOrReconfigureEngine = () => {
engineStreamActor.send({
type: EngineStreamTransition.StartOrReconfigureEngine,
modelingMachineActorSend,
settings: settingsEngine,
setAppState,
// It's possible a reconnect happens as we drag the window :')
onMediaStream(mediaStream: MediaStream) {
engineStreamActor.send({
type: EngineStreamTransition.SetMediaStream,
mediaStream,
})
},
})
}
useEffect(() => {
const play = () => {
engineStreamActor.send({
type: EngineStreamTransition.Play,
})
}
engineCommandManager.addEventListener(
EngineCommandManagerEvents.SceneReady,
play
)
return () => {
engineCommandManager.removeEventListener(
EngineCommandManagerEvents.SceneReady,
play
)
}
}, [])
useEffect(() => {
const video = engineStreamState.context.videoRef?.current
if (!video) return
const canvas = engineStreamState.context.canvasRef?.current
if (!canvas) return
new ResizeObserver(() => {
if (Date.now() - last.current < REASONABLE_TIME_TO_REFRESH_STREAM_SIZE)
return
last.current = Date.now()
if (
Math.abs(video.width - window.innerWidth) > 4 ||
Math.abs(video.height - window.innerHeight) > 4
) {
timeoutStart.current = Date.now()
startOrReconfigureEngine()
}
}).observe(document.body)
}, [engineStreamState.value])
// When the video and canvas element references are set, start the engine.
useEffect(() => {
if (
engineStreamState.context.canvasRef.current &&
engineStreamState.context.videoRef.current
) {
startOrReconfigureEngine()
}
}, [
engineStreamState.context.canvasRef.current,
engineStreamState.context.videoRef.current,
])
// On settings change, reconfigure the engine. When paused this gets really tricky,
// and also requires onMediaStream to be set!
useEffect(() => {
if (engineStreamState.value === EngineStreamState.Playing) {
startOrReconfigureEngine()
}
}, [settings.context, engineStreamState.value])
/**
* Subscribe to execute code when the file changes
* but only if the scene is already ready.
* See onSceneReady for the initial scene setup.
*/
useEffect(() => {
if (engineCommandManager.engineConnection?.isReady() && file?.path) {
console.log('execute on file change')
void kclManager.executeCode(true).catch(trap)
}
}, [file?.path, engineCommandManager.engineConnection])
const IDLE_TIME_MS = Number(streamIdleMode)
// When streamIdleMode is changed, setup or teardown the timeouts
const timeoutStart = useRef<number | null>(null)
useEffect(() => {
timeoutStart.current = streamIdleMode ? Date.now() : null
}, [streamIdleMode])
useEffect(() => {
let frameId: ReturnType<typeof window.requestAnimationFrame> = 0
const frameLoop = () => {
// Do not pause if the user is in the middle of an operation
if (!modelingMachineState.matches('idle')) {
// In fact, stop the timeout, because we don't want to trigger the
// pause when we exit the operation.
timeoutStart.current = null
} else if (timeoutStart.current) {
const elapsed = Date.now() - timeoutStart.current
if (elapsed >= IDLE_TIME_MS) {
timeoutStart.current = null
engineStreamActor.send({ type: EngineStreamTransition.Pause })
}
}
frameId = window.requestAnimationFrame(frameLoop)
}
frameId = window.requestAnimationFrame(frameLoop)
return () => {
window.cancelAnimationFrame(frameId)
}
}, [modelingMachineState])
useEffect(() => {
if (!streamIdleMode) return
const onAnyInput = () => {
// Just in case it happens in the middle of the user turning off
// idle mode.
if (!streamIdleMode) {
timeoutStart.current = null
return
}
if (engineStreamState.value === EngineStreamState.Paused) {
engineStreamActor.send({
type: EngineStreamTransition.StartOrReconfigureEngine,
modelingMachineActorSend,
settings: settingsEngine,
setAppState,
onMediaStream(mediaStream: MediaStream) {
engineStreamActor.send({
type: EngineStreamTransition.SetMediaStream,
mediaStream,
})
},
})
}
timeoutStart.current = Date.now()
}
// It's possible after a reconnect, the user doesn't move their mouse at
// all, meaning the timer is not reset to run. We need to set it every
// time our effect dependencies change then.
timeoutStart.current = Date.now()
window.document.addEventListener('keydown', onAnyInput)
window.document.addEventListener('keyup', onAnyInput)
window.document.addEventListener('mousemove', onAnyInput)
window.document.addEventListener('mousedown', onAnyInput)
window.document.addEventListener('mouseup', onAnyInput)
window.document.addEventListener('scroll', onAnyInput)
window.document.addEventListener('touchstart', onAnyInput)
window.document.addEventListener('touchstop', onAnyInput)
return () => {
timeoutStart.current = null
window.document.removeEventListener('keydown', onAnyInput)
window.document.removeEventListener('keyup', onAnyInput)
window.document.removeEventListener('mousemove', onAnyInput)
window.document.removeEventListener('mousedown', onAnyInput)
window.document.removeEventListener('mouseup', onAnyInput)
window.document.removeEventListener('scroll', onAnyInput)
window.document.removeEventListener('touchstart', onAnyInput)
window.document.removeEventListener('touchstop', onAnyInput)
}
}, [streamIdleMode, engineStreamState.value])
const isNetworkOkay =
overallState === NetworkHealthState.Ok ||
overallState === NetworkHealthState.Weak
const handleMouseUp: MouseEventHandler<HTMLDivElement> = (e) => {
if (!isNetworkOkay) return
if (!engineStreamState.context.videoRef.current) return
// If we're in sketch mode, don't send a engine-side select event
if (modelingMachineState.matches('Sketch')) return
// Only respect default plane selection if we're on a selection command argument
if (
modelingMachineState.matches({ idle: 'showPlanes' }) &&
!(
commandBarState.matches('Gathering arguments') &&
commandBarState.context.currentArgument?.inputType === 'selection'
)
)
return
// If we're mousing up from a camera drag, don't send a select event
if (sceneInfra.camControls.wasDragging === true) return
if (btnName(e.nativeEvent).left) {
// eslint-disable-next-line @typescript-eslint/no-floating-promises
sendSelectEventToEngine(e, engineStreamState.context.videoRef.current)
}
}
/**
* On double-click of sketch entities we automatically enter sketch mode with the selected sketch,
* allowing for quick editing of sketches. TODO: This should be moved to a more central place.
*/
const enterSketchModeIfSelectingSketch: MouseEventHandler<HTMLDivElement> = (
e
) => {
if (
!isNetworkOkay ||
!engineStreamState.context.videoRef.current ||
modelingMachineState.matches('Sketch') ||
modelingMachineState.matches({ idle: 'showPlanes' }) ||
sceneInfra.camControls.wasDragging === true ||
!btnName(e.nativeEvent).left
) {
return
}
sendSelectEventToEngine(e, engineStreamState.context.videoRef.current)
.then(({ entity_id }) => {
if (!entity_id) {
// No entity selected. This is benign
return
}
const path = getArtifactOfTypes(
{ key: entity_id, types: ['path', 'solid2d', 'segment', 'helix'] },
engineCommandManager.artifactGraph
)
if (err(path)) {
return path
}
sceneInfra.modelingSend({ type: 'Enter sketch' })
})
.catch(reportRejection)
}
return (
// eslint-disable-next-line jsx-a11y/no-static-element-interactions
<div
ref={videoWrapperRef}
className="absolute inset-0 z-0"
id="stream"
data-testid="stream"
onMouseUp={handleMouseUp}
onDoubleClick={enterSketchModeIfSelectingSketch}
onContextMenu={(e) => e.preventDefault()}
onContextMenuCapture={(e) => e.preventDefault()}
>
<video
autoPlay
muted
key={engineStreamActor.id + 'video'}
ref={engineStreamState.context.videoRef}
controls={false}
className="w-full cursor-pointer h-full"
disablePictureInPicture
id="video-stream"
/>
<canvas
key={engineStreamActor.id + 'canvas'}
ref={engineStreamState.context.canvasRef}
className="cursor-pointer"
id="freeze-frame"
>
No canvas support
</canvas>
<ClientSideScene
cameraControls={settings.context.modeling.mouseControls.current}
/>
<ViewControlContextMenu
event="mouseup"
guard={(e) =>
sceneInfra.camControls.wasDragging === false &&
btnName(e).right === true
}
menuTargetElement={videoWrapperRef}
/>
</div>
)
}

36
src/components/ModelStateIndicator.tsx
View File

@ -4,24 +4,34 @@ import { Spinner } from '@src/components/Spinner'
export const ModelStateIndicator = () => {
const [commands] = useEngineCommands()
const [isDone, setIsDone] = useState<boolean>(false)
const engineStreamActor = useEngineStreamContext.useActorRef()
const engineStreamState = engineStreamActor.getSnapshot()
const lastCommandType = commands[commands.length - 1]?.type
useEffect(() => {
if (lastCommandType === CommandLogType.SetDefaultSystemProperties) {
setIsDone(false)
}
if (lastCommandType === CommandLogType.ExecutionDone) {
setIsDone(true)
}
}, [lastCommandType])
let className = 'w-6 h-6 '
let icon = <Spinner className={className} />
let icon = <div className={className}></div>
let dataTestId = 'model-state-indicator'
if (lastCommandType === 'receive-reliable') {
className +=
'bg-chalkboard-20 dark:bg-chalkboard-80 !group-disabled:bg-chalkboard-30 !dark:group-disabled:bg-chalkboard-80 rounded-sm bg-succeed-10/30 dark:bg-succeed'
icon = (
<CustomIcon
data-testid={dataTestId + '-receive-reliable'}
name="checkmark"
/>
)
} else if (lastCommandType === 'execution-done') {
className +=
'border-6 border border-solid border-chalkboard-60 dark:border-chalkboard-80 bg-chalkboard-20 dark:bg-chalkboard-80 !group-disabled:bg-chalkboard-30 !dark:group-disabled:bg-chalkboard-80 rounded-sm bg-succeed-10/30 dark:bg-succeed'
if (engineStreamState.value === EngineStreamState.Paused) {
className += 'text-secondary'
icon = <CustomIcon data-testid={dataTestId + '-paused'} name="parallel" />
} else if (engineStreamState.value === EngineStreamState.Resuming) {
className += 'text-secondary'
icon = <CustomIcon data-testid={dataTestId + '-resuming'} name="parallel" />
} else if (isDone) {
className += 'text-secondary'
icon = (
<CustomIcon
data-testid={dataTestId + '-execution-done'}

18
src/components/ModelingMachineProvider.tsx
View File

@ -147,6 +147,7 @@ export const ModelingMachineProvider = ({
showScaleGrid,
cameraOrbit,
enableSSAO,
cameraProjection,
},
} = useSettings()
const navigate = useNavigate()
@ -163,6 +164,7 @@ export const ModelingMachineProvider = ({
commandBarActor,
commandBarIsClosedSelector
)
// Settings machine setup
// const retrievedSettings = useRef(
// localStorage?.getItem(MODELING_PERSIST_KEY) || '{}'
@ -1913,22 +1915,6 @@ export const ModelingMachineProvider = ({
}
}, [modelingActor])
useSetupEngineManager(
streamRef,
modelingSend,
modelingState.context,
{
pool: pool,
theme: theme.current,
highlightEdges: highlightEdges.current,
enableSSAO: enableSSAO.current,
showScaleGrid: showScaleGrid.current,
cameraProjection: cameraProjection.current,
cameraOrbit: cameraOrbit.current,
},
token
)
useEffect(() => {
kclManager.registerExecuteCallback(() => {
modelingSend({ type: 'Re-execute' })

254
src/hooks/useEngineStreamContext.ts Normal file
View File

@ -0,0 +1,254 @@
import { makeDefaultPlanes, clearSceneAndBustCache } from 'lang/wasm'
import { MutableRefObject } from 'react'
import { setup, assign, fromPromise } from 'xstate'
import { createActorContext } from '@xstate/react'
import { kclManager, sceneInfra, engineCommandManager } from 'lib/singletons'
import { trap } from 'lib/trap'
export enum EngineStreamState {
Off = 'off',
On = 'on',
WaitForMediaStream = 'wait-for-media-stream',
Playing = 'playing',
Paused = 'paused',
// The is the state inbetween Paused and Playing *specifically that order*.
Resuming = 'resuming',
}
export enum EngineStreamTransition {
SetMediaStream = 'set-context',
Play = 'play',
Resume = 'resume',
Pause = 'pause',
StartOrReconfigureEngine = 'start-or-reconfigure-engine',
}
export interface EngineStreamContext {
pool: string | null
authToken: string | null
mediaStream: MediaStream | null
videoRef: MutableRefObject<HTMLVideoElement | null>
canvasRef: MutableRefObject<HTMLCanvasElement | null>
zoomToFit: boolean
}
export function getDimensions(streamWidth: number, streamHeight: number) {
const factorOf = 4
const maxResolution = 2160
const ratio = Math.min(
Math.min(maxResolution / streamWidth, maxResolution / streamHeight),
1.0
)
const quadWidth = Math.round((streamWidth * ratio) / factorOf) * factorOf
const quadHeight = Math.round((streamHeight * ratio) / factorOf) * factorOf
return { width: quadWidth, height: quadHeight }
}
const engineStreamMachine = setup({
types: {
context: {} as EngineStreamContext,
input: {} as EngineStreamContext,
},
actors: {
[EngineStreamTransition.Play]: fromPromise(
async ({
input: { context, params },
}: {
input: { context: EngineStreamContext; params: { zoomToFit: boolean } }
}) => {
const canvas = context.canvasRef.current
if (!canvas) return false
const video = context.videoRef.current
if (!video) return false
const mediaStream = context.mediaStream
if (!mediaStream) return false
video.style.display = 'block'
canvas.style.display = 'none'
await sceneInfra.camControls.restoreCameraPosition()
await clearSceneAndBustCache(kclManager.engineCommandManager)
video.srcObject = mediaStream
await video.play()
await kclManager.executeCode(params.zoomToFit)
}
),
[EngineStreamTransition.Pause]: fromPromise(
async ({
input: { context },
}: {
input: { context: EngineStreamContext }
}) => {
const video = context.videoRef.current
if (!video) return
video.pause()
const canvas = context.canvasRef.current
if (!canvas) return
canvas.width = video.videoWidth
canvas.height = video.videoHeight
canvas.style.width = video.videoWidth + 'px'
canvas.style.height = video.videoHeight + 'px'
canvas.style.display = 'block'
const ctx = canvas.getContext('2d')
if (!ctx) return
ctx.drawImage(video, 0, 0, canvas.width, canvas.height)
// Make sure we're on the next frame for no flickering between canvas
// and the video elements.
window.requestAnimationFrame(() => {
video.style.display = 'none'
// Destroy the media stream only. We will re-establish it. We could
// leave everything at pausing, preventing video decoders from running
// but we can do even better by significantly reducing network
// cards also.
context.mediaStream?.getVideoTracks()[0].stop()
video.srcObject = null
sceneInfra.camControls.old = {
camera: sceneInfra.camControls.camera.clone(),
target: sceneInfra.camControls.target.clone(),
}
engineCommandManager.tearDown({ idleMode: true })
})
}
),
[EngineStreamTransition.StartOrReconfigureEngine]: fromPromise(
async ({
input: { context, event },
}: {
input: { context: EngineStreamContext; event: any }
}) => {
if (!context.authToken) return
const video = context.videoRef.current
if (!video) return
const { width, height } = getDimensions(
window.innerWidth,
window.innerHeight
)
video.width = width
video.height = height
const settingsNext = {
// override the pool param (?pool=) to request a specific engine instance
// from a particular pool.
pool: context.pool,
...event.settings,
}
engineCommandManager.settings = settingsNext
engineCommandManager.start({
setMediaStream: event.onMediaStream,
setIsStreamReady: (isStreamReady) =>
event.setAppState({ isStreamReady }),
width,
height,
token: context.authToken,
settings: settingsNext,
makeDefaultPlanes: () => {
return makeDefaultPlanes(kclManager.engineCommandManager)
},
})
event.modelingMachineActorSend({
type: 'Set context',
data: {
streamDimensions: {
streamWidth: width,
streamHeight: height,
},
},
})
}
),
},
}).createMachine({
context: (initial) => initial.input,
initial: EngineStreamState.Off,
states: {
[EngineStreamState.Off]: {
on: {
[EngineStreamTransition.StartOrReconfigureEngine]: {
target: EngineStreamState.On,
},
},
},
[EngineStreamState.On]: {
reenter: true,
invoke: {
src: EngineStreamTransition.StartOrReconfigureEngine,
input: (args) => args,
},
on: {
[EngineStreamTransition.SetMediaStream]: {
target: EngineStreamState.On,
actions: [
assign({ mediaStream: ({ context, event }) => event.mediaStream }),
],
},
[EngineStreamTransition.Play]: {
target: EngineStreamState.Playing,
actions: [assign({ zoomToFit: () => true })],
},
},
},
[EngineStreamState.Playing]: {
invoke: {
src: EngineStreamTransition.Play,
input: (args) => ({
context: args.context,
params: { zoomToFit: args.context.zoomToFit },
}),
},
on: {
[EngineStreamTransition.Pause]: {
target: EngineStreamState.Paused,
},
},
},
[EngineStreamState.Paused]: {
invoke: {
src: EngineStreamTransition.Pause,
input: (args) => args,
},
on: {
[EngineStreamTransition.StartOrReconfigureEngine]: {
target: EngineStreamState.Resuming,
},
},
},
[EngineStreamState.Resuming]: {
invoke: {
src: EngineStreamTransition.StartOrReconfigureEngine,
input: (args) => args,
},
on: {
[EngineStreamTransition.SetMediaStream]: {
actions: [
assign({ mediaStream: ({ context, event }) => event.mediaStream }),
],
},
[EngineStreamTransition.Play]: {
target: EngineStreamState.Playing,
actions: [assign({ zoomToFit: () => false })],
},
},
},
},
})
export default createActorContext(engineStreamMachine)

4
src/lang/KclSingleton.ts
View File

@ -473,7 +473,7 @@ export class KclManager {
sceneInfra.modelingSend({ type: 'code edit during sketch' })
}
this.engineCommandManager.addCommandLog({
type: 'execution-done',
type: CommandLogType.ExecutionDone,
data: null,
})
@ -492,7 +492,7 @@ export class KclManager {
this.isExecuting = false
this.executeIsStale = null
this.engineCommandManager.addCommandLog({
type: 'execution-done',
type: CommandLogType.ExecutionDone,
data: null,
})
markOnce('code/endExecuteAst')

58
src/lang/std/engineConnection.ts
View File

@ -394,13 +394,14 @@ class EngineConnection extends EventTarget {
default:
if (this.isConnecting()) break
// Means we never could do an initial connection. Reconnect everything.
if (!this.pingPongSpan.ping) this.connect().catch(reportRejection)
if (!this.pingPongSpan.ping)
this.connect({ reconnect: false }).catch(reportRejection)
break
}
}, pingIntervalMs)
// eslint-disable-next-line @typescript-eslint/no-floating-promises
this.connect()
this.connect({ reconnect: false })
}
// SHOULD ONLY BE USED FOR VITESTS
@ -511,7 +512,9 @@ class EngineConnection extends EventTarget {
this.idleMode = opts?.idleMode ?? false
clearInterval(this.pingIntervalId)
if (opts?.idleMode) {
this.disconnectAll()
if (this.idleMode) {
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: {
@ -530,8 +533,6 @@ class EngineConnection extends EventTarget {
type: DisconnectingType.Quit,
},
}
this.disconnectAll()
}
initiateConnectionExclusive(): boolean {
@ -585,8 +586,7 @@ class EngineConnection extends EventTarget {
* This will attempt the full handshake, and retry if the connection
* did not establish.
*/
connect(reconnecting?: boolean): Promise<void> {
// eslint-disable-next-line @typescript-eslint/no-this-alias
connect(args: { reconnect: boolean }): Promise<void> {
const that = this
return new Promise((resolve) => {
if (this.isConnecting() || this.isReady()) {
@ -1197,7 +1197,7 @@ class EngineConnection extends EventTarget {
this.websocket.addEventListener('message', this.onWebSocketMessage)
}
if (reconnecting) {
if (args.reconnect) {
createWebSocketConnection()
} else {
this.onNetworkStatusReady = () => {
@ -1210,6 +1210,7 @@ class EngineConnection extends EventTarget {
}
})
}
// Do not change this back to an object or any, we should only be sending the
// WebSocketRequest type!
unreliableSend(message: Models['WebSocketRequest_type']) {
@ -1261,8 +1262,17 @@ class EngineConnection extends EventTarget {
this.websocket?.readyState === 3
if (closedPc && closedUDC && closedWS) {
// Do not notify the rest of the program that we have cut off anything.
this.state = { type: EngineConnectionStateType.Disconnected }
if (!this.idleMode) {
// Do not notify the rest of the program that we have cut off anything.
this.state = { type: EngineConnectionStateType.Disconnected }
} else {
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: {
type: DisconnectingType.Pause,
},
}
}
this.triggeredStart = false
}
}
@ -1288,23 +1298,32 @@ export interface Subscription<T extends ModelTypes> {
) => void
}
export enum CommandLogType {
SendModeling = 'send-modeling',
SendScene = 'send-scene',
ReceiveReliable = 'receive-reliable',
ExecutionDone = 'execution-done',
ExportDone = 'export-done',
SetDefaultSystemProperties = 'set_default_system_properties',
}
export type CommandLog =
| {
type: 'send-modeling'
type: CommandLogType.SendModeling
data: EngineCommand
}
| {
type: 'send-scene'
type: CommandLogType.SendScene
data: EngineCommand
}
| {
type: 'receive-reliable'
type: CommandLogType.ReceiveReliable
data: OkWebSocketResponseData
id: string
cmd_type?: string
}
| {
type: 'execution-done'
type: CommandLogType.ExecutionDone
data: null
}
@ -1646,7 +1665,7 @@ export class EngineCommandManager extends EventTarget {
message.request_id
) {
this.addCommandLog({
type: 'receive-reliable',
type: CommandLogType.ReceiveReliable,
data: message.resp,
id: message?.request_id || '',
cmd_type: pending?.command?.cmd?.type,
@ -1680,7 +1699,7 @@ export class EngineCommandManager extends EventTarget {
if (!command) return
if (command.type === 'modeling_cmd_req')
this.addCommandLog({
type: 'receive-reliable',
type: CommandLogType.ReceiveReliable,
data: {
type: 'modeling',
data: {
@ -1722,7 +1741,7 @@ export class EngineCommandManager extends EventTarget {
)
// eslint-disable-next-line @typescript-eslint/no-floating-promises
this.engineConnection?.connect()
this.engineConnection?.connect({ reconnect: false })
}
this.engineConnection.addEventListener(
EngineConnectionEvents.ConnectionStarted,
@ -1782,6 +1801,7 @@ export class EngineCommandManager extends EventTarget {
)
this.engineConnection?.tearDown(opts)
this.engineConnection = undefined
// Our window.engineCommandManager.tearDown assignment causes this case to happen which is
// only really for tests.
@ -1789,6 +1809,8 @@ export class EngineCommandManager extends EventTarget {
} else if (this.engineCommandManager?.engineConnection) {
// @ts-ignore
this.engineCommandManager?.engineConnection?.tearDown(opts)
// @ts-ignore
this.engineCommandManager.engineConnection = null
}
}
async startNewSession() {
@ -1864,7 +1886,7 @@ export class EngineCommandManager extends EventTarget {
) {
// highlight_set_entity, mouse_move and camera_drag_move are sent over the unreliable channel and are too noisy
this.addCommandLog({
type: 'send-scene',
type: CommandLogType.SendScene,
data: command,
})
}

59
src/lib/settings/initialSettings.tsx
View File

@ -123,6 +123,8 @@ export class Setting<T = unknown> {
}
}
const MS_IN_MINUTE = 1000 * 60
export function createSettings() {
return {
/** Settings that affect the behavior of the entire app,
@ -208,13 +210,58 @@ export function createSettings() {
/**
* Stream resource saving behavior toggle
*/
streamIdleMode: new Setting<boolean>({
defaultValue: false,
streamIdleMode: new Setting<number | undefined>({
defaultValue: undefined,
description: 'Toggle stream idling, saving bandwidth and battery',
validate: (v) => typeof v === 'boolean',
commandConfig: {
inputType: 'boolean',
},
validate: (v) =>
v === undefined ||
(typeof v === 'number' &&
v >= 1 * MS_IN_MINUTE &&
v <= 60 * MS_IN_MINUTE),
Component: ({ value, updateValue }) => (
<div className="flex item-center gap-4 px-2 m-0 py-0">
<div className="flex flex-col">
<input
type="checkbox"
checked={value !== undefined}
onChange={(e) =>
updateValue(
!e.currentTarget.checked ? undefined : 5 * MS_IN_MINUTE
)
}
className="block w-4 h-4"
/>
<div></div>
</div>
<div className="flex flex-col grow">
<input
type="range"
onChange={(e) =>
updateValue(Number(e.currentTarget.value) * MS_IN_MINUTE)
}
disabled={value === undefined}
value={
value !== null && value !== undefined
? value / MS_IN_MINUTE
: 5
}
min={1}
max={60}
step={1}
className="block flex-1"
/>
{value !== undefined && value !== null && (
<div>
{value / MS_IN_MINUTE === 60
? '1 hour'
: value / MS_IN_MINUTE === 1
? '1 minute'
: value / MS_IN_MINUTE + ' minutes'}
</div>
)}
</div>
</div>
),
}),
allowOrbitInSketchMode: new Setting<boolean>({
defaultValue: false,

12
src/lib/settings/settingsUtils.ts
View File

@ -33,6 +33,10 @@ import { appThemeToTheme } from '@src/lib/theme'
import { err } from '@src/lib/trap'
import type { DeepPartial } from '@src/lib/types'
type OmitNull<T> = T extends null ? undefined : T
const toUndefinedIfNull = (a: any): OmitNull<any> =>
a === null ? undefined : a
/**
* Convert from a rust settings struct into the JS settings struct.
* We do this because the JS settings type has all the fancy shit
@ -49,7 +53,9 @@ export function configurationToSettingsPayload(
: undefined,
onboardingStatus: configuration?.settings?.app?.onboarding_status,
dismissWebBanner: configuration?.settings?.app?.dismiss_web_banner,
streamIdleMode: configuration?.settings?.app?.stream_idle_mode,
streamIdleMode: toUndefinedIfNull(
configuration?.settings?.app?.stream_idle_mode
),
allowOrbitInSketchMode:
configuration?.settings?.app?.allow_orbit_in_sketch_mode,
projectDirectory: configuration?.settings?.project?.directory,
@ -128,7 +134,9 @@ export function projectConfigurationToSettingsPayload(
: undefined,
onboardingStatus: configuration?.settings?.app?.onboarding_status,
dismissWebBanner: configuration?.settings?.app?.dismiss_web_banner,
streamIdleMode: configuration?.settings?.app?.stream_idle_mode,
streamIdleMode: toUndefinedIfNull(
configuration?.settings?.app?.stream_idle_mode
),
allowOrbitInSketchMode:
configuration?.settings?.app?.allow_orbit_in_sketch_mode,
namedViews: deepPartialNamedViewsToNamedViews(

3
src/lib/timings.ts Normal file
View File

@ -0,0 +1,3 @@
// 0.25s is the average visual reaction time for humans so we'll go a bit less
// so those exception people don't see.
export const REASONABLE_TIME_TO_REFRESH_STREAM_SIZE = 100