From deb83cf62caa31bf4f4ff6e9aa8fe739f8ea7d7b Mon Sep 17 00:00:00 2001 From: 49lf Date: Mon, 17 Feb 2025 10:38:15 -0500 Subject: [PATCH] tsc lint fmt --- src/App.tsx | 2 +- src/clientSideScene/CameraControls.ts | 6 + src/components/EngineStream.tsx | 24 ++-- src/hooks/useEngineStreamContext.ts | 175 +++++++++++++++++++------- src/lang/std/engineConnection.ts | 7 +- 5 files changed, 156 insertions(+), 58 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index 336456f32..8e1f3edfc 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -163,7 +163,7 @@ export function App() { videoRef, canvasRef, mediaStream: null, - authToken: token ?? null, + authToken: token, pool, zoomToFit: true, }, diff --git a/src/clientSideScene/CameraControls.ts b/src/clientSideScene/CameraControls.ts index c036f8de9..63dfbc436 100644 --- a/src/clientSideScene/CameraControls.ts +++ b/src/clientSideScene/CameraControls.ts @@ -280,6 +280,11 @@ export class CameraControls { >[0] const cb = ({ data, type }: CallBackParam) => { + // We're reconnecting, so ignore this init proces. + if (this.old) { + return + } + const camSettings = data.settings this.camera.position.set( camSettings.pos.x, @@ -291,6 +296,7 @@ export class CameraControls { camSettings.center.y, camSettings.center.z ) + const orientation = new Quaternion( camSettings.orientation.x, camSettings.orientation.y, diff --git a/src/components/EngineStream.tsx b/src/components/EngineStream.tsx index 9f22043e1..294622b69 100644 --- a/src/components/EngineStream.tsx +++ b/src/components/EngineStream.tsx @@ -14,6 +14,7 @@ import { PATHS } from 'lib/paths' import { IndexLoaderData } from 'lib/types' import { err, reportRejection, trap } from 'lib/trap' import { getArtifactOfTypes } from 'lang/std/artifactGraph' +import { clearSceneAndBustCache } from 'lang/wasm' import { ViewControlContextMenu } from './ViewControlMenu' import { commandBarActor, useCommandBarState } from 'machines/commandBarMachine' import { useSelector } from '@xstate/react' @@ -67,18 +68,20 @@ export const EngineStream = () => { }) } + const play = () => { + engineStreamActor.send({ + type: EngineStreamTransition.Play, + }) + } + useEffect(() => { - const play = () => { - engineStreamActor.send({ - type: EngineStreamTransition.Play, - }) - } engineCommandManager.addEventListener( EngineCommandManagerEvents.SceneReady, play ) return () => { + engineCommandManager.tearDown() engineCommandManager.removeEventListener( EngineCommandManagerEvents.SceneReady, play @@ -87,6 +90,7 @@ export const EngineStream = () => { }, []) useEffect(() => { + if (engineStreamState.value === EngineStreamState.Reconfiguring) return const video = engineStreamState.context.videoRef?.current if (!video) return const canvas = engineStreamState.context.canvasRef?.current @@ -123,10 +127,8 @@ export const EngineStream = () => { // On settings change, reconfigure the engine. When paused this gets really tricky, // and also requires onMediaStream to be set! useEffect(() => { - if (engineStreamState.value === EngineStreamState.Playing) { - startOrReconfigureEngine() - } - }, [settings.context, engineStreamState.value]) + startOrReconfigureEngine() + }, Object.values(settingsEngine)) /** * Subscribe to execute code when the file changes @@ -135,8 +137,8 @@ export const EngineStream = () => { */ useEffect(() => { if (engineCommandManager.engineConnection?.isReady() && file?.path) { - console.log('execute on file change') - void kclManager.executeCode(true).catch(trap) + console.log('file changed, executing code') + kclManager.executeCode(true).catch(trap) } }, [file?.path, engineCommandManager.engineConnection]) diff --git a/src/hooks/useEngineStreamContext.ts b/src/hooks/useEngineStreamContext.ts index 1783b962c..a89ac22b0 100644 --- a/src/hooks/useEngineStreamContext.ts +++ b/src/hooks/useEngineStreamContext.ts @@ -1,15 +1,18 @@ +import { uuidv4 } from 'lib/utils' import { makeDefaultPlanes, clearSceneAndBustCache } from 'lang/wasm' import { MutableRefObject } from 'react' import { setup, assign, fromPromise } from 'xstate' import { createActorContext } from '@xstate/react' import { kclManager, sceneInfra, engineCommandManager } from 'lib/singletons' import { trap } from 'lib/trap' +import { Vector3, Vector4 } from 'three' export enum EngineStreamState { Off = 'off', On = 'on', WaitForMediaStream = 'wait-for-media-stream', Playing = 'playing', + Reconfiguring = 'reconfiguring', Paused = 'paused', // The is the state in-between Paused and Playing *specifically that order*. Resuming = 'resuming', @@ -25,7 +28,7 @@ export enum EngineStreamTransition { export interface EngineStreamContext { pool: string | null - authToken: string | null + authToken: string | undefined mediaStream: MediaStream | null videoRef: MutableRefObject canvasRef: MutableRefObject @@ -44,6 +47,23 @@ export function getDimensions(streamWidth: number, streamHeight: number) { return { width: quadWidth, height: quadHeight } } +export function holdOntoVideoFrameInCanvas( + video: HTMLVideoElement, + canvas: HTMLCanvasElement +) { + video.pause() + canvas.width = video.videoWidth + canvas.height = video.videoHeight + canvas.style.width = video.videoWidth + 'px' + canvas.style.height = video.videoHeight + 'px' + canvas.style.display = 'block' + + const ctx = canvas.getContext('2d') + if (!ctx) return + + ctx.drawImage(video, 0, 0, canvas.width, canvas.height) +} + const engineStreamMachine = setup({ types: { context: {} as EngineStreamContext, @@ -69,7 +89,6 @@ const engineStreamMachine = setup({ canvas.style.display = 'none' await sceneInfra.camControls.restoreCameraPosition() - await clearSceneAndBustCache(kclManager.engineCommandManager) video.srcObject = mediaStream await video.play() @@ -91,36 +110,76 @@ const engineStreamMachine = setup({ const canvas = context.canvasRef.current if (!canvas) return - canvas.width = video.videoWidth - canvas.height = video.videoHeight - canvas.style.width = video.videoWidth + 'px' - canvas.style.height = video.videoHeight + 'px' - canvas.style.display = 'block' - - const ctx = canvas.getContext('2d') - if (!ctx) return - - ctx.drawImage(video, 0, 0, canvas.width, canvas.height) + holdOntoVideoFrameInCanvas(video, canvas) // Make sure we're on the next frame for no flickering between canvas // and the video elements. - window.requestAnimationFrame(() => { - video.style.display = 'none' + window.requestAnimationFrame( + () => + void (async () => { + video.style.display = 'none' - // Destroy the media stream only. We will re-establish it. We could - // leave everything at pausing, preventing video decoders from running - // but we can do even better by significantly reducing network - // cards also. - context.mediaStream?.getVideoTracks()[0].stop() - video.srcObject = null + // Destroy the media stream. We will re-establish it. We could + // leave everything at pausing, preventing video decoders from running + // but we can do even better by significantly reducing network + // cards also. + context.mediaStream?.getVideoTracks()[0].stop() + video.srcObject = null - sceneInfra.camControls.old = { - camera: sceneInfra.camControls.camera.clone(), - target: sceneInfra.camControls.target.clone(), - } + const reqDefaultCameraGetSettings = + await engineCommandManager.sendSceneCommand({ + // CameraControls subscribes to default_camera_get_settings response events + // firing this at connection ensure the camera's are synced initially + type: 'modeling_cmd_req', + cmd_id: uuidv4(), + cmd: { + type: 'default_camera_get_settings', + }, + }) - engineCommandManager.tearDown({ idleMode: true }) - }) + if ( + reqDefaultCameraGetSettings === null || + !reqDefaultCameraGetSettings.success + ) + return + if ( + !('modeling_response' in reqDefaultCameraGetSettings.resp.data) + ) + return + if ( + reqDefaultCameraGetSettings.resp.data.modeling_response.type === + 'empty' + ) + return + if ( + !( + 'settings' in + reqDefaultCameraGetSettings.resp.data.modeling_response.data + ) + ) + return + + const center = + reqDefaultCameraGetSettings.resp.data.modeling_response.data + .settings.center + const pos = + reqDefaultCameraGetSettings.resp.data.modeling_response.data + .settings.pos + + sceneInfra.camControls.old = { + camera: sceneInfra.camControls.camera.clone(), + target: new Vector3(center.x, center.y, center.z), + } + + sceneInfra.camControls.old.camera.position.set( + pos.x, + pos.y, + pos.z + ) + + engineCommandManager.tearDown({ idleMode: true }) + })() + ) } ), [EngineStreamTransition.StartOrReconfigureEngine]: fromPromise( @@ -134,6 +193,9 @@ const engineStreamMachine = setup({ const video = context.videoRef.current if (!video) return + const canvas = context.canvasRef.current + if (!canvas) return + const { width, height } = getDimensions( window.innerWidth, window.innerHeight @@ -151,27 +213,38 @@ const engineStreamMachine = setup({ engineCommandManager.settings = settingsNext - engineCommandManager.start({ - setMediaStream: event.onMediaStream, - setIsStreamReady: (isStreamReady) => - event.setAppState({ isStreamReady }), - width, - height, - token: context.authToken, - settings: settingsNext, - makeDefaultPlanes: () => { - return makeDefaultPlanes(kclManager.engineCommandManager) - }, - }) + // If we don't pause there could be a really bad flicker + // on reconfiguration (resize, for example) + holdOntoVideoFrameInCanvas(video, canvas) + canvas.style.display = 'block' - event.modelingMachineActorSend({ - type: 'Set context', - data: { - streamDimensions: { - streamWidth: width, - streamHeight: height, + window.requestAnimationFrame(() => { + video.style.display = 'none' + engineCommandManager.start({ + setMediaStream: event.onMediaStream, + setIsStreamReady: (isStreamReady) => { + video.style.display = 'block' + canvas.style.display = 'none' + event.setAppState({ isStreamReady }) }, - }, + width, + height, + token: context.authToken, + settings: settingsNext, + makeDefaultPlanes: () => { + return makeDefaultPlanes(kclManager.engineCommandManager) + }, + }) + + event.modelingMachineActorSend({ + type: 'Set context', + data: { + streamDimensions: { + streamWidth: width, + streamHeight: height, + }, + }, + }) }) } ), @@ -215,11 +288,23 @@ const engineStreamMachine = setup({ }), }, on: { + [EngineStreamTransition.StartOrReconfigureEngine]: { + target: EngineStreamState.Reconfiguring, + }, [EngineStreamTransition.Pause]: { target: EngineStreamState.Paused, }, }, }, + [EngineStreamState.Reconfiguring]: { + invoke: { + src: EngineStreamTransition.StartOrReconfigureEngine, + input: (args) => args, + onDone: { + target: EngineStreamState.Playing, + }, + }, + }, [EngineStreamState.Paused]: { invoke: { src: EngineStreamTransition.Pause, diff --git a/src/lang/std/engineConnection.ts b/src/lang/std/engineConnection.ts index 208076d95..ca1c54da5 100644 --- a/src/lang/std/engineConnection.ts +++ b/src/lang/std/engineConnection.ts @@ -839,6 +839,11 @@ class EngineConnection extends EventTarget { this.engineCommandManager.inSequence = 1 + // Bust the cache before anything + ;(async () => { + await clearSceneAndBustCache(kclManager.engineCommandManager) + })().catch(reportRejection) + this.dispatchEvent( new CustomEvent(EngineConnectionEvents.Opened, { detail: this }) ) @@ -1801,7 +1806,6 @@ export class EngineCommandManager extends EventTarget { ) this.engineConnection?.tearDown(opts) - this.engineConnection = undefined // Our window.engineCommandManager.tearDown assignment causes this case to happen which is // only really for tests. @@ -1812,6 +1816,7 @@ export class EngineCommandManager extends EventTarget { // @ts-ignore this.engineCommandManager.engineConnection = null } + this.engineConnection = undefined } async startNewSession() { this.responseMap = {}