diff --git a/src/clientSideScene/CameraControls.ts b/src/clientSideScene/CameraControls.ts index 761dc4772..82c766bc1 100644 --- a/src/clientSideScene/CameraControls.ts +++ b/src/clientSideScene/CameraControls.ts @@ -105,12 +105,7 @@ export class CameraControls { wasDragging: boolean mouseDownPosition: Vector2 mouseNewPosition: Vector2 - old: - | { - camera: PerspectiveCamera | OrthographicCamera - target: Vector3 - } - | undefined + oldCameraState: undefined | CameraViewState_type rotationSpeed = 0.3 enableRotate = true enablePan = true @@ -281,7 +276,7 @@ export class CameraControls { const cb = ({ data, type }: CallBackParam) => { // We're reconnecting, so ignore this init proces. - if (this.old) { + if (this.oldCameraState) { return } @@ -969,26 +964,40 @@ export class CameraControls { }) } - async restoreCameraPosition(): Promise { - if (!this.old) return + async restoreRemoteCameraStateAndTriggerSync() { + if (!this.oldCameraState) return - this.camera = this.old.camera.clone() - this.target = this.old.target.clone() - - void this.engineCommandManager.sendSceneCommand({ + await this.engineCommandManager.sendSceneCommand({ type: 'modeling_cmd_req', cmd_id: uuidv4(), cmd: { - type: 'default_camera_look_at', - ...convertThreeCamValuesToEngineCam({ - isPerspective: true, - position: this.camera.position, - quaternion: this.camera.quaternion, - zoom: this.camera.zoom, - target: this.target, - }), + type: 'default_camera_set_view', + view: this.oldCameraState, }, }) + + await this.engineCommandManager.sendSceneCommand({ + type: 'modeling_cmd_req', + cmd_id: uuidv4(), + cmd: { + type: 'default_camera_get_settings', + }, + }) + } + + async saveRemoteCameraState() { + const cameraViewStateResponse = await this.engineCommandManager.sendSceneCommand({ + type: 'modeling_cmd_req', + cmd_id: uuidv4(), + cmd: { type: 'default_camera_get_view' }, + }) + if (!cameraViewStateResponse) return + if ('resp' in cameraViewStateResponse + && 'modeling_response' in cameraViewStateResponse.resp.data + && 'data' in cameraViewStateResponse.resp.data.modeling_response + && 'view' in cameraViewStateResponse.resp.data.modeling_response.data) { + this.oldCameraState = cameraViewStateResponse.resp.data.modeling_response.data.view + } } async tweenCameraToQuaternion( diff --git a/src/components/EngineStream.tsx b/src/components/EngineStream.tsx index 46b69c0c9..28da4e68c 100644 --- a/src/components/EngineStream.tsx +++ b/src/components/EngineStream.tsx @@ -16,13 +16,16 @@ import { getArtifactOfTypes } from 'lang/std/artifactGraph' import { ViewControlContextMenu } from './ViewControlMenu' import { useSettings, engineStreamActor } from 'machines/appMachine' import { commandBarActor, useCommandBarState } from 'machines/commandBarMachine' -import { EngineStreamState, EngineStreamTransition } from 'machines/engineStreamMachine' +import { + EngineStreamState, + EngineStreamTransition, +} from 'machines/engineStreamMachine' import { useSelector } from '@xstate/react' import { REASONABLE_TIME_TO_REFRESH_STREAM_SIZE } from 'lib/timings' export const EngineStream = (props: { - pool: string | null, - authToken: string | undefined, + pool: string | null + authToken: string | undefined }) => { const { setAppState } = useAppState() @@ -97,6 +100,22 @@ export const EngineStream = (props: { } }, []) + // In the past we'd try to play immediately, but the proper thing is to way + // for the 'canplay' event to tell us data is ready. + useEffect(() => { + const videoRef = engineStreamState.context.videoRef.current + if (!videoRef) { + return + } + const play = () => { + videoRef.play().catch(console.error) + } + videoRef.addEventListener('canplay', play) + return () => { + videoRef.removeEventListener('canplay', play) + } + }, [engineStreamState.context.videoRef.current]) + useEffect(() => { if (engineStreamState.value === EngineStreamState.Reconfiguring) return const video = engineStreamState.context.videoRef?.current @@ -105,17 +124,21 @@ export const EngineStream = (props: { if (!canvas) return new ResizeObserver(() => { - if (Date.now() - last.current < REASONABLE_TIME_TO_REFRESH_STREAM_SIZE) - return - last.current = Date.now() + // Prevents: + // `Uncaught ResizeObserver loop completed with undelivered notifications` + window.requestAnimationFrame(() => { + if (Date.now() - last.current < REASONABLE_TIME_TO_REFRESH_STREAM_SIZE) + return + last.current = Date.now() - if ( - Math.abs(video.width - window.innerWidth) > 4 || - Math.abs(video.height - window.innerHeight) > 4 - ) { - timeoutStart.current = Date.now() - startOrReconfigureEngine() - } + if ( + Math.abs(video.width - window.innerWidth) > 4 || + Math.abs(video.height - window.innerHeight) > 4 + ) { + timeoutStart.current = Date.now() + startOrReconfigureEngine() + } + }) }).observe(document.body) }, [engineStreamState.value]) @@ -262,7 +285,7 @@ export const EngineStream = (props: { if (btnName(e.nativeEvent).left) { // eslint-disable-next-line @typescript-eslint/no-floating-promises - sendSelectEventToEngine(e, engineStreamState.context.videoRef.current) + sendSelectEventToEngine(e) } } @@ -284,7 +307,7 @@ export const EngineStream = (props: { return } - sendSelectEventToEngine(e, engineStreamState.context.videoRef.current) + sendSelectEventToEngine(e) .then(({ entity_id }) => { if (!entity_id) { // No entity selected. This is benign diff --git a/src/hooks/useSetupEngineManager.ts b/src/hooks/useSetupEngineManager.ts deleted file mode 100644 index 689a3c624..000000000 --- a/src/hooks/useSetupEngineManager.ts +++ /dev/null @@ -1,183 +0,0 @@ -import { useAppState, useAppStream } from '@src/AppState' -import { useEffect, useLayoutEffect, useRef } from 'react' - -import type { useModelingContext } from '@src/hooks/useModelingContext' -import { useNetworkContext } from '@src/hooks/useNetworkContext' -import { - DisconnectingType, - EngineConnectionStateType, -} from '@src/lang/std/engineConnection' -import type { SettingsViaQueryString } from '@src/lib/settings/settingsTypes' -import { engineCommandManager } from '@src/lib/singletons' -import { Themes } from '@src/lib/theme' -import { deferExecution } from '@src/lib/utils' - -export function useSetupEngineManager( - streamRef: React.RefObject, - modelingSend: ReturnType['send'], - modelingContext: ReturnType['context'], - settings: SettingsViaQueryString = { - pool: null, - theme: Themes.System, - highlightEdges: true, - enableSSAO: true, - showScaleGrid: false, - cameraProjection: 'perspective', - cameraOrbit: 'spherical', - }, - token?: string -) { - const networkContext = useNetworkContext() - const { pingPongHealth, immediateState } = networkContext - const { setAppState } = useAppState() - const { setMediaStream } = useAppStream() - - const hasSetNonZeroDimensions = useRef(false) - - if (settings.pool) { - // override the pool param (?pool=) to request a specific engine instance - // from a particular pool. - engineCommandManager.settings.pool = settings.pool - } - - const startEngineInstance = () => { - // Load the engine command manager once with the initial width and height, - // then we do not want to reload it. - const { width: quadWidth, height: quadHeight } = getDimensions( - streamRef?.current?.offsetWidth ?? 0, - streamRef?.current?.offsetHeight ?? 0 - ) - engineCommandManager.start({ - setMediaStream: (mediaStream) => setMediaStream(mediaStream), - setIsStreamReady: (isStreamReady) => setAppState({ isStreamReady }), - width: quadWidth, - height: quadHeight, - token, - settings, - }) - hasSetNonZeroDimensions.current = true - } - - useLayoutEffect(() => { - const { width: quadWidth, height: quadHeight } = getDimensions( - streamRef?.current?.offsetWidth ?? 0, - streamRef?.current?.offsetHeight ?? 0 - ) - if (!hasSetNonZeroDimensions.current && quadHeight && quadWidth) { - startEngineInstance() - } - }, [ - streamRef?.current?.offsetWidth, - streamRef?.current?.offsetHeight, - modelingSend, - ]) - - useEffect(() => { - if (pingPongHealth === 'TIMEOUT') { - engineCommandManager.tearDown() - } - }, [pingPongHealth]) - - useEffect(() => { - const intervalId = setInterval(() => { - if (immediateState.type === EngineConnectionStateType.Disconnected) { - engineCommandManager.engineConnection = undefined - startEngineInstance() - } - }, 3000) - return () => { - clearInterval(intervalId) - } - }, [immediateState]) - - useEffect(() => { - engineCommandManager.settings = settings - - const handleResize = deferExecution(() => { - engineCommandManager.handleResize( - getDimensions( - streamRef?.current?.offsetWidth ?? 0, - streamRef?.current?.offsetHeight ?? 0 - ) - ) - }, 500) - - const onOnline = () => { - startEngineInstance() - } - - const onVisibilityChange = () => { - if (window.document.visibilityState === 'visible') { - if ( - !engineCommandManager.engineConnection?.isReady() && - !engineCommandManager.engineConnection?.isConnecting() - ) { - startEngineInstance() - } - } - } - window.document.addEventListener('visibilitychange', onVisibilityChange) - - const onAnyInput = () => { - const isEngineNotReadyOrConnecting = - !engineCommandManager.engineConnection?.isReady() && - !engineCommandManager.engineConnection?.isConnecting() - - const conn = engineCommandManager.engineConnection - - const isStreamPaused = - conn?.state.type === EngineConnectionStateType.Disconnecting && - conn?.state.value.type === DisconnectingType.Pause - - if (isEngineNotReadyOrConnecting || isStreamPaused) { - engineCommandManager.engineConnection = undefined - startEngineInstance() - } - } - window.document.addEventListener('keydown', onAnyInput) - window.document.addEventListener('mousemove', onAnyInput) - window.document.addEventListener('mousedown', onAnyInput) - window.document.addEventListener('scroll', onAnyInput) - window.document.addEventListener('touchstart', onAnyInput) - - const onOffline = () => { - engineCommandManager.tearDown() - } - - window.addEventListener('online', onOnline) - window.addEventListener('offline', onOffline) - window.addEventListener('resize', handleResize) - return () => { - window.document.removeEventListener( - 'visibilitychange', - onVisibilityChange - ) - window.document.removeEventListener('keydown', onAnyInput) - window.document.removeEventListener('mousemove', onAnyInput) - window.document.removeEventListener('mousedown', onAnyInput) - window.document.removeEventListener('scroll', onAnyInput) - window.document.removeEventListener('touchstart', onAnyInput) - window.removeEventListener('online', onOnline) - window.removeEventListener('offline', onOffline) - window.removeEventListener('resize', handleResize) - } - - // Engine relies on many settings so we should rebind events when it changes - // We have to list out the ones we care about because the settings object holds - // non-settings too... - }, [...Object.values(settings)]) -} - -function getDimensions(streamWidth?: number, streamHeight?: number) { - const factorOf = 4 - const maxResolution = 2000 - const width = streamWidth ? streamWidth : 0 - const height = streamHeight ? streamHeight : 0 - const ratio = Math.min( - Math.min(maxResolution / width, maxResolution / height), - 1.0 - ) - const quadWidth = Math.round((width * ratio) / factorOf) * factorOf - const quadHeight = Math.round((height * ratio) / factorOf) * factorOf - return { width: quadWidth, height: quadHeight } -} diff --git a/src/lang/std/engineConnection.ts b/src/lang/std/engineConnection.ts index 6bd54164f..ccd7e3ff5 100644 --- a/src/lang/std/engineConnection.ts +++ b/src/lang/std/engineConnection.ts @@ -839,13 +839,6 @@ class EngineConnection extends EventTarget { this.engineCommandManager.inSequence = 1 - // Bust the cache before anything - ;(async () => { - await rustContext.clearSceneAndBustCache( - kclManager.engineCommandManager.settings - ) - })().catch(reportRejection) - this.dispatchEvent( new CustomEvent(EngineConnectionEvents.Opened, { detail: this }) ) diff --git a/src/lib/settings/settingsUtils.test.ts b/src/lib/settings/settingsUtils.test.ts index f8840d165..a6c31ed51 100644 --- a/src/lib/settings/settingsUtils.test.ts +++ b/src/lib/settings/settingsUtils.test.ts @@ -43,11 +43,10 @@ describe(`testing settings initialization`, () => { }, }, } - const projectConfiguration: DeepPartial = { + const projectConfiguration: DeepPartial = { settings: { app: { appearance: { - theme: 'light', color: 200, }, }, @@ -82,11 +81,10 @@ describe(`testing getAllCurrentSettings`, () => { }, }, } - const projectConfiguration: DeepPartial = { + const projectConfiguration: DeepPartial = { settings: { app: { appearance: { - theme: 'light', color: 200, }, }, diff --git a/src/machines/appMachine.ts b/src/machines/appMachine.ts index 4423ce9df..2178b3d03 100644 --- a/src/machines/appMachine.ts +++ b/src/machines/appMachine.ts @@ -68,6 +68,6 @@ export const useSettings = () => return settings }) -export const engineStreamActor = appActor.system.get(ENGINE_STREAM) as ActorRefFrom< - typeof engineStreamMachine -> +export const engineStreamActor = appActor.system.get( + ENGINE_STREAM +) as ActorRefFrom diff --git a/src/machines/engineStreamMachine.ts b/src/machines/engineStreamMachine.ts index 190dd91d3..22b74379b 100644 --- a/src/machines/engineStreamMachine.ts +++ b/src/machines/engineStreamMachine.ts @@ -1,7 +1,13 @@ +import { jsAppSettings } from 'lang/wasm' import { uuidv4 } from 'lib/utils' import { MutableRefObject } from 'react' import { setup, assign, fromPromise } from 'xstate' -import { rustContext, kclManager, sceneInfra, engineCommandManager } from 'lib/singletons' +import { + rustContext, + kclManager, + sceneInfra, + engineCommandManager, +} from 'lib/singletons' import { trap } from 'lib/trap' import { Vector3, Vector4 } from 'three' @@ -39,7 +45,7 @@ export const engineStreamContextCreate = (): EngineStreamContext => ({ pool: null, authToken: undefined, mediaStream: null, - videoRef: { current: null }, + videoRef: { current: null }, canvasRef: { current: null }, zoomToFit: true, }) @@ -94,13 +100,28 @@ export const engineStreamMachine = setup({ const mediaStream = context.mediaStream if (!mediaStream) return false + // If the video is already playing it means we're doing a reconfigure. + // We don't want to re-run the KCL or touch the video element at all. + if (!video.paused) { + return + } + + await sceneInfra.camControls.restoreRemoteCameraStateAndTriggerSync() + video.style.display = 'block' canvas.style.display = 'none' - // await sceneInfra.camControls.restoreCameraPosition() - video.srcObject = mediaStream - await video.play() + + // Bust the cache before trying to execute since this may + // be a reconnection and if cache is not cleared, it + // will not reexecute. + // When calling cache before _any_ executions it errors, but non-fatal. + await rustContext + .clearSceneAndBustCache( + { settings: await jsAppSettings() }, + ) + .catch(console.warn) await kclManager.executeCode(params.zoomToFit) } @@ -120,19 +141,21 @@ export const engineStreamMachine = setup({ if (!canvas) return await holdOntoVideoFrameInCanvas(video, canvas) + video.style.display = 'none' + + await sceneInfra.camControls.saveRemoteCameraState() // Make sure we're on the next frame for no flickering between canvas // and the video elements. window.requestAnimationFrame( () => void (async () => { - video.style.display = 'none' - // Destroy the media stream. We will re-establish it. We could // leave everything at pausing, preventing video decoders from running // but we can do even better by significantly reducing network // cards also. context.mediaStream?.getVideoTracks()[0].stop() + context.mediaStream = null video.srcObject = null engineCommandManager.tearDown({ idleMode: true }) @@ -171,18 +194,10 @@ export const engineStreamMachine = setup({ engineCommandManager.settings = settingsNext - // If we don't pause there could be a really bad flicker - // on reconfiguration (resize, for example) - await holdOntoVideoFrameInCanvas(video, canvas) - canvas.style.display = 'block' - window.requestAnimationFrame(() => { - video.style.display = 'none' engineCommandManager.start({ setMediaStream: event.onMediaStream, setIsStreamReady: (isStreamReady) => { - video.style.display = 'block' - canvas.style.display = 'none' event.setAppState({ isStreamReady }) }, width, @@ -212,13 +227,11 @@ export const engineStreamMachine = setup({ reenter: true, on: { [EngineStreamTransition.SetPool]: { - target: EngineStreamState.Setup, - actions: [ - assign({ pool: ({ context, event }) => event.data.pool }), - ], + target: EngineStreamState.Off, + actions: [assign({ pool: ({ context, event }) => event.data.pool })], }, [EngineStreamTransition.SetAuthToken]: { - target: EngineStreamState.Setup, + target: EngineStreamState.Off, actions: [ assign({ authToken: ({ context, event }) => event.data.authToken }), ],