ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Backoff Reconnect (#6358)

Browse Source 
* Add a Stop command that does a bit of cleanup in the engineStateMachine

* Major network code startup refactor; backoff reconnect; many more logs for us

* Save camera state after every user interaction in case of disconnection and restoral

* Translate basic WebSocket error numbers into useful text

* Add a RestartRequest event to the engineCommandManager

* Adjust for a rebase

* Add E2E test for stream pause behavior

* Fix snapshot test

* fmt, lint

* small issue

* Fix tests

* Fix up idle test

* One last fix

* fixes

* Remove circ dep

* fix test

* use a const for time

* TEST -> isPlaywright instead

* whoops
This commit is contained in:
Zookeeper Lee
2025-05-15 11:58:00 -04:00
committed by GitHub
parent d3a4fd8b55
commit 3f00e7186c
12 changed files with 923 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

252
src/components/EngineStream.tsx
View File

@ -1,3 +1,4 @@
import { isPlaywright } from '@src/lib/isPlaywright'
import { useAppState } from '@src/AppState'
import { ClientSideScene } from '@src/clientSideScene/ClientSideSceneComp'
import { ViewControlContextMenu } from '@src/components/ViewControlMenu'
@ -5,7 +6,10 @@ import { useModelingContext } from '@src/hooks/useModelingContext'
import { useNetworkContext } from '@src/hooks/useNetworkContext'
import { NetworkHealthState } from '@src/hooks/useNetworkStatus'
import { getArtifactOfTypes } from '@src/lang/std/artifactGraph'
import { EngineCommandManagerEvents } from '@src/lang/std/engineConnection'
import {
EngineCommandManagerEvents,
EngineConnectionStateType,
} from '@src/lang/std/engineConnection'
import { btnName } from '@src/lib/cameraControls'
import { PATHS } from '@src/lib/paths'
import { sendSelectEventToEngine } from '@src/lib/selections'
@ -33,22 +37,38 @@ import { createThumbnailPNGOnDesktop } from '@src/lib/screenshot'
import type { SettingsViaQueryString } from '@src/lib/settings/settingsTypes'
import { resetCameraPosition } from '@src/lib/resetCameraPosition'
const TIME_1_SECOND = 1000
export const EngineStream = (props: {
pool: string | null
authToken: string | undefined
}) => {
const { setAppState } = useAppState()
const [firstPlay, setFirstPlay] = useState(true)
const { overallState } = useNetworkContext()
const settings = useSettings()
const engineStreamState = useSelector(engineStreamActor, (state) => state)
const { state: modelingMachineState, send: modelingMachineActorSend } =
useModelingContext()
const { file, project } = useRouteLoaderData(PATHS.FILE) as IndexLoaderData
const last = useRef<number>(Date.now())
const [firstPlay, setFirstPlay] = useState(true)
const [isRestartRequestStarting, setIsRestartRequestStarting] =
useState(false)
const [attemptTimes, setAttemptTimes] = useState<[number, number]>([
0,
TIME_1_SECOND,
])
// These will be passed to the engineStreamActor to handle.
const videoRef = useRef<HTMLVideoElement>(null)
const canvasRef = useRef<HTMLCanvasElement>(null)
// For attaching right-click menu events
const videoWrapperRef = useRef<HTMLDivElement>(null)
const { overallState } = useNetworkContext()
const engineStreamState = useSelector(engineStreamActor, (state) => state)
/**
* We omit `pool` here because `engineStreamMachine` will override it anyway
* within the `EngineStreamTransition.StartOrReconfigureEngine` Promise actor.
@ -62,19 +82,46 @@ export const EngineStream = (props: {
cameraOrbit: settings.modeling.cameraOrbit.current,
}
const { state: modelingMachineState, send: modelingMachineActorSend } =
useModelingContext()
const streamIdleMode = settings.app.streamIdleMode.current
useEffect(() => {
engineStreamActor.send({
type: EngineStreamTransition.SetVideoRef,
videoRef: { current: videoRef.current },
})
}, [videoRef.current])
useEffect(() => {
engineStreamActor.send({
type: EngineStreamTransition.SetCanvasRef,
canvasRef: { current: canvasRef.current },
})
}, [canvasRef.current])
useEffect(() => {
engineStreamActor.send({
type: EngineStreamTransition.SetPool,
pool: props.pool,
})
}, [props.pool])
useEffect(() => {
engineStreamActor.send({
type: EngineStreamTransition.SetAuthToken,
authToken: props.authToken,
})
}, [props.authToken])
// We have to call this here because of the dependencies:
// modelingMachineActorSend, setAppState, settingsEngine
// It's possible to pass these in earlier but I (lee) don't want to
// restructure this further at the moment.
const startOrReconfigureEngine = () => {
engineStreamActor.send({
type: EngineStreamTransition.StartOrReconfigureEngine,
modelingMachineActorSend,
settings: settingsEngine,
setAppState,
// It's possible a reconnect happens as we drag the window :')
onMediaStream(mediaStream: MediaStream) {
engineStreamActor.send({
type: EngineStreamTransition.SetMediaStream,
@ -84,18 +131,47 @@ export const EngineStream = (props: {
})
}
// When the scene is ready play the stream and execute!
useEffect(() => {
if (
engineStreamState.value !== EngineStreamState.WaitingForDependencies &&
engineStreamState.value !== EngineStreamState.Stopped
)
return
startOrReconfigureEngine()
}, [engineStreamState, setAppState])
// I would inline this but it needs to be a function for removeEventListener.
const play = () => {
engineStreamActor.send({
type: EngineStreamTransition.Play,
})
}
useEffect(() => {
engineCommandManager.addEventListener(
EngineCommandManagerEvents.SceneReady,
play
)
return () => {
engineCommandManager.removeEventListener(
EngineCommandManagerEvents.SceneReady,
play
)
}
}, [])
// When the scene is ready, execute kcl!
const executeKcl = () => {
console.log('scene is ready, execute kcl')
const kmp = kclManager.executeCode().catch(trap)
if (!firstPlay) return
setFirstPlay(false)
console.log('scene is ready, fire!')
setFirstPlay(false)
// Reset the restart timeouts
setAttemptTimes([0, TIME_1_SECOND])
console.log('firstPlay true, zoom to fit')
kmp
.then(async () => {
await resetCameraPosition()
@ -112,51 +188,65 @@ export const EngineStream = (props: {
useEffect(() => {
engineCommandManager.addEventListener(
EngineCommandManagerEvents.SceneReady,
play
executeKcl
)
return () => {
engineCommandManager.removeEventListener(
EngineCommandManagerEvents.SceneReady,
play
executeKcl
)
}
}, [firstPlay])
useEffect(() => {
// We do a back-off restart, using a fibonacci sequence, since it
// has a nice retry time curve (somewhat quick then exponential)
const attemptRestartIfNecessary = () => {
if (isRestartRequestStarting) return
setIsRestartRequestStarting(true)
setTimeout(() => {
engineStreamState.context.videoRef.current?.pause()
engineCommandManager.tearDown()
startOrReconfigureEngine()
setFirstPlay(false)
setIsRestartRequestStarting(false)
}, attemptTimes[0] + attemptTimes[1])
setAttemptTimes([attemptTimes[1], attemptTimes[0] + attemptTimes[1]])
}
// Poll that we're connected. If not, send a reset signal.
// Do not restart if we're in idle mode.
const connectionCheckIntervalId = setInterval(() => {
// SKIP DURING TESTS BECAUSE IT WILL MESS WITH REUSING THE
// ELECTRON INSTANCE.
if (isPlaywright()) {
return
}
// Don't try try to restart if we're already connected!
const hasEngineConnectionInst = engineCommandManager.engineConnection
const isDisconnected =
engineCommandManager.engineConnection?.state.type ===
EngineConnectionStateType.Disconnected
const inIdleMode = engineStreamState.value === EngineStreamState.Paused
if ((hasEngineConnectionInst && !isDisconnected) || inIdleMode) return
attemptRestartIfNecessary()
}, TIME_1_SECOND)
engineCommandManager.addEventListener(
EngineCommandManagerEvents.SceneReady,
play
EngineCommandManagerEvents.EngineRestartRequest,
attemptRestartIfNecessary
)
engineStreamActor.send({
type: EngineStreamTransition.SetPool,
data: { pool: props.pool },
})
engineStreamActor.send({
type: EngineStreamTransition.SetAuthToken,
data: { authToken: props.authToken },
})
return () => {
engineCommandManager.tearDown()
}
}, [])
clearInterval(connectionCheckIntervalId)
// In the past we'd try to play immediately, but the proper thing is to way
// for the 'canplay' event to tell us data is ready.
useEffect(() => {
const videoRef = engineStreamState.context.videoRef.current
if (!videoRef) {
return
engineCommandManager.removeEventListener(
EngineCommandManagerEvents.EngineRestartRequest,
attemptRestartIfNecessary
)
}
const play = () => {
videoRef.play().catch(console.error)
}
videoRef.addEventListener('canplay', play)
return () => {
videoRef.removeEventListener('canplay', play)
}
}, [engineStreamState.context.videoRef.current])
}, [engineStreamState, attemptTimes, isRestartRequestStarting])
useEffect(() => {
if (engineStreamState.value === EngineStreamState.Reconfiguring) return
@ -184,25 +274,6 @@ export const EngineStream = (props: {
}).observe(document.body)
}, [engineStreamState.value])
// When the video and canvas element references are set, start the engine.
useEffect(() => {
if (
engineStreamState.context.canvasRef.current &&
engineStreamState.context.videoRef.current
) {
startOrReconfigureEngine()
}
}, [
engineStreamState.context.canvasRef.current,
engineStreamState.context.videoRef.current,
])
// On settings change, reconfigure the engine. When paused this gets really tricky,
// and also requires onMediaStream to be set!
useEffect(() => {
startOrReconfigureEngine()
}, Object.values(settingsEngine))
/**
* Subscribe to execute code when the file changes
* but only if the scene is already ready.
@ -285,18 +356,7 @@ export const EngineStream = (props: {
}
if (engineStreamState.value === EngineStreamState.Paused) {
engineStreamActor.send({
type: EngineStreamTransition.StartOrReconfigureEngine,
modelingMachineActorSend,
settings: settingsEngine,
setAppState,
onMediaStream(mediaStream: MediaStream) {
engineStreamActor.send({
type: EngineStreamTransition.SetMediaStream,
mediaStream,
})
},
})
startOrReconfigureEngine()
}
timeoutStart.current = Date.now()
@ -314,7 +374,7 @@ export const EngineStream = (props: {
window.document.addEventListener('mouseup', onAnyInput)
window.document.addEventListener('scroll', onAnyInput)
window.document.addEventListener('touchstart', onAnyInput)
window.document.addEventListener('touchstop', onAnyInput)
window.document.addEventListener('touchend', onAnyInput)
return () => {
timeoutStart.current = null
@ -325,10 +385,34 @@ export const EngineStream = (props: {
window.document.removeEventListener('mouseup', onAnyInput)
window.document.removeEventListener('scroll', onAnyInput)
window.document.removeEventListener('touchstart', onAnyInput)
window.document.removeEventListener('touchstop', onAnyInput)
window.document.removeEventListener('touchend', onAnyInput)
}
}, [streamIdleMode, engineStreamState.value])
// On various inputs save the camera state, in case we get disconnected.
useEffect(() => {
const onInput = () => {
// Save the remote camera state to restore on stream restore.
// Fire-and-forget because we don't know when a camera movement is
// completed on the engine side (there are no responses to data channel
// mouse movements.)
sceneInfra.camControls.saveRemoteCameraState().catch(trap)
}
// These usually signal a user is done some sort of operation.
window.document.addEventListener('keyup', onInput)
window.document.addEventListener('mouseup', onInput)
window.document.addEventListener('scroll', onInput)
window.document.addEventListener('touchend', onInput)
return () => {
window.document.removeEventListener('keyup', onInput)
window.document.removeEventListener('mouseup', onInput)
window.document.removeEventListener('scroll', onInput)
window.document.removeEventListener('touchend', onInput)
}
}, [])
const isNetworkOkay =
overallState === NetworkHealthState.Ok ||
overallState === NetworkHealthState.Weak
@ -399,7 +483,7 @@ export const EngineStream = (props: {
autoPlay
muted
key={engineStreamActor.id + 'video'}
ref={engineStreamState.context.videoRef}
ref={videoRef}
controls={false}
className="w-full cursor-pointer h-full"
disablePictureInPicture
@ -407,7 +491,7 @@ export const EngineStream = (props: {
/>
<canvas
key={engineStreamActor.id + 'canvas'}
ref={engineStreamState.context.canvasRef}
ref={canvasRef}
className="cursor-pointer"
id="freeze-frame"
>
@ -424,9 +508,11 @@ export const EngineStream = (props: {
}
menuTargetElement={videoWrapperRef}
/>
{![EngineStreamState.Playing, EngineStreamState.Paused].some(
(s) => s === engineStreamState.value
) && (
{![
EngineStreamState.Playing,
EngineStreamState.Paused,
EngineStreamState.Resuming,
].some((s) => s === engineStreamState.value) && (
<Loading dataTestId="loading-engine" className="fixed inset-0 h-screen">
Connecting to engine
</Loading>