This commit is contained in:
49lf
2024-09-13 16:45:36 -04:00
parent 6dc87aa4fe
commit b11772b27c
6 changed files with 305 additions and 164 deletions

View File

@ -4,6 +4,7 @@ import { Stream } from './components/Stream'
import { AppHeader } from './components/AppHeader'
import { useHotkeys } from 'react-hotkeys-hook'
import { useLoaderData, useNavigate } from 'react-router-dom'
import { getNormalisedCoordinates } from './lib/utils'
import { type IndexLoaderData } from 'lib/types'
import { PATHS } from 'lib/paths'
import { useSettingsAuthContext } from 'hooks/useSettingsAuthContext'
@ -33,6 +34,12 @@ export function App() {
// the coredump.
const ref = useRef<HTMLDivElement>(null)
// Stream related refs and data
const videoRef = useRef<HTMLVideoElement>(null)
const canvasRef = useRef<HTMLCanvasElement>(null)
let [searchParams] = useSearchParams()
const pool = searchParams.get('pool')
const projectName = project?.name || null
const projectPath = project?.path || null
useEffect(() => {
@ -73,21 +80,35 @@ export function App() {
useEngineConnectionSubscriptions()
return (
<div className="relative h-full flex flex-col" ref={ref}>
<AppHeader
className={'transition-opacity transition-duration-75 ' + paneOpacity}
project={{ project, file }}
enableMenu={true}
/>
<ModalContainer />
<ModelingSidebar paneOpacity={paneOpacity} />
<Stream />
{/* <CamToggle /> */}
<LowerRightControls coreDumpManager={coreDumpManager}>
<UnitsMenu />
<Gizmo />
<CameraProjectionToggle />
</LowerRightControls>
<div
className="relative h-full flex flex-col"
onMouseMove={handleMouseMove}
ref={ref}
>
<AppHeader
className={'transition-opacity transition-duration-75 ' + paneOpacity}
project={{ project, file }}
enableMenu={true}
/>
<ModalContainer />
<ModelingSidebar paneOpacity={paneOpacity} />
<EngineStreamContext.Provider options={{
input: {
videoRef,
canvasRef,
mediaStream: null,
authToken: auth?.context?.token ?? null,
pool
}
}}>
<EngineStream />
</EngineStreamContext.Provider>
{/* <CamToggle /> */}
<LowerRightControls coreDumpManager={coreDumpManager}>
<UnitsMenu />
<Gizmo />
<CameraProjectionToggle />
</LowerRightControls>
</div>
)
}

View File

@ -1,4 +1,4 @@
import { MouseEventHandler, useEffect, useRef, useState, MutableRefObject } from 'react'
import { MouseEventHandler, useEffect, useRef, useState, MutableRefObject, useCallback } from 'react'
import { useAppState } from 'AppState'
import { createMachine, createActor, setup } from 'xstate'
import { createActorContext } from '@xstate/react'
@ -26,11 +26,15 @@ export const EngineStream = () => {
const [clickCoords, setClickCoords] = useState<{ x: number; y: number }>()
const { setAppState } = useAppState()
const videoRef = useRef<HTMLVideoElement>(null)
const canvasRef = useRef<HTMLCanvasElement>(null)
const { overallState } = useNetworkContext()
const { auth, settings } = useSettingsAuthContext()
const { settings } = useSettingsAuthContext()
const settingsEngine = {
theme: settings.context.app.theme.current,
enableSSAO: settings.context.app.enableSSAO.current,
highlightEdges: settings.context.modeling.highlightEdges.current,
showScaleGrid: settings.context.modeling.showScaleGrid.current,
}
const {
state: modelingMachineState,
@ -41,17 +45,7 @@ export const EngineStream = () => {
const engineStreamActor = useEngineStreamContext.useActorRef()
const engineStreamState = engineStreamActor.getSnapshot()
useEffect(() => {
engineStreamActor.send({ type: EngineStreamTransition.SetContextProperty, value: { authToken: auth?.context?.token } } )
}, [])
useEffect(() => {
engineStreamActor.send({ type: EngineStreamTransition.SetContextProperty, value: { videoRef } })
}, [videoRef.current])
useEffect(() => {
engineStreamActor.send({ type: EngineStreamTransition.SetContextProperty, value: { canvasRef } })
}, [canvasRef.current])
const streamIdleMode = settings.context.app.streamIdleMode.current
useEffect(() => {
let timestampNext: number | null = null
@ -63,8 +57,22 @@ export const EngineStream = () => {
// so those exception people don't see.
const REASONABLE_TIME_TO_REFRESH_STREAM_SIZE = 200
const configure = () => {
engineStreamActor.send({ type: EngineStreamTransition.StartOrReconfigureEngine, modelingMachineActorSend, settings, setAppState })
engineStreamActor.send({
type: EngineStreamTransition.StartOrReconfigureEngine,
modelingMachineActorSend,
settings: settingsEngine,
setAppState,
// It's possible a reconnect happens as we drag the window :')
onMediaStream(mediaStream: MediaStream) {
engineStreamActor.send({
type: EngineStreamTransition.SetMediaStream,
mediaStream
})
}
})
}
// setTimeout is avoided here because there is no need to create and destroy
@ -82,7 +90,9 @@ export const EngineStream = () => {
totalDelta = 0
timestampLast = null
needsResize = false
configure()
window.requestAnimationFrame(() => {
configure()
})
} else {
window.requestAnimationFrame(() => {
needsResize = true
@ -93,12 +103,116 @@ export const EngineStream = () => {
}
window.addEventListener('resize', onResize)
const play = () => {
engineStreamActor.send({
type: EngineStreamTransition.Play,
})
}
engineCommandManager.addEventListener(
EngineCommandManagerEvents.SceneReady,
play
)
return () => {
window.removeEventListener('resize', onResize)
}
}, [settings])
engineCommandManager.removeEventListener(
EngineCommandManagerEvents.SceneReady,
play
)
const isNetworkOkay =
}
}, [])
// When the video and canvas element references are set, start the engine.
useEffect(() => {
if (engineStreamState.context.canvasRef.current && engineStreamState.context.videoRef.current) {
engineStreamActor.send({
type: EngineStreamTransition.StartOrReconfigureEngine,
modelingMachineActorSend,
settings: settingsEngine,
setAppState,
onMediaStream(mediaStream: MediaStream) {
engineStreamActor.send({
type: EngineStreamTransition.SetMediaStream,
mediaStream
})
}
})
}
}, [engineStreamState.context.canvasRef.current, engineStreamState.context.videoRef.current])
// On settings change, reconfigure the engine.
useEffect(() => {
engineStreamActor.send({ type: EngineStreamTransition.StartOrReconfigureEngine, modelingMachineActorSend, settings: settingsEngine, setAppState })
}, [settings.context])
const IDLE_TIME_MS = 1000 * 6
// When streamIdleMode is changed, setup or teardown the timeouts
const timeoutId = useRef<ReturnType<typeof setTimeout> | undefined>(undefined)
useEffect(() => {
// If timeoutId is falsey, then reset it if steamIdleMode is true
if (streamIdleMode && !timeoutId.current) {
timeoutId.current = setTimeout(() => {
engineStreamActor.send({ type: EngineStreamTransition.Pause })
}, IDLE_TIME_MS)
} else if (!streamIdleMode) {
clearTimeout(timeoutId.current)
timeoutId.current = undefined
}
}, [streamIdleMode])
useEffect(() => {
if (!timeoutId.current) return
const onAnyInput = () => {
// Just in case it happens in the middle of the user turning off
// idle mode.
if (!streamIdleMode) {
clearTimeout(timeoutId.current)
return
}
if (engineStreamState.value === EngineStreamState.Paused) {
engineStreamActor.send({
type: EngineStreamTransition.StartOrReconfigureEngine,
modelingMachineActorSend,
settings: settingsEngine,
setAppState,
onMediaStream(mediaStream: MediaStream) {
engineStreamActor.send({
type: EngineStreamTransition.SetMediaStream,
mediaStream
})
}
})
}
clearTimeout(timeoutId.current)
timeoutId.current = setTimeout(() => {
engineStreamActor.send({ type: EngineStreamTransition.Pause })
}, IDLE_TIME_MS)
}
window.document.addEventListener('keydown', onAnyInput)
window.document.addEventListener('mousemove', onAnyInput)
window.document.addEventListener('mousedown', onAnyInput)
window.document.addEventListener('scroll', onAnyInput)
window.document.addEventListener('touchstart', onAnyInput)
return () => {
window.document.removeEventListener('keydown', onAnyInput)
window.document.removeEventListener('mousemove', onAnyInput)
window.document.removeEventListener('mousedown', onAnyInput)
window.document.removeEventListener('scroll', onAnyInput)
window.document.removeEventListener('touchstart', onAnyInput)
}
}, [streamIdleMode, engineStreamState.value])
const isNetworkOkay =
overallState === NetworkHealthState.Ok ||
overallState === NetworkHealthState.Weak
@ -106,7 +220,7 @@ export const EngineStream = () => {
const handleMouseUp: MouseEventHandler<HTMLDivElement> = (e) => {
if (!isNetworkOkay) return
if (!videoRef.current) return
if (!engineStreamState.context.videoRef.current) return
modelingMachineActorSend({
type: 'Set context',
@ -120,7 +234,7 @@ export const EngineStream = () => {
if (!modelingMachineContext.store?.didDragInStream && btnName(e).left) {
sendSelectEventToEngine(
e,
videoRef.current,
engineStreamState.context.videoRef.current,
modelingMachineContext.store?.streamDimensions
)
}
@ -136,7 +250,7 @@ export const EngineStream = () => {
const handleMouseDown: MouseEventHandler<HTMLDivElement> = (e) => {
if (!isNetworkOkay) return
if (!videoRef.current) return
if (!engineStreamState.context.videoRef.current) return
if (modelingMachineState.matches('Sketch')) return
if (modelingMachineState.matches('Sketch no face')) return
@ -144,7 +258,7 @@ export const EngineStream = () => {
const { x, y } = getNormalisedCoordinates({
clientX: e.clientX,
clientY: e.clientY,
el: videoRef.current,
el: engineStreamState.context.videoRef.current,
...modelingMachineContext.store?.streamDimensions,
})
@ -188,60 +302,20 @@ export const EngineStream = () => {
onContextMenuCapture={(e) => e.preventDefault()}
>
<video
ref={videoRef}
key={engineStreamActor.id + 'video'}
ref={engineStreamState.context.videoRef}
controls={false}
onMouseMoveCapture={handleMouseMove}
className="w-full cursor-pointer h-full"
className="cursor-pointer"
disablePictureInPicture
id="video-stream"
/>
<canvas ref={canvasRef} className="w-full cursor-pointer h-full" id="freeze-frame">No canvas support</canvas>
<canvas
key={engineStreamActor.id + 'canvas'}
ref={engineStreamState.context.canvasRef} className="cursor-pointer" id="freeze-frame">No canvas support</canvas>
<ClientSideScene
cameraControls={settings.context.modeling.mouseControls.current}
/>
{(engineStreamState.value === EngineStreamState.Paused ||
engineStreamState.value === EngineStreamState.Resuming) && (
<div className="text-center absolute inset-0">
<div
className="flex flex-col items-center justify-center h-screen"
data-testid="paused"
>
<div className="border-primary border p-2 rounded-sm">
<svg
width="8"
height="12"
viewBox="0 0 8 12"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
fillRule="evenodd"
clipRule="evenodd"
d="M2 12V0H0V12H2ZM8 12V0H6V12H8Z"
fill="var(--primary)"
/>
</svg>
</div>
<p className="text-base mt-2 text-primary bold">
{engineStreamState.value === EngineStreamState.Paused && 'Paused'}
{engineStreamState.value === EngineStreamState.Resuming && 'Resuming'}
</p>
</div>
</div>
)}
{(!isNetworkOkay || isLoading) && (
<div className="text-center absolute inset-0">
<Loading>
{!isNetworkOkay && !isLoading ? (
<span data-testid="loading-stream">Stream disconnected...</span>
) : (
!isLoading && (
<span data-testid="loading-stream">Loading stream...</span>
)
)}
</Loading>
</div>
)}
</div>
)
}

View File

@ -128,9 +128,6 @@ export const ModelingMachineProvider = ({
const streamRef = useRef<HTMLDivElement>(null)
const persistedContext = useMemo(() => getPersistedContext(), [])
let [searchParams] = useSearchParams()
const pool = searchParams.get('pool')
const { commandBarState, commandBarSend } = useCommandsContext()
// Settings machine setup

View File

@ -21,38 +21,39 @@ import { PATHS } from 'lib/paths'
import { IndexLoaderData } from 'lib/types'
export enum EngineStreamState {
Off = 'off',
On = 'on',
Playing = 'playing',
Paused = 'paused',
Resuming = 'resuming',
NotSetup = 'not-setup',
IsSetup = 'is-setup',
EngineStartedOrReconfigured = 'engine-started-or-reconfigured',
}
export enum EngineStreamTransition {
SetContextProperty= 'set-context-property',
SetMediaStream = 'set-context',
Play = 'play',
Resume = 'resume',
Pause = 'pause',
StartOrReconfigureEngine = 'start-or-reconfigure-engine',
IsSetup = 'is-setup',
}
export interface EngineStreamContext {
pool: string | null,
authToken: string | null,
mediaStreamRef: MutableRefObject<MediaStream | null>,
mediaStream: MediaStream | null,
videoRef: MutableRefObject<HTMLVideoElement | null>,
canvasRef: MutableRefObject<HTMLCanvasElement | null>,
}
function getDimensions(streamWidth: number, streamHeight: number) {
// Scaling for either portrait or landscape
const maxHeightResolution = streamWidth > streamHeight ? 1080 : 1920
const aspectRatio = 16/9
const height = Math.min(streamHeight, maxHeightResolution)
const width = height * aspectRatio
return { width, height }
const factorOf = 4
const maxResolution = 2160
const ratio = Math.min(
Math.min(maxResolution / streamWidth, maxResolution / streamHeight),
1.0
)
const quadWidth = Math.round((streamWidth * ratio) / factorOf) * factorOf
const quadHeight = Math.round((streamHeight * ratio) / factorOf) * factorOf
return { width: quadWidth, height: quadHeight }
}
const engineStreamMachine = setup({
@ -61,22 +62,14 @@ const engineStreamMachine = setup({
input: {} as EngineStreamContext,
},
actions: {
[EngineStreamTransition.SetContextProperty]({ context, event }) {
const nextContext = {
...context,
...event.value,
}
assign(nextContext)
return nextContext.authToken && nextContext.videoRef.current && nextContext.canvasRef.current
},
[EngineStreamTransition.Play]({ context }, params: { reconnect: boolean }) {
[EngineStreamTransition.Play]({ context }) {
const canvas = context.canvasRef.current
if (!canvas) return false
const video = context.videoRef.current
if (!video) return false
const mediaStream = context.mediaStreamRef.current
const mediaStream = context.mediaStream
if (!mediaStream) return false
video.style.display = "block"
@ -86,7 +79,6 @@ const engineStreamMachine = setup({
video.play().catch((e) => {
console.warn('Video playing was prevented', e, video)
}).then(() => {
if (params.reconnect) return
kclManager.executeCode(true)
})
},
@ -117,28 +109,42 @@ const engineStreamMachine = setup({
// leave everything at pausing, preventing video decoders from running
// but we can do even better by significantly reducing network
// cards also.
context.mediaStreamRef.current?.getVideoTracks()[0].stop()
context.mediaStream?.getVideoTracks()[0].stop()
video.srcObject = null
context.mediaStreamRef.current = null
engineCommandManager.tearDown({ idleMode: true })
})
},
async [EngineStreamTransition.StartOrReconfigureEngine]({ context, event: { modelingMachineActorSend, settings, setAppState } }) {
async [EngineStreamTransition.StartOrReconfigureEngine]({ context, event }) {
if (!context.authToken) return
const video = context.videoRef.current
if (!video) return
const { width, height } = getDimensions(
window.innerWidth,
window.innerHeight,
)
engineCommandManager.settings = settings
video.width = width
video.height = height
const settingsNext = {
// override the pool param (?pool=) to request a specific engine instance
// from a particular pool.
pool: context.pool,
...event.settings,
}
engineCommandManager.settings = settingsNext
engineCommandManager.start({
setMediaStream: (mediaStream) => context.mediaStreamRef.current = mediaStream,
setIsStreamReady: (isStreamReady) => setAppState({ isStreamReady }),
setMediaStream: event.onMediaStream,
setIsStreamReady: (isStreamReady) => event.setAppState({ isStreamReady }),
width,
height,
token: context.authToken,
settings,
settings: settingsNext,
makeDefaultPlanes: () => {
return makeDefaultPlanes(kclManager.engineCommandManager)
},
@ -147,7 +153,7 @@ const engineStreamMachine = setup({
},
})
modelingMachineActorSend({
event.modelingMachineActorSend({
type: 'Set context',
data: {
streamDimensions: {
@ -157,38 +163,41 @@ const engineStreamMachine = setup({
},
})
},
async [EngineStreamTransition.Resume]({ context, event }) {
// engineCommandManager.engineConnection?.reattachMediaStream()
},
}
}).createMachine({
context: {
mediaStreamRef: { current: null },
videoRef: { current: null },
canvasRef: { current: null },
authToken: null,
},
initial: EngineStreamState.NotSetup,
context: (initial) => initial.input,
initial: EngineStreamState.Off,
states: {
[EngineStreamState.NotSetup]: {
[EngineStreamState.Off]: {
on: {
[EngineStreamTransition.SetContextProperty]: {
target: EngineStreamState.IsSetup,
actions: { type: EngineStreamTransition.SetContextProperty },
[EngineStreamTransition.StartOrReconfigureEngine]: {
target: EngineStreamState.On,
actions: [{ type: EngineStreamTransition.StartOrReconfigureEngine } ]
}
}
},
[EngineStreamState.On]: {
on: {
[EngineStreamTransition.SetMediaStream]: {
target: EngineStreamState.On,
actions: [ assign({ mediaStream: ({ context, event }) => event.mediaStream }) ]
},
}
},
[EngineStreamState.IsSetup]: {
always: {
target: EngineStreamState.EngineStartedOrReconfigured,
actions: [ { type: EngineStreamTransition.StartOrReconfigureEngine } ]
}
},
[EngineStreamState.EngineStartedOrReconfigured]: {
always: {
target: EngineStreamState.Playing,
actions: [ { type: EngineStreamTransition.Play , params: { reconnect: false } } ]
[EngineStreamTransition.Play]: {
target: EngineStreamState.Playing,
actions: [ { type: EngineStreamTransition.Play } ]
}
}
},
[EngineStreamState.Playing]: {
on: {
[EngineStreamTransition.StartOrReconfigureEngine]: {
target: EngineStreamState.Playing,
reenter: true,
actions: [{ type: EngineStreamTransition.StartOrReconfigureEngine } ]
},
[EngineStreamTransition.Pause]: {
target: EngineStreamState.Paused,
actions: [ { type: EngineStreamTransition.Pause } ]
@ -197,15 +206,22 @@ const engineStreamMachine = setup({
},
[EngineStreamState.Paused]: {
on: {
[EngineStreamTransition.Resume]: {
[EngineStreamTransition.StartOrReconfigureEngine]: {
target: EngineStreamState.Resuming,
actions: [{ type: EngineStreamTransition.StartOrReconfigureEngine } ]
},
}
},
[EngineStreamState.Resuming]: {
always: {
target: EngineStreamState.Playing,
actions: [ { type: EngineStreamTransition.Play, params: { reconnect: true } } ]
on: {
[EngineStreamTransition.SetMediaStream]: {
target: EngineStreamState.Resuming,
actions: [ assign({ mediaStream: ({ context, event }) => event.mediaStream }) ]
},
[EngineStreamTransition.Play]: {
target: EngineStreamState.Playing,
actions: [ { type: EngineStreamTransition.Play } ]
}
}
},
}

View File

@ -404,13 +404,13 @@ class EngineConnection extends EventTarget {
default:
if (this.isConnecting()) break
// Means we never could do an initial connection. Reconnect everything.
if (!this.pingPongSpan.ping) this.connect().catch(reportRejection)
if (!this.pingPongSpan.ping) this.connect({ reconnect: false }).catch(reportRejection)
break
}
}, pingIntervalMs)
// eslint-disable-next-line @typescript-eslint/no-floating-promises
this.connect()
this.connect({ reconnect: false })
}
// SHOULD ONLY BE USED FOR VITESTS
@ -521,7 +521,9 @@ class EngineConnection extends EventTarget {
this.idleMode = opts?.idleMode ?? false
clearInterval(this.pingIntervalId)
if (opts?.idleMode) {
this.disconnectAll()
if (this.idleMode) {
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: {
@ -541,7 +543,6 @@ class EngineConnection extends EventTarget {
},
}
this.disconnectAll()
}
/**
@ -551,7 +552,7 @@ class EngineConnection extends EventTarget {
* This will attempt the full handshake, and retry if the connection
* did not establish.
*/
connect(reconnecting?: boolean): Promise<void> {
connect(args: { reconnect: boolean }): Promise<void> {
return new Promise((resolve) => {
if (this.isConnecting() || this.isReady()) {
return
@ -1162,8 +1163,8 @@ class EngineConnection extends EventTarget {
this.websocket.addEventListener('message', this.onWebSocketMessage)
}
if (reconnecting) {
createWebSocketConnection()
if (args.reconnect) {
createWebSocketConnection()
} else {
this.onNetworkStatusReady = () => {
createWebSocketConnection()
@ -1175,6 +1176,32 @@ class EngineConnection extends EventTarget {
}
})
}
reattachMediaStream() {
this.pc
?.createOffer({ iceRestart: true })
.then((offer: RTCSessionDescriptionInit) => {
this.state = {
type: EngineConnectionStateType.Connecting,
value: {
type: ConnectingType.SetLocalDescription,
},
}
return this.pc?.setLocalDescription(offer).then(() => {
this.send({
type: 'sdp_offer',
offer: offer as Models['RtcSessionDescription_type'],
})
this.state = {
type: EngineConnectionStateType.Connecting,
value: {
type: ConnectingType.OfferedSdp,
},
}
})
})
}
// Do not change this back to an object or any, we should only be sending the
// WebSocketRequest type!
unreliableSend(message: Models['WebSocketRequest_type']) {
@ -1226,8 +1253,17 @@ class EngineConnection extends EventTarget {
this.websocket?.readyState === 3
if (closedPc && closedUDC && closedWS) {
// Do not notify the rest of the program that we have cut off anything.
this.state = { type: EngineConnectionStateType.Disconnected }
if (!this.idleMode) {
// Do not notify the rest of the program that we have cut off anything.
this.state = { type: EngineConnectionStateType.Disconnected }
} else {
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: {
type: DisconnectingType.Pause,
},
}
}
}
}
}
@ -1759,7 +1795,7 @@ export class EngineCommandManager extends EventTarget {
)
// eslint-disable-next-line @typescript-eslint/no-floating-promises
this.engineConnection?.connect()
this.engineConnection?.connect({ reconnect: false })
}
this.engineConnection.addEventListener(
EngineConnectionEvents.ConnectionStarted,
@ -1821,6 +1857,7 @@ export class EngineCommandManager extends EventTarget {
)
this.engineConnection?.tearDown(opts)
this.engineConnection = undefined
// Our window.tearDown assignment causes this case to happen which is
// only really for tests.
@ -1828,6 +1865,8 @@ export class EngineCommandManager extends EventTarget {
} else if (this.engineCommandManager?.engineConnection) {
// @ts-ignore
this.engineCommandManager?.engineConnection?.tearDown(opts)
// @ts-ignore
this.engineCommandManager.engineConnection = null
}
}
async startNewSession() {

View File

@ -139,12 +139,6 @@ export const fileLoader: LoaderFunction = async (
? await getProjectInfo(projectPath)
: null
console.log('maybeProjectInfo', {
maybeProjectInfo,
defaultProjectData,
projectPathData,
})
const projectData: IndexLoaderData = {
code,
project: maybeProjectInfo ?? defaultProjectData,