Lf94/save settings between reconnects (#2997)

* Keep settings between reconnects

* Set idle timeout to 2 minutes

* Put idle behind flags

* Remove pauses

* Fix online->offline->online

* Revert "Remove pauses"

This reverts commit 267ef4ff4b86f2d8014bfb2a8e8a633adc8001dc.

* ci

* call correct setmediastream
This commit is contained in:
49fl
2024-07-12 16:42:23 -04:00
committed by GitHub
parent 5a5fe3bb95
commit e81b614523
3 changed files with 437 additions and 245 deletions

View File

@ -1,3 +1,4 @@
import { DEV } from 'env'
import { MouseEventHandler, useEffect, useRef, useState } from 'react' import { MouseEventHandler, useEffect, useRef, useState } from 'react'
import { getNormalisedCoordinates } from '../lib/utils' import { getNormalisedCoordinates } from '../lib/utils'
import Loading from './Loading' import Loading from './Loading'
@ -22,6 +23,8 @@ export const Stream = () => {
const { overallState } = useNetworkContext() const { overallState } = useNetworkContext()
const [isFreezeFrame, setIsFreezeFrame] = useState(false) const [isFreezeFrame, setIsFreezeFrame] = useState(false)
const IDLE = true
const isNetworkOkay = const isNetworkOkay =
overallState === NetworkHealthState.Ok || overallState === NetworkHealthState.Ok ||
overallState === NetworkHealthState.Weak overallState === NetworkHealthState.Weak
@ -53,7 +56,7 @@ export const Stream = () => {
capture: true, capture: true,
}) })
const IDLE_TIME_MS = 1000 * 20 const IDLE_TIME_MS = 1000 * 60 * 2
let timeoutIdIdleA: ReturnType<typeof setTimeout> | undefined = undefined let timeoutIdIdleA: ReturnType<typeof setTimeout> | undefined = undefined
const teardown = () => { const teardown = () => {
@ -62,19 +65,21 @@ export const Stream = () => {
sceneInfra.modelingSend({ type: 'Cancel' }) sceneInfra.modelingSend({ type: 'Cancel' })
// Give video time to pause // Give video time to pause
window.requestAnimationFrame(() => { window.requestAnimationFrame(() => {
engineCommandManager.engineConnection?.tearDown({ freeze: true }) engineCommandManager.tearDown()
}) })
} }
// Teardown everything if we go hidden or reconnect // Teardown everything if we go hidden or reconnect
if (globalThis?.window?.document) { if (IDLE && DEV) {
globalThis.window.document.onvisibilitychange = () => { if (globalThis?.window?.document) {
if (globalThis.window.document.visibilityState === 'hidden') { globalThis.window.document.onvisibilitychange = () => {
clearTimeout(timeoutIdIdleA) if (globalThis.window.document.visibilityState === 'hidden') {
timeoutIdIdleA = setTimeout(teardown, IDLE_TIME_MS) clearTimeout(timeoutIdIdleA)
} else if (!engineCommandManager.engineConnection?.isReady()) { timeoutIdIdleA = setTimeout(teardown, IDLE_TIME_MS)
clearTimeout(timeoutIdIdleA) } else if (!engineCommandManager.engineConnection?.isReady()) {
engineCommandManager.engineConnection?.connect(true) clearTimeout(timeoutIdIdleA)
engineCommandManager.engineConnection?.connect(true)
}
} }
} }
} }
@ -82,35 +87,44 @@ export const Stream = () => {
let timeoutIdIdleB: ReturnType<typeof setTimeout> | undefined = undefined let timeoutIdIdleB: ReturnType<typeof setTimeout> | undefined = undefined
const onAnyInput = () => { const onAnyInput = () => {
if (!engineCommandManager.engineConnection?.isReady()) {
engineCommandManager.engineConnection?.connect(true)
}
// Clear both timers // Clear both timers
clearTimeout(timeoutIdIdleA) clearTimeout(timeoutIdIdleA)
clearTimeout(timeoutIdIdleB) clearTimeout(timeoutIdIdleB)
timeoutIdIdleB = setTimeout(teardown, IDLE_TIME_MS) timeoutIdIdleB = setTimeout(teardown, IDLE_TIME_MS)
} }
globalThis?.window?.document?.addEventListener('keydown', onAnyInput) if (IDLE && DEV) {
globalThis?.window?.document?.addEventListener('mousemove', onAnyInput) globalThis?.window?.document?.addEventListener('keydown', onAnyInput)
globalThis?.window?.document?.addEventListener('mousedown', onAnyInput) globalThis?.window?.document?.addEventListener('mousemove', onAnyInput)
globalThis?.window?.document?.addEventListener('scroll', onAnyInput) globalThis?.window?.document?.addEventListener('mousedown', onAnyInput)
globalThis?.window?.document?.addEventListener('touchstart', onAnyInput) globalThis?.window?.document?.addEventListener('scroll', onAnyInput)
globalThis?.window?.document?.addEventListener('touchstart', onAnyInput)
}
timeoutIdIdleB = setTimeout(teardown, IDLE_TIME_MS) if (IDLE && DEV) {
timeoutIdIdleB = setTimeout(teardown, IDLE_TIME_MS)
}
return () => { return () => {
globalThis?.window?.document?.removeEventListener('paste', handlePaste, { globalThis?.window?.document?.removeEventListener('paste', handlePaste, {
capture: true, capture: true,
}) })
globalThis?.window?.document?.removeEventListener('keydown', onAnyInput) if (IDLE && DEV) {
globalThis?.window?.document?.removeEventListener('mousemove', onAnyInput) globalThis?.window?.document?.removeEventListener('keydown', onAnyInput)
globalThis?.window?.document?.removeEventListener('mousedown', onAnyInput) globalThis?.window?.document?.removeEventListener(
globalThis?.window?.document?.removeEventListener('scroll', onAnyInput) 'mousemove',
globalThis?.window?.document?.removeEventListener( onAnyInput
'touchstart', )
onAnyInput globalThis?.window?.document?.removeEventListener(
) 'mousedown',
onAnyInput
)
globalThis?.window?.document?.removeEventListener('scroll', onAnyInput)
globalThis?.window?.document?.removeEventListener(
'touchstart',
onAnyInput
)
}
} }
}, []) }, [])

View File

@ -1,4 +1,4 @@
import { useLayoutEffect, useEffect, useRef } from 'react' import { useLayoutEffect, useEffect, useRef, useState } from 'react'
import { engineCommandManager, kclManager } from 'lib/singletons' import { engineCommandManager, kclManager } from 'lib/singletons'
import { deferExecution } from 'lib/utils' import { deferExecution } from 'lib/utils'
import { Themes } from 'lib/theme' import { Themes } from 'lib/theme'
@ -30,9 +30,6 @@ export function useSetupEngineManager(
const { setAppState } = useAppState() const { setAppState } = useAppState()
const { setMediaStream } = useAppStream() const { setMediaStream } = useAppStream()
const streamWidth = streamRef?.current?.offsetWidth
const streamHeight = streamRef?.current?.offsetHeight
const hasSetNonZeroDimensions = useRef<boolean>(false) const hasSetNonZeroDimensions = useRef<boolean>(false)
if (settings.pool) { if (settings.pool) {
@ -41,55 +38,60 @@ export function useSetupEngineManager(
engineCommandManager.pool = settings.pool engineCommandManager.pool = settings.pool
} }
const startEngineInstance = () => { const startEngineInstance = (restart: boolean = false) => {
// Load the engine command manager once with the initial width and height, // Load the engine command manager once with the initial width and height,
// then we do not want to reload it. // then we do not want to reload it.
const { width: quadWidth, height: quadHeight } = getDimensions( const { width: quadWidth, height: quadHeight } = getDimensions(
streamWidth, streamRef?.current?.offsetWidth ?? 0,
streamHeight streamRef?.current?.offsetHeight ?? 0
) )
if ( if (restart) {
!hasSetNonZeroDimensions.current && kclManager.isFirstRender = false
quadHeight &&
quadWidth &&
settings.modelingSend
) {
engineCommandManager.start({
setMediaStream: setMediaStream,
setIsStreamReady: (isStreamReady) => setAppState({ isStreamReady }),
width: quadWidth,
height: quadHeight,
executeCode: () => {
// We only want to execute the code here that we already have set.
// Nothing else.
kclManager.isFirstRender = true
return kclManager.executeCode(true, true).then(() => {
kclManager.isFirstRender = false
})
},
token,
settings,
makeDefaultPlanes: () => {
return makeDefaultPlanes(kclManager.engineCommandManager)
},
modifyGrid: (hidden: boolean) => {
return modifyGrid(kclManager.engineCommandManager, hidden)
},
})
settings.modelingSend({
type: 'Set context',
data: {
streamDimensions: {
streamWidth: quadWidth,
streamHeight: quadHeight,
},
},
})
hasSetNonZeroDimensions.current = true
} }
engineCommandManager.start({
restart,
setMediaStream: (mediaStream) => setMediaStream(mediaStream),
setIsStreamReady: (isStreamReady) => setAppState({ isStreamReady }),
width: quadWidth,
height: quadHeight,
executeCode: () => {
// We only want to execute the code here that we already have set.
// Nothing else.
kclManager.isFirstRender = true
return kclManager.executeCode(true, true).then(() => {
kclManager.isFirstRender = false
})
},
token,
settings,
makeDefaultPlanes: () => {
return makeDefaultPlanes(kclManager.engineCommandManager)
},
modifyGrid: (hidden: boolean) => {
return modifyGrid(kclManager.engineCommandManager, hidden)
},
})
settings.modelingSend({
type: 'Set context',
data: {
streamDimensions: {
streamWidth: quadWidth,
streamHeight: quadHeight,
},
},
})
hasSetNonZeroDimensions.current = true
} }
useLayoutEffect(startEngineInstance, [ useLayoutEffect(() => {
const { width: quadWidth, height: quadHeight } = getDimensions(
streamRef?.current?.offsetWidth ?? 0,
streamRef?.current?.offsetHeight ?? 0
)
if (!hasSetNonZeroDimensions.current && quadHeight && quadWidth) {
startEngineInstance()
}
}, [
streamRef?.current?.offsetWidth, streamRef?.current?.offsetWidth,
streamRef?.current?.offsetHeight, streamRef?.current?.offsetHeight,
settings.modelingSend, settings.modelingSend,
@ -98,8 +100,8 @@ export function useSetupEngineManager(
useEffect(() => { useEffect(() => {
const handleResize = deferExecution(() => { const handleResize = deferExecution(() => {
const { width, height } = getDimensions( const { width, height } = getDimensions(
streamRef?.current?.offsetWidth, streamRef?.current?.offsetWidth ?? 0,
streamRef?.current?.offsetHeight streamRef?.current?.offsetHeight ?? 0
) )
if ( if (
settings.modelingContext.store.streamDimensions.streamWidth !== width || settings.modelingContext.store.streamDimensions.streamWidth !== width ||
@ -122,10 +124,37 @@ export function useSetupEngineManager(
}, 500) }, 500)
const onOnline = () => { const onOnline = () => {
startEngineInstance() startEngineInstance(true)
} }
const onVisibilityChange = () => {
if (window.document.visibilityState === 'visible') {
if (
!engineCommandManager.engineConnection?.isReady() &&
!engineCommandManager.engineConnection?.isConnecting()
) {
startEngineInstance()
}
}
}
window.document.addEventListener('visibilitychange', onVisibilityChange)
const onAnyInput = () => {
if (
!engineCommandManager.engineConnection?.isReady() &&
!engineCommandManager.engineConnection?.isConnecting()
) {
startEngineInstance()
}
}
window.document.addEventListener('keydown', onAnyInput)
window.document.addEventListener('mousemove', onAnyInput)
window.document.addEventListener('mousedown', onAnyInput)
window.document.addEventListener('scroll', onAnyInput)
window.document.addEventListener('touchstart', onAnyInput)
const onOffline = () => { const onOffline = () => {
kclManager.isFirstRender = true
engineCommandManager.tearDown() engineCommandManager.tearDown()
} }
@ -133,11 +162,30 @@ export function useSetupEngineManager(
window.addEventListener('offline', onOffline) window.addEventListener('offline', onOffline)
window.addEventListener('resize', handleResize) window.addEventListener('resize', handleResize)
return () => { return () => {
window.document.removeEventListener(
'visibilitychange',
onVisibilityChange
)
window.document.removeEventListener('keydown', onAnyInput)
window.document.removeEventListener('mousemove', onAnyInput)
window.document.removeEventListener('mousedown', onAnyInput)
window.document.removeEventListener('scroll', onAnyInput)
window.document.removeEventListener('touchstart', onAnyInput)
window.removeEventListener('online', onOnline) window.removeEventListener('online', onOnline)
window.removeEventListener('offline', onOffline) window.removeEventListener('offline', onOffline)
window.removeEventListener('resize', handleResize) window.removeEventListener('resize', handleResize)
} }
}, [])
// Engine relies on many settings so we should rebind events when it changes
// We have to list out the ones we care about because the settings object holds
// non-settings too...
}, [
settings.enableSSAO,
settings.highlightEdges,
settings.showScaleGrid,
settings.theme,
settings.pool,
])
} }
function getDimensions(streamWidth?: number, streamHeight?: number) { function getDimensions(streamWidth?: number, streamHeight?: number) {

View File

@ -302,6 +302,30 @@ class EngineConnection extends EventTarget {
mediaStream?: MediaStream mediaStream?: MediaStream
freezeFrame: boolean = false freezeFrame: boolean = false
onIceCandidate = function (
this: RTCPeerConnection,
event: RTCPeerConnectionIceEvent
) {}
onIceCandidateError = function (
this: RTCPeerConnection,
event: RTCPeerConnectionIceErrorEvent
) {}
onConnectionStateChange = function (this: RTCPeerConnection, event: Event) {}
onDataChannelOpen = function (this: RTCDataChannel, event: Event) {}
onDataChannelClose = function (this: RTCDataChannel, event: Event) {}
onDataChannelError = function (this: RTCDataChannel, event: Event) {}
onDataChannelMessage = function (this: RTCDataChannel, event: MessageEvent) {}
onDataChannel = function (
this: RTCPeerConnection,
event: RTCDataChannelEvent
) {}
onTrack = function (this: RTCPeerConnection, event: RTCTrackEvent) {}
onWebSocketOpen = function (event: Event) {}
onWebSocketClose = function (event: Event) {}
onWebSocketError = function (event: Event) {}
onWebSocketMessage = function (event: MessageEvent) {}
onNetworkStatusReady = () => {}
private _state: EngineConnectionState = { private _state: EngineConnectionState = {
type: EngineConnectionStateType.Fresh, type: EngineConnectionStateType.Fresh,
} }
@ -346,6 +370,7 @@ class EngineConnection extends EventTarget {
private engineCommandManager: EngineCommandManager private engineCommandManager: EngineCommandManager
private pingPongSpan: { ping?: Date; pong?: Date } private pingPongSpan: { ping?: Date; pong?: Date }
private pingIntervalId: ReturnType<typeof setInterval>
constructor({ constructor({
engineCommandManager, engineCommandManager,
@ -368,7 +393,7 @@ class EngineConnection extends EventTarget {
// Without an interval ping, our connection will timeout. // Without an interval ping, our connection will timeout.
// If this.freezeFrame is true we skip this logic so only reconnect // If this.freezeFrame is true we skip this logic so only reconnect
// happens on mouse move // happens on mouse move
setInterval(() => { this.pingIntervalId = setInterval(() => {
if (this.freezeFrame) return if (this.freezeFrame) return
switch (this.state.type as EngineConnectionStateType) { switch (this.state.type as EngineConnectionStateType) {
@ -434,6 +459,44 @@ class EngineConnection extends EventTarget {
tearDown(opts?: { freeze: boolean }) { tearDown(opts?: { freeze: boolean }) {
this.freezeFrame = opts?.freeze ?? false this.freezeFrame = opts?.freeze ?? false
this.disconnectAll() this.disconnectAll()
clearInterval(this.pingIntervalId)
this.pc?.removeEventListener('icecandidate', this.onIceCandidate)
this.pc?.removeEventListener('icecandidateerror', this.onIceCandidateError)
this.pc?.removeEventListener(
'connectionstatechange',
this.onConnectionStateChange
)
this.pc?.removeEventListener('track', this.onTrack)
this.unreliableDataChannel?.removeEventListener(
'open',
this.onDataChannelOpen
)
this.unreliableDataChannel?.removeEventListener(
'close',
this.onDataChannelClose
)
this.unreliableDataChannel?.removeEventListener(
'error',
this.onDataChannelError
)
this.unreliableDataChannel?.removeEventListener(
'message',
this.onDataChannelMessage
)
this.pc?.removeEventListener('datachannel', this.onDataChannel)
this.websocket?.removeEventListener('open', this.onWebSocketOpen)
this.websocket?.removeEventListener('close', this.onWebSocketClose)
this.websocket?.removeEventListener('error', this.onWebSocketError)
this.websocket?.removeEventListener('message', this.onWebSocketMessage)
window.removeEventListener(
'use-network-status-ready',
this.onNetworkStatusReady
)
this.state = { this.state = {
type: EngineConnectionStateType.Disconnecting, type: EngineConnectionStateType.Disconnecting,
value: { type: DisconnectingType.Quit }, value: { type: DisconnectingType.Quit },
@ -477,7 +540,7 @@ class EngineConnection extends EventTarget {
}, },
} }
this.pc.addEventListener('icecandidate', (event) => { this.onIceCandidate = (event: RTCPeerConnectionIceEvent) => {
if (event.candidate === null) { if (event.candidate === null) {
return return
} }
@ -499,18 +562,20 @@ class EngineConnection extends EventTarget {
usernameFragment: event.candidate.usernameFragment || undefined, usernameFragment: event.candidate.usernameFragment || undefined,
}, },
}) })
}) }
this.pc.addEventListener('icecandidate', this.onIceCandidate)
this.pc.addEventListener('icecandidateerror', (_event: Event) => { this.onIceCandidateError = (_event: Event) => {
const event = _event as RTCPeerConnectionIceErrorEvent const event = _event as RTCPeerConnectionIceErrorEvent
console.warn( console.warn(
`ICE candidate returned an error: ${event.errorCode}: ${event.errorText} for ${event.url}` `ICE candidate returned an error: ${event.errorCode}: ${event.errorText} for ${event.url}`
) )
}) }
this.pc.addEventListener('icecandidateerror', this.onIceCandidateError)
// https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/connectionstatechange_event // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/connectionstatechange_event
// Event type: generic Event type... // Event type: generic Event type...
this.pc.addEventListener('connectionstatechange', (event: any) => { this.onConnectionStateChange = (event: any) => {
console.log('connectionstatechange: ' + event.target?.connectionState) console.log('connectionstatechange: ' + event.target?.connectionState)
switch (event.target?.connectionState) { switch (event.target?.connectionState) {
// From what I understand, only after have we done the ICE song and // From what I understand, only after have we done the ICE song and
@ -539,9 +604,13 @@ class EngineConnection extends EventTarget {
default: default:
break break
} }
}) }
this.pc.addEventListener(
'connectionstatechange',
this.onConnectionStateChange
)
this.pc.addEventListener('track', (event) => { this.onTrack = (event) => {
const mediaStream = event.streams[0] const mediaStream = event.streams[0]
this.state = { this.state = {
@ -625,9 +694,10 @@ class EngineConnection extends EventTarget {
// to pass it to the rest of the application. // to pass it to the rest of the application.
this.mediaStream = mediaStream this.mediaStream = mediaStream
}) }
this.pc.addEventListener('track', this.onTrack)
this.pc.addEventListener('datachannel', (event) => { this.onDataChannel = (event) => {
this.unreliableDataChannel = event.channel this.unreliableDataChannel = event.channel
this.state = { this.state = {
@ -638,7 +708,7 @@ class EngineConnection extends EventTarget {
}, },
} }
this.unreliableDataChannel.addEventListener('open', (event) => { this.onDataChannelOpen = (event) => {
this.state = { this.state = {
type: EngineConnectionStateType.Connecting, type: EngineConnectionStateType.Connecting,
value: { value: {
@ -654,14 +724,22 @@ class EngineConnection extends EventTarget {
this.dispatchEvent( this.dispatchEvent(
new CustomEvent(EngineConnectionEvents.Opened, { detail: this }) new CustomEvent(EngineConnectionEvents.Opened, { detail: this })
) )
}) }
this.unreliableDataChannel?.addEventListener(
'open',
this.onDataChannelOpen
)
this.unreliableDataChannel.addEventListener('close', (event) => { this.onDataChannelClose = (event) => {
this.disconnectAll() this.disconnectAll()
this.finalizeIfAllConnectionsClosed() this.finalizeIfAllConnectionsClosed()
}) }
this.unreliableDataChannel?.addEventListener(
'close',
this.onDataChannelClose
)
this.unreliableDataChannel.addEventListener('error', (event) => { this.onDataChannelError = (event) => {
this.disconnectAll() this.disconnectAll()
this.state = { this.state = {
@ -674,8 +752,13 @@ class EngineConnection extends EventTarget {
}, },
}, },
} }
}) }
this.unreliableDataChannel.addEventListener('message', (event) => { this.unreliableDataChannel?.addEventListener(
'error',
this.onDataChannelError
)
this.onDataChannelMessage = (event) => {
const result: UnreliableResponses = JSON.parse(event.data) const result: UnreliableResponses = JSON.parse(event.data)
Object.values( Object.values(
this.engineCommandManager.unreliableSubscriptions[result.type] || {} this.engineCommandManager.unreliableSubscriptions[result.type] || {}
@ -697,8 +780,13 @@ class EngineConnection extends EventTarget {
} }
} }
) )
}) }
}) this.unreliableDataChannel.addEventListener(
'message',
this.onDataChannelMessage
)
}
this.pc.addEventListener('datachannel', this.onDataChannel)
} }
const createWebSocketConnection = () => { const createWebSocketConnection = () => {
@ -712,7 +800,7 @@ class EngineConnection extends EventTarget {
this.websocket = new WebSocket(this.url, []) this.websocket = new WebSocket(this.url, [])
this.websocket.binaryType = 'arraybuffer' this.websocket.binaryType = 'arraybuffer'
this.websocket.addEventListener('open', (event) => { this.onWebSocketOpen = (event) => {
this.state = { this.state = {
type: EngineConnectionStateType.Connecting, type: EngineConnectionStateType.Connecting,
value: { value: {
@ -733,14 +821,16 @@ class EngineConnection extends EventTarget {
// Send an initial ping // Send an initial ping
this.send({ type: 'ping' }) this.send({ type: 'ping' })
this.pingPongSpan.ping = new Date() this.pingPongSpan.ping = new Date()
}) }
this.websocket.addEventListener('open', this.onWebSocketOpen)
this.websocket.addEventListener('close', (event) => { this.onWebSocketClose = (event) => {
this.disconnectAll() this.disconnectAll()
this.finalizeIfAllConnectionsClosed() this.finalizeIfAllConnectionsClosed()
}) }
this.websocket.addEventListener('close', this.onWebSocketClose)
this.websocket.addEventListener('error', (event) => { this.onWebSocketError = (event) => {
this.disconnectAll() this.disconnectAll()
this.state = { this.state = {
@ -753,9 +843,10 @@ class EngineConnection extends EventTarget {
}, },
}, },
} }
}) }
this.websocket.addEventListener('error', this.onWebSocketError)
this.websocket.addEventListener('message', (event) => { this.onWebSocketMessage = (event) => {
// In the EngineConnection, we're looking for messages to/from // In the EngineConnection, we're looking for messages to/from
// the server that relate to the ICE handshake, or WebRTC // the server that relate to the ICE handshake, or WebRTC
// negotiation. There may be other messages (including ArrayBuffer // negotiation. There may be other messages (including ArrayBuffer
@ -960,15 +1051,20 @@ class EngineConnection extends EventTarget {
}) })
break break
} }
}) }
this.websocket.addEventListener('message', this.onWebSocketMessage)
} }
if (reconnecting) { if (reconnecting) {
createWebSocketConnection() createWebSocketConnection()
} else { } else {
window.addEventListener('use-network-status-ready', () => { this.onNetworkStatusReady = () => {
createWebSocketConnection() createWebSocketConnection()
}) }
window.addEventListener(
'use-network-status-ready',
this.onNetworkStatusReady
)
} }
} }
// Do not change this back to an object or any, we should only be sending the // Do not change this back to an object or any, we should only be sending the
@ -1154,7 +1250,15 @@ export class EngineCommandManager extends EventTarget {
private makeDefaultPlanes: () => Promise<DefaultPlanes> | null = () => null private makeDefaultPlanes: () => Promise<DefaultPlanes> | null = () => null
private modifyGrid: (hidden: boolean) => Promise<void> | null = () => null private modifyGrid: (hidden: boolean) => Promise<void> | null = () => null
private onEngineConnectionOpened = () => {}
private onEngineConnectionClosed = () => {}
private onEngineConnectionStarted = ({ detail: engineConnection }: any) => {}
private onEngineConnectionNewTrack = ({
detail,
}: CustomEvent<NewTrackArgs>) => {}
start({ start({
restart,
setMediaStream, setMediaStream,
setIsStreamReady, setIsStreamReady,
width, width,
@ -1170,6 +1274,7 @@ export class EngineCommandManager extends EventTarget {
showScaleGrid: false, showScaleGrid: false,
}, },
}: { }: {
restart?: boolean
setMediaStream: (stream: MediaStream) => void setMediaStream: (stream: MediaStream) => void
setIsStreamReady: (isStreamReady: boolean) => void setIsStreamReady: (isStreamReady: boolean) => void
width: number width: number
@ -1215,162 +1320,168 @@ export class EngineCommandManager extends EventTarget {
}) })
) )
this.onEngineConnectionOpened = () => {
// Set the stream background color
// This takes RGBA values from 0-1
// So we convert from the conventional 0-255 found in Figma
void this.sendSceneCommand({
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'set_background_color',
color: getThemeColorForEngine(settings.theme),
},
})
// Sets the default line colors
const opposingTheme = getOppositeTheme(settings.theme)
this.sendSceneCommand({
cmd_id: uuidv4(),
type: 'modeling_cmd_req',
cmd: {
type: 'set_default_system_properties',
color: getThemeColorForEngine(opposingTheme),
},
})
// Set the edge lines visibility
this.sendSceneCommand({
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'edge_lines_visible' as any, // TODO: update kittycad.ts to use the correct type
hidden: !settings.highlightEdges,
},
})
this._camControlsCameraChange()
this.sendSceneCommand({
// CameraControls subscribes to default_camera_get_settings response events
// firing this at connection ensure the camera's are synced initially
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'default_camera_get_settings',
},
})
// We want modify the grid first because we don't want it to flash.
// Ideally these would already be default hidden in engine (TODO do
// that) https://github.com/KittyCAD/engine/issues/2282
this.modifyGrid(!settings.showScaleGrid)?.then(async () => {
await this.initPlanes()
this.resolveReady()
setIsStreamReady(true)
await executeCode()
})
}
this.engineConnection.addEventListener( this.engineConnection.addEventListener(
EngineConnectionEvents.Opened, EngineConnectionEvents.Opened,
() => { this.onEngineConnectionOpened
// Set the stream background color
// This takes RGBA values from 0-1
// So we convert from the conventional 0-255 found in Figma
void this.sendSceneCommand({
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'set_background_color',
color: getThemeColorForEngine(settings.theme),
},
})
// Sets the default line colors
const opposingTheme = getOppositeTheme(settings.theme)
this.sendSceneCommand({
cmd_id: uuidv4(),
type: 'modeling_cmd_req',
cmd: {
type: 'set_default_system_properties',
color: getThemeColorForEngine(opposingTheme),
},
})
// Set the edge lines visibility
this.sendSceneCommand({
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'edge_lines_visible' as any, // TODO: update kittycad.ts to use the correct type
hidden: !settings.highlightEdges,
},
})
this._camControlsCameraChange()
this.sendSceneCommand({
// CameraControls subscribes to default_camera_get_settings response events
// firing this at connection ensure the camera's are synced initially
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'default_camera_get_settings',
},
})
// We want modify the grid first because we don't want it to flash.
// Ideally these would already be default hidden in engine (TODO do
// that) https://github.com/KittyCAD/engine/issues/2282
this.modifyGrid(!settings.showScaleGrid)?.then(async () => {
await this.initPlanes()
this.resolveReady()
setIsStreamReady(true)
await executeCode()
})
}
) )
this.onEngineConnectionClosed = () => {
setIsStreamReady(false)
}
this.engineConnection.addEventListener( this.engineConnection.addEventListener(
EngineConnectionEvents.Closed, EngineConnectionEvents.Closed,
() => { this.onEngineConnectionClosed
setIsStreamReady(false)
}
) )
this.engineConnection.addEventListener( this.onEngineConnectionStarted = ({ detail: engineConnection }: any) => {
EngineConnectionEvents.ConnectionStarted, engineConnection?.pc?.addEventListener(
({ detail: engineConnection }: any) => { 'datachannel',
engineConnection?.pc?.addEventListener( (event: RTCDataChannelEvent) => {
'datachannel', let unreliableDataChannel = event.channel
(event: RTCDataChannelEvent) => {
let unreliableDataChannel = event.channel
unreliableDataChannel.addEventListener( unreliableDataChannel.addEventListener(
'message', 'message',
(event: MessageEvent) => { (event: MessageEvent) => {
const result: UnreliableResponses = JSON.parse(event.data) const result: UnreliableResponses = JSON.parse(event.data)
Object.values( Object.values(
this.unreliableSubscriptions[result.type] || {} this.unreliableSubscriptions[result.type] || {}
).forEach( ).forEach(
// TODO: There is only one response that uses the unreliable channel atm, // TODO: There is only one response that uses the unreliable channel atm,
// highlight_set_entity, if there are more it's likely they will all have the same // highlight_set_entity, if there are more it's likely they will all have the same
// sequence logic, but I'm not sure if we use a single global sequence or a sequence // sequence logic, but I'm not sure if we use a single global sequence or a sequence
// per unreliable subscription. // per unreliable subscription.
(callback) => { (callback) => {
let data = result?.data let data = result?.data
if (isHighlightSetEntity_type(data)) { if (isHighlightSetEntity_type(data)) {
if ( if (
data.sequence !== undefined && data.sequence !== undefined &&
data.sequence > this.inSequence data.sequence > this.inSequence
) { ) {
this.inSequence = data.sequence this.inSequence = data.sequence
callback(result) callback(result)
}
} }
} }
) }
}
)
}
)
// When the EngineConnection starts a connection, we want to register
// callbacks into the WebSocket/PeerConnection.
engineConnection.websocket?.addEventListener('message', ((
event: MessageEvent
) => {
if (event.data instanceof ArrayBuffer) {
// If the data is an ArrayBuffer, it's the result of an export command,
// because in all other cases we send JSON strings. But in the case of
// export we send a binary blob.
// Pass this to our export function.
exportSave(event.data).then(() => {
this.pendingExport?.resolve()
}, this.pendingExport?.reject)
} else {
const message: Models['WebSocketResponse_type'] = JSON.parse(
event.data
)
if (
message.success &&
(message.resp.type === 'modeling' ||
message.resp.type === 'modeling_batch') &&
message.request_id
) {
this.handleModelingCommand(
message.resp,
message.request_id,
message
) )
} else if (
!message.success &&
message.request_id &&
this.artifactMap[message.request_id]
) {
this.handleFailedModelingCommand(message.request_id, message)
} }
)
}
)
// When the EngineConnection starts a connection, we want to register
// callbacks into the WebSocket/PeerConnection.
engineConnection.websocket?.addEventListener('message', ((
event: MessageEvent
) => {
if (event.data instanceof ArrayBuffer) {
// If the data is an ArrayBuffer, it's the result of an export command,
// because in all other cases we send JSON strings. But in the case of
// export we send a binary blob.
// Pass this to our export function.
exportSave(event.data).then(() => {
this.pendingExport?.resolve()
}, this.pendingExport?.reject)
} else {
const message: Models['WebSocketResponse_type'] = JSON.parse(
event.data
)
if (
message.success &&
(message.resp.type === 'modeling' ||
message.resp.type === 'modeling_batch') &&
message.request_id
) {
this.handleModelingCommand(
message.resp,
message.request_id,
message
)
} else if (
!message.success &&
message.request_id &&
this.artifactMap[message.request_id]
) {
this.handleFailedModelingCommand(message.request_id, message)
} }
}) as EventListener) }
}) as EventListener)
this.engineConnection?.addEventListener( this.onEngineConnectionNewTrack = ({
EngineConnectionEvents.NewTrack, detail: { mediaStream },
(({ detail: { mediaStream } }: CustomEvent<NewTrackArgs>) => { }: CustomEvent<NewTrackArgs>) => {
mediaStream.getVideoTracks()[0].addEventListener('mute', () => { mediaStream.getVideoTracks()[0].addEventListener('mute', () => {
console.error( console.error(
'video track mute: check webrtc internals -> inbound rtp' 'video track mute: check webrtc internals -> inbound rtp'
) )
}) })
setMediaStream(mediaStream) setMediaStream(mediaStream)
}) as EventListener
)
this.engineConnection?.connect()
} }
this.engineConnection?.addEventListener(
EngineConnectionEvents.NewTrack,
this.onEngineConnectionNewTrack as EventListener
)
this.engineConnection?.connect()
}
this.engineConnection.addEventListener(
EngineConnectionEvents.ConnectionStarted,
this.onEngineConnectionStarted
) )
} }
@ -1629,7 +1740,26 @@ export class EngineCommandManager extends EventTarget {
} }
tearDown() { tearDown() {
if (this.engineConnection) { if (this.engineConnection) {
this.engineConnection.removeEventListener(
EngineConnectionEvents.Opened,
this.onEngineConnectionOpened
)
this.engineConnection.removeEventListener(
EngineConnectionEvents.Closed,
this.onEngineConnectionClosed
)
this.engineConnection.removeEventListener(
EngineConnectionEvents.ConnectionStarted,
this.onEngineConnectionStarted
)
this.engineConnection.removeEventListener(
EngineConnectionEvents.NewTrack,
this.onEngineConnectionNewTrack as EventListener
)
this.engineConnection?.tearDown() this.engineConnection?.tearDown()
this.engineConnection = undefined
// Our window.tearDown assignment causes this case to happen which is // Our window.tearDown assignment causes this case to happen which is
// only really for tests. // only really for tests.
// @ts-ignore // @ts-ignore