ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lf94/save settings between reconnects (#2997)

Browse Source 
* Keep settings between reconnects

* Set idle timeout to 2 minutes

* Put idle behind flags

* Remove pauses

* Fix online->offline->online

* Revert "Remove pauses"

This reverts commit 267ef4ff4b86f2d8014bfb2a8e8a633adc8001dc.

* ci

* call correct setmediastream
This commit is contained in:
49fl
2024-07-12 16:42:23 -04:00
committed by GitHub
parent 5a5fe3bb95
commit e81b614523
3 changed files with 437 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/components/Stream.tsx
View File

@ -1,3 +1,4 @@
import { DEV } from 'env'
import { MouseEventHandler, useEffect, useRef, useState } from 'react'
import { getNormalisedCoordinates } from '../lib/utils'
import Loading from './Loading'
@ -22,6 +23,8 @@ export const Stream = () => {
const { overallState } = useNetworkContext()
const [isFreezeFrame, setIsFreezeFrame] = useState(false)
const IDLE = true
const isNetworkOkay =
overallState === NetworkHealthState.Ok ||
overallState === NetworkHealthState.Weak
@ -53,7 +56,7 @@ export const Stream = () => {
capture: true,
})
const IDLE_TIME_MS = 1000 * 20
const IDLE_TIME_MS = 1000 * 60 * 2
let timeoutIdIdleA: ReturnType<typeof setTimeout> | undefined = undefined
const teardown = () => {
@ -62,11 +65,12 @@ export const Stream = () => {
sceneInfra.modelingSend({ type: 'Cancel' })
// Give video time to pause
window.requestAnimationFrame(() => {
engineCommandManager.engineConnection?.tearDown({ freeze: true })
engineCommandManager.tearDown()
})
}
// Teardown everything if we go hidden or reconnect
if (IDLE && DEV) {
if (globalThis?.window?.document) {
globalThis.window.document.onvisibilitychange = () => {
if (globalThis.window.document.visibilityState === 'hidden') {
@ -78,40 +82,50 @@ export const Stream = () => {
}
}
}
}
let timeoutIdIdleB: ReturnType<typeof setTimeout> | undefined = undefined
const onAnyInput = () => {
if (!engineCommandManager.engineConnection?.isReady()) {
engineCommandManager.engineConnection?.connect(true)
}
// Clear both timers
clearTimeout(timeoutIdIdleA)
clearTimeout(timeoutIdIdleB)
timeoutIdIdleB = setTimeout(teardown, IDLE_TIME_MS)
}
if (IDLE && DEV) {
globalThis?.window?.document?.addEventListener('keydown', onAnyInput)
globalThis?.window?.document?.addEventListener('mousemove', onAnyInput)
globalThis?.window?.document?.addEventListener('mousedown', onAnyInput)
globalThis?.window?.document?.addEventListener('scroll', onAnyInput)
globalThis?.window?.document?.addEventListener('touchstart', onAnyInput)
}
if (IDLE && DEV) {
timeoutIdIdleB = setTimeout(teardown, IDLE_TIME_MS)
}
return () => {
globalThis?.window?.document?.removeEventListener('paste', handlePaste, {
capture: true,
})
if (IDLE && DEV) {
globalThis?.window?.document?.removeEventListener('keydown', onAnyInput)
globalThis?.window?.document?.removeEventListener('mousemove', onAnyInput)
globalThis?.window?.document?.removeEventListener('mousedown', onAnyInput)
globalThis?.window?.document?.removeEventListener(
'mousemove',
onAnyInput
)
globalThis?.window?.document?.removeEventListener(
'mousedown',
onAnyInput
)
globalThis?.window?.document?.removeEventListener('scroll', onAnyInput)
globalThis?.window?.document?.removeEventListener(
'touchstart',
onAnyInput
)
}
}
}, [])
useEffect(() => {

88
src/hooks/useSetupEngineManager.ts
View File

@ -1,4 +1,4 @@
import { useLayoutEffect, useEffect, useRef } from 'react'
import { useLayoutEffect, useEffect, useRef, useState } from 'react'
import { engineCommandManager, kclManager } from 'lib/singletons'
import { deferExecution } from 'lib/utils'
import { Themes } from 'lib/theme'
@ -30,9 +30,6 @@ export function useSetupEngineManager(
const { setAppState } = useAppState()
const { setMediaStream } = useAppStream()
const streamWidth = streamRef?.current?.offsetWidth
const streamHeight = streamRef?.current?.offsetHeight
const hasSetNonZeroDimensions = useRef<boolean>(false)
if (settings.pool) {
@ -41,21 +38,19 @@ export function useSetupEngineManager(
engineCommandManager.pool = settings.pool
}
const startEngineInstance = () => {
const startEngineInstance = (restart: boolean = false) => {
// Load the engine command manager once with the initial width and height,
// then we do not want to reload it.
const { width: quadWidth, height: quadHeight } = getDimensions(
streamWidth,
streamHeight
streamRef?.current?.offsetWidth ?? 0,
streamRef?.current?.offsetHeight ?? 0
)
if (
!hasSetNonZeroDimensions.current &&
quadHeight &&
quadWidth &&
settings.modelingSend
) {
if (restart) {
kclManager.isFirstRender = false
}
engineCommandManager.start({
setMediaStream: setMediaStream,
restart,
setMediaStream: (mediaStream) => setMediaStream(mediaStream),
setIsStreamReady: (isStreamReady) => setAppState({ isStreamReady }),
width: quadWidth,
height: quadHeight,
@ -87,9 +82,16 @@ export function useSetupEngineManager(
})
hasSetNonZeroDimensions.current = true
}
}
useLayoutEffect(startEngineInstance, [
useLayoutEffect(() => {
const { width: quadWidth, height: quadHeight } = getDimensions(
streamRef?.current?.offsetWidth ?? 0,
streamRef?.current?.offsetHeight ?? 0
)
if (!hasSetNonZeroDimensions.current && quadHeight && quadWidth) {
startEngineInstance()
}
}, [
streamRef?.current?.offsetWidth,
streamRef?.current?.offsetHeight,
settings.modelingSend,
@ -98,8 +100,8 @@ export function useSetupEngineManager(
useEffect(() => {
const handleResize = deferExecution(() => {
const { width, height } = getDimensions(
streamRef?.current?.offsetWidth,
streamRef?.current?.offsetHeight
streamRef?.current?.offsetWidth ?? 0,
streamRef?.current?.offsetHeight ?? 0
)
if (
settings.modelingContext.store.streamDimensions.streamWidth !== width ||
@ -122,10 +124,37 @@ export function useSetupEngineManager(
}, 500)
const onOnline = () => {
startEngineInstance()
startEngineInstance(true)
}
const onVisibilityChange = () => {
if (window.document.visibilityState === 'visible') {
if (
!engineCommandManager.engineConnection?.isReady() &&
!engineCommandManager.engineConnection?.isConnecting()
) {
startEngineInstance()
}
}
}
window.document.addEventListener('visibilitychange', onVisibilityChange)
const onAnyInput = () => {
if (
!engineCommandManager.engineConnection?.isReady() &&
!engineCommandManager.engineConnection?.isConnecting()
) {
startEngineInstance()
}
}
window.document.addEventListener('keydown', onAnyInput)
window.document.addEventListener('mousemove', onAnyInput)
window.document.addEventListener('mousedown', onAnyInput)
window.document.addEventListener('scroll', onAnyInput)
window.document.addEventListener('touchstart', onAnyInput)
const onOffline = () => {
kclManager.isFirstRender = true
engineCommandManager.tearDown()
}
@ -133,11 +162,30 @@ export function useSetupEngineManager(
window.addEventListener('offline', onOffline)
window.addEventListener('resize', handleResize)
return () => {
window.document.removeEventListener(
'visibilitychange',
onVisibilityChange
)
window.document.removeEventListener('keydown', onAnyInput)
window.document.removeEventListener('mousemove', onAnyInput)
window.document.removeEventListener('mousedown', onAnyInput)
window.document.removeEventListener('scroll', onAnyInput)
window.document.removeEventListener('touchstart', onAnyInput)
window.removeEventListener('online', onOnline)
window.removeEventListener('offline', onOffline)
window.removeEventListener('resize', handleResize)
}
}, [])
// Engine relies on many settings so we should rebind events when it changes
// We have to list out the ones we care about because the settings object holds
// non-settings too...
}, [
settings.enableSSAO,
settings.highlightEdges,
settings.showScaleGrid,
settings.theme,
settings.pool,
])
}
function getDimensions(streamWidth?: number, streamHeight?: number) {

214
src/lang/std/engineConnection.ts
View File

@ -302,6 +302,30 @@ class EngineConnection extends EventTarget {
mediaStream?: MediaStream
freezeFrame: boolean = false
onIceCandidate = function (
this: RTCPeerConnection,
event: RTCPeerConnectionIceEvent
) {}
onIceCandidateError = function (
this: RTCPeerConnection,
event: RTCPeerConnectionIceErrorEvent
) {}
onConnectionStateChange = function (this: RTCPeerConnection, event: Event) {}
onDataChannelOpen = function (this: RTCDataChannel, event: Event) {}
onDataChannelClose = function (this: RTCDataChannel, event: Event) {}
onDataChannelError = function (this: RTCDataChannel, event: Event) {}
onDataChannelMessage = function (this: RTCDataChannel, event: MessageEvent) {}
onDataChannel = function (
this: RTCPeerConnection,
event: RTCDataChannelEvent
) {}
onTrack = function (this: RTCPeerConnection, event: RTCTrackEvent) {}
onWebSocketOpen = function (event: Event) {}
onWebSocketClose = function (event: Event) {}
onWebSocketError = function (event: Event) {}
onWebSocketMessage = function (event: MessageEvent) {}
onNetworkStatusReady = () => {}
private _state: EngineConnectionState = {
type: EngineConnectionStateType.Fresh,
}
@ -346,6 +370,7 @@ class EngineConnection extends EventTarget {
private engineCommandManager: EngineCommandManager
private pingPongSpan: { ping?: Date; pong?: Date }
private pingIntervalId: ReturnType<typeof setInterval>
constructor({
engineCommandManager,
@ -368,7 +393,7 @@ class EngineConnection extends EventTarget {
// Without an interval ping, our connection will timeout.
// If this.freezeFrame is true we skip this logic so only reconnect
// happens on mouse move
setInterval(() => {
this.pingIntervalId = setInterval(() => {
if (this.freezeFrame) return
switch (this.state.type as EngineConnectionStateType) {
@ -434,6 +459,44 @@ class EngineConnection extends EventTarget {
tearDown(opts?: { freeze: boolean }) {
this.freezeFrame = opts?.freeze ?? false
this.disconnectAll()
clearInterval(this.pingIntervalId)
this.pc?.removeEventListener('icecandidate', this.onIceCandidate)
this.pc?.removeEventListener('icecandidateerror', this.onIceCandidateError)
this.pc?.removeEventListener(
'connectionstatechange',
this.onConnectionStateChange
)
this.pc?.removeEventListener('track', this.onTrack)
this.unreliableDataChannel?.removeEventListener(
'open',
this.onDataChannelOpen
)
this.unreliableDataChannel?.removeEventListener(
'close',
this.onDataChannelClose
)
this.unreliableDataChannel?.removeEventListener(
'error',
this.onDataChannelError
)
this.unreliableDataChannel?.removeEventListener(
'message',
this.onDataChannelMessage
)
this.pc?.removeEventListener('datachannel', this.onDataChannel)
this.websocket?.removeEventListener('open', this.onWebSocketOpen)
this.websocket?.removeEventListener('close', this.onWebSocketClose)
this.websocket?.removeEventListener('error', this.onWebSocketError)
this.websocket?.removeEventListener('message', this.onWebSocketMessage)
window.removeEventListener(
'use-network-status-ready',
this.onNetworkStatusReady
)
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: { type: DisconnectingType.Quit },
@ -477,7 +540,7 @@ class EngineConnection extends EventTarget {
},
}
this.pc.addEventListener('icecandidate', (event) => {
this.onIceCandidate = (event: RTCPeerConnectionIceEvent) => {
if (event.candidate === null) {
return
}
@ -499,18 +562,20 @@ class EngineConnection extends EventTarget {
usernameFragment: event.candidate.usernameFragment || undefined,
},
})
})
}
this.pc.addEventListener('icecandidate', this.onIceCandidate)
this.pc.addEventListener('icecandidateerror', (_event: Event) => {
this.onIceCandidateError = (_event: Event) => {
const event = _event as RTCPeerConnectionIceErrorEvent
console.warn(
`ICE candidate returned an error: ${event.errorCode}: ${event.errorText} for ${event.url}`
)
})
}
this.pc.addEventListener('icecandidateerror', this.onIceCandidateError)
// https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/connectionstatechange_event
// Event type: generic Event type...
this.pc.addEventListener('connectionstatechange', (event: any) => {
this.onConnectionStateChange = (event: any) => {
console.log('connectionstatechange: ' + event.target?.connectionState)
switch (event.target?.connectionState) {
// From what I understand, only after have we done the ICE song and
@ -539,9 +604,13 @@ class EngineConnection extends EventTarget {
default:
break
}
})
}
this.pc.addEventListener(
'connectionstatechange',
this.onConnectionStateChange
)
this.pc.addEventListener('track', (event) => {
this.onTrack = (event) => {
const mediaStream = event.streams[0]
this.state = {
@ -625,9 +694,10 @@ class EngineConnection extends EventTarget {
// to pass it to the rest of the application.
this.mediaStream = mediaStream
})
}
this.pc.addEventListener('track', this.onTrack)
this.pc.addEventListener('datachannel', (event) => {
this.onDataChannel = (event) => {
this.unreliableDataChannel = event.channel
this.state = {
@ -638,7 +708,7 @@ class EngineConnection extends EventTarget {
},
}
this.unreliableDataChannel.addEventListener('open', (event) => {
this.onDataChannelOpen = (event) => {
this.state = {
type: EngineConnectionStateType.Connecting,
value: {
@ -654,14 +724,22 @@ class EngineConnection extends EventTarget {
this.dispatchEvent(
new CustomEvent(EngineConnectionEvents.Opened, { detail: this })
)
})
}
this.unreliableDataChannel?.addEventListener(
'open',
this.onDataChannelOpen
)
this.unreliableDataChannel.addEventListener('close', (event) => {
this.onDataChannelClose = (event) => {
this.disconnectAll()
this.finalizeIfAllConnectionsClosed()
})
}
this.unreliableDataChannel?.addEventListener(
'close',
this.onDataChannelClose
)
this.unreliableDataChannel.addEventListener('error', (event) => {
this.onDataChannelError = (event) => {
this.disconnectAll()
this.state = {
@ -674,8 +752,13 @@ class EngineConnection extends EventTarget {
},
},
}
})
this.unreliableDataChannel.addEventListener('message', (event) => {
}
this.unreliableDataChannel?.addEventListener(
'error',
this.onDataChannelError
)
this.onDataChannelMessage = (event) => {
const result: UnreliableResponses = JSON.parse(event.data)
Object.values(
this.engineCommandManager.unreliableSubscriptions[result.type] || {}
@ -697,8 +780,13 @@ class EngineConnection extends EventTarget {
}
}
)
})
})
}
this.unreliableDataChannel.addEventListener(
'message',
this.onDataChannelMessage
)
}
this.pc.addEventListener('datachannel', this.onDataChannel)
}
const createWebSocketConnection = () => {
@ -712,7 +800,7 @@ class EngineConnection extends EventTarget {
this.websocket = new WebSocket(this.url, [])
this.websocket.binaryType = 'arraybuffer'
this.websocket.addEventListener('open', (event) => {
this.onWebSocketOpen = (event) => {
this.state = {
type: EngineConnectionStateType.Connecting,
value: {
@ -733,14 +821,16 @@ class EngineConnection extends EventTarget {
// Send an initial ping
this.send({ type: 'ping' })
this.pingPongSpan.ping = new Date()
})
}
this.websocket.addEventListener('open', this.onWebSocketOpen)
this.websocket.addEventListener('close', (event) => {
this.onWebSocketClose = (event) => {
this.disconnectAll()
this.finalizeIfAllConnectionsClosed()
})
}
this.websocket.addEventListener('close', this.onWebSocketClose)
this.websocket.addEventListener('error', (event) => {
this.onWebSocketError = (event) => {
this.disconnectAll()
this.state = {
@ -753,9 +843,10 @@ class EngineConnection extends EventTarget {
},
},
}
})
}
this.websocket.addEventListener('error', this.onWebSocketError)
this.websocket.addEventListener('message', (event) => {
this.onWebSocketMessage = (event) => {
// In the EngineConnection, we're looking for messages to/from
// the server that relate to the ICE handshake, or WebRTC
// negotiation. There may be other messages (including ArrayBuffer
@ -960,15 +1051,20 @@ class EngineConnection extends EventTarget {
})
break
}
})
}
this.websocket.addEventListener('message', this.onWebSocketMessage)
}
if (reconnecting) {
createWebSocketConnection()
} else {
window.addEventListener('use-network-status-ready', () => {
this.onNetworkStatusReady = () => {
createWebSocketConnection()
})
}
window.addEventListener(
'use-network-status-ready',
this.onNetworkStatusReady
)
}
}
// Do not change this back to an object or any, we should only be sending the
@ -1154,7 +1250,15 @@ export class EngineCommandManager extends EventTarget {
private makeDefaultPlanes: () => Promise<DefaultPlanes> | null = () => null
private modifyGrid: (hidden: boolean) => Promise<void> | null = () => null
private onEngineConnectionOpened = () => {}
private onEngineConnectionClosed = () => {}
private onEngineConnectionStarted = ({ detail: engineConnection }: any) => {}
private onEngineConnectionNewTrack = ({
detail,
}: CustomEvent<NewTrackArgs>) => {}
start({
restart,
setMediaStream,
setIsStreamReady,
width,
@ -1170,6 +1274,7 @@ export class EngineCommandManager extends EventTarget {
showScaleGrid: false,
},
}: {
restart?: boolean
setMediaStream: (stream: MediaStream) => void
setIsStreamReady: (isStreamReady: boolean) => void
width: number
@ -1215,9 +1320,7 @@ export class EngineCommandManager extends EventTarget {
})
)
this.engineConnection.addEventListener(
EngineConnectionEvents.Opened,
() => {
this.onEngineConnectionOpened = () => {
// Set the stream background color
// This takes RGBA values from 0-1
// So we convert from the conventional 0-255 found in Figma
@ -1272,18 +1375,20 @@ export class EngineCommandManager extends EventTarget {
await executeCode()
})
}
this.engineConnection.addEventListener(
EngineConnectionEvents.Opened,
this.onEngineConnectionOpened
)
this.engineConnection.addEventListener(
EngineConnectionEvents.Closed,
() => {
this.onEngineConnectionClosed = () => {
setIsStreamReady(false)
}
this.engineConnection.addEventListener(
EngineConnectionEvents.Closed,
this.onEngineConnectionClosed
)
this.engineConnection.addEventListener(
EngineConnectionEvents.ConnectionStarted,
({ detail: engineConnection }: any) => {
this.onEngineConnectionStarted = ({ detail: engineConnection }: any) => {
engineConnection?.pc?.addEventListener(
'datachannel',
(event: RTCDataChannelEvent) => {
@ -1356,9 +1461,9 @@ export class EngineCommandManager extends EventTarget {
}
}) as EventListener)
this.engineConnection?.addEventListener(
EngineConnectionEvents.NewTrack,
(({ detail: { mediaStream } }: CustomEvent<NewTrackArgs>) => {
this.onEngineConnectionNewTrack = ({
detail: { mediaStream },
}: CustomEvent<NewTrackArgs>) => {
mediaStream.getVideoTracks()[0].addEventListener('mute', () => {
console.error(
'video track mute: check webrtc internals -> inbound rtp'
@ -1366,11 +1471,17 @@ export class EngineCommandManager extends EventTarget {
})
setMediaStream(mediaStream)
}) as EventListener
}
this.engineConnection?.addEventListener(
EngineConnectionEvents.NewTrack,
this.onEngineConnectionNewTrack as EventListener
)
this.engineConnection?.connect()
}
this.engineConnection.addEventListener(
EngineConnectionEvents.ConnectionStarted,
this.onEngineConnectionStarted
)
}
@ -1629,7 +1740,26 @@ export class EngineCommandManager extends EventTarget {
}
tearDown() {
if (this.engineConnection) {
this.engineConnection.removeEventListener(
EngineConnectionEvents.Opened,
this.onEngineConnectionOpened
)
this.engineConnection.removeEventListener(
EngineConnectionEvents.Closed,
this.onEngineConnectionClosed
)
this.engineConnection.removeEventListener(
EngineConnectionEvents.ConnectionStarted,
this.onEngineConnectionStarted
)
this.engineConnection.removeEventListener(
EngineConnectionEvents.NewTrack,
this.onEngineConnectionNewTrack as EventListener
)
this.engineConnection?.tearDown()
this.engineConnection = undefined
// Our window.tearDown assignment causes this case to happen which is
// only really for tests.
// @ts-ignore