ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Backoff Reconnect (#6358)

Browse Source 
* Add a Stop command that does a bit of cleanup in the engineStateMachine

* Major network code startup refactor; backoff reconnect; many more logs for us

* Save camera state after every user interaction in case of disconnection and restoral

* Translate basic WebSocket error numbers into useful text

* Add a RestartRequest event to the engineCommandManager

* Adjust for a rebase

* Add E2E test for stream pause behavior

* Fix snapshot test

* fmt, lint

* small issue

* Fix tests

* Fix up idle test

* One last fix

* fixes

* Remove circ dep

* fix test

* use a const for time

* TEST -> isPlaywright instead

* whoops
This commit is contained in:
Zookeeper Lee
2025-05-15 11:58:00 -04:00
committed by GitHub
parent d3a4fd8b55
commit 3f00e7186c
12 changed files with 923 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
e2e/playwright/snapshot-tests.spec.ts
View File

@ -766,7 +766,7 @@ test.describe('Grid visibility', { tag: '@snapshot' }, () => {
})
})
test('theme persists', async ({ page, context }) => {
test('theme persists', async ({ page, context, homePage }) => {
const u = await getUtils(page)
await context.addInitScript(async () => {
localStorage.setItem(
@ -784,7 +784,7 @@ test('theme persists', async ({ page, context }) => {
await page.setViewportSize({ width: 1200, height: 500 })
await u.waitForAuthSkipAppStart()
await homePage.goToModelingScene()
await page.waitForTimeout(500)
// await page.getByRole('link', { name: 'Settings Settings (tooltip)' }).click()
@ -812,7 +812,7 @@ test('theme persists', async ({ page, context }) => {
// Disconnect and reconnect to check the theme persists through a reload
// Expect the network to be down
await expect(networkToggle).toContainText('Offline')
await expect(networkToggle).toContainText('Problem')
// simulate network up
await u.emulateNetworkConditions({

BIN
e2e/playwright/snapshot-tests.spec.ts-snapshots/theme-persists-1.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 47 KiB

498
e2e/playwright/test-network-and-connection-issues.spec.ts
View File

@ -1,236 +1,257 @@
import type { EngineCommand } from '@src/lang/std/artifactGraph'
import { uuidv4 } from '@src/lib/utils'
import { commonPoints, getUtils } from '@e2e/playwright/test-utils'
import {
commonPoints,
getUtils,
TEST_COLORS,
circleMove,
} from '@e2e/playwright/test-utils'
import { expect, test } from '@e2e/playwright/zoo-test'
test.describe(
'Test network and connection issues',
{
tag: ['@macos', '@windows'],
},
() => {
test(
'simulate network down and network little widget',
{ tag: '@skipLocalEngine' },
async ({ page, homePage }) => {
const u = await getUtils(page)
await page.setBodyDimensions({ width: 1200, height: 500 })
test.describe('Test network related behaviors', () => {
test(
'simulate network down and network little widget',
{ tag: '@skipLocalEngine' },
async ({ page, homePage }) => {
const networkToggleConnectedText = page.getByText('Connected')
const networkToggleWeakText = page.getByText('Network health (Weak)')
await homePage.goToModelingScene()
const u = await getUtils(page)
await page.setBodyDimensions({ width: 1200, height: 500 })
const networkToggle = page.getByTestId('network-toggle')
await homePage.goToModelingScene()
// This is how we wait until the stream is online
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).not.toBeDisabled({ timeout: 15000 })
const networkToggle = page.getByTestId('network-toggle')
const networkWidget = page.locator('[data-testid="network-toggle"]')
await expect(networkWidget).toBeVisible()
await networkWidget.hover()
// This is how we wait until the stream is online
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).not.toBeDisabled({ timeout: 15000 })
const networkPopover = page.locator('[data-testid="network-popover"]')
await expect(networkPopover).not.toBeVisible()
const networkWidget = page.locator('[data-testid="network-toggle"]')
await expect(networkWidget).toBeVisible()
await networkWidget.hover()
// (First check) Expect the network to be up
await expect(networkToggle).toContainText('Connected')
const networkPopover = page.locator('[data-testid="network-popover"]')
await expect(networkPopover).not.toBeVisible()
// Click the network widget
await networkWidget.click()
// (First check) Expect the network to be up
await expect(
networkToggleConnectedText.or(networkToggleWeakText)
).toBeVisible()
// Check the modal opened.
await expect(networkPopover).toBeVisible()
// Click the network widget
await networkWidget.click()
// Click off the modal.
await page.mouse.click(100, 100)
await expect(networkPopover).not.toBeVisible()
// Check the modal opened.
await expect(networkPopover).toBeVisible()
// Turn off the network
await u.emulateNetworkConditions({
offline: true,
// values of 0 remove any active throttling. crbug.com/456324#c9
latency: 0,
downloadThroughput: -1,
uploadThroughput: -1,
})
// Click off the modal.
await page.mouse.click(100, 100)
await expect(networkPopover).not.toBeVisible()
// Expect the network to be down
await expect(networkToggle).toContainText('Problem')
// Turn off the network
await u.emulateNetworkConditions({
offline: true,
// values of 0 remove any active throttling. crbug.com/456324#c9
latency: 0,
downloadThroughput: -1,
uploadThroughput: -1,
})
// Click the network widget
await networkWidget.click()
// Expect the network to be down
await expect(networkToggle).toContainText('Problem')
// Check the modal opened.
await expect(networkPopover).toBeVisible()
// Click the network widget
await networkWidget.click()
// Click off the modal.
await page.mouse.click(0, 0)
await expect(networkPopover).not.toBeVisible()
// Check the modal opened.
await expect(networkPopover).toBeVisible()
// Turn back on the network
await u.emulateNetworkConditions({
offline: false,
// values of 0 remove any active throttling. crbug.com/456324#c9
latency: 0,
downloadThroughput: -1,
uploadThroughput: -1,
})
// Click off the modal.
await page.mouse.click(0, 0)
await expect(networkPopover).not.toBeVisible()
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).not.toBeDisabled({ timeout: 15000 })
// Turn back on the network
await u.emulateNetworkConditions({
offline: false,
// values of 0 remove any active throttling. crbug.com/456324#c9
latency: 0,
downloadThroughput: -1,
uploadThroughput: -1,
})
// (Second check) expect the network to be up
await expect(networkToggle).toContainText('Connected')
}
)
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).not.toBeDisabled({ timeout: 15000 })
test(
'Engine disconnect & reconnect in sketch mode',
{ tag: '@skipLocalEngine' },
async ({ page, homePage, toolbar, scene, cmdBar }) => {
const networkToggle = page.getByTestId('network-toggle')
// (Second check) expect the network to be up
await expect(
networkToggleConnectedText.or(networkToggleWeakText)
).toBeVisible()
}
)
const u = await getUtils(page)
await page.setBodyDimensions({ width: 1200, height: 500 })
const PUR = 400 / 37.5 //pixeltoUnitRatio
test(
'Engine disconnect & reconnect in sketch mode',
{ tag: '@skipLocalEngine' },
async ({ page, homePage, toolbar, scene, cmdBar }) => {
const networkToggle = page.getByTestId('network-toggle')
const networkToggleConnectedText = page.getByText('Connected')
const networkToggleWeakText = page.getByText('Network health (Weak)')
await homePage.goToModelingScene()
await u.waitForPageLoad()
const u = await getUtils(page)
await page.setBodyDimensions({ width: 1200, height: 500 })
const PUR = 400 / 37.5 //pixeltoUnitRatio
await u.openDebugPanel()
// click on "Start Sketch" button
await u.clearCommandLogs()
await page.getByRole('button', { name: 'Start Sketch' }).click()
await page.waitForTimeout(100)
await homePage.goToModelingScene()
await u.waitForPageLoad()
// select a plane
await page.mouse.click(700, 200)
await u.openDebugPanel()
// click on "Start Sketch" button
await u.clearCommandLogs()
await page.getByRole('button', { name: 'Start Sketch' }).click()
await page.waitForTimeout(100)
await expect(page.locator('.cm-content')).toHaveText(
`sketch001 = startSketchOn(XZ)`
)
await u.closeDebugPanel()
// select a plane
await page.mouse.click(700, 200)
await page.waitForTimeout(500) // TODO detect animation ending, or disable animation
await expect(page.locator('.cm-content')).toHaveText(
`@settings(defaultLengthUnit = in)sketch001 = startSketchOn(XZ)`
)
await u.closeDebugPanel()
const startXPx = 600
await page.mouse.click(startXPx + PUR * 10, 500 - PUR * 10)
await expect(page.locator('.cm-content')).toHaveText(
`sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt})`
)
await page.waitForTimeout(100)
await page.waitForTimeout(500) // TODO detect animation ending, or disable animation
await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 10)
await page.waitForTimeout(100)
const startXPx = 600
await page.mouse.click(startXPx + PUR * 10, 500 - PUR * 10)
await expect(page.locator('.cm-content')).toHaveText(
`@settings(defaultLengthUnit = in)sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt})`
)
await page.waitForTimeout(100)
await expect(
page.locator('.cm-content')
).toHaveText(`sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt})
await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 10)
await page.waitForTimeout(100)
await expect(
page.locator('.cm-content')
).toHaveText(`@settings(defaultLengthUnit = in)sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt})
|> xLine(length = ${commonPoints.num1})`)
// Expect the network to be up
await expect(networkToggle).toContainText('Connected')
// Expect the network to be up
await networkToggle.hover()
await expect(
networkToggleConnectedText.or(networkToggleWeakText)
).toBeVisible()
// simulate network down
await u.emulateNetworkConditions({
offline: true,
// values of 0 remove any active throttling. crbug.com/456324#c9
latency: 0,
downloadThroughput: -1,
uploadThroughput: -1,
})
// simulate network down
await u.emulateNetworkConditions({
offline: true,
// values of 0 remove any active throttling. crbug.com/456324#c9
latency: 0,
downloadThroughput: -1,
uploadThroughput: -1,
})
// Expect the network to be down
await expect(networkToggle).toContainText('Problem')
// Expect the network to be down
await networkToggle.hover()
await expect(networkToggle).toContainText('Problem')
// Ensure we are not in sketch mode
await expect(
page.getByRole('button', { name: 'Exit Sketch' })
).not.toBeVisible()
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).toBeVisible()
// Ensure we are not in sketch mode
await expect(
page.getByRole('button', { name: 'Exit Sketch' })
).not.toBeVisible()
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).toBeVisible()
// simulate network up
await u.emulateNetworkConditions({
offline: false,
// values of 0 remove any active throttling. crbug.com/456324#c9
latency: 0,
downloadThroughput: -1,
uploadThroughput: -1,
})
// simulate network up
await u.emulateNetworkConditions({
offline: false,
// values of 0 remove any active throttling. crbug.com/456324#c9
latency: 0,
downloadThroughput: -1,
uploadThroughput: -1,
})
// Wait for the app to be ready for use
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).not.toBeDisabled({ timeout: 15000 })
// Wait for the app to be ready for use
await expect(
page.getByRole('button', { name: 'Start Sketch' })
).not.toBeDisabled({ timeout: 15000 })
// Expect the network to be up
await expect(networkToggle).toContainText('Connected')
await scene.settled(cmdBar)
// Expect the network to be up
await networkToggle.hover()
await expect(
networkToggleConnectedText.or(networkToggleWeakText)
).toBeVisible()
// Click off the code pane.
await page.mouse.click(100, 100)
await scene.settled(cmdBar)
// select a line
await page
.getByText(`startProfile(sketch001, at = ${commonPoints.startAt})`)
.click()
// Click off the code pane.
await page.mouse.click(100, 100)
// enter sketch again
await toolbar.editSketch()
// select a line
await page
.getByText(`startProfile(sketch001, at = ${commonPoints.startAt})`)
.click()
// Click the line tool
await page
.getByRole('button', { name: 'line Line', exact: true })
.click()
// enter sketch again
await toolbar.editSketch()
await page.waitForTimeout(150)
// Click the line tool
await page.getByRole('button', { name: 'line Line', exact: true }).click()
const camCommand: EngineCommand = {
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'default_camera_look_at',
center: { x: 109, y: 0, z: -152 },
vantage: { x: 115, y: -505, z: -152 },
up: { x: 0, y: 0, z: 1 },
},
}
const updateCamCommand: EngineCommand = {
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'default_camera_get_settings',
},
}
await toolbar.openPane('debug')
await u.sendCustomCmd(camCommand)
await page.waitForTimeout(100)
await u.sendCustomCmd(updateCamCommand)
await page.waitForTimeout(100)
await page.waitForTimeout(150)
// click to continue profile
await page.mouse.click(1007, 400)
await page.waitForTimeout(100)
// Ensure we can continue sketching
await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 20)
await expect
.poll(u.normalisedEditorCode)
.toBe(`sketch001 = startSketchOn(XZ)
const camCommand: EngineCommand = {
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'default_camera_look_at',
center: { x: 109, y: 0, z: -152 },
vantage: { x: 115, y: -505, z: -152 },
up: { x: 0, y: 0, z: 1 },
},
}
const updateCamCommand: EngineCommand = {
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'default_camera_get_settings',
},
}
await toolbar.openPane('debug')
await u.sendCustomCmd(camCommand)
await page.waitForTimeout(100)
await u.sendCustomCmd(updateCamCommand)
await page.waitForTimeout(100)
// click to continue profile
await page.mouse.click(1007, 400)
await page.waitForTimeout(100)
// Ensure we can continue sketching
await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 20)
await expect
.poll(u.normalisedEditorCode)
.toBe(`@settings(defaultLengthUnit = in)
sketch001 = startSketchOn(XZ)
profile001 = startProfile(sketch001, at = [12.34, -12.34])
|> xLine(length = 12.34)
|> line(end = [-12.34, 12.34])
`)
await page.waitForTimeout(100)
await page.mouse.click(startXPx, 500 - PUR * 20)
await page.waitForTimeout(100)
await page.mouse.click(startXPx, 500 - PUR * 20)
await expect
.poll(u.normalisedEditorCode)
.toBe(`sketch001 = startSketchOn(XZ)
await expect
.poll(u.normalisedEditorCode)
.toBe(`@settings(defaultLengthUnit = in)
sketch001 = startSketchOn(XZ)
profile001 = startProfile(sketch001, at = [12.34, -12.34])
|> xLine(length = 12.34)
|> line(end = [-12.34, 12.34])
@ -238,22 +259,105 @@ profile001 = startProfile(sketch001, at = [12.34, -12.34])
`)
// Unequip line tool
await page.keyboard.press('Escape')
// Make sure we didn't pop out of sketch mode.
await expect(
page.getByRole('button', { name: 'Exit Sketch' })
).toBeVisible()
await expect(
page.getByRole('button', { name: 'line Line', exact: true })
).not.toHaveAttribute('aria-pressed', 'true')
// Unequip line tool
await page.keyboard.press('Escape')
// Make sure we didn't pop out of sketch mode.
await expect(
page.getByRole('button', { name: 'Exit Sketch' })
).toBeVisible()
await expect(
page.getByRole('button', { name: 'line Line', exact: true })
).not.toHaveAttribute('aria-pressed', 'true')
// Exit sketch
await page.keyboard.press('Escape')
await expect(
page.getByRole('button', { name: 'Exit Sketch' })
).not.toBeVisible()
// Exit sketch
await page.keyboard.press('Escape')
await expect(
page.getByRole('button', { name: 'Exit Sketch' })
).not.toBeVisible()
}
)
test(
'Paused stream freezes view frame, unpause reconnect is seamless to user',
{ tag: ['@electron', '@skipLocalEngine'] },
async ({ page, homePage, scene, cmdBar, toolbar, tronApp }) => {
const networkToggle = page.getByTestId('network-toggle')
const networkToggleConnectedText = page.getByText('Connected')
const networkToggleWeakText = page.getByText('Network health (Weak)')
if (!tronApp) {
fail()
}
)
}
)
await tronApp.cleanProjectDir({
app: {
stream_idle_mode: 5000,
},
})
await page.addInitScript(async () => {
localStorage.setItem(
'persistCode',
`sketch001 = startSketchOn(XY)
profile001 = startProfile(sketch001, at = [0.0, 0.0])
|> line(end = [10.0, 0])
|> line(end = [0, 10.0])
|> close()`
)
})
const dim = { width: 1200, height: 500 }
await page.setBodyDimensions(dim)
await test.step('Go to modeling scene', async () => {
await homePage.goToModelingScene()
await scene.settled(cmdBar)
})
await test.step('Verify pausing behavior', async () => {
// Wait 5s + 1s to pause.
await page.waitForTimeout(6000)
// We should now be paused. To the user, it should appear we're still
// connected.
await networkToggle.hover()
await expect(
networkToggleConnectedText.or(networkToggleWeakText)
).toBeVisible()
const center = {
x: dim.width / 2,
y: dim.height / 2,
}
let probe = { x: 0, y: 0 }
// ... and the model's still visibly there
probe.x = center.x + dim.width / 100
probe.y = center.y
await scene.expectPixelColor(TEST_COLORS.GREY, probe, 15)
probe = { ...center }
// Now move the mouse around to unpause!
await circleMove(page, probe.x, probe.y, 20, 10)
// ONCE AGAIN! Check the view area hasn't changed at all.
// Check the pixel a couple times as it reconnects.
// NOTE: Remember, idle behavior is still on at this point -
// if this test takes longer than 5s shit WILL go south!
probe.x = center.x + dim.width / 100
probe.y = center.y
await scene.expectPixelColor(TEST_COLORS.GREY, probe, 15)
await page.waitForTimeout(1000)
await scene.expectPixelColor(TEST_COLORS.GREY, probe, 15)
probe = { ...center }
// Ensure we're still connected
await networkToggle.hover()
await expect(
networkToggleConnectedText.or(networkToggleWeakText)
).toBeVisible()
})
}
)
})

2
e2e/playwright/test-utils.ts
View File

@ -44,6 +44,8 @@ export const lowerRightMasks = (page: Page) => [
export type TestColor = [number, number, number]
export const TEST_COLORS: { [key: string]: TestColor } = {
WHITE: [249, 249, 249],
OFFWHITE: [237, 237, 237],
GREY: [142, 142, 142],
YELLOW: [255, 255, 0],
BLUE: [0, 0, 255],
DARK_MODE_BKGD: [27, 27, 27],

4
src/App.tsx
View File

@ -117,7 +117,9 @@ export function App() {
// When leaving the modeling scene, cut the engine stream.
return () => {
engineStreamActor.send({ type: EngineStreamTransition.Pause })
// When leaving the modeling scene, cut the engine stream.
// Stop is more serious than Pause
engineStreamActor.send({ type: EngineStreamTransition.Stop })
}
}, [])

1
src/clientSideScene/CameraControls.ts
View File

@ -975,7 +975,6 @@ export class CameraControls {
},
})
}
await this.engineCommandManager.sendSceneCommand({
type: 'modeling_cmd_req',
cmd_id: uuidv4(),

252
src/components/EngineStream.tsx
View File

@ -1,3 +1,4 @@
import { isPlaywright } from '@src/lib/isPlaywright'
import { useAppState } from '@src/AppState'
import { ClientSideScene } from '@src/clientSideScene/ClientSideSceneComp'
import { ViewControlContextMenu } from '@src/components/ViewControlMenu'
@ -5,7 +6,10 @@ import { useModelingContext } from '@src/hooks/useModelingContext'
import { useNetworkContext } from '@src/hooks/useNetworkContext'
import { NetworkHealthState } from '@src/hooks/useNetworkStatus'
import { getArtifactOfTypes } from '@src/lang/std/artifactGraph'
import { EngineCommandManagerEvents } from '@src/lang/std/engineConnection'
import {
EngineCommandManagerEvents,
EngineConnectionStateType,
} from '@src/lang/std/engineConnection'
import { btnName } from '@src/lib/cameraControls'
import { PATHS } from '@src/lib/paths'
import { sendSelectEventToEngine } from '@src/lib/selections'
@ -33,22 +37,38 @@ import { createThumbnailPNGOnDesktop } from '@src/lib/screenshot'
import type { SettingsViaQueryString } from '@src/lib/settings/settingsTypes'
import { resetCameraPosition } from '@src/lib/resetCameraPosition'
const TIME_1_SECOND = 1000
export const EngineStream = (props: {
pool: string | null
authToken: string | undefined
}) => {
const { setAppState } = useAppState()
const [firstPlay, setFirstPlay] = useState(true)
const { overallState } = useNetworkContext()
const settings = useSettings()
const engineStreamState = useSelector(engineStreamActor, (state) => state)
const { state: modelingMachineState, send: modelingMachineActorSend } =
useModelingContext()
const { file, project } = useRouteLoaderData(PATHS.FILE) as IndexLoaderData
const last = useRef<number>(Date.now())
const [firstPlay, setFirstPlay] = useState(true)
const [isRestartRequestStarting, setIsRestartRequestStarting] =
useState(false)
const [attemptTimes, setAttemptTimes] = useState<[number, number]>([
0,
TIME_1_SECOND,
])
// These will be passed to the engineStreamActor to handle.
const videoRef = useRef<HTMLVideoElement>(null)
const canvasRef = useRef<HTMLCanvasElement>(null)
// For attaching right-click menu events
const videoWrapperRef = useRef<HTMLDivElement>(null)
const { overallState } = useNetworkContext()
const engineStreamState = useSelector(engineStreamActor, (state) => state)
/**
* We omit `pool` here because `engineStreamMachine` will override it anyway
* within the `EngineStreamTransition.StartOrReconfigureEngine` Promise actor.
@ -62,19 +82,46 @@ export const EngineStream = (props: {
cameraOrbit: settings.modeling.cameraOrbit.current,
}
const { state: modelingMachineState, send: modelingMachineActorSend } =
useModelingContext()
const streamIdleMode = settings.app.streamIdleMode.current
useEffect(() => {
engineStreamActor.send({
type: EngineStreamTransition.SetVideoRef,
videoRef: { current: videoRef.current },
})
}, [videoRef.current])
useEffect(() => {
engineStreamActor.send({
type: EngineStreamTransition.SetCanvasRef,
canvasRef: { current: canvasRef.current },
})
}, [canvasRef.current])
useEffect(() => {
engineStreamActor.send({
type: EngineStreamTransition.SetPool,
pool: props.pool,
})
}, [props.pool])
useEffect(() => {
engineStreamActor.send({
type: EngineStreamTransition.SetAuthToken,
authToken: props.authToken,
})
}, [props.authToken])
// We have to call this here because of the dependencies:
// modelingMachineActorSend, setAppState, settingsEngine
// It's possible to pass these in earlier but I (lee) don't want to
// restructure this further at the moment.
const startOrReconfigureEngine = () => {
engineStreamActor.send({
type: EngineStreamTransition.StartOrReconfigureEngine,
modelingMachineActorSend,
settings: settingsEngine,
setAppState,
// It's possible a reconnect happens as we drag the window :')
onMediaStream(mediaStream: MediaStream) {
engineStreamActor.send({
type: EngineStreamTransition.SetMediaStream,
@ -84,18 +131,47 @@ export const EngineStream = (props: {
})
}
// When the scene is ready play the stream and execute!
useEffect(() => {
if (
engineStreamState.value !== EngineStreamState.WaitingForDependencies &&
engineStreamState.value !== EngineStreamState.Stopped
)
return
startOrReconfigureEngine()
}, [engineStreamState, setAppState])
// I would inline this but it needs to be a function for removeEventListener.
const play = () => {
engineStreamActor.send({
type: EngineStreamTransition.Play,
})
}
useEffect(() => {
engineCommandManager.addEventListener(
EngineCommandManagerEvents.SceneReady,
play
)
return () => {
engineCommandManager.removeEventListener(
EngineCommandManagerEvents.SceneReady,
play
)
}
}, [])
// When the scene is ready, execute kcl!
const executeKcl = () => {
console.log('scene is ready, execute kcl')
const kmp = kclManager.executeCode().catch(trap)
if (!firstPlay) return
setFirstPlay(false)
console.log('scene is ready, fire!')
setFirstPlay(false)
// Reset the restart timeouts
setAttemptTimes([0, TIME_1_SECOND])
console.log('firstPlay true, zoom to fit')
kmp
.then(async () => {
await resetCameraPosition()
@ -112,51 +188,65 @@ export const EngineStream = (props: {
useEffect(() => {
engineCommandManager.addEventListener(
EngineCommandManagerEvents.SceneReady,
play
executeKcl
)
return () => {
engineCommandManager.removeEventListener(
EngineCommandManagerEvents.SceneReady,
play
executeKcl
)
}
}, [firstPlay])
useEffect(() => {
// We do a back-off restart, using a fibonacci sequence, since it
// has a nice retry time curve (somewhat quick then exponential)
const attemptRestartIfNecessary = () => {
if (isRestartRequestStarting) return
setIsRestartRequestStarting(true)
setTimeout(() => {
engineStreamState.context.videoRef.current?.pause()
engineCommandManager.tearDown()
startOrReconfigureEngine()
setFirstPlay(false)
setIsRestartRequestStarting(false)
}, attemptTimes[0] + attemptTimes[1])
setAttemptTimes([attemptTimes[1], attemptTimes[0] + attemptTimes[1]])
}
// Poll that we're connected. If not, send a reset signal.
// Do not restart if we're in idle mode.
const connectionCheckIntervalId = setInterval(() => {
// SKIP DURING TESTS BECAUSE IT WILL MESS WITH REUSING THE
// ELECTRON INSTANCE.
if (isPlaywright()) {
return
}
// Don't try try to restart if we're already connected!
const hasEngineConnectionInst = engineCommandManager.engineConnection
const isDisconnected =
engineCommandManager.engineConnection?.state.type ===
EngineConnectionStateType.Disconnected
const inIdleMode = engineStreamState.value === EngineStreamState.Paused
if ((hasEngineConnectionInst && !isDisconnected) || inIdleMode) return
attemptRestartIfNecessary()
}, TIME_1_SECOND)
engineCommandManager.addEventListener(
EngineCommandManagerEvents.SceneReady,
play
EngineCommandManagerEvents.EngineRestartRequest,
attemptRestartIfNecessary
)
engineStreamActor.send({
type: EngineStreamTransition.SetPool,
data: { pool: props.pool },
})
engineStreamActor.send({
type: EngineStreamTransition.SetAuthToken,
data: { authToken: props.authToken },
})
return () => {
engineCommandManager.tearDown()
}
}, [])
clearInterval(connectionCheckIntervalId)
// In the past we'd try to play immediately, but the proper thing is to way
// for the 'canplay' event to tell us data is ready.
useEffect(() => {
const videoRef = engineStreamState.context.videoRef.current
if (!videoRef) {
return
engineCommandManager.removeEventListener(
EngineCommandManagerEvents.EngineRestartRequest,
attemptRestartIfNecessary
)
}
const play = () => {
videoRef.play().catch(console.error)
}
videoRef.addEventListener('canplay', play)
return () => {
videoRef.removeEventListener('canplay', play)
}
}, [engineStreamState.context.videoRef.current])
}, [engineStreamState, attemptTimes, isRestartRequestStarting])
useEffect(() => {
if (engineStreamState.value === EngineStreamState.Reconfiguring) return
@ -184,25 +274,6 @@ export const EngineStream = (props: {
}).observe(document.body)
}, [engineStreamState.value])
// When the video and canvas element references are set, start the engine.
useEffect(() => {
if (
engineStreamState.context.canvasRef.current &&
engineStreamState.context.videoRef.current
) {
startOrReconfigureEngine()
}
}, [
engineStreamState.context.canvasRef.current,
engineStreamState.context.videoRef.current,
])
// On settings change, reconfigure the engine. When paused this gets really tricky,
// and also requires onMediaStream to be set!
useEffect(() => {
startOrReconfigureEngine()
}, Object.values(settingsEngine))
/**
* Subscribe to execute code when the file changes
* but only if the scene is already ready.
@ -285,18 +356,7 @@ export const EngineStream = (props: {
}
if (engineStreamState.value === EngineStreamState.Paused) {
engineStreamActor.send({
type: EngineStreamTransition.StartOrReconfigureEngine,
modelingMachineActorSend,
settings: settingsEngine,
setAppState,
onMediaStream(mediaStream: MediaStream) {
engineStreamActor.send({
type: EngineStreamTransition.SetMediaStream,
mediaStream,
})
},
})
startOrReconfigureEngine()
}
timeoutStart.current = Date.now()
@ -314,7 +374,7 @@ export const EngineStream = (props: {
window.document.addEventListener('mouseup', onAnyInput)
window.document.addEventListener('scroll', onAnyInput)
window.document.addEventListener('touchstart', onAnyInput)
window.document.addEventListener('touchstop', onAnyInput)
window.document.addEventListener('touchend', onAnyInput)
return () => {
timeoutStart.current = null
@ -325,10 +385,34 @@ export const EngineStream = (props: {
window.document.removeEventListener('mouseup', onAnyInput)
window.document.removeEventListener('scroll', onAnyInput)
window.document.removeEventListener('touchstart', onAnyInput)
window.document.removeEventListener('touchstop', onAnyInput)
window.document.removeEventListener('touchend', onAnyInput)
}
}, [streamIdleMode, engineStreamState.value])
// On various inputs save the camera state, in case we get disconnected.
useEffect(() => {
const onInput = () => {
// Save the remote camera state to restore on stream restore.
// Fire-and-forget because we don't know when a camera movement is
// completed on the engine side (there are no responses to data channel
// mouse movements.)
sceneInfra.camControls.saveRemoteCameraState().catch(trap)
}
// These usually signal a user is done some sort of operation.
window.document.addEventListener('keyup', onInput)
window.document.addEventListener('mouseup', onInput)
window.document.addEventListener('scroll', onInput)
window.document.addEventListener('touchend', onInput)
return () => {
window.document.removeEventListener('keyup', onInput)
window.document.removeEventListener('mouseup', onInput)
window.document.removeEventListener('scroll', onInput)
window.document.removeEventListener('touchend', onInput)
}
}, [])
const isNetworkOkay =
overallState === NetworkHealthState.Ok ||
overallState === NetworkHealthState.Weak
@ -399,7 +483,7 @@ export const EngineStream = (props: {
autoPlay
muted
key={engineStreamActor.id + 'video'}
ref={engineStreamState.context.videoRef}
ref={videoRef}
controls={false}
className="w-full cursor-pointer h-full"
disablePictureInPicture
@ -407,7 +491,7 @@ export const EngineStream = (props: {
/>
<canvas
key={engineStreamActor.id + 'canvas'}
ref={engineStreamState.context.canvasRef}
ref={canvasRef}
className="cursor-pointer"
id="freeze-frame"
>
@ -424,9 +508,11 @@ export const EngineStream = (props: {
}
menuTargetElement={videoWrapperRef}
/>
{![EngineStreamState.Playing, EngineStreamState.Paused].some(
(s) => s === engineStreamState.value
) && (
{![
EngineStreamState.Playing,
EngineStreamState.Paused,
EngineStreamState.Resuming,
].some((s) => s === engineStreamState.value) && (
<Loading dataTestId="loading-engine" className="fixed inset-0 h-screen">
Connecting to engine
</Loading>

4
src/components/Loading.tsx
View File

@ -138,7 +138,9 @@ const Loading = ({ children, className, dataTestId }: LoadingProps) => {
CONNECTION_ERROR_TEXT[error.error] +
(error.context
? '\n\nThe error details are: ' +
JSON.stringify(error.context)
(error.context instanceof Object
? JSON.stringify(error.context)
: error.context)
: ''),
{
renderer: new SafeRenderer(markedOptions),

3
src/components/ModelingMachineProvider.tsx
View File

@ -204,12 +204,13 @@ export const ModelingMachineProvider = ({
sceneInfra.camControls.syncDirection = 'engineToClient'
// TODO: Re-evaluate if this pause/play logic is needed.
store.videoElement?.pause()
return kclManager
.executeCode()
.then(() => {
if (engineCommandManager.engineConnection?.idleMode) return
if (engineCommandManager.idleMode) return
store.videoElement?.play().catch((e) => {
console.warn('Video playing was prevented', e)

360
src/lang/std/engineConnection.ts
View File

@ -1,3 +1,4 @@
import { TEST } from '@src/env'
import type { Models } from '@kittycad/lib'
import { VITE_KC_API_WS_MODELING_URL, VITE_KC_DEV_TOKEN } from '@src/env'
import { jsAppSettings } from '@src/lib/settings/settingsUtils'
@ -7,6 +8,7 @@ import type { MachineManager } from '@src/components/MachineManagerProvider'
import type { useModelingContext } from '@src/hooks/useModelingContext'
import type { KclManager } from '@src/lang/KclSingleton'
import type CodeManager from '@src/lang/codeManager'
import type { SceneInfra } from '@src/clientSideScene/sceneInfra'
import type { EngineCommand, ResponseMap } from '@src/lang/std/artifactGraph'
import type { CommandLog } from '@src/lang/std/commandLog'
import { CommandLogType } from '@src/lang/std/commandLog'
@ -109,6 +111,13 @@ export const CONNECTION_ERROR_TEXT: Record<ConnectionError, string> = {
'An unexpected error occurred. Please report this to us.',
}
export const WEBSOCKET_READYSTATE_TEXT: Record<number, string> = {
[WebSocket.CONNECTING]: 'WebSocket.CONNECTING',
[WebSocket.OPEN]: 'WebSocket.OPEN',
[WebSocket.CLOSING]: 'WebSocket.CLOSING',
[WebSocket.CLOSED]: 'WebSocket.CLOSED',
}
export interface ErrorType {
// The error we've encountered.
error: ConnectionError
@ -208,6 +217,9 @@ export enum EngineConnectionEvents {
// We can eventually use it for more, but one step at a time.
ConnectionStateChanged = 'connection-state-changed', // (state: EngineConnectionState) => void
// There are various failure scenarios where we want to try a restart.
RestartRequest = 'restart-request',
// These are used for the EngineCommandManager and were created
// before onConnectionStateChange existed.
ConnectionStarted = 'connection-started', // (engineConnection: EngineConnection) => void
@ -238,11 +250,23 @@ class EngineConnection extends EventTarget {
pc?: RTCPeerConnection
unreliableDataChannel?: RTCDataChannel
mediaStream?: MediaStream
idleMode: boolean = false
promise?: Promise<void>
sdpAnswer?: RTCSessionDescriptionInit
triggeredStart = false
onWebSocketOpen = function (event: Event) {}
onWebSocketClose = function (event: Event) {}
onWebSocketError = function (event: Event) {}
onWebSocketMessage = function (event: MessageEvent) {}
onIceGatheringStateChange = function (
this: RTCPeerConnection,
event: Event
) {}
onIceConnectionStateChange = function (
this: RTCPeerConnection,
event: Event
) {}
onNegotiationNeeded = function (this: RTCPeerConnection, event: Event) {}
onIceCandidate = function (
this: RTCPeerConnection,
event: RTCPeerConnectionIceEvent
@ -252,6 +276,9 @@ class EngineConnection extends EventTarget {
event: RTCPeerConnectionIceErrorEvent
) {}
onConnectionStateChange = function (this: RTCPeerConnection, event: Event) {}
onSignalingStateChange = function (this: RTCDataChannel, event: Event) {}
onTrack = function (this: RTCPeerConnection, event: RTCTrackEvent) {}
onDataChannelOpen = function (this: RTCDataChannel, event: Event) {}
onDataChannelClose = function (this: RTCDataChannel, event: Event) {}
onDataChannelError = function (this: RTCDataChannel, event: Event) {}
@ -260,11 +287,7 @@ class EngineConnection extends EventTarget {
this: RTCPeerConnection,
event: RTCDataChannelEvent
) {}
onTrack = function (this: RTCPeerConnection, event: RTCTrackEvent) {}
onWebSocketOpen = function (event: Event) {}
onWebSocketClose = function (event: Event) {}
onWebSocketError = function (event: Event) {}
onWebSocketMessage = function (event: MessageEvent) {}
onNetworkStatusReady = () => {}
private _state: EngineConnectionState = {
@ -309,10 +332,10 @@ class EngineConnection extends EventTarget {
private engineCommandManager: EngineCommandManager
private pingPongSpan: { ping?: number; pong?: number }
private pingIntervalId: ReturnType<typeof setInterval> | null = null
private pingIntervalId: ReturnType<typeof setInterval> | undefined = undefined
isUsingConnectionLite: boolean = false
timeoutToForceConnectId: ReturnType<typeof setTimeout> | null = null
timeoutToForceConnectId: ReturnType<typeof setTimeout> | undefined = undefined
constructor({
engineCommandManager,
@ -416,18 +439,18 @@ class EngineConnection extends EventTarget {
return this.state.type === EngineConnectionStateType.ConnectionEstablished
}
tearDown(opts?: { idleMode: boolean }) {
this.idleMode = opts?.idleMode ?? false
if (this.pingIntervalId) {
clearInterval(this.pingIntervalId)
}
if (this.timeoutToForceConnectId) {
clearTimeout(this.timeoutToForceConnectId)
}
tearDown() {
clearInterval(this.pingIntervalId)
clearTimeout(this.timeoutToForceConnectId)
// As each network connection (websocket, webrtc, peer connection) is
// closed, they will handle removing their own event listeners.
// If they didn't then it'd be possible we stop listened to close events
// which is what we want to do in the first place :)
this.disconnectAll()
if (this.idleMode) {
if (this.engineCommandManager.idleMode) {
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: {
@ -435,6 +458,7 @@ class EngineConnection extends EventTarget {
},
}
}
// Pass the state along
if (this.state.type === EngineConnectionStateType.Disconnecting) return
if (this.state.type === EngineConnectionStateType.Disconnected) return
@ -568,30 +592,41 @@ class EngineConnection extends EventTarget {
}, 3000)
}
this.pc?.addEventListener?.('icecandidate', this.onIceCandidate)
// Watch out human! The names of the next couple events are really similar!
this.onIceGatheringStateChange = (event) => {
console.log('icegatheringstatechange', event)
that.initiateConnectionExclusive()
}
this.pc?.addEventListener?.(
'icegatheringstatechange',
function (_event) {
console.log('icegatheringstatechange', this.iceGatheringState)
if (this.iceGatheringState !== 'complete') return
that.initiateConnectionExclusive()
}
this.onIceGatheringStateChange
)
this.onIceConnectionStateChange = (event: Event) => {
console.log('iceconnectionstatechange', event)
}
this.pc?.addEventListener?.(
'iceconnectionstatechange',
function (_event) {
console.log('iceconnectionstatechange', this.iceConnectionState)
console.log('iceconnectionstatechange', this.iceGatheringState)
}
this.onIceConnectionStateChange
)
this.onNegotiationNeeded = (event: Event) => {
console.log('negotiationneeded', event)
}
this.pc?.addEventListener?.(
'negotiationneeded',
this.onNegotiationNeeded
)
this.onSignalingStateChange = (event) => {
console.log('signalingstatechange', event)
}
this.pc?.addEventListener?.(
'signalingstatechange',
this.onSignalingStateChange
)
this.pc?.addEventListener?.('negotiationneeded', function (_event) {
console.log('negotiationneeded', this.iceConnectionState)
console.log('negotiationneeded', this.iceGatheringState)
})
this.pc?.addEventListener?.('signalingstatechange', function (event) {
console.log('signalingstatechange', this.signalingState)
})
this.onIceCandidateError = (_event: Event) => {
const event = _event as RTCPeerConnectionIceErrorEvent
@ -618,38 +653,12 @@ class EngineConnection extends EventTarget {
detail: { conn: this, mediaStream: this.mediaStream! },
})
)
setTimeout(() => {
// Everything is now connected.
this.state = {
type: EngineConnectionStateType.ConnectionEstablished,
}
this.engineCommandManager.inSequence = 1
this.dispatchEvent(
new CustomEvent(EngineConnectionEvents.Opened, {
detail: this,
})
)
markOnce('code/endInitialEngineConnect')
}, 2000)
break
case 'connecting':
break
case 'disconnected':
case 'failed':
this.pc?.removeEventListener('icecandidate', this.onIceCandidate)
this.pc?.removeEventListener(
'icecandidateerror',
this.onIceCandidateError
)
this.pc?.removeEventListener(
'connectionstatechange',
this.onConnectionStateChange
)
this.pc?.removeEventListener('track', this.onTrack)
case 'failed':
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: {
@ -662,6 +671,43 @@ class EngineConnection extends EventTarget {
}
this.disconnectAll()
break
// The remote end broke up with us! :(
case 'disconnected':
this.dispatchEvent(
new CustomEvent(EngineConnectionEvents.RestartRequest, {})
)
break
case 'closed':
this.pc?.removeEventListener('icecandidate', this.onIceCandidate)
this.pc?.removeEventListener(
'icegatheringstatechange',
this.onIceGatheringStateChange
)
this.pc?.removeEventListener(
'iceconnectionstatechange',
this.onIceConnectionStateChange
)
this.pc?.removeEventListener(
'negotiationneeded',
this.onNegotiationNeeded
)
this.pc?.removeEventListener(
'signalingstatechange',
this.onSignalingStateChange
)
this.pc?.removeEventListener(
'icecandidateerror',
this.onIceCandidateError
)
this.pc?.removeEventListener(
'connectionstatechange',
this.onConnectionStateChange
)
this.pc?.removeEventListener('track', this.onTrack)
this.pc?.removeEventListener('datachannel', this.onDataChannel)
break
default:
break
}
@ -735,9 +781,7 @@ class EngineConnection extends EventTarget {
// The app is eager to use the MediaStream; as soon as onNewTrack is
// called, the following sequence happens:
// EngineConnection.onNewTrack -> StoreState.setMediaStream ->
// Stream.tsx reacts to mediaStream change, setting a video element.
// We wait until connectionstatechange changes to "connected"
// to pass it to the rest of the application.
// EngineStream.tsx reacts to mediaStream change, setting a video element.
this.mediaStream = mediaStream
}
@ -761,6 +805,25 @@ class EngineConnection extends EventTarget {
type: ConnectingType.DataChannelEstablished,
},
}
// Start firing off engine commands at this point.
// They could be fired at an earlier time, onWebSocketOpen,
// but DataChannel can offer some benefits like speed,
// and it's nice to say everything's connected before interacting
// with the server.
this.state = {
type: EngineConnectionStateType.ConnectionEstablished,
}
this.engineCommandManager.inSequence = 1
this.dispatchEvent(
new CustomEvent(EngineConnectionEvents.Opened, {
detail: this,
})
)
markOnce('code/endInitialEngineConnect')
}
this.unreliableDataChannel?.addEventListener(
'open',
@ -784,7 +847,6 @@ class EngineConnection extends EventTarget {
'message',
this.onDataChannelMessage
)
this.pc?.removeEventListener('datachannel', this.onDataChannel)
this.disconnectAll()
}
@ -898,16 +960,19 @@ class EngineConnection extends EventTarget {
}
this.websocket.addEventListener('close', this.onWebSocketClose)
this.onWebSocketError = (event) => {
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: {
type: DisconnectingType.Error,
this.onWebSocketError = (event: Event) => {
if (event.target instanceof WebSocket) {
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: {
error: ConnectionError.WebSocketError,
context: event,
type: DisconnectingType.Error,
value: {
error: ConnectionError.WebSocketError,
context:
WEBSOCKET_READYSTATE_TEXT[event.target.readyState] ?? event,
},
},
},
}
}
this.disconnectAll()
@ -1213,20 +1278,27 @@ class EngineConnection extends EventTarget {
!this.websocket ||
this.websocket?.readyState === 3
if (closedPc && closedUDC && closedWS) {
if (!this.idleMode) {
// Do not notify the rest of the program that we have cut off anything.
this.state = { type: EngineConnectionStateType.Disconnected }
} else {
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: {
type: DisconnectingType.Pause,
},
}
}
this.triggeredStart = false
if (!(closedPc && closedUDC && closedWS)) {
return
}
// Clean up all the event listeners.
if (!this.engineCommandManager.idleMode) {
// Do not notify the rest of the program that we have cut off anything.
this.state = { type: EngineConnectionStateType.Disconnected }
this.dispatchEvent(
new CustomEvent(EngineConnectionEvents.RestartRequest, {})
)
} else {
this.state = {
type: EngineConnectionStateType.Disconnecting,
value: {
type: DisconnectingType.Pause,
},
}
}
this.triggeredStart = false
}
}
@ -1254,6 +1326,9 @@ export enum EngineCommandManagerEvents {
// engineConnection is available but scene setup may not have run
EngineAvailable = 'engine-available',
// request a restart of engineConnection
EngineRestartRequest = 'engine-restart-request',
// the whole scene is ready (settings loaded)
SceneReady = 'scene-ready',
}
@ -1366,10 +1441,29 @@ export class EngineCommandManager extends EventTarget {
kclManager: null | KclManager = null
codeManager?: CodeManager
rustContext?: RustContext
sceneInfra?: SceneInfra
// The current "manufacturing machine" aka 3D printer, CNC, etc.
public machineManager: MachineManager | null = null
// Dispatch to the application the engine needs a restart.
private onEngineConnectionRestartRequest = () => {
this.dispatchEvent(
new CustomEvent(EngineCommandManagerEvents.EngineRestartRequest, {})
)
}
private onOffline = () => {
console.log('Browser reported network is offline')
if (TEST) {
console.warn('DURING TESTS ENGINECONNECTION.ONOFFLINE WILL DO NOTHING.')
return
}
this.onEngineConnectionRestartRequest()
}
idleMode: boolean = false
start({
setMediaStream,
setIsStreamReady,
@ -1415,6 +1509,8 @@ export class EngineCommandManager extends EventTarget {
return
}
window.addEventListener('offline', this.onOffline)
let additionalSettings = this.settings.enableSSAO ? '&post_effect=ssao' : ''
additionalSettings +=
'&show_grid=' + (this.settings.showScaleGrid ? 'true' : 'false')
@ -1436,29 +1532,51 @@ export class EngineCommandManager extends EventTarget {
})
)
this.engineConnection.addEventListener(
EngineConnectionEvents.RestartRequest,
this.onEngineConnectionRestartRequest as EventListener
)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
this.onEngineConnectionOpened = async () => {
await this.rustContext?.clearSceneAndBustCache(
await jsAppSettings(),
this.codeManager?.currentFilePath || undefined
)
console.log('onEngineConnectionOpened')
try {
console.log('clearing scene and busting cache')
await this.rustContext?.clearSceneAndBustCache(
await jsAppSettings(),
this.codeManager?.currentFilePath || undefined
)
} catch (e) {
// If this happens shit's actually gone south aka the websocket closed.
// Let's restart.
console.warn("shit's gone south")
console.warn(e)
this.engineConnection?.dispatchEvent(
new CustomEvent(EngineConnectionEvents.RestartRequest, {})
)
return
}
// Set the stream's camera projection type
// We don't send a command to the engine if in perspective mode because
// for now it's the engine's default.
if (settings.cameraProjection === 'orthographic') {
this.sendSceneCommand({
console.log('Setting camera to orthographic')
await this.sendSceneCommand({
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'default_camera_set_orthographic',
},
}).catch(reportRejection)
})
}
// Set the theme
this.setTheme(this.settings.theme).catch(reportRejection)
console.log('Setting theme', this.settings.theme)
await this.setTheme(this.settings.theme)
// Set up a listener for the dark theme media query
console.log('Setup theme media query change')
darkModeMatcher?.addEventListener(
'change',
this.onDarkThemeMediaQueryChange
@ -1466,7 +1584,8 @@ export class EngineCommandManager extends EventTarget {
// Set the edge lines visibility
// eslint-disable-next-line @typescript-eslint/no-floating-promises
this.sendSceneCommand({
console.log('setting edge_lines_visible')
await this.sendSceneCommand({
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
@ -1475,21 +1594,30 @@ export class EngineCommandManager extends EventTarget {
},
})
console.log('camControlsCameraChange')
this._camControlsCameraChange()
// eslint-disable-next-line @typescript-eslint/no-floating-promises
this.sendSceneCommand({
// CameraControls subscribes to default_camera_get_settings response events
// firing this at connection ensure the camera's are synced initially
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'default_camera_get_settings',
},
})
// We should eventually only have 1 restoral call.
if (this.idleMode) {
await this.sceneInfra?.camControls.restoreRemoteCameraStateAndTriggerSync()
} else {
// NOTE: This code is old. It uses the old hack to restore camera.
console.log('call default_camera_get_settings')
// eslint-disable-next-line @typescript-eslint/no-floating-promises
await this.sendSceneCommand({
// CameraControls subscribes to default_camera_get_settings response events
// firing this at connection ensure the camera's are synced initially
type: 'modeling_cmd_req',
cmd_id: uuidv4(),
cmd: {
type: 'default_camera_get_settings',
},
})
}
setIsStreamReady(true)
console.log('Dispatching SceneReady')
// Other parts of the application should use this to react on scene ready.
this.dispatchEvent(
new CustomEvent(EngineCommandManagerEvents.SceneReady, {
@ -1555,7 +1683,7 @@ export class EngineCommandManager extends EventTarget {
}) as EventListener)
this.onVideoTrackMute = () => {
console.error('video track mute: check webrtc internals -> inbound rtp')
console.warn('video track mute - potentially lost stream for a moment')
}
this.onEngineConnectionNewTrack = ({
@ -1746,9 +1874,11 @@ export class EngineCommandManager extends EventTarget {
this.engineConnection?.send(resizeCmd)
}
tearDown(opts?: {
idleMode: boolean
}) {
tearDown(opts?: { idleMode: boolean }) {
this.idleMode = opts?.idleMode ?? false
window.removeEventListener('offline', this.onOffline)
if (this.engineConnection) {
for (const [cmdId, pending] of Object.entries(this.pendingCommands)) {
pending.reject([
@ -1786,14 +1916,14 @@ export class EngineCommandManager extends EventTarget {
this.onDarkThemeMediaQueryChange
)
this.engineConnection?.tearDown(opts)
this.engineConnection?.tearDown()
// Our window.engineCommandManager.tearDown assignment causes this case to happen which is
// only really for tests.
// @ts-ignore
} else if (this.engineCommandManager?.engineConnection) {
// @ts-ignore
this.engineCommandManager?.engineConnection?.tearDown(opts)
this.engineCommandManager?.engineConnection?.tearDown()
// @ts-ignore
this.engineCommandManager.engineConnection = null
}
@ -2112,25 +2242,25 @@ export class EngineCommandManager extends EventTarget {
// Set the stream background color
// This takes RGBA values from 0-1
// So we convert from the conventional 0-255 found in Figma
this.sendSceneCommand({
await this.sendSceneCommand({
cmd_id: uuidv4(),
type: 'modeling_cmd_req',
cmd: {
type: 'set_background_color',
color: getThemeColorForEngine(theme),
},
}).catch(reportRejection)
})
// Sets the default line colors
const opposingTheme = getOppositeTheme(theme)
this.sendSceneCommand({
await this.sendSceneCommand({
cmd_id: uuidv4(),
type: 'modeling_cmd_req',
cmd: {
type: 'set_default_system_properties',
color: getThemeColorForEngine(opposingTheme),
},
}).catch(reportRejection)
})
}
}

1
src/lib/singletons.ts
View File

@ -74,6 +74,7 @@ editorManager.kclManager = kclManager
// TODO: proper dependency injection.
engineCommandManager.kclManager = kclManager
engineCommandManager.codeManager = codeManager
engineCommandManager.sceneInfra = sceneInfra
engineCommandManager.rustContext = rustContext
kclManager.sceneInfraBaseUnitMultiplierSetter = (unit: BaseUnit) => {

301
src/machines/engineStreamMachine.ts
View File

@ -4,41 +4,53 @@ import { assign, fromPromise, setup } from 'xstate'
import type { AppMachineContext } from '@src/lib/types'
export enum EngineStreamState {
Off = 'off',
On = 'on',
WaitForMediaStream = 'wait-for-media-stream',
WaitingForDependencies = 'waiting-for-dependencies',
WaitingForMediaStream = 'waiting-for-media-stream',
WaitingToPlay = 'waiting-to-play',
Playing = 'playing',
Reconfiguring = 'reconfiguring',
Paused = 'paused',
Stopped = 'stopped',
// The is the state in-between Paused and Playing *specifically that order*.
Resuming = 'resuming',
}
export enum EngineStreamTransition {
SetMediaStream = 'set-media-stream',
// This brings us back to the configuration loop
WaitForDependencies = 'wait-for-dependencies',
// Our dependencies to set
SetPool = 'set-pool',
SetAuthToken = 'set-auth-token',
SetVideoRef = 'set-video-ref',
SetCanvasRef = 'set-canvas-ref',
SetMediaStream = 'set-media-stream',
// Stream operations
Play = 'play',
Resume = 'resume',
Pause = 'pause',
Stop = 'stop',
// Used to reconfigure the stream during connection
StartOrReconfigureEngine = 'start-or-reconfigure-engine',
}
export interface EngineStreamContext {
pool: string | null
authToken: string | undefined
mediaStream: MediaStream | null
videoRef: MutableRefObject<HTMLVideoElement | null>
canvasRef: MutableRefObject<HTMLCanvasElement | null>
mediaStream: MediaStream | null
zoomToFit: boolean
}
export const engineStreamContextCreate = (): EngineStreamContext => ({
pool: null,
authToken: undefined,
mediaStream: null,
videoRef: { current: null },
canvasRef: { current: null },
mediaStream: null,
zoomToFit: true,
})
@ -77,76 +89,6 @@ export const engineStreamMachine = setup({
input: {} as EngineStreamContext,
},
actors: {
[EngineStreamTransition.Play]: fromPromise(
async ({
input: { context, params, rootContext },
}: {
input: {
context: EngineStreamContext
params: { zoomToFit: boolean }
rootContext: AppMachineContext
}
}) => {
const canvas = context.canvasRef.current
if (!canvas) return false
const video = context.videoRef.current
if (!video) return false
const mediaStream = context.mediaStream
if (!mediaStream) return false
// If the video is already playing it means we're doing a reconfigure.
// We don't want to re-run the KCL or touch the video element at all.
if (!video.paused) {
return
}
await rootContext.sceneInfra.camControls.restoreRemoteCameraStateAndTriggerSync()
video.style.display = 'block'
canvas.style.display = 'none'
video.srcObject = mediaStream
}
),
[EngineStreamTransition.Pause]: fromPromise(
async ({
input: { context, rootContext },
}: {
input: { context: EngineStreamContext; rootContext: AppMachineContext }
}) => {
const video = context.videoRef.current
if (!video) return
video.pause()
const canvas = context.canvasRef.current
if (!canvas) return
await holdOntoVideoFrameInCanvas(video, canvas)
video.style.display = 'none'
await rootContext.sceneInfra.camControls.saveRemoteCameraState()
// Make sure we're on the next frame for no flickering between canvas
// and the video elements.
window.requestAnimationFrame(
() =>
void (async () => {
// Destroy the media stream. We will re-establish it. We could
// leave everything at pausing, preventing video decoders from running
// but we can do even better by significantly reducing network
// cards also.
context.mediaStream?.getVideoTracks()[0].stop()
context.mediaStream = null
video.srcObject = null
rootContext.engineCommandManager.tearDown({ idleMode: true })
})()
)
}
),
[EngineStreamTransition.StartOrReconfigureEngine]: fromPromise(
async ({
input: { context, event, rootContext },
@ -157,21 +99,17 @@ export const engineStreamMachine = setup({
rootContext: AppMachineContext
}
}) => {
if (!context.authToken) return
const video = context.videoRef.current
if (!video) return
const canvas = context.canvasRef.current
if (!canvas) return
if (!context.authToken) return Promise.reject()
if (!context.videoRef.current) return Promise.reject()
if (!context.canvasRef.current) return Promise.reject()
const { width, height } = getDimensions(
window.innerWidth,
window.innerHeight
)
video.width = width
video.height = height
context.videoRef.current.width = width
context.videoRef.current.height = height
const settingsNext = {
// override the pool param (?pool=) to request a specific engine instance
@ -206,51 +144,183 @@ export const engineStreamMachine = setup({
})
}
),
[EngineStreamTransition.Play]: fromPromise(
async ({
input: { context, params },
}: {
input: { context: EngineStreamContext; params: { zoomToFit: boolean } }
}) => {
if (!context.canvasRef.current) return
if (!context.videoRef.current) return
if (!context.mediaStream) return
// If the video is already playing it means we're doing a reconfigure.
// We don't want to re-run the KCL or touch the video element at all.
if (!context.videoRef.current.paused) {
return
}
// In the past we'd try to play immediately, but the proper thing is to way
// for the 'canplay' event to tell us data is ready.
const onCanPlay = () => {
if (!context.videoRef.current) {
return
}
context.videoRef.current.play().catch(console.error)
// Yes, event listeners can remove themselves because of the
// lazy nature of interpreted languages :D
context.videoRef.current.removeEventListener('canplay', onCanPlay)
}
// We're receiving video frames, so show the video now.
const onPlay = () => {
// We have to give engine time to crunch all the scene setup we
// ask it to do. As far as I can tell it doesn't block until
// they are done, so we must wait.
setTimeout(() => {
if (!context.videoRef.current) {
return
}
if (!context.canvasRef.current) {
return
}
context.videoRef.current.style.display = 'block'
context.canvasRef.current.style.display = 'none'
context.videoRef.current.removeEventListener('play', onPlay)
// I've tried < 400ms and sometimes it's possible to see a flash
// and the camera snap.
}, 400)
}
context.videoRef.current.addEventListener('canplay', onCanPlay)
context.videoRef.current.addEventListener('play', onPlay)
// THIS ASSIGNMENT IS *EXTREMELY* EFFECTFUL! The amount of logic
// this triggers is quite far and wide. It drives the above events.
context.videoRef.current.srcObject = context.mediaStream
}
),
// Pause is also called when leaving the modeling scene. It's possible
// then videoRef and canvasRef are now null due to their DOM elements
// being destroyed.
[EngineStreamTransition.Pause]: fromPromise(
async ({
input: { context, rootContext },
}: {
input: {
context: EngineStreamContext
rootContext: AppMachineContext
}
}) => {
if (context.videoRef.current && context.canvasRef.current) {
await context.videoRef.current.pause()
await holdOntoVideoFrameInCanvas(
context.videoRef.current,
context.canvasRef.current
)
context.videoRef.current.style.display = 'none'
}
await rootContext.sceneInfra.camControls.saveRemoteCameraState()
// Make sure we're on the next frame for no flickering between canvas
// and the video elements.
window.requestAnimationFrame(
() =>
void (async () => {
// Destroy the media stream. We will re-establish it. We could
// leave everything at pausing, preventing video decoders from running
// but we can do even better by significantly reducing network
// cards also.
context.mediaStream?.getVideoTracks()[0].stop()
context.mediaStream = null
if (context.videoRef.current) {
context.videoRef.current.srcObject = null
}
rootContext.engineCommandManager.tearDown({ idleMode: true })
})()
)
}
),
},
}).createMachine({
initial: EngineStreamState.Off,
initial: EngineStreamState.WaitingForDependencies,
context: (initial) => initial.input,
states: {
[EngineStreamState.Off]: {
reenter: true,
[EngineStreamState.WaitingForDependencies]: {
on: {
[EngineStreamTransition.SetPool]: {
target: EngineStreamState.Off,
actions: [assign({ pool: ({ context, event }) => event.data.pool })],
target: EngineStreamState.WaitingForDependencies,
actions: [assign({ pool: ({ context, event }) => event.pool })],
},
[EngineStreamTransition.SetAuthToken]: {
target: EngineStreamState.Off,
target: EngineStreamState.WaitingForDependencies,
actions: [
assign({ authToken: ({ context, event }) => event.data.authToken }),
assign({ authToken: ({ context, event }) => event.authToken }),
],
},
[EngineStreamTransition.SetVideoRef]: {
target: EngineStreamState.WaitingForDependencies,
actions: [
assign({ videoRef: ({ context, event }) => event.videoRef }),
],
},
[EngineStreamTransition.SetCanvasRef]: {
target: EngineStreamState.WaitingForDependencies,
actions: [
assign({ canvasRef: ({ context, event }) => event.canvasRef }),
],
},
[EngineStreamTransition.StartOrReconfigureEngine]: {
target: EngineStreamState.On,
target: EngineStreamState.WaitingForMediaStream,
},
},
},
[EngineStreamState.On]: {
reenter: true,
[EngineStreamState.WaitingForMediaStream]: {
invoke: {
src: EngineStreamTransition.StartOrReconfigureEngine,
input: (args) => ({
context: args.context,
rootContext: args.self.system.get('root').getSnapshot().context,
params: { zoomToFit: args.context.zoomToFit },
event: args.event,
}),
onError: [
{
target: EngineStreamState.WaitingForDependencies,
reenter: true,
},
],
},
on: {
// Transition requested by engineConnection
[EngineStreamTransition.StartOrReconfigureEngine]: {
target: EngineStreamState.WaitingForMediaStream,
reenter: true,
},
[EngineStreamTransition.SetMediaStream]: {
target: EngineStreamState.On,
target: EngineStreamState.WaitingToPlay,
actions: [
assign({ mediaStream: ({ context, event }) => event.mediaStream }),
],
},
},
},
[EngineStreamState.WaitingToPlay]: {
on: {
[EngineStreamTransition.Play]: {
target: EngineStreamState.Playing,
actions: [assign({ zoomToFit: () => true })],
},
// We actually failed inbetween needing to play and sending commands.
[EngineStreamTransition.StartOrReconfigureEngine]: {
target: EngineStreamState.WaitingForMediaStream,
reenter: true,
},
},
},
@ -270,6 +340,9 @@ export const engineStreamMachine = setup({
[EngineStreamTransition.Pause]: {
target: EngineStreamState.Paused,
},
[EngineStreamTransition.Stop]: {
target: EngineStreamState.Stopped,
},
},
},
[EngineStreamState.Reconfiguring]: {
@ -280,9 +353,7 @@ export const engineStreamMachine = setup({
rootContext: args.self.system.get('root').getSnapshot().context,
event: args.event,
}),
onDone: {
target: EngineStreamState.Playing,
},
onDone: [{ target: EngineStreamState.Playing }],
},
},
[EngineStreamState.Paused]: {
@ -299,8 +370,27 @@ export const engineStreamMachine = setup({
},
},
},
[EngineStreamState.Stopped]: {
invoke: {
src: EngineStreamTransition.Pause,
input: (args) => ({
context: args.context,
rootContext: args.self.system.get('root').getSnapshot().context,
}),
onDone: [
{
target: EngineStreamState.WaitingForDependencies,
actions: [
assign({
videoRef: { current: null },
canvasRef: { current: null },
}),
],
},
],
},
},
[EngineStreamState.Resuming]: {
reenter: true,
invoke: {
src: EngineStreamTransition.StartOrReconfigureEngine,
input: (args) => ({
@ -315,14 +405,11 @@ export const engineStreamMachine = setup({
target: EngineStreamState.Paused,
},
[EngineStreamTransition.SetMediaStream]: {
target: EngineStreamState.Playing,
actions: [
assign({ mediaStream: ({ context, event }) => event.mediaStream }),
],
},
[EngineStreamTransition.Play]: {
target: EngineStreamState.Playing,
actions: [assign({ zoomToFit: () => false })],
},
},
},
},