2023-06-22 16:43:33 +10:00
|
|
|
import { SourceRange } from '../executor'
|
|
|
|
import { Selections } from '../../useStore'
|
2023-08-01 09:36:40 -05:00
|
|
|
import { VITE_KC_API_WS_MODELING_URL } from '../../env'
|
2023-08-02 15:41:59 +10:00
|
|
|
import { Models } from '@kittycad/lib'
|
2023-08-02 16:23:17 -07:00
|
|
|
import { exportSave } from '../../lib/exportSave'
|
2023-06-22 16:43:33 +10:00
|
|
|
|
|
|
|
interface ResultCommand {
|
|
|
|
type: 'result'
|
|
|
|
data: any
|
|
|
|
}
|
|
|
|
interface PendingCommand {
|
|
|
|
type: 'pending'
|
|
|
|
promise: Promise<any>
|
|
|
|
resolve: (val: any) => void
|
|
|
|
}
|
|
|
|
|
|
|
|
export interface ArtifactMap {
|
|
|
|
[key: string]: ResultCommand | PendingCommand
|
|
|
|
}
|
|
|
|
export interface SourceRangeMap {
|
|
|
|
[key: string]: SourceRange
|
|
|
|
}
|
|
|
|
|
|
|
|
interface SelectionsArgs {
|
|
|
|
id: string
|
|
|
|
type: Selections['codeBasedSelections'][number]['type']
|
|
|
|
}
|
|
|
|
|
|
|
|
interface CursorSelectionsArgs {
|
|
|
|
otherSelections: Selections['otherSelections']
|
|
|
|
idBasedSelections: { type: string; id: string }[]
|
|
|
|
}
|
|
|
|
|
2023-08-02 18:46:40 +10:00
|
|
|
type _EngineCommand = Models['ModelingCmdReq_type']
|
2023-08-02 15:41:59 +10:00
|
|
|
|
|
|
|
// TODO extending this type to add the type property is a work around
|
|
|
|
// see https://github.com/KittyCAD/api-deux/issues/1096
|
|
|
|
export interface EngineCommand extends _EngineCommand {
|
|
|
|
type: 'modeling_cmd_req'
|
2023-06-22 16:43:33 +10:00
|
|
|
}
|
|
|
|
|
|
|
|
export class EngineCommandManager {
|
|
|
|
artifactMap: ArtifactMap = {}
|
|
|
|
sourceRangeMap: SourceRangeMap = {}
|
2023-06-23 10:54:19 +10:00
|
|
|
sequence = 0
|
2023-06-22 16:43:33 +10:00
|
|
|
socket?: WebSocket
|
|
|
|
pc?: RTCPeerConnection
|
2023-06-23 09:56:37 +10:00
|
|
|
lossyDataChannel?: RTCDataChannel
|
2023-07-10 15:15:07 +10:00
|
|
|
waitForReady: Promise<void> = new Promise(() => {})
|
|
|
|
private resolveReady = () => {}
|
2023-06-22 16:43:33 +10:00
|
|
|
onHoverCallback: (id?: string) => void = () => {}
|
|
|
|
onClickCallback: (selection: SelectionsArgs) => void = () => {}
|
|
|
|
onCursorsSelectedCallback: (selections: CursorSelectionsArgs) => void =
|
|
|
|
() => {}
|
2023-06-23 14:19:15 +10:00
|
|
|
constructor({
|
|
|
|
setMediaStream,
|
|
|
|
setIsStreamReady,
|
2023-07-11 20:34:09 +10:00
|
|
|
token,
|
2023-06-23 14:19:15 +10:00
|
|
|
}: {
|
|
|
|
setMediaStream: (stream: MediaStream) => void
|
|
|
|
setIsStreamReady: (isStreamReady: boolean) => void
|
2023-07-11 20:34:09 +10:00
|
|
|
token?: string
|
2023-06-23 14:19:15 +10:00
|
|
|
}) {
|
2023-07-10 15:15:07 +10:00
|
|
|
this.waitForReady = new Promise((resolve) => {
|
|
|
|
this.resolveReady = resolve
|
|
|
|
})
|
|
|
|
|
2023-08-01 09:36:40 -05:00
|
|
|
this.socket = new WebSocket(VITE_KC_API_WS_MODELING_URL, [])
|
2023-08-02 16:23:17 -07:00
|
|
|
|
|
|
|
// Change binary type from "blob" to "arraybuffer"
|
|
|
|
this.socket.binaryType = 'arraybuffer'
|
|
|
|
|
2023-06-22 16:43:33 +10:00
|
|
|
this.pc = new RTCPeerConnection()
|
2023-06-23 10:54:19 +10:00
|
|
|
this.pc.createDataChannel('unreliable_modeling_cmds')
|
2023-06-22 16:43:33 +10:00
|
|
|
this.socket.addEventListener('open', (event) => {
|
|
|
|
console.log('Connected to websocket, waiting for ICE servers')
|
2023-07-11 20:34:09 +10:00
|
|
|
if (token) {
|
|
|
|
this.socket?.send(
|
|
|
|
JSON.stringify({ headers: { Authorization: `Bearer ${token}` } })
|
|
|
|
)
|
|
|
|
}
|
2023-06-22 16:43:33 +10:00
|
|
|
})
|
|
|
|
|
|
|
|
this.socket.addEventListener('close', (event) => {
|
2023-08-02 13:51:05 -05:00
|
|
|
console.log('websocket connection closed', event)
|
2023-06-22 16:43:33 +10:00
|
|
|
})
|
|
|
|
|
|
|
|
this.socket.addEventListener('error', (event) => {
|
2023-08-02 13:51:05 -05:00
|
|
|
console.log('websocket connection error', event)
|
2023-06-22 16:43:33 +10:00
|
|
|
})
|
|
|
|
|
|
|
|
this?.socket?.addEventListener('message', (event) => {
|
2023-07-11 20:34:09 +10:00
|
|
|
if (!this.socket || !this.pc) return
|
2023-06-22 16:43:33 +10:00
|
|
|
|
2023-08-02 16:23:17 -07:00
|
|
|
// console.log('Message from server ', event.data);
|
|
|
|
if (event.data instanceof ArrayBuffer) {
|
|
|
|
// If the data is an ArrayBuffer, it's the result of an export command,
|
|
|
|
// because in all other cases we send JSON strings. But in the case of
|
|
|
|
// export we send a binary blob.
|
|
|
|
// Pass this to our export function.
|
|
|
|
exportSave(event.data)
|
2023-06-22 16:43:33 +10:00
|
|
|
} else if (
|
|
|
|
typeof event.data === 'string' &&
|
|
|
|
event.data.toLocaleLowerCase().startsWith('error')
|
|
|
|
) {
|
|
|
|
console.warn('something went wrong: ', event.data)
|
|
|
|
} else {
|
|
|
|
const message = JSON.parse(event.data)
|
2023-08-03 05:51:52 +10:00
|
|
|
if (message.type === 'sdp_answer') {
|
2023-07-10 15:15:07 +10:00
|
|
|
this.pc?.setRemoteDescription(
|
2023-06-22 16:43:33 +10:00
|
|
|
new RTCSessionDescription(message.answer)
|
|
|
|
)
|
2023-08-02 17:27:14 -04:00
|
|
|
} else if (message.type === 'trickle_ice') {
|
|
|
|
this.pc?.addIceCandidate(message.candidate)
|
2023-08-02 15:41:59 +10:00
|
|
|
} else if (message.type === 'ice_server_info' && this.pc) {
|
|
|
|
console.log('received ice_server_info')
|
2023-08-02 17:27:14 -04:00
|
|
|
if (message.ice_servers.length > 0) {
|
|
|
|
this.pc?.setConfiguration({
|
|
|
|
iceServers: message.ice_servers,
|
|
|
|
iceTransportPolicy: 'relay',
|
|
|
|
})
|
|
|
|
} else {
|
|
|
|
this.pc?.setConfiguration({})
|
|
|
|
}
|
2023-06-22 16:43:33 +10:00
|
|
|
this.pc.addEventListener('track', (event) => {
|
|
|
|
console.log('received track', event)
|
|
|
|
const mediaStream = event.streams[0]
|
|
|
|
setMediaStream(mediaStream)
|
|
|
|
})
|
|
|
|
|
|
|
|
this.pc.addEventListener('connectionstatechange', (e) =>
|
|
|
|
console.log(this?.pc?.iceConnectionState)
|
|
|
|
)
|
|
|
|
this.pc.addEventListener('icecandidate', (event) => {
|
|
|
|
if (!this.pc || !this.socket) return
|
|
|
|
if (event.candidate === null) {
|
2023-08-03 05:51:52 +10:00
|
|
|
console.log('sent sdp_offer')
|
2023-06-22 16:43:33 +10:00
|
|
|
this.socket.send(
|
|
|
|
JSON.stringify({
|
2023-08-03 05:51:52 +10:00
|
|
|
type: 'sdp_offer',
|
2023-06-22 16:43:33 +10:00
|
|
|
offer: this.pc.localDescription,
|
|
|
|
})
|
|
|
|
)
|
2023-08-02 17:27:14 -04:00
|
|
|
} else {
|
|
|
|
console.log('sending trickle ice candidate')
|
|
|
|
const { candidate } = event
|
|
|
|
this.socket?.send(
|
|
|
|
JSON.stringify({
|
|
|
|
type: 'trickle_ice',
|
|
|
|
candidate: candidate.toJSON(),
|
|
|
|
})
|
|
|
|
)
|
2023-06-22 16:43:33 +10:00
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
// Offer to receive 1 video track
|
|
|
|
this.pc.addTransceiver('video', {
|
|
|
|
direction: 'sendrecv',
|
|
|
|
})
|
|
|
|
this.pc
|
|
|
|
.createOffer()
|
|
|
|
.then(async (descriptionInit) => {
|
|
|
|
await this?.pc?.setLocalDescription(descriptionInit)
|
2023-08-03 05:51:52 +10:00
|
|
|
console.log('sent sdp_offer begin')
|
2023-06-22 16:43:33 +10:00
|
|
|
const msg = JSON.stringify({
|
2023-08-03 05:51:52 +10:00
|
|
|
type: 'sdp_offer',
|
2023-06-22 16:43:33 +10:00
|
|
|
offer: this.pc?.localDescription,
|
|
|
|
})
|
|
|
|
this.socket?.send(msg)
|
|
|
|
})
|
|
|
|
.catch(console.log)
|
2023-06-23 09:56:37 +10:00
|
|
|
|
|
|
|
this.pc.addEventListener('datachannel', (event) => {
|
|
|
|
this.lossyDataChannel = event.channel
|
|
|
|
console.log('accepted lossy data channel', event.channel.label)
|
|
|
|
this.lossyDataChannel.addEventListener('open', (event) => {
|
2023-07-11 20:34:09 +10:00
|
|
|
setIsStreamReady(true)
|
|
|
|
this.resolveReady()
|
2023-06-23 09:56:37 +10:00
|
|
|
console.log('lossy data channel opened', event)
|
|
|
|
})
|
|
|
|
this.lossyDataChannel.addEventListener('close', (event) => {
|
|
|
|
console.log('lossy data channel closed')
|
|
|
|
})
|
|
|
|
this.lossyDataChannel.addEventListener('error', (event) => {
|
|
|
|
console.log('lossy data channel error')
|
|
|
|
})
|
|
|
|
this.lossyDataChannel.addEventListener('message', (event) => {
|
|
|
|
console.log('lossy data channel message: ', event)
|
|
|
|
})
|
|
|
|
})
|
2023-07-10 15:15:07 +10:00
|
|
|
} else if (message.cmd_id) {
|
|
|
|
const id = message.cmd_id
|
|
|
|
const command = this.artifactMap[id]
|
|
|
|
if (command && command.type === 'pending') {
|
|
|
|
const resolve = command.resolve
|
|
|
|
this.artifactMap[id] = {
|
|
|
|
type: 'result',
|
|
|
|
data: message.result,
|
|
|
|
}
|
|
|
|
resolve({
|
|
|
|
id,
|
|
|
|
})
|
|
|
|
} else {
|
|
|
|
this.artifactMap[id] = {
|
|
|
|
type: 'result',
|
|
|
|
data: message.result,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// TODO talk to the gang about this
|
|
|
|
// the following message types are made up
|
|
|
|
// and are placeholders
|
|
|
|
} else if (message.type === 'hover') {
|
2023-06-22 16:43:33 +10:00
|
|
|
this.onHoverCallback(message.id)
|
|
|
|
} else if (message.type === 'click') {
|
|
|
|
this.onClickCallback(message)
|
|
|
|
} else {
|
2023-08-02 16:23:17 -07:00
|
|
|
console.log('received message', message)
|
2023-06-22 16:43:33 +10:00
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
2023-06-23 09:56:37 +10:00
|
|
|
tearDown() {
|
|
|
|
// close all channels, sockets and WebRTC connections
|
|
|
|
this.lossyDataChannel?.close()
|
|
|
|
this.socket?.close()
|
|
|
|
this.pc?.close()
|
|
|
|
}
|
2023-06-22 16:43:33 +10:00
|
|
|
|
|
|
|
startNewSession() {
|
|
|
|
this.artifactMap = {}
|
|
|
|
this.sourceRangeMap = {}
|
|
|
|
}
|
|
|
|
endSession() {
|
|
|
|
// this.socket?.close()
|
|
|
|
// socket.off('command')
|
|
|
|
}
|
|
|
|
onHover(callback: (id?: string) => void) {
|
|
|
|
// It's when the user hovers over a part in the 3d scene, and so the engine should tell the
|
|
|
|
// frontend about that (with it's id) so that the FE can highlight code associated with that id
|
|
|
|
this.onHoverCallback = callback
|
|
|
|
}
|
|
|
|
onClick(callback: (selection: SelectionsArgs) => void) {
|
|
|
|
// TODO talk to the gang about this
|
|
|
|
// It's when the user clicks on a part in the 3d scene, and so the engine should tell the
|
|
|
|
// frontend about that (with it's id) so that the FE can put the user's cursor on the right
|
|
|
|
// line of code
|
|
|
|
this.onClickCallback = callback
|
|
|
|
}
|
|
|
|
cusorsSelected(selections: {
|
|
|
|
otherSelections: Selections['otherSelections']
|
|
|
|
idBasedSelections: { type: string; id: string }[]
|
|
|
|
}) {
|
|
|
|
// TODO talk to the gang about this
|
|
|
|
// Really idBasedSelections is the only part that's relevant to the server, but it's when
|
|
|
|
// the user puts their cursor over a line of code, and there is a engine-id associated with
|
|
|
|
// it, so we want to tell the engine to change it's color or something
|
|
|
|
if (this.socket?.readyState === 0) {
|
|
|
|
console.log('socket not open')
|
|
|
|
return
|
|
|
|
}
|
|
|
|
console.log('sending cursorsSelected')
|
|
|
|
this.socket?.send(
|
|
|
|
JSON.stringify({ command: 'cursorsSelected', body: selections })
|
|
|
|
)
|
|
|
|
}
|
|
|
|
sendSceneCommand(command: EngineCommand) {
|
|
|
|
if (this.socket?.readyState === 0) {
|
|
|
|
console.log('socket not ready')
|
|
|
|
return
|
|
|
|
}
|
2023-08-02 15:41:59 +10:00
|
|
|
const cmd = command.cmd
|
|
|
|
if (cmd.type === 'camera_drag_move' && this.lossyDataChannel) {
|
2023-06-23 09:56:37 +10:00
|
|
|
console.log('sending lossy command', command, this.lossyDataChannel)
|
2023-08-02 15:41:59 +10:00
|
|
|
cmd.sequence = this.sequence
|
2023-06-23 10:54:19 +10:00
|
|
|
this.sequence++
|
2023-06-23 09:56:37 +10:00
|
|
|
this.lossyDataChannel.send(JSON.stringify(command))
|
|
|
|
return
|
|
|
|
}
|
2023-08-02 16:23:17 -07:00
|
|
|
console.log('sending command', command)
|
2023-06-22 16:43:33 +10:00
|
|
|
this.socket?.send(JSON.stringify(command))
|
|
|
|
}
|
|
|
|
sendModellingCommand({
|
|
|
|
id,
|
|
|
|
params,
|
|
|
|
range,
|
|
|
|
command,
|
|
|
|
}: {
|
|
|
|
id: string
|
|
|
|
params: any
|
|
|
|
range: SourceRange
|
|
|
|
command: EngineCommand
|
|
|
|
}): Promise<any> {
|
|
|
|
this.sourceRangeMap[id] = range
|
|
|
|
|
|
|
|
if (this.socket?.readyState === 0) {
|
|
|
|
console.log('socket not ready')
|
|
|
|
return new Promise(() => {})
|
|
|
|
}
|
|
|
|
this.socket?.send(JSON.stringify(command))
|
|
|
|
let resolve: (val: any) => void = () => {}
|
|
|
|
const promise = new Promise((_resolve, reject) => {
|
|
|
|
resolve = _resolve
|
|
|
|
})
|
|
|
|
this.artifactMap[id] = {
|
|
|
|
type: 'pending',
|
|
|
|
promise,
|
|
|
|
resolve,
|
|
|
|
}
|
|
|
|
return promise
|
|
|
|
}
|
|
|
|
commandResult(id: string): Promise<any> {
|
|
|
|
const command = this.artifactMap[id]
|
|
|
|
if (!command) {
|
|
|
|
throw new Error('No command found')
|
|
|
|
}
|
|
|
|
if (command.type === 'result') {
|
|
|
|
return command.data
|
|
|
|
}
|
|
|
|
return command.promise
|
|
|
|
}
|
|
|
|
async waitForAllCommands(): Promise<{
|
|
|
|
artifactMap: ArtifactMap
|
|
|
|
sourceRangeMap: SourceRangeMap
|
|
|
|
}> {
|
|
|
|
const pendingCommands = Object.values(this.artifactMap).filter(
|
|
|
|
({ type }) => type === 'pending'
|
|
|
|
) as PendingCommand[]
|
|
|
|
const proms = pendingCommands.map(({ promise }) => promise)
|
|
|
|
await Promise.all(proms)
|
|
|
|
return {
|
|
|
|
artifactMap: this.artifactMap,
|
|
|
|
sourceRangeMap: this.sourceRangeMap,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|