diff --git a/package.json b/package.json index b15baace8..4964da44e 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "@fortawesome/free-solid-svg-icons": "^6.4.0", "@fortawesome/react-fontawesome": "^0.2.0", "@headlessui/react": "^1.7.13", - "@kittycad/lib": "^0.0.24", + "@kittycad/lib": "^0.0.27", "@react-hook/resize-observer": "^1.2.6", "@tauri-apps/api": "^1.3.0", "@testing-library/jest-dom": "^5.14.1", diff --git a/src/App.tsx b/src/App.tsx index d6107a0b9..28f6f1ca9 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,6 +1,7 @@ import { useRef, useEffect, + useLayoutEffect, useMemo, useCallback, MouseEventHandler, @@ -40,9 +41,10 @@ import { } from '@fortawesome/free-solid-svg-icons' import { useHotkeys } from 'react-hotkeys-hook' import { TEST } from './env' +import { getNormalisedCoordinates } from './lib/utils' export function App() { - const cam = useRef() + const streamRef = useRef(null) useHotKeyListener() const { editorView, @@ -60,7 +62,7 @@ export function App() { resetKCLErrors, selectionRangeTypeMap, setArtifactMap, - engineCommandManager: _engineCommandManager, + engineCommandManager, setEngineCommandManager, setHighlightRange, setCursor2, @@ -79,6 +81,9 @@ export function App() { openPanes, setOpenPanes, onboardingStatus, + setDidDragInStream, + setStreamDimensions, + streamDimensions, } = useStore((s) => ({ editorView: s.editorView, setEditorView: s.setEditorView, @@ -117,6 +122,9 @@ export function App() { openPanes: s.openPanes, setOpenPanes: s.setOpenPanes, onboardingStatus: s.onboardingStatus, + setDidDragInStream: s.setDidDragInStream, + setStreamDimensions: s.setStreamDimensions, + streamDimensions: s.streamDimensions, })) // Pane toggling keyboard shortcuts @@ -193,7 +201,7 @@ export function App() { }) .filter(Boolean) as any - _engineCommandManager?.cusorsSelected({ + engineCommandManager?.cusorsSelected({ otherSelections: [], idBasedSelections, }) @@ -203,18 +211,33 @@ export function App() { codeBasedSelections, }) } - const engineCommandManager = useMemo(() => { - return new EngineCommandManager({ + const pixelDensity = window.devicePixelRatio + const streamWidth = streamRef?.current?.offsetWidth + const streamHeight = streamRef?.current?.offsetHeight + + const width = streamWidth ? streamWidth * pixelDensity : 0 + const quadWidth = Math.round(width / 4) * 4 + const height = streamHeight ? streamHeight * pixelDensity : 0 + const quadHeight = Math.round(height / 4) * 4 + + useLayoutEffect(() => { + setStreamDimensions({ + streamWidth: quadWidth, + streamHeight: quadHeight, + }) + if (!width || !height) return + const eng = new EngineCommandManager({ setMediaStream, setIsStreamReady, + width: quadWidth, + height: quadHeight, token, }) - }, [token]) - useEffect(() => { + setEngineCommandManager(eng) return () => { - engineCommandManager?.tearDown() + eng?.tearDown() } - }, [engineCommandManager]) + }, [quadWidth, quadHeight]) useEffect(() => { if (!isStreamReady) return @@ -229,11 +252,11 @@ export function App() { setAst(_ast) resetLogs() resetKCLErrors() - if (_engineCommandManager) { - _engineCommandManager.endSession() + if (engineCommandManager) { + engineCommandManager.endSession() + engineCommandManager.startNewSession() } - engineCommandManager.startNewSession() - setEngineCommandManager(engineCommandManager) + if (!engineCommandManager) return const programMemory = await _executor( _ast, { @@ -290,7 +313,12 @@ export function App() { setHighlightRange(sourceRange) } }) - engineCommandManager.onClick(({ id, type }) => { + engineCommandManager.onClick((selections) => { + if (!selections) { + setCursor2() + return + } + const { id, type } = selections setCursor2({ range: sourceRangeMap[id], type }) }) if (programMemory !== undefined) { @@ -314,19 +342,29 @@ export function App() { const debounceSocketSend = throttle((message) => { engineCommandManager?.sendSceneCommand(message) }, 16) - const handleMouseMove = useCallback>( - ({ clientX, clientY, ctrlKey, currentTarget }) => { - if (!cmdId) return - if (!isMouseDownInStream) return + const handleMouseMove: MouseEventHandler = ({ + clientX, + clientY, + ctrlKey, + currentTarget, + }) => { + if (isMouseDownInStream) { + setDidDragInStream(true) + } - const { left, top } = currentTarget.getBoundingClientRect() - const x = clientX - left - const y = clientY - top - const interaction = ctrlKey ? 'pan' : 'rotate' + const { x, y } = getNormalisedCoordinates({ + clientX, + clientY, + el: currentTarget, + ...streamDimensions, + }) - const newCmdId = uuidv4() - setCmdId(newCmdId) + const interaction = ctrlKey ? 'pan' : 'rotate' + const newCmdId = uuidv4() + setCmdId(newCmdId) + + if (cmdId && isMouseDownInStream) { debounceSocketSend({ type: 'modeling_cmd_req', cmd: { @@ -337,9 +375,18 @@ export function App() { cmd_id: newCmdId, file_id: fileId, }) - }, - [debounceSocketSend, isMouseDownInStream, cmdId, fileId, setCmdId] - ) + } else { + debounceSocketSend({ + type: 'modeling_cmd_req', + cmd: { + type: 'highlight_set_entity', + selected_at_window: { x, y }, + }, + cmd_id: newCmdId, + file_id: fileId, + }) + } + } const extraExtensions = useMemo(() => { if (TEST) return [] @@ -355,6 +402,7 @@ export function App() {
{ const [zoom, setZoom] = useState(0) @@ -20,6 +21,9 @@ export const Stream = ({ className = '' }) => { fileId, setFileId, setCmdId, + didDragInStream, + setDidDragInStream, + streamDimensions, } = useStore((s) => ({ mediaStream: s.mediaStream, engineCommandManager: s.engineCommandManager, @@ -28,6 +32,9 @@ export const Stream = ({ className = '' }) => { fileId: s.fileId, setFileId: s.setFileId, setCmdId: s.setCmdId, + didDragInStream: s.didDragInStream, + setDidDragInStream: s.setDidDragInStream, + streamDimensions: s.streamDimensions, })) useEffect(() => { @@ -49,9 +56,12 @@ export const Stream = ({ className = '' }) => { ctrlKey, }) => { if (!videoRef.current) return - const { left, top } = videoRef.current.getBoundingClientRect() - const x = clientX - left - const y = clientY - top + const { x, y } = getNormalisedCoordinates({ + clientX, + clientY, + el: videoRef.current, + ...streamDimensions, + }) console.log('click', x, y) const newId = uuidv4() @@ -100,13 +110,14 @@ export const Stream = ({ className = '' }) => { ctrlKey, }) => { if (!videoRef.current) return - const { left, top } = videoRef.current.getBoundingClientRect() - const x = clientX - left - const y = clientY - top + const { x, y } = getNormalisedCoordinates({ + clientX, + clientY, + el: videoRef.current, + ...streamDimensions, + }) const newCmdId = uuidv4() - setCmdId(newCmdId) - const interaction = ctrlKey ? 'pan' : 'rotate' engineCommandManager?.sendSceneCommand({ @@ -120,9 +131,20 @@ export const Stream = ({ className = '' }) => { file_id: fileId, }) - setCmdId('') - setIsMouseDownInStream(false) + if (!didDragInStream) { + engineCommandManager?.sendSceneCommand({ + type: 'modeling_cmd_req', + cmd: { + type: 'select_with_point', + selection_type: 'add', + selected_at_window: { x, y }, + }, + cmd_id: uuidv4(), + file_id: fileId, + }) + } + setDidDragInStream(false) } return ( diff --git a/src/lang/std/engineConnection.ts b/src/lang/std/engineConnection.ts index 0c79e79b3..f618bbaa9 100644 --- a/src/lang/std/engineConnection.ts +++ b/src/lang/std/engineConnection.ts @@ -3,6 +3,7 @@ import { Selections } from '../../useStore' import { VITE_KC_API_WS_MODELING_URL } from '../../env' import { Models } from '@kittycad/lib' import { exportSave } from '../../lib/exportSave' +import { v4 as uuidv4 } from 'uuid' interface ResultCommand { type: 'result' @@ -39,33 +40,40 @@ export interface EngineCommand extends _EngineCommand { type: 'modeling_cmd_req' } +type WSResponse = Models['OkModelingCmdResponse_type'] + export class EngineCommandManager { artifactMap: ArtifactMap = {} sourceRangeMap: SourceRangeMap = {} - sequence = 0 + outSequence = 1 + inSequence = 1 socket?: WebSocket pc?: RTCPeerConnection lossyDataChannel?: RTCDataChannel waitForReady: Promise = new Promise(() => {}) private resolveReady = () => {} onHoverCallback: (id?: string) => void = () => {} - onClickCallback: (selection: SelectionsArgs) => void = () => {} + onClickCallback: (selection?: SelectionsArgs) => void = () => {} onCursorsSelectedCallback: (selections: CursorSelectionsArgs) => void = () => {} constructor({ setMediaStream, setIsStreamReady, + width, + height, token, }: { setMediaStream: (stream: MediaStream) => void setIsStreamReady: (isStreamReady: boolean) => void + width: number + height: number token?: string }) { this.waitForReady = new Promise((resolve) => { this.resolveReady = resolve }) - - this.socket = new WebSocket(VITE_KC_API_WS_MODELING_URL, []) + const url = `${VITE_KC_API_WS_MODELING_URL}?video_res_width=${width}&video_res_height=${height}` + this.socket = new WebSocket(url, []) // Change binary type from "blob" to "arraybuffer" this.socket.binaryType = 'arraybuffer' @@ -185,12 +193,33 @@ export class EngineCommandManager { console.log('lossy data channel error') }) this.lossyDataChannel.addEventListener('message', (event) => { - console.log('lossy data channel message: ', event) + const result: WSResponse = JSON.parse(event.data) + if ( + result.type === 'highlight_set_entity' && + result.sequence && + result.sequence > this.inSequence + ) { + this.onHoverCallback(result.entity_id) + this.inSequence = result.sequence + } }) }) } else if (message.cmd_id) { const id = message.cmd_id const command = this.artifactMap[id] + if (message?.result?.ok) { + const result: WSResponse = message.result.ok + if (result.type === 'select_with_point') { + if (result.entity_id) { + this.onClickCallback({ + id: result.entity_id, + type: 'default', + }) + } else { + this.onClickCallback() + } + } + } if (command && command.type === 'pending') { const resolve = command.resolve this.artifactMap[id] = { @@ -206,15 +235,6 @@ export class EngineCommandManager { data: message.result, } } - // TODO talk to the gang about this - // the following message types are made up - // and are placeholders - } else if (message.type === 'hover') { - this.onHoverCallback(message.id) - } else if (message.type === 'click') { - this.onClickCallback(message) - } else { - console.log('received message', message) } } }) @@ -239,7 +259,7 @@ export class EngineCommandManager { // frontend about that (with it's id) so that the FE can highlight code associated with that id this.onHoverCallback = callback } - onClick(callback: (selection: SelectionsArgs) => void) { + onClick(callback: (selection?: SelectionsArgs) => void) { // TODO talk to the gang about this // It's when the user clicks on a part in the 3d scene, and so the engine should tell the // frontend about that (with it's id) so that the FE can put the user's cursor on the right @@ -250,18 +270,27 @@ export class EngineCommandManager { otherSelections: Selections['otherSelections'] idBasedSelections: { type: string; id: string }[] }) { - // TODO talk to the gang about this - // Really idBasedSelections is the only part that's relevant to the server, but it's when - // the user puts their cursor over a line of code, and there is a engine-id associated with - // it, so we want to tell the engine to change it's color or something if (this.socket?.readyState === 0) { console.log('socket not open') return } - console.log('sending cursorsSelected') - this.socket?.send( - JSON.stringify({ command: 'cursorsSelected', body: selections }) - ) + this.sendSceneCommand({ + type: 'modeling_cmd_req', + cmd: { + type: 'select_clear', + }, + cmd_id: uuidv4(), + file_id: uuidv4(), + }) + this.sendSceneCommand({ + type: 'modeling_cmd_req', + cmd: { + type: 'select_add', + entities: selections.idBasedSelections.map((s) => s.id), + }, + cmd_id: uuidv4(), + file_id: uuidv4(), + }) } sendSceneCommand(command: EngineCommand) { if (this.socket?.readyState === 0) { @@ -270,9 +299,13 @@ export class EngineCommandManager { } const cmd = command.cmd if (cmd.type === 'camera_drag_move' && this.lossyDataChannel) { - console.log('sending lossy command', command, this.lossyDataChannel) - cmd.sequence = this.sequence - this.sequence++ + cmd.sequence = this.outSequence + this.outSequence++ + this.lossyDataChannel.send(JSON.stringify(command)) + return + } else if (cmd.type === 'highlight_set_entity' && this.lossyDataChannel) { + cmd.sequence = this.outSequence + this.outSequence++ this.lossyDataChannel.send(JSON.stringify(command)) return } diff --git a/src/lib/testHelpers.ts b/src/lib/testHelpers.ts index 92e2a966d..4eaaac866 100644 --- a/src/lib/testHelpers.ts +++ b/src/lib/testHelpers.ts @@ -36,6 +36,8 @@ export async function executor( const engineCommandManager = new EngineCommandManager({ setIsStreamReady: () => {}, setMediaStream: () => {}, + width: 100, + height: 100, }) await engineCommandManager.waitForReady engineCommandManager.startNewSession() diff --git a/src/lib/utils.ts b/src/lib/utils.ts index b9bd92401..37732e0b8 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -55,3 +55,25 @@ export function throttle( return throttled } + +export function getNormalisedCoordinates({ + clientX, + clientY, + streamWidth, + streamHeight, + el, +}: { + clientX: number + clientY: number + streamWidth: number + streamHeight: number + el: HTMLElement +}) { + const { left, top, width, height } = el?.getBoundingClientRect() + const browserX = clientX - left + const browserY = clientY - top + return { + x: Math.round((browserX / width) * streamWidth), + y: Math.round((browserY / height) * streamHeight), + } +} diff --git a/src/useStore.ts b/src/useStore.ts index f1e1d98b0..83c5c3515 100644 --- a/src/useStore.ts +++ b/src/useStore.ts @@ -130,7 +130,7 @@ export interface StoreState { highlightRange: [number, number] setHighlightRange: (range: Selection['range']) => void setCursor: (selections: Selections) => void - setCursor2: (a: Selection) => void + setCursor2: (a?: Selection) => void selectionRanges: Selections selectionRangeTypeMap: { [key: number]: Selection['type'] } setSelectionRanges: (range: Selections) => void @@ -179,10 +179,17 @@ export interface StoreState { setIsStreamReady: (isStreamReady: boolean) => void isMouseDownInStream: boolean setIsMouseDownInStream: (isMouseDownInStream: boolean) => void + didDragInStream: boolean + setDidDragInStream: (didDragInStream: boolean) => void cmdId?: string setCmdId: (cmdId: string) => void fileId: string setFileId: (fileId: string) => void + streamDimensions: { streamWidth: number; streamHeight: number } + setStreamDimensions: (dimensions: { + streamWidth: number + streamHeight: number + }) => void // tauri specific app settings defaultDir: DefaultDir @@ -254,6 +261,16 @@ export const useStore = create()( }, setCursor2: (codeSelections) => { const currestSelections = get().selectionRanges + const code = get().code + if (!codeSelections) { + get().setCursor({ + otherSelections: currestSelections.otherSelections, + codeBasedSelections: [ + { range: [0, code.length - 1], type: 'default' }, + ], + }) + return + } const selections: Selections = { ...currestSelections, codeBasedSelections: get().isShiftDown @@ -366,11 +383,17 @@ export const useStore = create()( setIsMouseDownInStream: (isMouseDownInStream) => { set({ isMouseDownInStream }) }, + didDragInStream: false, + setDidDragInStream: (didDragInStream) => { + set({ didDragInStream }) + }, // For stream event handling cmdId: undefined, setCmdId: (cmdId) => set({ cmdId }), fileId: '', setFileId: (fileId) => set({ fileId }), + streamDimensions: { streamWidth: 1280, streamHeight: 720 }, + setStreamDimensions: (streamDimensions) => set({ streamDimensions }), // tauri specific app settings defaultDir: { diff --git a/yarn.lock b/yarn.lock index 227a3abb6..77e7b84e2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1740,10 +1740,10 @@ resolved "https://registry.yarnpkg.com/@juggle/resize-observer/-/resize-observer-3.4.0.tgz#08d6c5e20cf7e4cc02fd181c4b0c225cd31dbb60" integrity sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA== -"@kittycad/lib@^0.0.24": - version "0.0.24" - resolved "https://registry.yarnpkg.com/@kittycad/lib/-/lib-0.0.24.tgz#afc2f0baf8b742344e86332a60e89e85804b4b30" - integrity sha512-RT3EThq0s7DQFT9+8HQ9yRgIb5+Q2xwYFYl/aNel5DdkfAya3WGlwhjv2YOtMXsy981JVpabo0HXD3tLfm/9QA== +"@kittycad/lib@^0.0.27": + version "0.0.27" + resolved "https://registry.yarnpkg.com/@kittycad/lib/-/lib-0.0.27.tgz#6619494e365b6dfe1e400835d52b56a5d5128caa" + integrity sha512-oRISiGJghEVm9hUs8y8ZYGRw4HFDkvK51E71ADKrNp+Mtmrzwjzt2sEj0C3yBQ8kKjeUiCeJU2EHSf2kYfD9bw== dependencies: node-fetch "3.3.2" openapi-types "^12.0.0"