re implement selections (#243)

package.json

@@ -7,7 +7,7 @@
     "@fortawesome/free-solid-svg-icons": "^6.4.0",
     "@fortawesome/react-fontawesome": "^0.2.0",
     "@headlessui/react": "^1.7.13",
-    "@kittycad/lib": "^0.0.24",
+    "@kittycad/lib": "^0.0.27",
     "@react-hook/resize-observer": "^1.2.6",
     "@tauri-apps/api": "^1.3.0",
     "@testing-library/jest-dom": "^5.14.1",

src/App.tsx

@@ -1,6 +1,7 @@
 import {
   useRef,
   useEffect,
+  useLayoutEffect,
   useMemo,
   useCallback,
   MouseEventHandler,
@@ -40,9 +41,10 @@ import {
 } from '@fortawesome/free-solid-svg-icons'
 import { useHotkeys } from 'react-hotkeys-hook'
 import { TEST } from './env'
+import { getNormalisedCoordinates } from './lib/utils'
 
 export function App() {
-  const cam = useRef()
+  const streamRef = useRef<HTMLDivElement>(null)
   useHotKeyListener()
   const {
     editorView,
@@ -60,7 +62,7 @@ export function App() {
     resetKCLErrors,
     selectionRangeTypeMap,
     setArtifactMap,
-    engineCommandManager: _engineCommandManager,
+    engineCommandManager,
     setEngineCommandManager,
     setHighlightRange,
     setCursor2,
@@ -79,6 +81,9 @@ export function App() {
     openPanes,
     setOpenPanes,
     onboardingStatus,
+    setDidDragInStream,
+    setStreamDimensions,
+    streamDimensions,
   } = useStore((s) => ({
     editorView: s.editorView,
     setEditorView: s.setEditorView,
@@ -117,6 +122,9 @@ export function App() {
     openPanes: s.openPanes,
     setOpenPanes: s.setOpenPanes,
     onboardingStatus: s.onboardingStatus,
+    setDidDragInStream: s.setDidDragInStream,
+    setStreamDimensions: s.setStreamDimensions,
+    streamDimensions: s.streamDimensions,
   }))
 
   // Pane toggling keyboard shortcuts
@@ -193,7 +201,7 @@ export function App() {
       })
       .filter(Boolean) as any
 
-    _engineCommandManager?.cusorsSelected({
+    engineCommandManager?.cusorsSelected({
       otherSelections: [],
       idBasedSelections,
     })
@@ -203,18 +211,33 @@ export function App() {
       codeBasedSelections,
     })
   }
-  const engineCommandManager = useMemo(() => {
+  const pixelDensity = window.devicePixelRatio
-    return new EngineCommandManager({
+  const streamWidth = streamRef?.current?.offsetWidth
+  const streamHeight = streamRef?.current?.offsetHeight
+
+  const width = streamWidth ? streamWidth * pixelDensity : 0
+  const quadWidth = Math.round(width / 4) * 4
+  const height = streamHeight ? streamHeight * pixelDensity : 0
+  const quadHeight = Math.round(height / 4) * 4
+
+  useLayoutEffect(() => {
+    setStreamDimensions({
+      streamWidth: quadWidth,
+      streamHeight: quadHeight,
+    })
+    if (!width || !height) return
+    const eng = new EngineCommandManager({
       setMediaStream,
       setIsStreamReady,
+      width: quadWidth,
+      height: quadHeight,
       token,
     })
-  }, [token])
+    setEngineCommandManager(eng)
-  useEffect(() => {
     return () => {
-      engineCommandManager?.tearDown()
+      eng?.tearDown()
     }
-  }, [engineCommandManager])
+  }, [quadWidth, quadHeight])
 
   useEffect(() => {
     if (!isStreamReady) return
@@ -229,11 +252,11 @@ export function App() {
         setAst(_ast)
         resetLogs()
         resetKCLErrors()
-        if (_engineCommandManager) {
+        if (engineCommandManager) {
-          _engineCommandManager.endSession()
+          engineCommandManager.endSession()
+          engineCommandManager.startNewSession()
         }
-        engineCommandManager.startNewSession()
+        if (!engineCommandManager) return
-        setEngineCommandManager(engineCommandManager)
         const programMemory = await _executor(
           _ast,
           {
@@ -290,7 +313,12 @@ export function App() {
             setHighlightRange(sourceRange)
           }
         })
-        engineCommandManager.onClick(({ id, type }) => {
+        engineCommandManager.onClick((selections) => {
+          if (!selections) {
+            setCursor2()
+            return
+          }
+          const { id, type } = selections
           setCursor2({ range: sourceRangeMap[id], type })
         })
         if (programMemory !== undefined) {
@@ -314,19 +342,29 @@ export function App() {
   const debounceSocketSend = throttle<EngineCommand>((message) => {
     engineCommandManager?.sendSceneCommand(message)
   }, 16)
-  const handleMouseMove = useCallback<MouseEventHandler<HTMLDivElement>>(
+  const handleMouseMove: MouseEventHandler<HTMLDivElement> = ({
-    ({ clientX, clientY, ctrlKey, currentTarget }) => {
+    clientX,
-      if (!cmdId) return
+    clientY,
-      if (!isMouseDownInStream) return
+    ctrlKey,
+    currentTarget,
+  }) => {
+    if (isMouseDownInStream) {
+      setDidDragInStream(true)
+    }
 
-      const { left, top } = currentTarget.getBoundingClientRect()
+    const { x, y } = getNormalisedCoordinates({
-      const x = clientX - left
+      clientX,
-      const y = clientY - top
+      clientY,
-      const interaction = ctrlKey ? 'pan' : 'rotate'
+      el: currentTarget,
+      ...streamDimensions,
+    })
 
-      const newCmdId = uuidv4()
+    const interaction = ctrlKey ? 'pan' : 'rotate'
-      setCmdId(newCmdId)
 
+    const newCmdId = uuidv4()
+    setCmdId(newCmdId)
+
+    if (cmdId && isMouseDownInStream) {
       debounceSocketSend({
         type: 'modeling_cmd_req',
         cmd: {
@@ -337,9 +375,18 @@ export function App() {
         cmd_id: newCmdId,
         file_id: fileId,
       })
-    },
+    } else {
-    [debounceSocketSend, isMouseDownInStream, cmdId, fileId, setCmdId]
+      debounceSocketSend({
-  )
+        type: 'modeling_cmd_req',
+        cmd: {
+          type: 'highlight_set_entity',
+          selected_at_window: { x, y },
+        },
+        cmd_id: newCmdId,
+        file_id: fileId,
+      })
+    }
+  }
 
   const extraExtensions = useMemo(() => {
     if (TEST) return []
@@ -355,6 +402,7 @@ export function App() {
     <div
       className="h-screen overflow-hidden relative flex flex-col"
       onMouseMove={handleMouseMove}
+      ref={streamRef}
     >
       <AppHeader
         className={

src/components/Stream.tsx

@@ -9,6 +9,7 @@ import { v4 as uuidv4 } from 'uuid'
 import { useStore } from '../useStore'
 import { throttle } from '../lib/utils'
 import { EngineCommand } from '../lang/std/engineConnection'
+import { getNormalisedCoordinates } from '../lib/utils'
 
 export const Stream = ({ className = '' }) => {
   const [zoom, setZoom] = useState(0)
@@ -20,6 +21,9 @@ export const Stream = ({ className = '' }) => {
     fileId,
     setFileId,
     setCmdId,
+    didDragInStream,
+    setDidDragInStream,
+    streamDimensions,
   } = useStore((s) => ({
     mediaStream: s.mediaStream,
     engineCommandManager: s.engineCommandManager,
@@ -28,6 +32,9 @@ export const Stream = ({ className = '' }) => {
     fileId: s.fileId,
     setFileId: s.setFileId,
     setCmdId: s.setCmdId,
+    didDragInStream: s.didDragInStream,
+    setDidDragInStream: s.setDidDragInStream,
+    streamDimensions: s.streamDimensions,
   }))
 
   useEffect(() => {
@@ -49,9 +56,12 @@ export const Stream = ({ className = '' }) => {
     ctrlKey,
   }) => {
     if (!videoRef.current) return
-    const { left, top } = videoRef.current.getBoundingClientRect()
+    const { x, y } = getNormalisedCoordinates({
-    const x = clientX - left
+      clientX,
-    const y = clientY - top
+      clientY,
+      el: videoRef.current,
+      ...streamDimensions,
+    })
     console.log('click', x, y)
 
     const newId = uuidv4()
@@ -100,13 +110,14 @@ export const Stream = ({ className = '' }) => {
     ctrlKey,
   }) => {
     if (!videoRef.current) return
-    const { left, top } = videoRef.current.getBoundingClientRect()
+    const { x, y } = getNormalisedCoordinates({
-    const x = clientX - left
+      clientX,
-    const y = clientY - top
+      clientY,
+      el: videoRef.current,
+      ...streamDimensions,
+    })
 
     const newCmdId = uuidv4()
-    setCmdId(newCmdId)
-
     const interaction = ctrlKey ? 'pan' : 'rotate'
 
     engineCommandManager?.sendSceneCommand({
@@ -120,9 +131,20 @@ export const Stream = ({ className = '' }) => {
       file_id: fileId,
     })
 
-    setCmdId('')
-
     setIsMouseDownInStream(false)
+    if (!didDragInStream) {
+      engineCommandManager?.sendSceneCommand({
+        type: 'modeling_cmd_req',
+        cmd: {
+          type: 'select_with_point',
+          selection_type: 'add',
+          selected_at_window: { x, y },
+        },
+        cmd_id: uuidv4(),
+        file_id: fileId,
+      })
+    }
+    setDidDragInStream(false)
   }
 
   return (

src/lang/std/engineConnection.ts

@@ -3,6 +3,7 @@ import { Selections } from '../../useStore'
 import { VITE_KC_API_WS_MODELING_URL } from '../../env'
 import { Models } from '@kittycad/lib'
 import { exportSave } from '../../lib/exportSave'
+import { v4 as uuidv4 } from 'uuid'
 
 interface ResultCommand {
   type: 'result'
@@ -39,33 +40,40 @@ export interface EngineCommand extends _EngineCommand {
   type: 'modeling_cmd_req'
 }
 
+type WSResponse = Models['OkModelingCmdResponse_type']
+
 export class EngineCommandManager {
   artifactMap: ArtifactMap = {}
   sourceRangeMap: SourceRangeMap = {}
-  sequence = 0
+  outSequence = 1
+  inSequence = 1
   socket?: WebSocket
   pc?: RTCPeerConnection
   lossyDataChannel?: RTCDataChannel
   waitForReady: Promise<void> = new Promise(() => {})
   private resolveReady = () => {}
   onHoverCallback: (id?: string) => void = () => {}
-  onClickCallback: (selection: SelectionsArgs) => void = () => {}
+  onClickCallback: (selection?: SelectionsArgs) => void = () => {}
   onCursorsSelectedCallback: (selections: CursorSelectionsArgs) => void =
     () => {}
   constructor({
     setMediaStream,
     setIsStreamReady,
+    width,
+    height,
     token,
   }: {
     setMediaStream: (stream: MediaStream) => void
     setIsStreamReady: (isStreamReady: boolean) => void
+    width: number
+    height: number
     token?: string
   }) {
     this.waitForReady = new Promise((resolve) => {
       this.resolveReady = resolve
     })
-
+    const url = `${VITE_KC_API_WS_MODELING_URL}?video_res_width=${width}&video_res_height=${height}`
-    this.socket = new WebSocket(VITE_KC_API_WS_MODELING_URL, [])
+    this.socket = new WebSocket(url, [])
 
     // Change binary type from "blob" to "arraybuffer"
     this.socket.binaryType = 'arraybuffer'
@@ -185,12 +193,33 @@ export class EngineCommandManager {
               console.log('lossy data channel error')
             })
             this.lossyDataChannel.addEventListener('message', (event) => {
-              console.log('lossy data channel message: ', event)
+              const result: WSResponse = JSON.parse(event.data)
+              if (
+                result.type === 'highlight_set_entity' &&
+                result.sequence &&
+                result.sequence > this.inSequence
+              ) {
+                this.onHoverCallback(result.entity_id)
+                this.inSequence = result.sequence
+              }
             })
           })
         } else if (message.cmd_id) {
           const id = message.cmd_id
           const command = this.artifactMap[id]
+          if (message?.result?.ok) {
+            const result: WSResponse = message.result.ok
+            if (result.type === 'select_with_point') {
+              if (result.entity_id) {
+                this.onClickCallback({
+                  id: result.entity_id,
+                  type: 'default',
+                })
+              } else {
+                this.onClickCallback()
+              }
+            }
+          }
           if (command && command.type === 'pending') {
             const resolve = command.resolve
             this.artifactMap[id] = {
@@ -206,15 +235,6 @@ export class EngineCommandManager {
               data: message.result,
             }
           }
-          // TODO talk to the gang about this
-          // the following message types are made up
-          // and are placeholders
-        } else if (message.type === 'hover') {
-          this.onHoverCallback(message.id)
-        } else if (message.type === 'click') {
-          this.onClickCallback(message)
-        } else {
-          console.log('received message', message)
         }
       }
     })
@@ -239,7 +259,7 @@ export class EngineCommandManager {
     // frontend about that (with it's id) so that the FE can highlight code associated with that id
     this.onHoverCallback = callback
   }
-  onClick(callback: (selection: SelectionsArgs) => void) {
+  onClick(callback: (selection?: SelectionsArgs) => void) {
     // TODO talk to the gang about this
     // It's when the user clicks on a part in the 3d scene, and so the engine should tell the
     // frontend about that (with it's id) so that the FE can put the user's cursor on the right
@@ -250,18 +270,27 @@ export class EngineCommandManager {
     otherSelections: Selections['otherSelections']
     idBasedSelections: { type: string; id: string }[]
   }) {
-    // TODO talk to the gang about this
-    // Really idBasedSelections is the only part that's relevant to the server, but it's when
-    // the user puts their cursor over a line of code, and there is a engine-id associated with
-    // it, so we want to tell the engine to change it's color or something
     if (this.socket?.readyState === 0) {
       console.log('socket not open')
       return
     }
-    console.log('sending cursorsSelected')
+    this.sendSceneCommand({
-    this.socket?.send(
+      type: 'modeling_cmd_req',
-      JSON.stringify({ command: 'cursorsSelected', body: selections })
+      cmd: {
-    )
+        type: 'select_clear',
+      },
+      cmd_id: uuidv4(),
+      file_id: uuidv4(),
+    })
+    this.sendSceneCommand({
+      type: 'modeling_cmd_req',
+      cmd: {
+        type: 'select_add',
+        entities: selections.idBasedSelections.map((s) => s.id),
+      },
+      cmd_id: uuidv4(),
+      file_id: uuidv4(),
+    })
   }
   sendSceneCommand(command: EngineCommand) {
     if (this.socket?.readyState === 0) {
@@ -270,9 +299,13 @@ export class EngineCommandManager {
     }
     const cmd = command.cmd
     if (cmd.type === 'camera_drag_move' && this.lossyDataChannel) {
-      console.log('sending lossy command', command, this.lossyDataChannel)
+      cmd.sequence = this.outSequence
-      cmd.sequence = this.sequence
+      this.outSequence++
-      this.sequence++
+      this.lossyDataChannel.send(JSON.stringify(command))
+      return
+    } else if (cmd.type === 'highlight_set_entity' && this.lossyDataChannel) {
+      cmd.sequence = this.outSequence
+      this.outSequence++
       this.lossyDataChannel.send(JSON.stringify(command))
       return
     }

src/lib/testHelpers.ts

@@ -36,6 +36,8 @@ export async function executor(
   const engineCommandManager = new EngineCommandManager({
     setIsStreamReady: () => {},
     setMediaStream: () => {},
+    width: 100,
+    height: 100,
   })
   await engineCommandManager.waitForReady
   engineCommandManager.startNewSession()

src/lib/utils.ts

@@ -55,3 +55,25 @@ export function throttle<T>(
 
   return throttled
 }
+
+export function getNormalisedCoordinates({
+  clientX,
+  clientY,
+  streamWidth,
+  streamHeight,
+  el,
+}: {
+  clientX: number
+  clientY: number
+  streamWidth: number
+  streamHeight: number
+  el: HTMLElement
+}) {
+  const { left, top, width, height } = el?.getBoundingClientRect()
+  const browserX = clientX - left
+  const browserY = clientY - top
+  return {
+    x: Math.round((browserX / width) * streamWidth),
+    y: Math.round((browserY / height) * streamHeight),
+  }
+}

src/useStore.ts

@@ -130,7 +130,7 @@ export interface StoreState {
   highlightRange: [number, number]
   setHighlightRange: (range: Selection['range']) => void
   setCursor: (selections: Selections) => void
-  setCursor2: (a: Selection) => void
+  setCursor2: (a?: Selection) => void
   selectionRanges: Selections
   selectionRangeTypeMap: { [key: number]: Selection['type'] }
   setSelectionRanges: (range: Selections) => void
@@ -179,10 +179,17 @@ export interface StoreState {
   setIsStreamReady: (isStreamReady: boolean) => void
   isMouseDownInStream: boolean
   setIsMouseDownInStream: (isMouseDownInStream: boolean) => void
+  didDragInStream: boolean
+  setDidDragInStream: (didDragInStream: boolean) => void
   cmdId?: string
   setCmdId: (cmdId: string) => void
   fileId: string
   setFileId: (fileId: string) => void
+  streamDimensions: { streamWidth: number; streamHeight: number }
+  setStreamDimensions: (dimensions: {
+    streamWidth: number
+    streamHeight: number
+  }) => void
 
   // tauri specific app settings
   defaultDir: DefaultDir
@@ -254,6 +261,16 @@ export const useStore = create<StoreState>()(
       },
       setCursor2: (codeSelections) => {
         const currestSelections = get().selectionRanges
+        const code = get().code
+        if (!codeSelections) {
+          get().setCursor({
+            otherSelections: currestSelections.otherSelections,
+            codeBasedSelections: [
+              { range: [0, code.length - 1], type: 'default' },
+            ],
+          })
+          return
+        }
         const selections: Selections = {
           ...currestSelections,
           codeBasedSelections: get().isShiftDown
@@ -366,11 +383,17 @@ export const useStore = create<StoreState>()(
       setIsMouseDownInStream: (isMouseDownInStream) => {
         set({ isMouseDownInStream })
       },
+      didDragInStream: false,
+      setDidDragInStream: (didDragInStream) => {
+        set({ didDragInStream })
+      },
       // For stream event handling
       cmdId: undefined,
       setCmdId: (cmdId) => set({ cmdId }),
       fileId: '',
       setFileId: (fileId) => set({ fileId }),
+      streamDimensions: { streamWidth: 1280, streamHeight: 720 },
+      setStreamDimensions: (streamDimensions) => set({ streamDimensions }),
 
       // tauri specific app settings
       defaultDir: {

yarn.lock

@@ -1740,10 +1740,10 @@
   resolved "https://registry.yarnpkg.com/@juggle/resize-observer/-/resize-observer-3.4.0.tgz#08d6c5e20cf7e4cc02fd181c4b0c225cd31dbb60"
   integrity sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA==
 
-"@kittycad/lib@^0.0.24":
+"@kittycad/lib@^0.0.27":
-  version "0.0.24"
+  version "0.0.27"
-  resolved "https://registry.yarnpkg.com/@kittycad/lib/-/lib-0.0.24.tgz#afc2f0baf8b742344e86332a60e89e85804b4b30"
+  resolved "https://registry.yarnpkg.com/@kittycad/lib/-/lib-0.0.27.tgz#6619494e365b6dfe1e400835d52b56a5d5128caa"
-  integrity sha512-RT3EThq0s7DQFT9+8HQ9yRgIb5+Q2xwYFYl/aNel5DdkfAya3WGlwhjv2YOtMXsy981JVpabo0HXD3tLfm/9QA==
+  integrity sha512-oRISiGJghEVm9hUs8y8ZYGRw4HFDkvK51E71ADKrNp+Mtmrzwjzt2sEj0C3yBQ8kKjeUiCeJU2EHSf2kYfD9bw==
   dependencies:
     node-fetch "3.3.2"
     openapi-types "^12.0.0"