re implement selections (#243)
This commit is contained in:
@ -7,7 +7,7 @@
|
||||
"@fortawesome/free-solid-svg-icons": "^6.4.0",
|
||||
"@fortawesome/react-fontawesome": "^0.2.0",
|
||||
"@headlessui/react": "^1.7.13",
|
||||
"@kittycad/lib": "^0.0.24",
|
||||
"@kittycad/lib": "^0.0.27",
|
||||
"@react-hook/resize-observer": "^1.2.6",
|
||||
"@tauri-apps/api": "^1.3.0",
|
||||
"@testing-library/jest-dom": "^5.14.1",
|
||||
|
94
src/App.tsx
94
src/App.tsx
@ -1,6 +1,7 @@
|
||||
import {
|
||||
useRef,
|
||||
useEffect,
|
||||
useLayoutEffect,
|
||||
useMemo,
|
||||
useCallback,
|
||||
MouseEventHandler,
|
||||
@ -40,9 +41,10 @@ import {
|
||||
} from '@fortawesome/free-solid-svg-icons'
|
||||
import { useHotkeys } from 'react-hotkeys-hook'
|
||||
import { TEST } from './env'
|
||||
import { getNormalisedCoordinates } from './lib/utils'
|
||||
|
||||
export function App() {
|
||||
const cam = useRef()
|
||||
const streamRef = useRef<HTMLDivElement>(null)
|
||||
useHotKeyListener()
|
||||
const {
|
||||
editorView,
|
||||
@ -60,7 +62,7 @@ export function App() {
|
||||
resetKCLErrors,
|
||||
selectionRangeTypeMap,
|
||||
setArtifactMap,
|
||||
engineCommandManager: _engineCommandManager,
|
||||
engineCommandManager,
|
||||
setEngineCommandManager,
|
||||
setHighlightRange,
|
||||
setCursor2,
|
||||
@ -79,6 +81,9 @@ export function App() {
|
||||
openPanes,
|
||||
setOpenPanes,
|
||||
onboardingStatus,
|
||||
setDidDragInStream,
|
||||
setStreamDimensions,
|
||||
streamDimensions,
|
||||
} = useStore((s) => ({
|
||||
editorView: s.editorView,
|
||||
setEditorView: s.setEditorView,
|
||||
@ -117,6 +122,9 @@ export function App() {
|
||||
openPanes: s.openPanes,
|
||||
setOpenPanes: s.setOpenPanes,
|
||||
onboardingStatus: s.onboardingStatus,
|
||||
setDidDragInStream: s.setDidDragInStream,
|
||||
setStreamDimensions: s.setStreamDimensions,
|
||||
streamDimensions: s.streamDimensions,
|
||||
}))
|
||||
|
||||
// Pane toggling keyboard shortcuts
|
||||
@ -193,7 +201,7 @@ export function App() {
|
||||
})
|
||||
.filter(Boolean) as any
|
||||
|
||||
_engineCommandManager?.cusorsSelected({
|
||||
engineCommandManager?.cusorsSelected({
|
||||
otherSelections: [],
|
||||
idBasedSelections,
|
||||
})
|
||||
@ -203,18 +211,33 @@ export function App() {
|
||||
codeBasedSelections,
|
||||
})
|
||||
}
|
||||
const engineCommandManager = useMemo(() => {
|
||||
return new EngineCommandManager({
|
||||
const pixelDensity = window.devicePixelRatio
|
||||
const streamWidth = streamRef?.current?.offsetWidth
|
||||
const streamHeight = streamRef?.current?.offsetHeight
|
||||
|
||||
const width = streamWidth ? streamWidth * pixelDensity : 0
|
||||
const quadWidth = Math.round(width / 4) * 4
|
||||
const height = streamHeight ? streamHeight * pixelDensity : 0
|
||||
const quadHeight = Math.round(height / 4) * 4
|
||||
|
||||
useLayoutEffect(() => {
|
||||
setStreamDimensions({
|
||||
streamWidth: quadWidth,
|
||||
streamHeight: quadHeight,
|
||||
})
|
||||
if (!width || !height) return
|
||||
const eng = new EngineCommandManager({
|
||||
setMediaStream,
|
||||
setIsStreamReady,
|
||||
width: quadWidth,
|
||||
height: quadHeight,
|
||||
token,
|
||||
})
|
||||
}, [token])
|
||||
useEffect(() => {
|
||||
setEngineCommandManager(eng)
|
||||
return () => {
|
||||
engineCommandManager?.tearDown()
|
||||
eng?.tearDown()
|
||||
}
|
||||
}, [engineCommandManager])
|
||||
}, [quadWidth, quadHeight])
|
||||
|
||||
useEffect(() => {
|
||||
if (!isStreamReady) return
|
||||
@ -229,11 +252,11 @@ export function App() {
|
||||
setAst(_ast)
|
||||
resetLogs()
|
||||
resetKCLErrors()
|
||||
if (_engineCommandManager) {
|
||||
_engineCommandManager.endSession()
|
||||
}
|
||||
if (engineCommandManager) {
|
||||
engineCommandManager.endSession()
|
||||
engineCommandManager.startNewSession()
|
||||
setEngineCommandManager(engineCommandManager)
|
||||
}
|
||||
if (!engineCommandManager) return
|
||||
const programMemory = await _executor(
|
||||
_ast,
|
||||
{
|
||||
@ -290,7 +313,12 @@ export function App() {
|
||||
setHighlightRange(sourceRange)
|
||||
}
|
||||
})
|
||||
engineCommandManager.onClick(({ id, type }) => {
|
||||
engineCommandManager.onClick((selections) => {
|
||||
if (!selections) {
|
||||
setCursor2()
|
||||
return
|
||||
}
|
||||
const { id, type } = selections
|
||||
setCursor2({ range: sourceRangeMap[id], type })
|
||||
})
|
||||
if (programMemory !== undefined) {
|
||||
@ -314,19 +342,29 @@ export function App() {
|
||||
const debounceSocketSend = throttle<EngineCommand>((message) => {
|
||||
engineCommandManager?.sendSceneCommand(message)
|
||||
}, 16)
|
||||
const handleMouseMove = useCallback<MouseEventHandler<HTMLDivElement>>(
|
||||
({ clientX, clientY, ctrlKey, currentTarget }) => {
|
||||
if (!cmdId) return
|
||||
if (!isMouseDownInStream) return
|
||||
const handleMouseMove: MouseEventHandler<HTMLDivElement> = ({
|
||||
clientX,
|
||||
clientY,
|
||||
ctrlKey,
|
||||
currentTarget,
|
||||
}) => {
|
||||
if (isMouseDownInStream) {
|
||||
setDidDragInStream(true)
|
||||
}
|
||||
|
||||
const { x, y } = getNormalisedCoordinates({
|
||||
clientX,
|
||||
clientY,
|
||||
el: currentTarget,
|
||||
...streamDimensions,
|
||||
})
|
||||
|
||||
const { left, top } = currentTarget.getBoundingClientRect()
|
||||
const x = clientX - left
|
||||
const y = clientY - top
|
||||
const interaction = ctrlKey ? 'pan' : 'rotate'
|
||||
|
||||
const newCmdId = uuidv4()
|
||||
setCmdId(newCmdId)
|
||||
|
||||
if (cmdId && isMouseDownInStream) {
|
||||
debounceSocketSend({
|
||||
type: 'modeling_cmd_req',
|
||||
cmd: {
|
||||
@ -337,9 +375,18 @@ export function App() {
|
||||
cmd_id: newCmdId,
|
||||
file_id: fileId,
|
||||
})
|
||||
} else {
|
||||
debounceSocketSend({
|
||||
type: 'modeling_cmd_req',
|
||||
cmd: {
|
||||
type: 'highlight_set_entity',
|
||||
selected_at_window: { x, y },
|
||||
},
|
||||
[debounceSocketSend, isMouseDownInStream, cmdId, fileId, setCmdId]
|
||||
)
|
||||
cmd_id: newCmdId,
|
||||
file_id: fileId,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const extraExtensions = useMemo(() => {
|
||||
if (TEST) return []
|
||||
@ -355,6 +402,7 @@ export function App() {
|
||||
<div
|
||||
className="h-screen overflow-hidden relative flex flex-col"
|
||||
onMouseMove={handleMouseMove}
|
||||
ref={streamRef}
|
||||
>
|
||||
<AppHeader
|
||||
className={
|
||||
|
@ -9,6 +9,7 @@ import { v4 as uuidv4 } from 'uuid'
|
||||
import { useStore } from '../useStore'
|
||||
import { throttle } from '../lib/utils'
|
||||
import { EngineCommand } from '../lang/std/engineConnection'
|
||||
import { getNormalisedCoordinates } from '../lib/utils'
|
||||
|
||||
export const Stream = ({ className = '' }) => {
|
||||
const [zoom, setZoom] = useState(0)
|
||||
@ -20,6 +21,9 @@ export const Stream = ({ className = '' }) => {
|
||||
fileId,
|
||||
setFileId,
|
||||
setCmdId,
|
||||
didDragInStream,
|
||||
setDidDragInStream,
|
||||
streamDimensions,
|
||||
} = useStore((s) => ({
|
||||
mediaStream: s.mediaStream,
|
||||
engineCommandManager: s.engineCommandManager,
|
||||
@ -28,6 +32,9 @@ export const Stream = ({ className = '' }) => {
|
||||
fileId: s.fileId,
|
||||
setFileId: s.setFileId,
|
||||
setCmdId: s.setCmdId,
|
||||
didDragInStream: s.didDragInStream,
|
||||
setDidDragInStream: s.setDidDragInStream,
|
||||
streamDimensions: s.streamDimensions,
|
||||
}))
|
||||
|
||||
useEffect(() => {
|
||||
@ -49,9 +56,12 @@ export const Stream = ({ className = '' }) => {
|
||||
ctrlKey,
|
||||
}) => {
|
||||
if (!videoRef.current) return
|
||||
const { left, top } = videoRef.current.getBoundingClientRect()
|
||||
const x = clientX - left
|
||||
const y = clientY - top
|
||||
const { x, y } = getNormalisedCoordinates({
|
||||
clientX,
|
||||
clientY,
|
||||
el: videoRef.current,
|
||||
...streamDimensions,
|
||||
})
|
||||
console.log('click', x, y)
|
||||
|
||||
const newId = uuidv4()
|
||||
@ -100,13 +110,14 @@ export const Stream = ({ className = '' }) => {
|
||||
ctrlKey,
|
||||
}) => {
|
||||
if (!videoRef.current) return
|
||||
const { left, top } = videoRef.current.getBoundingClientRect()
|
||||
const x = clientX - left
|
||||
const y = clientY - top
|
||||
const { x, y } = getNormalisedCoordinates({
|
||||
clientX,
|
||||
clientY,
|
||||
el: videoRef.current,
|
||||
...streamDimensions,
|
||||
})
|
||||
|
||||
const newCmdId = uuidv4()
|
||||
setCmdId(newCmdId)
|
||||
|
||||
const interaction = ctrlKey ? 'pan' : 'rotate'
|
||||
|
||||
engineCommandManager?.sendSceneCommand({
|
||||
@ -120,9 +131,20 @@ export const Stream = ({ className = '' }) => {
|
||||
file_id: fileId,
|
||||
})
|
||||
|
||||
setCmdId('')
|
||||
|
||||
setIsMouseDownInStream(false)
|
||||
if (!didDragInStream) {
|
||||
engineCommandManager?.sendSceneCommand({
|
||||
type: 'modeling_cmd_req',
|
||||
cmd: {
|
||||
type: 'select_with_point',
|
||||
selection_type: 'add',
|
||||
selected_at_window: { x, y },
|
||||
},
|
||||
cmd_id: uuidv4(),
|
||||
file_id: fileId,
|
||||
})
|
||||
}
|
||||
setDidDragInStream(false)
|
||||
}
|
||||
|
||||
return (
|
||||
|
@ -3,6 +3,7 @@ import { Selections } from '../../useStore'
|
||||
import { VITE_KC_API_WS_MODELING_URL } from '../../env'
|
||||
import { Models } from '@kittycad/lib'
|
||||
import { exportSave } from '../../lib/exportSave'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
|
||||
interface ResultCommand {
|
||||
type: 'result'
|
||||
@ -39,33 +40,40 @@ export interface EngineCommand extends _EngineCommand {
|
||||
type: 'modeling_cmd_req'
|
||||
}
|
||||
|
||||
type WSResponse = Models['OkModelingCmdResponse_type']
|
||||
|
||||
export class EngineCommandManager {
|
||||
artifactMap: ArtifactMap = {}
|
||||
sourceRangeMap: SourceRangeMap = {}
|
||||
sequence = 0
|
||||
outSequence = 1
|
||||
inSequence = 1
|
||||
socket?: WebSocket
|
||||
pc?: RTCPeerConnection
|
||||
lossyDataChannel?: RTCDataChannel
|
||||
waitForReady: Promise<void> = new Promise(() => {})
|
||||
private resolveReady = () => {}
|
||||
onHoverCallback: (id?: string) => void = () => {}
|
||||
onClickCallback: (selection: SelectionsArgs) => void = () => {}
|
||||
onClickCallback: (selection?: SelectionsArgs) => void = () => {}
|
||||
onCursorsSelectedCallback: (selections: CursorSelectionsArgs) => void =
|
||||
() => {}
|
||||
constructor({
|
||||
setMediaStream,
|
||||
setIsStreamReady,
|
||||
width,
|
||||
height,
|
||||
token,
|
||||
}: {
|
||||
setMediaStream: (stream: MediaStream) => void
|
||||
setIsStreamReady: (isStreamReady: boolean) => void
|
||||
width: number
|
||||
height: number
|
||||
token?: string
|
||||
}) {
|
||||
this.waitForReady = new Promise((resolve) => {
|
||||
this.resolveReady = resolve
|
||||
})
|
||||
|
||||
this.socket = new WebSocket(VITE_KC_API_WS_MODELING_URL, [])
|
||||
const url = `${VITE_KC_API_WS_MODELING_URL}?video_res_width=${width}&video_res_height=${height}`
|
||||
this.socket = new WebSocket(url, [])
|
||||
|
||||
// Change binary type from "blob" to "arraybuffer"
|
||||
this.socket.binaryType = 'arraybuffer'
|
||||
@ -185,12 +193,33 @@ export class EngineCommandManager {
|
||||
console.log('lossy data channel error')
|
||||
})
|
||||
this.lossyDataChannel.addEventListener('message', (event) => {
|
||||
console.log('lossy data channel message: ', event)
|
||||
const result: WSResponse = JSON.parse(event.data)
|
||||
if (
|
||||
result.type === 'highlight_set_entity' &&
|
||||
result.sequence &&
|
||||
result.sequence > this.inSequence
|
||||
) {
|
||||
this.onHoverCallback(result.entity_id)
|
||||
this.inSequence = result.sequence
|
||||
}
|
||||
})
|
||||
})
|
||||
} else if (message.cmd_id) {
|
||||
const id = message.cmd_id
|
||||
const command = this.artifactMap[id]
|
||||
if (message?.result?.ok) {
|
||||
const result: WSResponse = message.result.ok
|
||||
if (result.type === 'select_with_point') {
|
||||
if (result.entity_id) {
|
||||
this.onClickCallback({
|
||||
id: result.entity_id,
|
||||
type: 'default',
|
||||
})
|
||||
} else {
|
||||
this.onClickCallback()
|
||||
}
|
||||
}
|
||||
}
|
||||
if (command && command.type === 'pending') {
|
||||
const resolve = command.resolve
|
||||
this.artifactMap[id] = {
|
||||
@ -206,15 +235,6 @@ export class EngineCommandManager {
|
||||
data: message.result,
|
||||
}
|
||||
}
|
||||
// TODO talk to the gang about this
|
||||
// the following message types are made up
|
||||
// and are placeholders
|
||||
} else if (message.type === 'hover') {
|
||||
this.onHoverCallback(message.id)
|
||||
} else if (message.type === 'click') {
|
||||
this.onClickCallback(message)
|
||||
} else {
|
||||
console.log('received message', message)
|
||||
}
|
||||
}
|
||||
})
|
||||
@ -239,7 +259,7 @@ export class EngineCommandManager {
|
||||
// frontend about that (with it's id) so that the FE can highlight code associated with that id
|
||||
this.onHoverCallback = callback
|
||||
}
|
||||
onClick(callback: (selection: SelectionsArgs) => void) {
|
||||
onClick(callback: (selection?: SelectionsArgs) => void) {
|
||||
// TODO talk to the gang about this
|
||||
// It's when the user clicks on a part in the 3d scene, and so the engine should tell the
|
||||
// frontend about that (with it's id) so that the FE can put the user's cursor on the right
|
||||
@ -250,18 +270,27 @@ export class EngineCommandManager {
|
||||
otherSelections: Selections['otherSelections']
|
||||
idBasedSelections: { type: string; id: string }[]
|
||||
}) {
|
||||
// TODO talk to the gang about this
|
||||
// Really idBasedSelections is the only part that's relevant to the server, but it's when
|
||||
// the user puts their cursor over a line of code, and there is a engine-id associated with
|
||||
// it, so we want to tell the engine to change it's color or something
|
||||
if (this.socket?.readyState === 0) {
|
||||
console.log('socket not open')
|
||||
return
|
||||
}
|
||||
console.log('sending cursorsSelected')
|
||||
this.socket?.send(
|
||||
JSON.stringify({ command: 'cursorsSelected', body: selections })
|
||||
)
|
||||
this.sendSceneCommand({
|
||||
type: 'modeling_cmd_req',
|
||||
cmd: {
|
||||
type: 'select_clear',
|
||||
},
|
||||
cmd_id: uuidv4(),
|
||||
file_id: uuidv4(),
|
||||
})
|
||||
this.sendSceneCommand({
|
||||
type: 'modeling_cmd_req',
|
||||
cmd: {
|
||||
type: 'select_add',
|
||||
entities: selections.idBasedSelections.map((s) => s.id),
|
||||
},
|
||||
cmd_id: uuidv4(),
|
||||
file_id: uuidv4(),
|
||||
})
|
||||
}
|
||||
sendSceneCommand(command: EngineCommand) {
|
||||
if (this.socket?.readyState === 0) {
|
||||
@ -270,9 +299,13 @@ export class EngineCommandManager {
|
||||
}
|
||||
const cmd = command.cmd
|
||||
if (cmd.type === 'camera_drag_move' && this.lossyDataChannel) {
|
||||
console.log('sending lossy command', command, this.lossyDataChannel)
|
||||
cmd.sequence = this.sequence
|
||||
this.sequence++
|
||||
cmd.sequence = this.outSequence
|
||||
this.outSequence++
|
||||
this.lossyDataChannel.send(JSON.stringify(command))
|
||||
return
|
||||
} else if (cmd.type === 'highlight_set_entity' && this.lossyDataChannel) {
|
||||
cmd.sequence = this.outSequence
|
||||
this.outSequence++
|
||||
this.lossyDataChannel.send(JSON.stringify(command))
|
||||
return
|
||||
}
|
||||
|
@ -36,6 +36,8 @@ export async function executor(
|
||||
const engineCommandManager = new EngineCommandManager({
|
||||
setIsStreamReady: () => {},
|
||||
setMediaStream: () => {},
|
||||
width: 100,
|
||||
height: 100,
|
||||
})
|
||||
await engineCommandManager.waitForReady
|
||||
engineCommandManager.startNewSession()
|
||||
|
@ -55,3 +55,25 @@ export function throttle<T>(
|
||||
|
||||
return throttled
|
||||
}
|
||||
|
||||
export function getNormalisedCoordinates({
|
||||
clientX,
|
||||
clientY,
|
||||
streamWidth,
|
||||
streamHeight,
|
||||
el,
|
||||
}: {
|
||||
clientX: number
|
||||
clientY: number
|
||||
streamWidth: number
|
||||
streamHeight: number
|
||||
el: HTMLElement
|
||||
}) {
|
||||
const { left, top, width, height } = el?.getBoundingClientRect()
|
||||
const browserX = clientX - left
|
||||
const browserY = clientY - top
|
||||
return {
|
||||
x: Math.round((browserX / width) * streamWidth),
|
||||
y: Math.round((browserY / height) * streamHeight),
|
||||
}
|
||||
}
|
||||
|
@ -130,7 +130,7 @@ export interface StoreState {
|
||||
highlightRange: [number, number]
|
||||
setHighlightRange: (range: Selection['range']) => void
|
||||
setCursor: (selections: Selections) => void
|
||||
setCursor2: (a: Selection) => void
|
||||
setCursor2: (a?: Selection) => void
|
||||
selectionRanges: Selections
|
||||
selectionRangeTypeMap: { [key: number]: Selection['type'] }
|
||||
setSelectionRanges: (range: Selections) => void
|
||||
@ -179,10 +179,17 @@ export interface StoreState {
|
||||
setIsStreamReady: (isStreamReady: boolean) => void
|
||||
isMouseDownInStream: boolean
|
||||
setIsMouseDownInStream: (isMouseDownInStream: boolean) => void
|
||||
didDragInStream: boolean
|
||||
setDidDragInStream: (didDragInStream: boolean) => void
|
||||
cmdId?: string
|
||||
setCmdId: (cmdId: string) => void
|
||||
fileId: string
|
||||
setFileId: (fileId: string) => void
|
||||
streamDimensions: { streamWidth: number; streamHeight: number }
|
||||
setStreamDimensions: (dimensions: {
|
||||
streamWidth: number
|
||||
streamHeight: number
|
||||
}) => void
|
||||
|
||||
// tauri specific app settings
|
||||
defaultDir: DefaultDir
|
||||
@ -254,6 +261,16 @@ export const useStore = create<StoreState>()(
|
||||
},
|
||||
setCursor2: (codeSelections) => {
|
||||
const currestSelections = get().selectionRanges
|
||||
const code = get().code
|
||||
if (!codeSelections) {
|
||||
get().setCursor({
|
||||
otherSelections: currestSelections.otherSelections,
|
||||
codeBasedSelections: [
|
||||
{ range: [0, code.length - 1], type: 'default' },
|
||||
],
|
||||
})
|
||||
return
|
||||
}
|
||||
const selections: Selections = {
|
||||
...currestSelections,
|
||||
codeBasedSelections: get().isShiftDown
|
||||
@ -366,11 +383,17 @@ export const useStore = create<StoreState>()(
|
||||
setIsMouseDownInStream: (isMouseDownInStream) => {
|
||||
set({ isMouseDownInStream })
|
||||
},
|
||||
didDragInStream: false,
|
||||
setDidDragInStream: (didDragInStream) => {
|
||||
set({ didDragInStream })
|
||||
},
|
||||
// For stream event handling
|
||||
cmdId: undefined,
|
||||
setCmdId: (cmdId) => set({ cmdId }),
|
||||
fileId: '',
|
||||
setFileId: (fileId) => set({ fileId }),
|
||||
streamDimensions: { streamWidth: 1280, streamHeight: 720 },
|
||||
setStreamDimensions: (streamDimensions) => set({ streamDimensions }),
|
||||
|
||||
// tauri specific app settings
|
||||
defaultDir: {
|
||||
|
@ -1740,10 +1740,10 @@
|
||||
resolved "https://registry.yarnpkg.com/@juggle/resize-observer/-/resize-observer-3.4.0.tgz#08d6c5e20cf7e4cc02fd181c4b0c225cd31dbb60"
|
||||
integrity sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA==
|
||||
|
||||
"@kittycad/lib@^0.0.24":
|
||||
version "0.0.24"
|
||||
resolved "https://registry.yarnpkg.com/@kittycad/lib/-/lib-0.0.24.tgz#afc2f0baf8b742344e86332a60e89e85804b4b30"
|
||||
integrity sha512-RT3EThq0s7DQFT9+8HQ9yRgIb5+Q2xwYFYl/aNel5DdkfAya3WGlwhjv2YOtMXsy981JVpabo0HXD3tLfm/9QA==
|
||||
"@kittycad/lib@^0.0.27":
|
||||
version "0.0.27"
|
||||
resolved "https://registry.yarnpkg.com/@kittycad/lib/-/lib-0.0.27.tgz#6619494e365b6dfe1e400835d52b56a5d5128caa"
|
||||
integrity sha512-oRISiGJghEVm9hUs8y8ZYGRw4HFDkvK51E71ADKrNp+Mtmrzwjzt2sEj0C3yBQ8kKjeUiCeJU2EHSf2kYfD9bw==
|
||||
dependencies:
|
||||
node-fetch "3.3.2"
|
||||
openapi-types "^12.0.0"
|
||||
|
Reference in New Issue
Block a user