ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

re implement selections (#243)

Browse Source
This commit is contained in:
Kurt Hutten
2023-08-09 20:49:10 +10:00
committed by GitHub
parent f9259aa869
commit 8ebb8b8b94
8 changed files with 219 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -7,7 +7,7 @@
"@fortawesome/free-solid-svg-icons": "^6.4.0",
"@fortawesome/react-fontawesome": "^0.2.0",
"@headlessui/react": "^1.7.13",
"@kittycad/lib": "^0.0.24",
"@kittycad/lib": "^0.0.27",
"@react-hook/resize-observer": "^1.2.6",
"@tauri-apps/api": "^1.3.0",
"@testing-library/jest-dom": "^5.14.1",

102
src/App.tsx
View File

@ -1,6 +1,7 @@
import {
useRef,
useEffect,
useLayoutEffect,
useMemo,
useCallback,
MouseEventHandler,
@ -40,9 +41,10 @@ import {
} from '@fortawesome/free-solid-svg-icons'
import { useHotkeys } from 'react-hotkeys-hook'
import { TEST } from './env'
import { getNormalisedCoordinates } from './lib/utils'
export function App() {
const cam = useRef()
const streamRef = useRef<HTMLDivElement>(null)
useHotKeyListener()
const {
editorView,
@ -60,7 +62,7 @@ export function App() {
resetKCLErrors,
selectionRangeTypeMap,
setArtifactMap,
engineCommandManager: _engineCommandManager,
engineCommandManager,
setEngineCommandManager,
setHighlightRange,
setCursor2,
@ -79,6 +81,9 @@ export function App() {
openPanes,
setOpenPanes,
onboardingStatus,
setDidDragInStream,
setStreamDimensions,
streamDimensions,
} = useStore((s) => ({
editorView: s.editorView,
setEditorView: s.setEditorView,
@ -117,6 +122,9 @@ export function App() {
openPanes: s.openPanes,
setOpenPanes: s.setOpenPanes,
onboardingStatus: s.onboardingStatus,
setDidDragInStream: s.setDidDragInStream,
setStreamDimensions: s.setStreamDimensions,
streamDimensions: s.streamDimensions,
}))
// Pane toggling keyboard shortcuts
@ -193,7 +201,7 @@ export function App() {
})
.filter(Boolean) as any
_engineCommandManager?.cusorsSelected({
engineCommandManager?.cusorsSelected({
otherSelections: [],
idBasedSelections,
})
@ -203,18 +211,33 @@ export function App() {
codeBasedSelections,
})
}
const engineCommandManager = useMemo(() => {
return new EngineCommandManager({
const pixelDensity = window.devicePixelRatio
const streamWidth = streamRef?.current?.offsetWidth
const streamHeight = streamRef?.current?.offsetHeight
const width = streamWidth ? streamWidth * pixelDensity : 0
const quadWidth = Math.round(width / 4) * 4
const height = streamHeight ? streamHeight * pixelDensity : 0
const quadHeight = Math.round(height / 4) * 4
useLayoutEffect(() => {
setStreamDimensions({
streamWidth: quadWidth,
streamHeight: quadHeight,
})
if (!width || !height) return
const eng = new EngineCommandManager({
setMediaStream,
setIsStreamReady,
width: quadWidth,
height: quadHeight,
token,
})
}, [token])
useEffect(() => {
setEngineCommandManager(eng)
return () => {
engineCommandManager?.tearDown()
eng?.tearDown()
}
}, [engineCommandManager])
}, [quadWidth, quadHeight])
useEffect(() => {
if (!isStreamReady) return
@ -229,11 +252,11 @@ export function App() {
setAst(_ast)
resetLogs()
resetKCLErrors()
if (_engineCommandManager) {
_engineCommandManager.endSession()
if (engineCommandManager) {
engineCommandManager.endSession()
engineCommandManager.startNewSession()
}
engineCommandManager.startNewSession()
setEngineCommandManager(engineCommandManager)
if (!engineCommandManager) return
const programMemory = await _executor(
_ast,
{
@ -290,7 +313,12 @@ export function App() {
setHighlightRange(sourceRange)
}
})
engineCommandManager.onClick(({ id, type }) => {
engineCommandManager.onClick((selections) => {
if (!selections) {
setCursor2()
return
}
const { id, type } = selections
setCursor2({ range: sourceRangeMap[id], type })
})
if (programMemory !== undefined) {
@ -314,19 +342,29 @@ export function App() {
const debounceSocketSend = throttle<EngineCommand>((message) => {
engineCommandManager?.sendSceneCommand(message)
}, 16)
const handleMouseMove = useCallback<MouseEventHandler<HTMLDivElement>>(
({ clientX, clientY, ctrlKey, currentTarget }) => {
if (!cmdId) return
if (!isMouseDownInStream) return
const handleMouseMove: MouseEventHandler<HTMLDivElement> = ({
clientX,
clientY,
ctrlKey,
currentTarget,
}) => {
if (isMouseDownInStream) {
setDidDragInStream(true)
}
const { left, top } = currentTarget.getBoundingClientRect()
const x = clientX - left
const y = clientY - top
const interaction = ctrlKey ? 'pan' : 'rotate'
const { x, y } = getNormalisedCoordinates({
clientX,
clientY,
el: currentTarget,
...streamDimensions,
})
const newCmdId = uuidv4()
setCmdId(newCmdId)
const interaction = ctrlKey ? 'pan' : 'rotate'
const newCmdId = uuidv4()
setCmdId(newCmdId)
if (cmdId && isMouseDownInStream) {
debounceSocketSend({
type: 'modeling_cmd_req',
cmd: {
@ -337,9 +375,18 @@ export function App() {
cmd_id: newCmdId,
file_id: fileId,
})
},
[debounceSocketSend, isMouseDownInStream, cmdId, fileId, setCmdId]
)
} else {
debounceSocketSend({
type: 'modeling_cmd_req',
cmd: {
type: 'highlight_set_entity',
selected_at_window: { x, y },
},
cmd_id: newCmdId,
file_id: fileId,
})
}
}
const extraExtensions = useMemo(() => {
if (TEST) return []
@ -355,6 +402,7 @@ export function App() {
<div
className="h-screen overflow-hidden relative flex flex-col"
onMouseMove={handleMouseMove}
ref={streamRef}
>
<AppHeader
className={

42
src/components/Stream.tsx
View File

@ -9,6 +9,7 @@ import { v4 as uuidv4 } from 'uuid'
import { useStore } from '../useStore'
import { throttle } from '../lib/utils'
import { EngineCommand } from '../lang/std/engineConnection'
import { getNormalisedCoordinates } from '../lib/utils'
export const Stream = ({ className = '' }) => {
const [zoom, setZoom] = useState(0)
@ -20,6 +21,9 @@ export const Stream = ({ className = '' }) => {
fileId,
setFileId,
setCmdId,
didDragInStream,
setDidDragInStream,
streamDimensions,
} = useStore((s) => ({
mediaStream: s.mediaStream,
engineCommandManager: s.engineCommandManager,
@ -28,6 +32,9 @@ export const Stream = ({ className = '' }) => {
fileId: s.fileId,
setFileId: s.setFileId,
setCmdId: s.setCmdId,
didDragInStream: s.didDragInStream,
setDidDragInStream: s.setDidDragInStream,
streamDimensions: s.streamDimensions,
}))
useEffect(() => {
@ -49,9 +56,12 @@ export const Stream = ({ className = '' }) => {
ctrlKey,
}) => {
if (!videoRef.current) return
const { left, top } = videoRef.current.getBoundingClientRect()
const x = clientX - left
const y = clientY - top
const { x, y } = getNormalisedCoordinates({
clientX,
clientY,
el: videoRef.current,
...streamDimensions,
})
console.log('click', x, y)
const newId = uuidv4()
@ -100,13 +110,14 @@ export const Stream = ({ className = '' }) => {
ctrlKey,
}) => {
if (!videoRef.current) return
const { left, top } = videoRef.current.getBoundingClientRect()
const x = clientX - left
const y = clientY - top
const { x, y } = getNormalisedCoordinates({
clientX,
clientY,
el: videoRef.current,
...streamDimensions,
})
const newCmdId = uuidv4()
setCmdId(newCmdId)
const interaction = ctrlKey ? 'pan' : 'rotate'
engineCommandManager?.sendSceneCommand({
@ -120,9 +131,20 @@ export const Stream = ({ className = '' }) => {
file_id: fileId,
})
setCmdId('')
setIsMouseDownInStream(false)
if (!didDragInStream) {
engineCommandManager?.sendSceneCommand({
type: 'modeling_cmd_req',
cmd: {
type: 'select_with_point',
selection_type: 'add',
selected_at_window: { x, y },
},
cmd_id: uuidv4(),
file_id: fileId,
})
}
setDidDragInStream(false)
}
return (

85
src/lang/std/engineConnection.ts
View File

@ -3,6 +3,7 @@ import { Selections } from '../../useStore'
import { VITE_KC_API_WS_MODELING_URL } from '../../env'
import { Models } from '@kittycad/lib'
import { exportSave } from '../../lib/exportSave'
import { v4 as uuidv4 } from 'uuid'
interface ResultCommand {
type: 'result'
@ -39,33 +40,40 @@ export interface EngineCommand extends _EngineCommand {
type: 'modeling_cmd_req'
}
type WSResponse = Models['OkModelingCmdResponse_type']
export class EngineCommandManager {
artifactMap: ArtifactMap = {}
sourceRangeMap: SourceRangeMap = {}
sequence = 0
outSequence = 1
inSequence = 1
socket?: WebSocket
pc?: RTCPeerConnection
lossyDataChannel?: RTCDataChannel
waitForReady: Promise<void> = new Promise(() => {})
private resolveReady = () => {}
onHoverCallback: (id?: string) => void = () => {}
onClickCallback: (selection: SelectionsArgs) => void = () => {}
onClickCallback: (selection?: SelectionsArgs) => void = () => {}
onCursorsSelectedCallback: (selections: CursorSelectionsArgs) => void =
() => {}
constructor({
setMediaStream,
setIsStreamReady,
width,
height,
token,
}: {
setMediaStream: (stream: MediaStream) => void
setIsStreamReady: (isStreamReady: boolean) => void
width: number
height: number
token?: string
}) {
this.waitForReady = new Promise((resolve) => {
this.resolveReady = resolve
})
this.socket = new WebSocket(VITE_KC_API_WS_MODELING_URL, [])
const url = `${VITE_KC_API_WS_MODELING_URL}?video_res_width=${width}&video_res_height=${height}`
this.socket = new WebSocket(url, [])
// Change binary type from "blob" to "arraybuffer"
this.socket.binaryType = 'arraybuffer'
@ -185,12 +193,33 @@ export class EngineCommandManager {
console.log('lossy data channel error')
})
this.lossyDataChannel.addEventListener('message', (event) => {
console.log('lossy data channel message: ', event)
const result: WSResponse = JSON.parse(event.data)
if (
result.type === 'highlight_set_entity' &&
result.sequence &&
result.sequence > this.inSequence
) {
this.onHoverCallback(result.entity_id)
this.inSequence = result.sequence
}
})
})
} else if (message.cmd_id) {
const id = message.cmd_id
const command = this.artifactMap[id]
if (message?.result?.ok) {
const result: WSResponse = message.result.ok
if (result.type === 'select_with_point') {
if (result.entity_id) {
this.onClickCallback({
id: result.entity_id,
type: 'default',
})
} else {
this.onClickCallback()
}
}
}
if (command && command.type === 'pending') {
const resolve = command.resolve
this.artifactMap[id] = {
@ -206,15 +235,6 @@ export class EngineCommandManager {
data: message.result,
}
}
// TODO talk to the gang about this
// the following message types are made up
// and are placeholders
} else if (message.type === 'hover') {
this.onHoverCallback(message.id)
} else if (message.type === 'click') {
this.onClickCallback(message)
} else {
console.log('received message', message)
}
}
})
@ -239,7 +259,7 @@ export class EngineCommandManager {
// frontend about that (with it's id) so that the FE can highlight code associated with that id
this.onHoverCallback = callback
}
onClick(callback: (selection: SelectionsArgs) => void) {
onClick(callback: (selection?: SelectionsArgs) => void) {
// TODO talk to the gang about this
// It's when the user clicks on a part in the 3d scene, and so the engine should tell the
// frontend about that (with it's id) so that the FE can put the user's cursor on the right
@ -250,18 +270,27 @@ export class EngineCommandManager {
otherSelections: Selections['otherSelections']
idBasedSelections: { type: string; id: string }[]
}) {
// TODO talk to the gang about this
// Really idBasedSelections is the only part that's relevant to the server, but it's when
// the user puts their cursor over a line of code, and there is a engine-id associated with
// it, so we want to tell the engine to change it's color or something
if (this.socket?.readyState === 0) {
console.log('socket not open')
return
}
console.log('sending cursorsSelected')
this.socket?.send(
JSON.stringify({ command: 'cursorsSelected', body: selections })
)
this.sendSceneCommand({
type: 'modeling_cmd_req',
cmd: {
type: 'select_clear',
},
cmd_id: uuidv4(),
file_id: uuidv4(),
})
this.sendSceneCommand({
type: 'modeling_cmd_req',
cmd: {
type: 'select_add',
entities: selections.idBasedSelections.map((s) => s.id),
},
cmd_id: uuidv4(),
file_id: uuidv4(),
})
}
sendSceneCommand(command: EngineCommand) {
if (this.socket?.readyState === 0) {
@ -270,9 +299,13 @@ export class EngineCommandManager {
}
const cmd = command.cmd
if (cmd.type === 'camera_drag_move' && this.lossyDataChannel) {
console.log('sending lossy command', command, this.lossyDataChannel)
cmd.sequence = this.sequence
this.sequence++
cmd.sequence = this.outSequence
this.outSequence++
this.lossyDataChannel.send(JSON.stringify(command))
return
} else if (cmd.type === 'highlight_set_entity' && this.lossyDataChannel) {
cmd.sequence = this.outSequence
this.outSequence++
this.lossyDataChannel.send(JSON.stringify(command))
return
}

2
src/lib/testHelpers.ts
View File

@ -36,6 +36,8 @@ export async function executor(
const engineCommandManager = new EngineCommandManager({
setIsStreamReady: () => {},
setMediaStream: () => {},
width: 100,
height: 100,
})
await engineCommandManager.waitForReady
engineCommandManager.startNewSession()

22
src/lib/utils.ts
View File

@ -55,3 +55,25 @@ export function throttle<T>(
return throttled
}
export function getNormalisedCoordinates({
clientX,
clientY,
streamWidth,
streamHeight,
el,
}: {
clientX: number
clientY: number
streamWidth: number
streamHeight: number
el: HTMLElement
}) {
const { left, top, width, height } = el?.getBoundingClientRect()
const browserX = clientX - left
const browserY = clientY - top
return {
x: Math.round((browserX / width) * streamWidth),
y: Math.round((browserY / height) * streamHeight),
}
}

25
src/useStore.ts
View File

@ -130,7 +130,7 @@ export interface StoreState {
highlightRange: [number, number]
setHighlightRange: (range: Selection['range']) => void
setCursor: (selections: Selections) => void
setCursor2: (a: Selection) => void
setCursor2: (a?: Selection) => void
selectionRanges: Selections
selectionRangeTypeMap: { [key: number]: Selection['type'] }
setSelectionRanges: (range: Selections) => void
@ -179,10 +179,17 @@ export interface StoreState {
setIsStreamReady: (isStreamReady: boolean) => void
isMouseDownInStream: boolean
setIsMouseDownInStream: (isMouseDownInStream: boolean) => void
didDragInStream: boolean
setDidDragInStream: (didDragInStream: boolean) => void
cmdId?: string
setCmdId: (cmdId: string) => void
fileId: string
setFileId: (fileId: string) => void
streamDimensions: { streamWidth: number; streamHeight: number }
setStreamDimensions: (dimensions: {
streamWidth: number
streamHeight: number
}) => void
// tauri specific app settings
defaultDir: DefaultDir
@ -254,6 +261,16 @@ export const useStore = create<StoreState>()(
},
setCursor2: (codeSelections) => {
const currestSelections = get().selectionRanges
const code = get().code
if (!codeSelections) {
get().setCursor({
otherSelections: currestSelections.otherSelections,
codeBasedSelections: [
{ range: [0, code.length - 1], type: 'default' },
],
})
return
}
const selections: Selections = {
...currestSelections,
codeBasedSelections: get().isShiftDown
@ -366,11 +383,17 @@ export const useStore = create<StoreState>()(
setIsMouseDownInStream: (isMouseDownInStream) => {
set({ isMouseDownInStream })
},
didDragInStream: false,
setDidDragInStream: (didDragInStream) => {
set({ didDragInStream })
},
// For stream event handling
cmdId: undefined,
setCmdId: (cmdId) => set({ cmdId }),
fileId: '',
setFileId: (fileId) => set({ fileId }),
streamDimensions: { streamWidth: 1280, streamHeight: 720 },
setStreamDimensions: (streamDimensions) => set({ streamDimensions }),
// tauri specific app settings
defaultDir: {

8
yarn.lock
View File

@ -1740,10 +1740,10 @@
resolved "https://registry.yarnpkg.com/@juggle/resize-observer/-/resize-observer-3.4.0.tgz#08d6c5e20cf7e4cc02fd181c4b0c225cd31dbb60"
integrity sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA==
"@kittycad/lib@^0.0.24":
version "0.0.24"
resolved "https://registry.yarnpkg.com/@kittycad/lib/-/lib-0.0.24.tgz#afc2f0baf8b742344e86332a60e89e85804b4b30"
integrity sha512-RT3EThq0s7DQFT9+8HQ9yRgIb5+Q2xwYFYl/aNel5DdkfAya3WGlwhjv2YOtMXsy981JVpabo0HXD3tLfm/9QA==
"@kittycad/lib@^0.0.27":
version "0.0.27"
resolved "https://registry.yarnpkg.com/@kittycad/lib/-/lib-0.0.27.tgz#6619494e365b6dfe1e400835d52b56a5d5128caa"
integrity sha512-oRISiGJghEVm9hUs8y8ZYGRw4HFDkvK51E71ADKrNp+Mtmrzwjzt2sEj0C3yBQ8kKjeUiCeJU2EHSf2kYfD9bw==
dependencies:
node-fetch "3.3.2"
openapi-types "^12.0.0"