Add Client-Side Gizmo (#2354)

* draft #2279

Add client side gizmo #2279, work in progress

* draft #2279

unreliableSubscriptions

* draft #2279

nice Gizmo

* blue ring

give the canvas a round shape and a border, wrapping rounded div element around the canvas

* Refactor Gizmo Component

Extracted reusable constants
Modularized the code
Simplified the useEffect logic
Added TypeScript type annotations
Improved overall code structure and readability

* remove old gizmo

* fmt

* styling and relocation

 Add className "pointer-events-none" to gizmo wrapper div (for now to prevent context menu)
 Make LowerRightControls container element have these classNames: flex flex-col items-end gap-3
 Move gizmo into LowerRightControls.tsx as the first child of the section element
 Remove the fixed styling from the gizmo div so it flows in flexbox

* fmt

* fix camera up problem

* A snapshot a day keeps the bugs away! 📷🐛 (OS: ubuntu)

* up tweak

* Revert "up tweak"

This reverts commit a53a0ef240.

* test tweak

* tweak test

---------

Co-authored-by: Kurt Hutten Irev-Dev <k.hutten@protonmail.ch>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Frank Noirot <frank@kittycad.io>

e2e/playwright/flow-tests.spec.ts

@@ -1,7 +1,7 @@
 import { test, expect } from '@playwright/test'
 import { makeTemplate, getUtils } from './test-utils'
 import waitOn from 'wait-on'
-import { roundOff } from 'lib/utils'
+import { roundOff, uuidv4 } from 'lib/utils'
 import { SaveSettingsPayload } from 'lib/settings/settingsTypes'
 import { secrets } from './secrets'
 import {
@@ -14,6 +14,7 @@ import {
 import * as TOML from '@iarna/toml'
 import { Coords2d } from 'lang/std/sketch'
 import { KCL_DEFAULT_LENGTH } from 'lib/constants'
+import { EngineCommand } from 'lang/std/engineConnection'
 
 /*
 debug helper: unfortunately we do rely on exact coord mouse clicks in a few places
@@ -165,7 +166,26 @@ test('Can moving camera', async ({ page, context }) => {
     // We could break them out into separate tests, but the longest past of the test is waiting
     // for the stream to start, so it can be good to bundle related things together.
 
-    await u.updateCamPosition(camPos)
+    const camCommand: EngineCommand = {
+      type: 'modeling_cmd_req',
+      cmd_id: uuidv4(),
+      cmd: {
+        type: 'default_camera_look_at',
+        center: { x: 0, y: 0, z: 0 },
+        vantage: { x: camPos[0], y: camPos[1], z: camPos[2] },
+        up: { x: 0, y: 0, z: 1 },
+      },
+    }
+    const updateCamCommand: EngineCommand = {
+      type: 'modeling_cmd_req',
+      cmd_id: uuidv4(),
+      cmd: {
+        type: 'default_camera_get_settings',
+      },
+    }
+    await u.sendCustomCmd(camCommand)
+    await page.waitForTimeout(100)
+    await u.sendCustomCmd(updateCamCommand)
     await page.waitForTimeout(100)
 
     // rotate
@@ -225,9 +245,29 @@ test('Can moving camera', async ({ page, context }) => {
     await page.mouse.move(700, 200, { steps: 2 })
     await page.mouse.up({ button: 'right' })
     await page.keyboard.up('Shift')
-  }, [-10, -85, -85])
+  }, [-19, -85, -85])
 
-  await u.updateCamPosition(camPos)
+  const camCommand: EngineCommand = {
+    type: 'modeling_cmd_req',
+    cmd_id: uuidv4(),
+    cmd: {
+      type: 'default_camera_look_at',
+      center: { x: 0, y: 0, z: 0 },
+      vantage: { x: camPos[0], y: camPos[1], z: camPos[2] },
+      up: { x: 0, y: 0, z: 1 },
+    },
+  }
+  const updateCamCommand: EngineCommand = {
+    type: 'modeling_cmd_req',
+    cmd_id: uuidv4(),
+    cmd: {
+      type: 'default_camera_get_settings',
+    },
+  }
+  await u.sendCustomCmd(camCommand)
+  await page.waitForTimeout(100)
+  await u.sendCustomCmd(updateCamCommand)
+  await page.waitForTimeout(100)
 
   await u.clearCommandLogs()
   await u.closeDebugPanel()
@@ -263,7 +303,7 @@ test('Can moving camera', async ({ page, context }) => {
   await bakeInRetries(async () => {
     await page.mouse.move(700, 400)
     await page.mouse.wheel(0, -100)
-  }, [1, -94, -94])
+  }, [1, -68, -68])
 })
 
 test('if you click the format button it formats your code', async ({
@@ -626,10 +666,24 @@ const sketchOnPlaneAndBackSideTest = async (
   await u.waitForAuthSkipAppStart()
   await u.openDebugPanel()
 
-  const camCmdBackSide: [number, number, number] = [-100, -100, -100]
+  const coord =
-  let camPos: [number, number, number] = [100, 100, 100]
+    plane === '-XY' || plane === '-YZ' || plane === 'XZ' ? -100 : 100
-  if (plane === '-XY' || plane === '-YZ' || plane === 'XZ') {
+  const camCommand: EngineCommand = {
-    camPos = camCmdBackSide
+    type: 'modeling_cmd_req',
+    cmd_id: uuidv4(),
+    cmd: {
+      type: 'default_camera_look_at',
+      center: { x: 0, y: 0, z: 0 },
+      vantage: { x: coord, y: coord, z: coord },
+      up: { x: 0, y: 0, z: 1 },
+    },
+  }
+  const updateCamCommand: EngineCommand = {
+    type: 'modeling_cmd_req',
+    cmd_id: uuidv4(),
+    cmd: {
+      type: 'default_camera_get_settings',
+    },
   }
 
   const code = `const part001 = startSketchOn('${plane}')
@@ -639,7 +693,10 @@ const sketchOnPlaneAndBackSideTest = async (
 
   await u.clearCommandLogs()
   await page.getByRole('button', { name: 'Start Sketch' }).click()
-  await u.updateCamPosition(camPos)
+
+  await u.sendCustomCmd(camCommand)
+  await page.waitForTimeout(100)
+  await u.sendCustomCmd(updateCamCommand)
 
   await u.closeDebugPanel()
   await page.mouse.click(clickCoords.x, clickCoords.y)

src/clientSideScene/CameraControls.ts

@@ -20,6 +20,7 @@ import {
   EngineCommand,
   Subscription,
   EngineCommandManager,
+  UnreliableSubscription,
 } from 'lang/std/engineConnection'
 import { uuidv4 } from 'lib/utils'
 import { deg2Rad } from 'lib/utils2d'
@@ -232,9 +233,18 @@ export class CameraControls {
     this.update()
     this._usePerspectiveCamera()
 
-    const cb: Subscription<
+    type CallBackParam = Parameters<
-      'default_camera_zoom' | 'camera_drag_end' | 'default_camera_get_settings'
+      (
-    >['callback'] = ({ data, type }) => {
+        | Subscription<
+            | 'default_camera_zoom'
+            | 'camera_drag_end'
+            | 'default_camera_get_settings'
+          >
+        | UnreliableSubscription<'camera_drag_move'>
+      )['callback']
+    >[0]
+
+    const cb = ({ data, type }: CallBackParam) => {
       const camSettings = data.settings
       this.camera.position.set(
         camSettings.pos.x,
@@ -246,7 +256,13 @@ export class CameraControls {
         camSettings.center.y,
         camSettings.center.z
       )
-      this.camera.up.set(camSettings.up.x, camSettings.up.y, camSettings.up.z)
+      const quat = new Quaternion(
+        camSettings.orientation.x,
+        camSettings.orientation.y,
+        camSettings.orientation.z,
+        camSettings.orientation.w
+      ).invert()
+      this.camera.up.copy(new Vector3(0, 1, 0).applyQuaternion(quat))
       if (this.camera instanceof PerspectiveCamera && camSettings.ortho) {
         this.useOrthographicCamera()
       }
@@ -287,6 +303,10 @@ export class CameraControls {
         event: 'default_camera_get_settings',
         callback: cb,
       })
+      this.engineCommandManager.subscribeToUnreliable({
+        event: 'camera_drag_move',
+        callback: cb,
+      })
     })
   }
 

src/components/Gizmo.tsx

@@ -0,0 +1,146 @@
+import { sceneInfra } from 'lib/singletons'
+import { useEffect, useRef } from 'react'
+import {
+  WebGLRenderer,
+  Scene,
+  OrthographicCamera,
+  BoxGeometry,
+  SphereGeometry,
+  MeshBasicMaterial,
+  Color,
+  Mesh,
+  Clock,
+  Quaternion,
+  ColorRepresentation,
+} from 'three'
+
+const CANVAS_SIZE = 80
+const FRUSTUM_SIZE = 0.5
+const AXIS_LENGTH = 0.35
+const AXIS_WIDTH = 0.02
+const AXIS_COLORS = {
+  x: '#fa6668',
+  y: '#11eb6b',
+  z: '#6689ef',
+  gray: '#c6c7c2',
+}
+
+export default function Gizmo() {
+  const canvasRef = useRef<HTMLCanvasElement | null>(null)
+
+  useEffect(() => {
+    if (!canvasRef.current) return
+
+    const canvas = canvasRef.current
+    const renderer = new WebGLRenderer({ canvas, antialias: true, alpha: true })
+    renderer.setSize(CANVAS_SIZE, CANVAS_SIZE)
+    renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2))
+
+    const scene = new Scene()
+    const camera = createCamera()
+    const { gizmoAxes, gizmoAxisHeads } = createGizmo()
+    scene.add(...gizmoAxes, ...gizmoAxisHeads)
+
+    const clock = new Clock()
+    const clientCamera = sceneInfra.camControls.camera
+    let currentQuaternion = new Quaternion().copy(clientCamera.quaternion)
+
+    const animate = () => {
+      requestAnimationFrame(animate)
+      updateCameraOrientation(
+        camera,
+        currentQuaternion,
+        sceneInfra.camControls.camera.quaternion,
+        clock.getDelta()
+      )
+      renderer.render(scene, camera)
+    }
+    animate()
+
+    return () => {
+      renderer.dispose()
+    }
+  }, [])
+
+  return (
+    <div className="grid place-content-center rounded-full overflow-hidden border border-solid border-primary/50 pointer-events-none">
+      <canvas ref={canvasRef} />
+    </div>
+  )
+}
+
+const createCamera = () => {
+  return new OrthographicCamera(
+    -FRUSTUM_SIZE,
+    FRUSTUM_SIZE,
+    FRUSTUM_SIZE,
+    -FRUSTUM_SIZE,
+    0.5,
+    3
+  )
+}
+
+const createGizmo = () => {
+  const gizmoAxes = [
+    createAxis(AXIS_LENGTH, AXIS_WIDTH, AXIS_COLORS.x, 0, 'z'),
+    createAxis(AXIS_LENGTH, AXIS_WIDTH, AXIS_COLORS.y, Math.PI / 2, 'z'),
+    createAxis(AXIS_LENGTH, AXIS_WIDTH, AXIS_COLORS.z, -Math.PI / 2, 'y'),
+    createAxis(AXIS_LENGTH, AXIS_WIDTH, AXIS_COLORS.gray, Math.PI, 'z'),
+    createAxis(AXIS_LENGTH, AXIS_WIDTH, AXIS_COLORS.gray, -Math.PI / 2, 'z'),
+    createAxis(AXIS_LENGTH, AXIS_WIDTH, AXIS_COLORS.gray, Math.PI / 2, 'y'),
+  ]
+
+  const gizmoAxisHeads = [
+    createAxisHead(AXIS_LENGTH, AXIS_COLORS.x, 0, 'z'),
+    createAxisHead(AXIS_LENGTH, AXIS_COLORS.y, Math.PI / 2, 'z'),
+    createAxisHead(AXIS_LENGTH, AXIS_COLORS.z, -Math.PI / 2, 'y'),
+    createAxisHead(AXIS_LENGTH, AXIS_COLORS.gray, Math.PI, 'z'),
+    createAxisHead(AXIS_LENGTH, AXIS_COLORS.gray, -Math.PI / 2, 'z'),
+    createAxisHead(AXIS_LENGTH, AXIS_COLORS.gray, Math.PI / 2, 'y'),
+  ]
+
+  return { gizmoAxes, gizmoAxisHeads }
+}
+
+const createAxis = (
+  length: number,
+  width: number,
+  color: ColorRepresentation,
+  rotation = 0,
+  axis = 'x'
+) => {
+  const geometry = new BoxGeometry(length, width, width).translate(
+    length / 2,
+    0,
+    0
+  )
+  const material = new MeshBasicMaterial({ color: new Color(color) })
+  const mesh = new Mesh(geometry, material)
+  mesh.rotation[axis as 'x' | 'y' | 'z'] = rotation
+  return mesh
+}
+
+const createAxisHead = (
+  length: number,
+  color: ColorRepresentation,
+  rotation = 0,
+  axis = 'x'
+) => {
+  const geometry = new SphereGeometry(0.065, 16, 8).translate(length, 0, 0)
+  const material = new MeshBasicMaterial({ color: new Color(color) })
+  const mesh = new Mesh(geometry, material)
+  mesh.rotation[axis as 'x' | 'y' | 'z'] = rotation
+  return mesh
+}
+
+const updateCameraOrientation = (
+  camera: OrthographicCamera,
+  currentQuaternion: Quaternion,
+  targetQuaternion: Quaternion,
+  deltaTime: number
+) => {
+  const slerpFactor = 1 - Math.exp(-30 * deltaTime)
+  currentQuaternion.slerp(targetQuaternion, slerpFactor).normalize()
+  camera.position.set(0, 0, 1).applyQuaternion(currentQuaternion)
+  camera.quaternion.copy(currentQuaternion)
+}

src/components/LowerRightControls.tsx

@@ -1,6 +1,7 @@
 import { APP_VERSION } from 'routes/Settings'
 import { CustomIcon } from 'components/CustomIcon'
 import Tooltip from 'components/Tooltip'
+import Gizmo from 'components/Gizmo'
 import { paths } from 'lib/paths'
 import { NetworkHealthIndicator } from 'components/NetworkHealthIndicator'
 import { HelpMenu } from './HelpMenu'
@@ -14,8 +15,9 @@ export function LowerRightControls(props: React.PropsWithChildren) {
     '!text-chalkboard-70 hover:!text-chalkboard-80 dark:!text-chalkboard-40 dark:hover:!text-chalkboard-30'
 
   return (
-    <section className="fixed bottom-2 right-2">
+    <section className="fixed bottom-2 right-2 flex flex-col items-end gap-3">
       {props.children}
+      <Gizmo />
       <menu className="flex items-center justify-end gap-3">
         <a
           href={`https://github.com/KittyCAD/modeling-app/releases/tag/v${APP_VERSION}`}

src/lang/std/engineConnection.ts

@@ -590,6 +590,8 @@ class EngineConnection {
               ) {
                 this.engineCommandManager.inSequence = result.data.sequence
                 callback(result)
+              } else if (result.type !== 'highlight_set_entity') {
+                callback(result)
               }
             }
           )
@@ -907,7 +909,7 @@ type UnreliableResponses = Extract<
   Models['OkModelingCmdResponse_type'],
   { type: 'highlight_set_entity' | 'camera_drag_move' }
 >
-interface UnreliableSubscription<T extends UnreliableResponses['type']> {
+export interface UnreliableSubscription<T extends UnreliableResponses['type']> {
   event: T
   callback: (data: Extract<UnreliableResponses, { type: T }>) => void
 }
@@ -1119,24 +1121,6 @@ export class EngineCommandManager {
           },
         })
 
-        // Make the axis gizmo.
-        // We do this after the connection opened to avoid a race condition.
-        // Connected opened is the last thing that happens when the stream
-        // is ready.
-        // We also do this here because we want to ensure we create the gizmo
-        // and execute the code everytime the stream is restarted.
-        const gizmoId = uuidv4()
-        void this.sendSceneCommand({
-          type: 'modeling_cmd_req',
-          cmd_id: gizmoId,
-          cmd: {
-            type: 'make_axes_gizmo',
-            clobber: false,
-            // If true, axes gizmo will be placed in the corner of the screen.
-            // If false, it will be placed at the origin of the scene.
-            gizmo_mode: true,
-          },
-        })
         this._camControlsCameraChange()
         this.sendSceneCommand({
           // CameraControls subscribes to default_camera_get_settings response events
@@ -1420,17 +1404,6 @@ export class EngineCommandManager {
     this.lastArtifactMap = this.artifactMap
     this.artifactMap = {}
     await this.initPlanes()
-    await this.sendSceneCommand({
-      type: 'modeling_cmd_req',
-      cmd_id: uuidv4(),
-      cmd: {
-        type: 'make_axes_gizmo',
-        clobber: false,
-        // If true, axes gizmo will be placed in the corner of the screen.
-        // If false, it will be placed at the origin of the scene.
-        gizmo_mode: true,
-      },
-    })
   }
   subscribeTo<T extends ModelTypes>({
     event,