skip network tests mac

* Trigger shutdown operations after each test

* Idle mode states

* Don't show the reconnect when coming back from tab

e2e/playwright/flow-tests.spec.ts

@@ -52,7 +52,24 @@ const commonPoints = {
   // num2: 19.19,
 }
 
-// Utilities for writing tests that depend on test values
+test.afterEach(async ({ context, page }, testInfo) => {
+  if (testInfo.status === 'skipped') return
+  if (testInfo.status === 'failed') return
+
+  const u = await getUtils(page)
+  // Kill the network so shutdown happens properly
+  await u.emulateNetworkConditions({
+    offline: true,
+    // values of 0 remove any active throttling. crbug.com/456324#c9
+    latency: 0,
+    downloadThroughput: -1,
+    uploadThroughput: -1,
+  })
+
+  // It seems it's best to give the browser about 3s to close things
+  // It's not super reliable but we have no real other choice for now
+  await page.waitForTimeout(3000)
+})
 
 test.beforeEach(async ({ context, page }) => {
   // wait for Vite preview server to be up
@@ -78,7 +95,7 @@ test.beforeEach(async ({ context, page }) => {
   await page.emulateMedia({ reducedMotion: 'reduce' })
 })
 
-test.setTimeout(60000)
+test.setTimeout(120000)
 
 async function doBasicSketch(page: Page, openPanes: string[]) {
   const u = await getUtils(page)
@@ -6705,6 +6722,11 @@ ${extraLine ? 'const myVar = segLen(seg01, part001)' : ''}`
 
 test.describe('Test network and connection issues', () => {
   test('simulate network down and network little widget', async ({ page }) => {
+    const browserType = page.context().browser()?.browserType().name()
+    test.skip(
+      browserType !== 'chromium',
+      'emulateNetworkConditions only works in chromium'
+    )
     const u = await getUtils(page)
     await page.setViewportSize({ width: 1200, height: 500 })
 
@@ -6775,6 +6797,11 @@ test.describe('Test network and connection issues', () => {
   })
 
   test('Engine disconnect & reconnect in sketch mode', async ({ page }) => {
+    const browserType = page.context().browser()?.browserType().name()
+    test.skip(
+      browserType !== 'chromium',
+      'emulateNetworkConditions only works in chromium'
+    )
     const u = await getUtils(page)
     await page.setViewportSize({ width: 1200, height: 500 })
     const PUR = 400 / 37.5 //pixeltoUnitRatio

e2e/playwright/test-utils.ts

@@ -312,9 +312,9 @@ export async function getUtils(page: Page) {
         fullPage: true,
       })
       const screenshot = await PNG.sync.read(buffer)
-      // most likely related to pixel density but the screenshots for webkit are 2x the size
-      // there might be a more robust way of doing this.
-      const pixMultiplier = browserType === 'webkit' ? 2 : 1
+      const pixMultiplier: number = await page.evaluate(
+        'window.devicePixelRatio'
+      )
       const index =
         (screenshot.width * coords.y * pixMultiplier +
           coords.x * pixMultiplier) *
@@ -377,11 +377,13 @@ export async function getUtils(page: Page) {
     emulateNetworkConditions: async (
       networkOptions: Protocol.Network.emulateNetworkConditionsParameters
     ) => {
-      // Skip on non-Chromium browsers, since we need to use the CDP.
-      test.skip(
-        cdpSession === null,
-        'Network emulation is only supported in Chromium'
-      )
+      if (browserType !== 'chromium') {
+        console.warn('emulateNetworkConditions will not work on this browser')
+      }
+      if (cdpSession === null) {
+        // Use a fail safe if we can't simulate disconnect (on Safari)
+        return page.evaluate('window.tearDown()')
+      }
 
       cdpSession?.send('Network.emulateNetworkConditions', networkOptions)
     },

src/clientSideScene/ClientSideSceneComp.tsx

@@ -19,10 +19,7 @@ import {
   PROFILE_START,
   getParentGroup,
 } from './sceneEntities'
-import {
-  SegmentOverlay,
-  ModelingMachineContext as ModelingContextType,
-} from 'machines/modelingMachine'
+import { SegmentOverlay, SketchDetails } from 'machines/modelingMachine'
 import { findUsesOfTagInPipe, getNodeFromPath } from 'lang/queryAst'
 import {
   CallExpression,
@@ -356,12 +353,11 @@ export const confirmModal = create<ConfirmModalProps, boolean, boolean>(
 
 export async function deleteSegment({
   pathToNode,
-  context,
+  sketchDetails,
 }: {
   pathToNode: PathToNode
-  context: ModelingContextType
+  sketchDetails: SketchDetails | null
 }) {
-  const { sketchDetails } = context
   let modifiedAst: Program | Error = kclManager.ast
   const dependentRanges = findUsesOfTagInPipe(modifiedAst, pathToNode)
 
@@ -398,7 +394,13 @@ export async function deleteSegment({
   }
 
   if (!sketchDetails) return
-  await sceneEntitiesManager.updateAstAndRejigSketch(modifiedAst, context)
+  await sceneEntitiesManager.updateAstAndRejigSketch(
+    pathToNode,
+    modifiedAst,
+    sketchDetails.zAxis,
+    sketchDetails.yAxis,
+    sketchDetails.origin
+  )
 
   // Now 'Set sketchDetails' is called with the modified pathToNode
 }

src/clientSideScene/sceneCallbacks.ts

@@ -1,362 +0,0 @@
-import { getEventForSegmentSelection } from 'lib/selections'
-import {
-  editorManager,
-  kclManager,
-  sceneEntitiesManager,
-  sceneInfra,
-} from 'lib/singletons'
-import {
-  Intersection,
-  Mesh,
-  MeshBasicMaterial,
-  Object3D,
-  Object3DEventMap,
-  OrthographicCamera,
-  Points,
-  Vector2,
-  Vector3,
-} from 'three'
-import {
-  EXTRA_SEGMENT_HANDLE,
-  PROFILE_START,
-  STRAIGHT_SEGMENT,
-  TANGENTIAL_ARC_TO_SEGMENT,
-  getParentGroup,
-  sketchGroupFromPathToNode,
-} from './sceneEntities'
-import { trap } from 'lib/trap'
-import { addNewSketchLn } from 'lang/std/sketch'
-import { CallExpression, PathToNode, parse, recast } from 'lang/wasm'
-import { ARROWHEAD, X_AXIS, Y_AXIS } from './sceneInfra'
-import { getNodeFromPath } from 'lang/queryAst'
-import { getThemeColorForThreeJs } from 'lib/theme'
-import { orthoScale, perspScale, quaternionFromUpNForward } from './helpers'
-import { ModelingMachineContext } from 'machines/modelingMachine'
-import { addStartProfileAt } from 'lang/modifyAst'
-
-export interface OnMouseEnterLeaveArgs {
-  selected: Object3D<Object3DEventMap>
-  dragSelected?: Object3D<Object3DEventMap>
-  mouseEvent: MouseEvent
-}
-
-export interface OnDragCallbackArgs extends OnMouseEnterLeaveArgs {
-  intersectionPoint: {
-    twoD: Vector2
-    threeD: Vector3
-  }
-  intersects: Intersection<Object3D<Object3DEventMap>>[]
-}
-export interface OnClickCallbackArgs {
-  mouseEvent: MouseEvent
-  intersectionPoint?: {
-    twoD: Vector2
-    threeD: Vector3
-  }
-  intersects: Intersection<Object3D<Object3DEventMap>>[]
-  selected?: Object3D<Object3DEventMap>
-}
-
-export interface OnMoveCallbackArgs {
-  mouseEvent: MouseEvent
-  intersectionPoint: {
-    twoD: Vector2
-    threeD: Vector3
-  }
-  intersects: Intersection<Object3D<Object3DEventMap>>[]
-  selected?: Object3D<Object3DEventMap>
-}
-
-interface Callbacks {
-  onDragStart: (arg: OnDragCallbackArgs) => void
-  onDragEnd: (arg: OnDragCallbackArgs) => void
-  onDrag: (arg: OnDragCallbackArgs) => void
-  onMove: (arg: OnMoveCallbackArgs) => void
-  onClick: (arg: OnClickCallbackArgs) => void
-  onMouseEnter: (arg: OnMouseEnterLeaveArgs) => void
-  onMouseLeave: (arg: OnMouseEnterLeaveArgs) => void
-}
-
-type SetCallbacksWithCtx = (context: ModelingMachineContext) => Callbacks
-
-const dummyListenersAll: Callbacks = {
-  onDragStart: () => {},
-  onDragEnd: () => {},
-  onDrag: () => {},
-  onMove: () => {},
-  onClick: () => {},
-  onMouseEnter: () => {},
-  onMouseLeave: () => {},
-}
-
-const onMousEnterLeaveCallbacks = {
-  onMouseEnter: ({ selected, dragSelected }: OnMouseEnterLeaveArgs) => {
-    if ([X_AXIS, Y_AXIS].includes(selected?.userData?.type)) {
-      const obj = selected as Mesh
-      const mat = obj.material as MeshBasicMaterial
-      mat.color.set(obj.userData.baseColor)
-      mat.color.offsetHSL(0, 0, 0.5)
-    }
-    const parent = getParentGroup(selected, [
-      STRAIGHT_SEGMENT,
-      TANGENTIAL_ARC_TO_SEGMENT,
-      PROFILE_START,
-    ])
-    if (parent?.userData?.pathToNode) {
-      const updatedAst = parse(recast(kclManager.ast))
-      if (trap(updatedAst)) return
-      const _node = getNodeFromPath<CallExpression>(
-        updatedAst,
-        parent.userData.pathToNode,
-        'CallExpression'
-      )
-      if (trap(_node, { suppress: true })) return
-      const node = _node.node
-      editorManager.setHighlightRange([node.start, node.end])
-      const yellow = 0xffff00
-      colorSegment(selected, yellow)
-      const extraSegmentGroup = parent.getObjectByName(EXTRA_SEGMENT_HANDLE)
-      if (extraSegmentGroup) {
-        extraSegmentGroup.traverse((child) => {
-          if (child instanceof Points || child instanceof Mesh) {
-            child.material.opacity = dragSelected ? 0 : 1
-          }
-        })
-      }
-      const orthoFactor = orthoScale(sceneInfra.camControls.camera)
-
-      const factor =
-        (sceneInfra.camControls.camera instanceof OrthographicCamera
-          ? orthoFactor
-          : perspScale(sceneInfra.camControls.camera, parent)) /
-        sceneInfra._baseUnitMultiplier
-      if (parent.name === STRAIGHT_SEGMENT) {
-        sceneEntitiesManager.updateStraightSegment({
-          from: parent.userData.from,
-          to: parent.userData.to,
-          group: parent,
-          scale: factor,
-        })
-      } else if (parent.name === TANGENTIAL_ARC_TO_SEGMENT) {
-        sceneEntitiesManager.updateTangentialArcToSegment({
-          prevSegment: parent.userData.prevSegment,
-          from: parent.userData.from,
-          to: parent.userData.to,
-          group: parent,
-          scale: factor,
-        })
-      }
-      return
-    }
-    editorManager.setHighlightRange([0, 0])
-  },
-  onMouseLeave: ({ selected, ...rest }: OnMouseEnterLeaveArgs) => {
-    editorManager.setHighlightRange([0, 0])
-    const parent = getParentGroup(selected, [
-      STRAIGHT_SEGMENT,
-      TANGENTIAL_ARC_TO_SEGMENT,
-      PROFILE_START,
-    ])
-    if (parent) {
-      const orthoFactor = orthoScale(sceneInfra.camControls.camera)
-
-      const factor =
-        (sceneInfra.camControls.camera instanceof OrthographicCamera
-          ? orthoFactor
-          : perspScale(sceneInfra.camControls.camera, parent)) /
-        sceneInfra._baseUnitMultiplier
-      if (parent.name === STRAIGHT_SEGMENT) {
-        sceneEntitiesManager.updateStraightSegment({
-          from: parent.userData.from,
-          to: parent.userData.to,
-          group: parent,
-          scale: factor,
-        })
-      } else if (parent.name === TANGENTIAL_ARC_TO_SEGMENT) {
-        sceneEntitiesManager.updateTangentialArcToSegment({
-          prevSegment: parent.userData.prevSegment,
-          from: parent.userData.from,
-          to: parent.userData.to,
-          group: parent,
-          scale: factor,
-        })
-      }
-    }
-    const isSelected = parent?.userData?.isSelected
-    colorSegment(
-      selected,
-      isSelected
-        ? 0x0000ff
-        : parent?.userData?.baseColor ||
-            getThemeColorForThreeJs(sceneInfra._theme)
-    )
-    const extraSegmentGroup = parent?.getObjectByName(EXTRA_SEGMENT_HANDLE)
-    if (extraSegmentGroup) {
-      extraSegmentGroup.traverse((child) => {
-        if (child instanceof Points || child instanceof Mesh) {
-          child.material.opacity = 0
-        }
-      })
-    }
-    if ([X_AXIS, Y_AXIS].includes(selected?.userData?.type)) {
-      const obj = selected as Mesh
-      const mat = obj.material as MeshBasicMaterial
-      mat.color.set(obj.userData.baseColor)
-      if (obj.userData.isSelected) mat.color.offsetHSL(0, 0, 0.2)
-    }
-  },
-} as const
-
-export const idleCallbacks: SetCallbacksWithCtx = (context) => {
-  let addingNewSegmentStatus: 'nothing' | 'pending' | 'added' = 'nothing'
-  return {
-    onDragStart: () => {},
-    onDragEnd: () => {},
-    onMove: () => {},
-    ...onMousEnterLeaveCallbacks,
-    onClick: (args) => {
-      if (args?.mouseEvent.which !== 1) return
-      if (!args || !args.selected) {
-        sceneInfra.modelingSend({
-          type: 'Set selection',
-          data: {
-            selectionType: 'singleCodeCursor',
-          },
-        })
-        return
-      }
-      const { selected } = args
-      const event = getEventForSegmentSelection(selected)
-      if (!event) return
-      sceneInfra.modelingSend(event)
-    },
-    onDrag: async ({ selected, intersectionPoint, mouseEvent, intersects }) => {
-      if (mouseEvent.which !== 1) return
-
-      const group = getParentGroup(selected, [EXTRA_SEGMENT_HANDLE])
-      if (group?.name === EXTRA_SEGMENT_HANDLE) {
-        const segGroup = getParentGroup(selected)
-        const pathToNode: PathToNode = segGroup?.userData?.pathToNode
-        const pathToNodeIndex = pathToNode.findIndex(
-          (x) => x[1] === 'PipeExpression'
-        )
-
-        const sketchGroup = sketchGroupFromPathToNode({
-          pathToNode,
-          ast: kclManager.ast,
-          programMemory: kclManager.programMemory,
-        })
-        if (trap(sketchGroup)) return
-
-        const pipeIndex = pathToNode[pathToNodeIndex + 1][0] as number
-        if (addingNewSegmentStatus === 'nothing') {
-          const prevSegment = sketchGroup.value[pipeIndex - 2]
-          const mod = addNewSketchLn({
-            node: kclManager.ast,
-            programMemory: kclManager.programMemory,
-            to: [intersectionPoint.twoD.x, intersectionPoint.twoD.y],
-            from: [prevSegment.from[0], prevSegment.from[1]],
-            // TODO assuming it's always a straight segments being added
-            // as this is easiest, and we'll need to add "tabbing" behavior
-            // to support other segment types
-            fnName: 'line',
-            pathToNode: pathToNode,
-            spliceBetween: true,
-          })
-          addingNewSegmentStatus = 'pending'
-          if (trap(mod)) return
-
-          await kclManager.executeAstMock(mod.modifiedAst)
-          await sceneEntitiesManager.tearDownSketch({ removeAxis: false })
-          sceneEntitiesManager.setupSketch({
-            sketchPathToNode: pathToNode,
-            maybeModdedAst: kclManager.ast,
-            up: context.sketchDetails?.yAxis || [0, 1, 0],
-            forward: context.sketchDetails?.zAxis || [0, 0, 1],
-            position: context.sketchDetails?.origin || [0, 0, 0],
-          })
-          addingNewSegmentStatus = 'added'
-        } else if (addingNewSegmentStatus === 'added') {
-          const pathToNodeForNewSegment = pathToNode.slice(0, pathToNodeIndex)
-          pathToNodeForNewSegment.push([pipeIndex - 2, 'index'])
-          sceneEntitiesManager.onDragSegment({
-            sketchPathToNode: pathToNodeForNewSegment,
-            object: selected,
-            intersection2d: intersectionPoint.twoD,
-            intersects,
-          })
-        }
-        return
-      }
-
-      sceneEntitiesManager.onDragSegment({
-        object: selected,
-        intersection2d: intersectionPoint.twoD,
-        intersects,
-        sketchPathToNode: context.sketchDetails?.sketchPathToNode || [],
-      })
-    },
-  }
-}
-
-export const Sketch_LineTool_NoPoints: SetCallbacksWithCtx = ({
-  sketchDetails,
-}) => {
-  if (!sketchDetails) return dummyListenersAll
-  sceneEntitiesManager.createIntersectionPlane()
-  const quaternion = quaternionFromUpNForward(
-    new Vector3(...sketchDetails.yAxis),
-    new Vector3(...sketchDetails.zAxis)
-  )
-  sceneEntitiesManager.intersectionPlane &&
-    sceneEntitiesManager.intersectionPlane.setRotationFromQuaternion(quaternion)
-  sceneEntitiesManager.intersectionPlane &&
-    sceneEntitiesManager.intersectionPlane.position.copy(
-      new Vector3(...(sketchDetails?.origin || [0, 0, 0]))
-    )
-  return {
-    ...dummyListenersAll,
-    onClick: async (args) => {
-      if (!args) return
-      if (args.mouseEvent.which !== 1) return
-      const { intersectionPoint } = args
-      if (!intersectionPoint?.twoD || !sketchDetails?.sketchPathToNode) return
-      const addStartProfileAtRes = addStartProfileAt(
-        kclManager.ast,
-        sketchDetails.sketchPathToNode,
-        [intersectionPoint.twoD.x, intersectionPoint.twoD.y]
-      )
-
-      if (trap(addStartProfileAtRes)) return
-      const { modifiedAst } = addStartProfileAtRes
-
-      await kclManager.updateAst(modifiedAst, false)
-      sceneEntitiesManager.removeIntersectionPlane()
-      sceneInfra.modelingSend('Add start point')
-    },
-  }
-}
-
-function colorSegment(object: any, color: number) {
-  const segmentHead = getParentGroup(object, [ARROWHEAD, PROFILE_START])
-  if (segmentHead) {
-    segmentHead.traverse((child) => {
-      if (child instanceof Mesh) {
-        child.material.color.set(color)
-      }
-    })
-    return
-  }
-  const straightSegmentBody = getParentGroup(object, [
-    STRAIGHT_SEGMENT,
-    TANGENTIAL_ARC_TO_SEGMENT,
-  ])
-  if (straightSegmentBody) {
-    straightSegmentBody.traverse((child) => {
-      if (child instanceof Mesh && !child.userData.ignoreColorChange) {
-        child.material.color.set(color)
-      }
-    })
-    return
-  }
-}

src/clientSideScene/sceneEntities.ts

@@ -22,8 +22,12 @@ import {
 import {
   ARROWHEAD,
   AXIS_GROUP,
+  DEFAULT_PLANES,
+  DefaultPlane,
+  defaultPlaneColor,
   getSceneScale,
   INTERSECTION_PLANE_LAYER,
+  OnMouseEnterLeaveArgs,
   RAYCASTABLE_PLANE,
   SKETCH_GROUP_SEGMENTS,
   SKETCH_LAYER,
@@ -78,16 +82,16 @@ import {
   createPipeSubstitution,
   findUniqueName,
 } from 'lang/modifyAst'
-import { Selections, sendSelectEventToEngine } from 'lib/selections'
+import {
+  Selections,
+  getEventForSegmentSelection,
+  sendSelectEventToEngine,
+} from 'lib/selections'
 import { getTangentPointFromPreviousArc } from 'lib/utils2d'
 import { createGridHelper, orthoScale, perspScale } from './helpers'
 import { Models } from '@kittycad/lib'
 import { uuidv4 } from 'lib/utils'
-import {
-  SegmentOverlayPayload,
-  SketchDetails,
-  ModelingMachineContext,
-} from 'machines/modelingMachine'
+import { SegmentOverlayPayload, SketchDetails } from 'machines/modelingMachine'
 import {
   ArtifactMapCommand,
   EngineCommandManager,
@@ -98,7 +102,6 @@ import {
 } from 'lib/rectangleTool'
 import { getThemeColorForThreeJs } from 'lib/theme'
 import { err, trap } from 'lib/trap'
-import { OnMouseEnterLeaveArgs, idleCallbacks } from './sceneCallbacks'
 
 type DraftSegment = 'line' | 'tangentialArcTo'
 
@@ -496,23 +499,31 @@ export class SceneEntities {
       variableDeclarationName,
     }
   }
-  updateAstAndRejigSketch: (
+  updateAstAndRejigSketch = async (
+    sketchPathToNode: PathToNode,
     modifiedAst: Program | Error,
-    context: ModelingMachineContext
-  ) => Promise<any> = async (modifiedAst, context) => {
+    forward: [number, number, number],
+    up: [number, number, number],
+    origin: [number, number, number]
+  ) => {
     if (err(modifiedAst)) return modifiedAst
 
     const nextAst = await kclManager.updateAst(modifiedAst, false)
     await this.tearDownSketch({ removeAxis: false })
     sceneInfra.resetMouseListeners()
     await this.setupSketch({
-      sketchPathToNode: context.sketchDetails?.sketchPathToNode || [],
-      forward: context.sketchDetails?.zAxis || [0, 0, 1],
-      up: context.sketchDetails?.yAxis || [0, 1, 0],
-      position: context.sketchDetails?.origin || [0, 0, 0],
+      sketchPathToNode,
+      forward,
+      up,
+      position: origin,
       maybeModdedAst: nextAst.newAst,
     })
-    this.setupSketchIdleCallbacks(context)
+    this.setupSketchIdleCallbacks({
+      forward,
+      up,
+      position: origin,
+      pathToNode: sketchPathToNode,
+    })
     return nextAst
   }
   setUpDraftSegment = async (
@@ -827,8 +838,128 @@ export class SceneEntities {
       },
     })
   }
-  setupSketchIdleCallbacks = (context: ModelingMachineContext) => {
-    sceneInfra.setCallbacks(idleCallbacks(context))
+  setupSketchIdleCallbacks = ({
+    pathToNode,
+    up,
+    forward,
+    position,
+  }: {
+    pathToNode: PathToNode
+    forward: [number, number, number]
+    up: [number, number, number]
+    position?: [number, number, number]
+  }) => {
+    let addingNewSegmentStatus: 'nothing' | 'pending' | 'added' = 'nothing'
+    sceneInfra.setCallbacks({
+      onDragEnd: async () => {
+        if (addingNewSegmentStatus !== 'nothing') {
+          await this.tearDownSketch({ removeAxis: false })
+          this.setupSketch({
+            sketchPathToNode: pathToNode,
+            maybeModdedAst: kclManager.ast,
+            up,
+            forward,
+            position,
+          })
+          // setting up the callbacks again resets value in closures
+          this.setupSketchIdleCallbacks({
+            pathToNode,
+            up,
+            forward,
+            position,
+          })
+        }
+      },
+      onDrag: async ({
+        selected,
+        intersectionPoint,
+        mouseEvent,
+        intersects,
+      }) => {
+        if (mouseEvent.which !== 1) return
+
+        const group = getParentGroup(selected, [EXTRA_SEGMENT_HANDLE])
+        if (group?.name === EXTRA_SEGMENT_HANDLE) {
+          const segGroup = getParentGroup(selected)
+          const pathToNode: PathToNode = segGroup?.userData?.pathToNode
+          const pathToNodeIndex = pathToNode.findIndex(
+            (x) => x[1] === 'PipeExpression'
+          )
+
+          const sketchGroup = sketchGroupFromPathToNode({
+            pathToNode,
+            ast: kclManager.ast,
+            programMemory: kclManager.programMemory,
+          })
+          if (trap(sketchGroup)) return
+
+          const pipeIndex = pathToNode[pathToNodeIndex + 1][0] as number
+          if (addingNewSegmentStatus === 'nothing') {
+            const prevSegment = sketchGroup.value[pipeIndex - 2]
+            const mod = addNewSketchLn({
+              node: kclManager.ast,
+              programMemory: kclManager.programMemory,
+              to: [intersectionPoint.twoD.x, intersectionPoint.twoD.y],
+              from: [prevSegment.from[0], prevSegment.from[1]],
+              // TODO assuming it's always a straight segments being added
+              // as this is easiest, and we'll need to add "tabbing" behavior
+              // to support other segment types
+              fnName: 'line',
+              pathToNode: pathToNode,
+              spliceBetween: true,
+            })
+            addingNewSegmentStatus = 'pending'
+            if (trap(mod)) return
+
+            await kclManager.executeAstMock(mod.modifiedAst)
+            await this.tearDownSketch({ removeAxis: false })
+            this.setupSketch({
+              sketchPathToNode: pathToNode,
+              maybeModdedAst: kclManager.ast,
+              up,
+              forward,
+              position,
+            })
+            addingNewSegmentStatus = 'added'
+          } else if (addingNewSegmentStatus === 'added') {
+            const pathToNodeForNewSegment = pathToNode.slice(0, pathToNodeIndex)
+            pathToNodeForNewSegment.push([pipeIndex - 2, 'index'])
+            this.onDragSegment({
+              sketchPathToNode: pathToNodeForNewSegment,
+              object: selected,
+              intersection2d: intersectionPoint.twoD,
+              intersects,
+            })
+          }
+          return
+        }
+
+        this.onDragSegment({
+          object: selected,
+          intersection2d: intersectionPoint.twoD,
+          intersects,
+          sketchPathToNode: pathToNode,
+        })
+      },
+      onMove: () => {},
+      onClick: (args) => {
+        if (args?.mouseEvent.which !== 1) return
+        if (!args || !args.selected) {
+          sceneInfra.modelingSend({
+            type: 'Set selection',
+            data: {
+              selectionType: 'singleCodeCursor',
+            },
+          })
+          return
+        }
+        const { selected } = args
+        const event = getEventForSegmentSelection(selected)
+        if (!event) return
+        sceneInfra.modelingSend(event)
+      },
+      ...this.mouseEnterLeaveCallbacks(),
+    })
   }
   prepareTruncatedMemoryAndAst = (
     sketchPathToNode: PathToNode,
@@ -1298,6 +1429,150 @@ export class SceneEntities {
       this._tearDownSketch(0, resolve, reject, { removeAxis })
     })
   }
+  setupDefaultPlaneHover() {
+    sceneInfra.setCallbacks({
+      onMouseEnter: ({ selected }) => {
+        if (!(selected instanceof Mesh && selected.parent)) return
+        if (selected.parent.userData.type !== DEFAULT_PLANES) return
+        const type: DefaultPlane = selected.userData.type
+        selected.material.color = defaultPlaneColor(type, 0.5, 1)
+      },
+      onMouseLeave: ({ selected }) => {
+        if (!(selected instanceof Mesh && selected.parent)) return
+        if (selected.parent.userData.type !== DEFAULT_PLANES) return
+        const type: DefaultPlane = selected.userData.type
+        selected.material.color = defaultPlaneColor(type)
+      },
+      onClick: async (args) => {
+        const { entity_id } = await sendSelectEventToEngine(
+          args?.mouseEvent,
+          document.getElementById('video-stream') as HTMLVideoElement,
+          sceneInfra._streamDimensions
+        )
+
+        let _entity_id = entity_id
+        if (!_entity_id) return
+        if (
+          engineCommandManager.defaultPlanes?.xy === _entity_id ||
+          engineCommandManager.defaultPlanes?.xz === _entity_id ||
+          engineCommandManager.defaultPlanes?.yz === _entity_id ||
+          engineCommandManager.defaultPlanes?.negXy === _entity_id ||
+          engineCommandManager.defaultPlanes?.negXz === _entity_id ||
+          engineCommandManager.defaultPlanes?.negYz === _entity_id
+        ) {
+          const defaultPlaneStrMap: Record<string, DefaultPlaneStr> = {
+            [engineCommandManager.defaultPlanes.xy]: 'XY',
+            [engineCommandManager.defaultPlanes.xz]: 'XZ',
+            [engineCommandManager.defaultPlanes.yz]: 'YZ',
+            [engineCommandManager.defaultPlanes.negXy]: '-XY',
+            [engineCommandManager.defaultPlanes.negXz]: '-XZ',
+            [engineCommandManager.defaultPlanes.negYz]: '-YZ',
+          }
+          // TODO can we get this information from rust land when it creates the default planes?
+          // maybe returned from make_default_planes (src/wasm-lib/src/wasm.rs)
+          let zAxis: [number, number, number] = [0, 0, 1]
+          let yAxis: [number, number, number] = [0, 1, 0]
+
+          // get unit vector from camera position to target
+          const camVector = sceneInfra.camControls.camera.position
+            .clone()
+            .sub(sceneInfra.camControls.target)
+
+          if (engineCommandManager.defaultPlanes?.xy === _entity_id) {
+            zAxis = [0, 0, 1]
+            yAxis = [0, 1, 0]
+            if (camVector.z < 0) {
+              zAxis = [0, 0, -1]
+              _entity_id = engineCommandManager.defaultPlanes?.negXy || ''
+            }
+          } else if (engineCommandManager.defaultPlanes?.yz === _entity_id) {
+            zAxis = [1, 0, 0]
+            yAxis = [0, 0, 1]
+            if (camVector.x < 0) {
+              zAxis = [-1, 0, 0]
+              _entity_id = engineCommandManager.defaultPlanes?.negYz || ''
+            }
+          } else if (engineCommandManager.defaultPlanes?.xz === _entity_id) {
+            zAxis = [0, 1, 0]
+            yAxis = [0, 0, 1]
+            _entity_id = engineCommandManager.defaultPlanes?.negXz || ''
+            if (camVector.y < 0) {
+              zAxis = [0, -1, 0]
+              _entity_id = engineCommandManager.defaultPlanes?.xz || ''
+            }
+          }
+
+          sceneInfra.modelingSend({
+            type: 'Select default plane',
+            data: {
+              type: 'defaultPlane',
+              planeId: _entity_id,
+              plane: defaultPlaneStrMap[_entity_id],
+              zAxis,
+              yAxis,
+            },
+          })
+          return
+        }
+        const artifact = this.engineCommandManager.artifactMap[_entity_id]
+        // If we clicked on an extrude wall, we climb up the parent Id
+        // to get the sketch profile's face ID. If we clicked on an endcap,
+        // we already have it.
+        const targetId =
+          'additionalData' in artifact &&
+          artifact.additionalData?.type === 'cap'
+            ? _entity_id
+            : artifact.parentId
+
+        // tsc cannot infer that target can have extrusions
+        // from the commandType (why?) so we need to cast it
+        const target = this.engineCommandManager.artifactMap?.[
+          targetId || ''
+        ] as ArtifactMapCommand & { extrusions?: string[] }
+
+        // TODO: We get the first extrusion command ID,
+        // which is fine while backend systems only support one extrusion.
+        // but we need to more robustly handle resolving to the correct extrusion
+        // if there are multiple.
+        const extrusions =
+          this.engineCommandManager.artifactMap?.[target?.extrusions?.[0] || '']
+
+        if (artifact?.commandType !== 'solid3d_get_extrusion_face_info') return
+
+        const faceInfo = await getFaceDetails(_entity_id)
+        if (!faceInfo?.origin || !faceInfo?.z_axis || !faceInfo?.y_axis) return
+        const { z_axis, y_axis, origin } = faceInfo
+        const sketchPathToNode = getNodePathFromSourceRange(
+          kclManager.ast,
+          artifact.range
+        )
+
+        const extrudePathToNode = extrusions?.range
+          ? getNodePathFromSourceRange(kclManager.ast, extrusions.range)
+          : []
+
+        sceneInfra.modelingSend({
+          type: 'Select default plane',
+          data: {
+            type: 'extrudeFace',
+            zAxis: [z_axis.x, z_axis.y, z_axis.z],
+            yAxis: [y_axis.x, y_axis.y, y_axis.z],
+            position: [origin.x, origin.y, origin.z].map(
+              (num) => num / sceneInfra._baseUnitMultiplier
+            ) as [number, number, number],
+            sketchPathToNode,
+            extrudePathToNode,
+            cap:
+              artifact?.additionalData?.type === 'cap'
+                ? artifact.additionalData.info
+                : 'none',
+            faceId: _entity_id,
+          },
+        })
+        return
+      },
+    })
+  }
   mouseEnterLeaveCallbacks() {
     return {
       onMouseEnter: ({ selected, dragSelected }: OnMouseEnterLeaveArgs) => {
@@ -1588,7 +1863,6 @@ export function sketchGroupFromPathToNode({
   return result as SketchGroup
 }
 
-// TODO delete
 function colorSegment(object: any, color: number) {
   const segmentHead = getParentGroup(object, [ARROWHEAD, PROFILE_START])
   if (segmentHead) {

src/clientSideScene/sceneInfra.ts

@@ -1,25 +1,25 @@
 import {
   AmbientLight,
   Color,
-  DoubleSide,
   GridHelper,
-  Group,
-  Intersection,
   LineBasicMaterial,
-  Mesh,
-  MeshBasicMaterial,
-  Object3D,
-  Object3DEventMap,
   OrthographicCamera,
   PerspectiveCamera,
-  PlaneGeometry,
-  Raycaster,
   Scene,
-  Texture,
-  TextureLoader,
-  Vector2,
   Vector3,
   WebGLRenderer,
+  Raycaster,
+  Vector2,
+  Group,
+  PlaneGeometry,
+  MeshBasicMaterial,
+  Mesh,
+  DoubleSide,
+  Intersection,
+  Object3D,
+  Object3DEventMap,
+  TextureLoader,
+  Texture,
 } from 'three'
 import { Coords2d, compareVec2Epsilon2 } from 'lang/std/sketch'
 import { useModelingContext } from 'hooks/useModelingContext'
@@ -31,12 +31,6 @@ import { EngineCommandManager } from 'lang/std/engineConnection'
 import { MouseState, SegmentOverlayPayload } from 'machines/modelingMachine'
 import { getAngle, throttle } from 'lib/utils'
 import { Themes } from 'lib/theme'
-import {
-  OnClickCallbackArgs,
-  OnDragCallbackArgs,
-  OnMouseEnterLeaveArgs,
-  OnMoveCallbackArgs,
-} from './sceneCallbacks'
 
 type SendType = ReturnType<typeof useModelingContext>['send']
 
@@ -61,6 +55,39 @@ export const AXIS_GROUP = 'axisGroup'
 export const SKETCH_GROUP_SEGMENTS = 'sketch-group-segments'
 export const ARROWHEAD = 'arrowhead'
 
+export interface OnMouseEnterLeaveArgs {
+  selected: Object3D<Object3DEventMap>
+  dragSelected?: Object3D<Object3DEventMap>
+  mouseEvent: MouseEvent
+}
+
+interface OnDragCallbackArgs extends OnMouseEnterLeaveArgs {
+  intersectionPoint: {
+    twoD: Vector2
+    threeD: Vector3
+  }
+  intersects: Intersection<Object3D<Object3DEventMap>>[]
+}
+interface OnClickCallbackArgs {
+  mouseEvent: MouseEvent
+  intersectionPoint?: {
+    twoD: Vector2
+    threeD: Vector3
+  }
+  intersects: Intersection<Object3D<Object3DEventMap>>[]
+  selected?: Object3D<Object3DEventMap>
+}
+
+interface OnMoveCallbackArgs {
+  mouseEvent: MouseEvent
+  intersectionPoint: {
+    twoD: Vector2
+    threeD: Vector3
+  }
+  intersects: Intersection<Object3D<Object3DEventMap>>[]
+  selected?: Object3D<Object3DEventMap>
+}
+
 // This singleton class is responsible for all of the under the hood setup for the client side scene.
 // That is the cameras and switching between them, raycasters for click mouse events and their abstractions (onClick etc), setting up controls.
 // Anything that added the the scene for the user to interact with is probably in SceneEntities.ts
@@ -591,6 +618,59 @@ export class SceneInfra {
       this.onClickCallback({ mouseEvent, intersects })
     }
   }
+  showDefaultPlanes() {
+    const addPlane = (
+      rotation: { x: number; y: number; z: number }, //
+      type: DefaultPlane
+    ): Mesh => {
+      const planeGeometry = new PlaneGeometry(100, 100)
+      const planeMaterial = new MeshBasicMaterial({
+        color: defaultPlaneColor(type),
+        transparent: true,
+        opacity: 0.0,
+        side: DoubleSide,
+        depthTest: false, // needed to avoid transparency issues
+      })
+      const plane = new Mesh(planeGeometry, planeMaterial)
+      plane.rotation.x = rotation.x
+      plane.rotation.y = rotation.y
+      plane.rotation.z = rotation.z
+      plane.userData.type = type
+      plane.name = type
+      return plane
+    }
+    const planes = [
+      addPlane({ x: 0, y: Math.PI / 2, z: 0 }, YZ_PLANE),
+      addPlane({ x: 0, y: 0, z: 0 }, XY_PLANE),
+      addPlane({ x: -Math.PI / 2, y: 0, z: 0 }, XZ_PLANE),
+    ]
+    const planesGroup = new Group()
+    planesGroup.userData.type = DEFAULT_PLANES
+    planesGroup.name = DEFAULT_PLANES
+    planesGroup.add(...planes)
+    planesGroup.traverse((child) => {
+      if (child instanceof Mesh) {
+        child.layers.enable(SKETCH_LAYER)
+      }
+    })
+    planesGroup.layers.enable(SKETCH_LAYER)
+    const sceneScale = getSceneScale(
+      this.camControls.camera,
+      this.camControls.target
+    )
+    planesGroup.scale.set(
+      sceneScale / this._baseUnitMultiplier,
+      sceneScale / this._baseUnitMultiplier,
+      sceneScale / this._baseUnitMultiplier
+    )
+    this.scene.add(planesGroup)
+  }
+  removeDefaultPlanes() {
+    const planesGroup = this.scene.children.find(
+      ({ userData }) => userData.type === DEFAULT_PLANES
+    )
+    if (planesGroup) this.scene.remove(planesGroup)
+  }
   updateOtherSelectionColors = (otherSelections: Axis[]) => {
     const axisGroup = this.scene.children.find(
       ({ userData }) => userData?.type === AXIS_GROUP
@@ -648,3 +728,28 @@ function baseUnitTomm(baseUnit: BaseUnit) {
       return 914.4
   }
 }
+
+export type DefaultPlane =
+  | 'xy-default-plane'
+  | 'xz-default-plane'
+  | 'yz-default-plane'
+
+export const XY_PLANE: DefaultPlane = 'xy-default-plane'
+export const XZ_PLANE: DefaultPlane = 'xz-default-plane'
+export const YZ_PLANE: DefaultPlane = 'yz-default-plane'
+
+export function defaultPlaneColor(
+  plane: DefaultPlane,
+  lowCh = 0.1,
+  highCh = 0.7
+): Color {
+  switch (plane) {
+    case XY_PLANE:
+      return new Color(highCh, lowCh, lowCh)
+    case XZ_PLANE:
+      return new Color(lowCh, lowCh, highCh)
+    case YZ_PLANE:
+      return new Color(lowCh, highCh, lowCh)
+  }
+  return new Color(lowCh, lowCh, lowCh)
+}

src/components/ModelingMachineProvider.tsx

@@ -163,6 +163,8 @@ export const ModelingMachineProvider = ({
 
             store.videoElement?.pause()
             kclManager.executeCode(true).then(() => {
+              if (engineCommandManager.engineConnection?.freezeFrame) return
+
               store.videoElement?.play()
             })
           })()
@@ -549,8 +551,10 @@ export const ModelingMachineProvider = ({
             ) as [number, number, number],
           }
         },
-        'Get horizontal info': async (context): Promise<SetSelections> => {
-          const { selectionRanges, sketchDetails } = context
+        'Get horizontal info': async ({
+          selectionRanges,
+          sketchDetails,
+        }): Promise<SetSelections> => {
           const { modifiedAst, pathToNodeMap } =
             await applyConstraintHorzVertDistance({
               constraint: 'setHorzDistance',
@@ -564,8 +568,11 @@ export const ModelingMachineProvider = ({
             pathToNodeMap
           )
           const updatedAst = await sceneEntitiesManager.updateAstAndRejigSketch(
+            updatedPathToNode,
             _modifiedAst,
-            context
+            sketchDetails.zAxis,
+            sketchDetails.yAxis,
+            sketchDetails.origin
           )
           if (err(updatedAst)) return Promise.reject(updatedAst)
           const selection = updateSelections(
@@ -580,8 +587,10 @@ export const ModelingMachineProvider = ({
             updatedPathToNode,
           }
         },
-        'Get vertical info': async (context): Promise<SetSelections> => {
-          const { selectionRanges, sketchDetails } = context
+        'Get vertical info': async ({
+          selectionRanges,
+          sketchDetails,
+        }): Promise<SetSelections> => {
           const { modifiedAst, pathToNodeMap } =
             await applyConstraintHorzVertDistance({
               constraint: 'setVertDistance',
@@ -595,8 +604,11 @@ export const ModelingMachineProvider = ({
             pathToNodeMap
           )
           const updatedAst = await sceneEntitiesManager.updateAstAndRejigSketch(
+            updatedPathToNode,
             _modifiedAst,
-            context
+            sketchDetails.zAxis,
+            sketchDetails.yAxis,
+            sketchDetails.origin
           )
           if (err(updatedAst)) return Promise.reject(updatedAst)
           const selection = updateSelections(
@@ -611,8 +623,10 @@ export const ModelingMachineProvider = ({
             updatedPathToNode,
           }
         },
-        'Get angle info': async (context): Promise<SetSelections> => {
-          const { selectionRanges, sketchDetails } = context
+        'Get angle info': async ({
+          selectionRanges,
+          sketchDetails,
+        }): Promise<SetSelections> => {
           const info = angleBetweenInfo({
             selectionRanges,
           })
@@ -635,8 +649,11 @@ export const ModelingMachineProvider = ({
             pathToNodeMap
           )
           const updatedAst = await sceneEntitiesManager.updateAstAndRejigSketch(
+            updatedPathToNode,
             _modifiedAst,
-            context
+            sketchDetails.zAxis,
+            sketchDetails.yAxis,
+            sketchDetails.origin
           )
           if (err(updatedAst)) return Promise.reject(updatedAst)
           const selection = updateSelections(
@@ -651,8 +668,10 @@ export const ModelingMachineProvider = ({
             updatedPathToNode,
           }
         },
-        'Get length info': async (context): Promise<SetSelections> => {
-          const { selectionRanges, sketchDetails } = context
+        'Get length info': async ({
+          selectionRanges,
+          sketchDetails,
+        }): Promise<SetSelections> => {
           const { modifiedAst, pathToNodeMap } =
             await applyConstraintAngleLength({ selectionRanges })
           const _modifiedAst = parse(recast(modifiedAst))
@@ -663,8 +682,11 @@ export const ModelingMachineProvider = ({
             pathToNodeMap
           )
           const updatedAst = await sceneEntitiesManager.updateAstAndRejigSketch(
+            updatedPathToNode,
             _modifiedAst,
-            context
+            sketchDetails.zAxis,
+            sketchDetails.yAxis,
+            sketchDetails.origin
           )
           if (err(updatedAst)) return Promise.reject(updatedAst)
           const selection = updateSelections(
@@ -679,10 +701,10 @@ export const ModelingMachineProvider = ({
             updatedPathToNode,
           }
         },
-        'Get perpendicular distance info': async (
-          context
-        ): Promise<SetSelections> => {
-          const { selectionRanges, sketchDetails } = context
+        'Get perpendicular distance info': async ({
+          selectionRanges,
+          sketchDetails,
+        }): Promise<SetSelections> => {
           const { modifiedAst, pathToNodeMap } = await applyConstraintIntersect(
             {
               selectionRanges,
@@ -696,8 +718,11 @@ export const ModelingMachineProvider = ({
             pathToNodeMap
           )
           const updatedAst = await sceneEntitiesManager.updateAstAndRejigSketch(
+            updatedPathToNode,
             _modifiedAst,
-            context
+            sketchDetails.zAxis,
+            sketchDetails.yAxis,
+            sketchDetails.origin
           )
           if (err(updatedAst)) return Promise.reject(updatedAst)
           const selection = updateSelections(
@@ -712,8 +737,10 @@ export const ModelingMachineProvider = ({
             updatedPathToNode,
           }
         },
-        'Get ABS X info': async (context): Promise<SetSelections> => {
-          const { selectionRanges, sketchDetails } = context
+        'Get ABS X info': async ({
+          selectionRanges,
+          sketchDetails,
+        }): Promise<SetSelections> => {
           const { modifiedAst, pathToNodeMap } =
             await applyConstraintAbsDistance({
               constraint: 'xAbs',
@@ -727,8 +754,11 @@ export const ModelingMachineProvider = ({
             pathToNodeMap
           )
           const updatedAst = await sceneEntitiesManager.updateAstAndRejigSketch(
+            updatedPathToNode,
             _modifiedAst,
-            context
+            sketchDetails.zAxis,
+            sketchDetails.yAxis,
+            sketchDetails.origin
           )
           if (err(updatedAst)) return Promise.reject(updatedAst)
           const selection = updateSelections(
@@ -743,8 +773,10 @@ export const ModelingMachineProvider = ({
             updatedPathToNode,
           }
         },
-        'Get ABS Y info': async (context): Promise<SetSelections> => {
-          const { selectionRanges, sketchDetails } = context
+        'Get ABS Y info': async ({
+          selectionRanges,
+          sketchDetails,
+        }): Promise<SetSelections> => {
           const { modifiedAst, pathToNodeMap } =
             await applyConstraintAbsDistance({
               constraint: 'yAbs',
@@ -758,8 +790,11 @@ export const ModelingMachineProvider = ({
             pathToNodeMap
           )
           const updatedAst = await sceneEntitiesManager.updateAstAndRejigSketch(
+            updatedPathToNode,
             _modifiedAst,
-            context
+            sketchDetails.zAxis,
+            sketchDetails.yAxis,
+            sketchDetails.origin
           )
           if (err(updatedAst)) return Promise.reject(updatedAst)
           const selection = updateSelections(
@@ -775,10 +810,9 @@ export const ModelingMachineProvider = ({
           }
         },
         'Get convert to variable info': async (
-          context,
+          { sketchDetails, selectionRanges },
           { data }
         ): Promise<SetSelections> => {
-          const { selectionRanges, sketchDetails } = context
           if (!sketchDetails)
             return Promise.reject(new Error('No sketch details'))
           const { variableName } = await getVarNameModal({
@@ -802,8 +836,11 @@ export const ModelingMachineProvider = ({
             return Promise.reject(new Error('No path to replaced node'))
 
           const updatedAst = await sceneEntitiesManager.updateAstAndRejigSketch(
+            pathToReplacedNode || [],
             parsed,
-            context
+            sketchDetails.zAxis,
+            sketchDetails.yAxis,
+            sketchDetails.origin
           )
           if (err(updatedAst)) return Promise.reject(updatedAst)
           const selection = updateSelections(

src/components/Stream.tsx

@@ -8,7 +8,7 @@ import { NetworkHealthState } from 'hooks/useNetworkStatus'
 import { ClientSideScene } from 'clientSideScene/ClientSideSceneComp'
 import { butName } from 'lib/cameraControls'
 import { sendSelectEventToEngine } from 'lib/selections'
-import { kclManager } from 'lib/singletons'
+import { kclManager, engineCommandManager } from 'lib/singletons'
 
 export const Stream = () => {
   const [isLoading, setIsLoading] = useState(true)
@@ -18,6 +18,7 @@ export const Stream = () => {
   const { settings } = useSettingsAuthContext()
   const { state, send, context } = useModelingContext()
   const { overallState } = useNetworkContext()
+  const [isFreezeFrame, setIsFreezeFrame] = useState(false)
 
   const isNetworkOkay =
     overallState === NetworkHealthState.Ok ||
@@ -49,14 +50,69 @@ export const Stream = () => {
     globalThis?.window?.document?.addEventListener('paste', handlePaste, {
       capture: true,
     })
-    return () =>
+
+    // Teardown everything if we go hidden or reconnect
+    if (globalThis?.window?.document) {
+      globalThis.window.document.onvisibilitychange = () => {
+        if (globalThis.window.document.visibilityState === 'hidden') {
+          videoRef.current?.pause()
+          setIsFreezeFrame(true)
+          window.requestAnimationFrame(() => {
+            engineCommandManager.engineConnection?.tearDown({ freeze: true })
+          })
+        } else {
+          engineCommandManager.engineConnection?.connect(true)
+        }
+      }
+    }
+
+    const IDLE_TIME_MS = 1000 * 20
+    let timeoutIdIdle: ReturnType<typeof setTimeout> | undefined = undefined
+
+    const onIdle = () => {
+      videoRef.current?.pause()
+      setIsFreezeFrame(true)
+      kclManager.isFirstRender = true
+      setIsFirstRender(true)
+      // Give video time to pause
+      window.requestAnimationFrame(() => {
+        engineCommandManager.engineConnection?.tearDown({ freeze: true })
+      })
+    }
+    const onAnyInput = () => {
+      if (!engineCommandManager.engineConnection?.isReady()) {
+        engineCommandManager.engineConnection?.connect(true)
+      }
+      clearTimeout(timeoutIdIdle)
+      timeoutIdIdle = setTimeout(onIdle, IDLE_TIME_MS)
+    }
+
+    globalThis?.window?.document?.addEventListener('keydown', onAnyInput)
+    globalThis?.window?.document?.addEventListener('mousemove', onAnyInput)
+    globalThis?.window?.document?.addEventListener('mousedown', onAnyInput)
+    globalThis?.window?.document?.addEventListener('scroll', onAnyInput)
+    globalThis?.window?.document?.addEventListener('touchstart', onAnyInput)
+
+    timeoutIdIdle = setTimeout(onIdle, IDLE_TIME_MS)
+
+    return () => {
       globalThis?.window?.document?.removeEventListener('paste', handlePaste, {
         capture: true,
       })
+      globalThis?.window?.document?.removeEventListener('keydown', onAnyInput)
+      globalThis?.window?.document?.removeEventListener('mousemove', onAnyInput)
+      globalThis?.window?.document?.removeEventListener('mousedown', onAnyInput)
+      globalThis?.window?.document?.removeEventListener('scroll', onAnyInput)
+      globalThis?.window?.document?.removeEventListener(
+        'touchstart',
+        onAnyInput
+      )
+    }
   }, [])
 
   useEffect(() => {
     setIsFirstRender(kclManager.isFirstRender)
+    if (!kclManager.isFirstRender) videoRef.current?.play()
   }, [kclManager.isFirstRender])
 
   useEffect(() => {
@@ -67,7 +123,10 @@ export const Stream = () => {
       return
     if (!videoRef.current) return
     if (!context.store?.mediaStream) return
+
+    // Do not immediately play the stream!
     videoRef.current.srcObject = context.store.mediaStream
+    videoRef.current.pause()
 
     send({
       type: 'Set context',
@@ -109,6 +168,7 @@ export const Stream = () => {
       },
     })
     if (state.matches('Sketch')) return
+    if (state.matches('Sketch no face')) return
 
     if (!context.store?.didDragInStream && butName(e).left) {
       sendSelectEventToEngine(
@@ -171,17 +231,12 @@ export const Stream = () => {
       <ClientSideScene
         cameraControls={settings.context.modeling.mouseControls.current}
       />
-      {!isNetworkOkay && !isLoading && (
+      {(!isNetworkOkay || isLoading || isFirstRender) && !isFreezeFrame && (
         <div className="text-center absolute inset-0">
           <Loading>
-            <span data-testid="loading-stream">Stream disconnected...</span>
-          </Loading>
-        </div>
-      )}
-      {(isLoading || isFirstRender) && (
-        <div className="text-center absolute inset-0">
-          <Loading>
-            {!isLoading && isFirstRender ? (
+            {!isNetworkOkay && !isLoading ? (
+              <span data-testid="loading-stream">Stream disconnected...</span>
+            ) : !isLoading && isFirstRender ? (
               <span data-testid="loading-stream">Building scene...</span>
             ) : (
               <span data-testid="loading-stream">Loading stream...</span>

src/hooks/useEngineConnectionSubscriptions.ts

@@ -4,7 +4,7 @@ import { useModelingContext } from './useModelingContext'
 import { getEventForSelectWithPoint } from 'lib/selections'
 
 export function useEngineConnectionSubscriptions() {
-  const { send, state } = useModelingContext()
+  const { send, context } = useModelingContext()
 
   useEffect(() => {
     if (!engineCommandManager) return
@@ -29,7 +29,9 @@ export function useEngineConnectionSubscriptions() {
     const unSubClick = engineCommandManager.subscribeTo({
       event: 'select_with_point',
       callback: async (engineEvent) => {
-        const event = await getEventForSelectWithPoint(engineEvent, state)
+        const event = await getEventForSelectWithPoint(engineEvent, {
+          sketchEnginePathId: context.sketchEnginePathId,
+        })
         event && send(event)
       },
     })
@@ -37,5 +39,5 @@ export function useEngineConnectionSubscriptions() {
       unSubHover()
       unSubClick()
     }
-  }, [engineCommandManager, state])
+  }, [engineCommandManager, context?.sketchEnginePathId])
 }

src/lang/std/engineConnection.ts

@@ -300,6 +300,7 @@ class EngineConnection extends EventTarget {
   pc?: RTCPeerConnection
   unreliableDataChannel?: RTCDataChannel
   mediaStream?: MediaStream
+  freezeFrame: boolean = false
 
   private _state: EngineConnectionState = {
     type: EngineConnectionStateType.Fresh,
@@ -365,7 +366,11 @@ class EngineConnection extends EventTarget {
     this.pingPongSpan = { ping: undefined, pong: undefined }
 
     // Without an interval ping, our connection will timeout.
+    // If this.freezeFrame is true we skip this logic so only reconnect
+    // happens on mouse move
     setInterval(() => {
+      if (this.freezeFrame) return
+
       switch (this.state.type as EngineConnectionStateType) {
         case EngineConnectionStateType.ConnectionEstablished:
           // If there was no reply to the last ping, report a timeout.
@@ -426,7 +431,8 @@ class EngineConnection extends EventTarget {
     return this.state.type === EngineConnectionStateType.ConnectionEstablished
   }
 
-  tearDown() {
+  tearDown(opts?: { freeze: boolean }) {
+    this.freezeFrame = opts?.freeze ?? false
     this.disconnectAll()
     this.state = {
       type: EngineConnectionStateType.Disconnecting,
@@ -996,6 +1002,9 @@ class EngineConnection extends EventTarget {
       this.pc?.connectionState === 'closed' &&
       this.unreliableDataChannel?.readyState === 'closed'
     if (allClosed) {
+      // Do not notify the rest of the program that we have cut off anything.
+      if (this.freezeFrame) return
+
       this.state = { type: EngineConnectionStateType.Disconnected }
     }
   }
@@ -1619,7 +1628,15 @@ export class EngineCommandManager extends EventTarget {
     }
   }
   tearDown() {
-    this.engineConnection?.tearDown()
+    if (this.engineConnection) {
+      this.engineConnection?.tearDown()
+      // Our window.tearDown assignment causes this case to happen which is
+      // only really for tests.
+      // @ts-ignore
+    } else if (this.engineCommandManager?.engineConnection) {
+      // @ts-ignore
+      this.engineCommandManager?.engineConnection?.tearDown()
+    }
   }
   async startNewSession() {
     this.lastArtifactMap = this.artifactMap

src/lib/selections.ts

@@ -4,7 +4,6 @@ import {
   engineCommandManager,
   kclManager,
   sceneEntitiesManager,
-  sceneInfra,
 } from 'lib/singletons'
 import { CallExpression, SourceRange, Value, parse, recast } from 'lang/wasm'
 import { ModelingMachineEvent } from 'machines/modelingMachine'
@@ -16,7 +15,6 @@ import { Program } from 'lang/wasm'
 import {
   doesPipeHaveCallExp,
   getNodeFromPath,
-  getNodePathFromSourceRange,
   hasSketchPipeBeenExtruded,
   isSingleCursorInPipe,
 } from 'lang/queryAst'
@@ -26,15 +24,11 @@ import {
   TANGENTIAL_ARC_TO_SEGMENT,
   getParentGroup,
   PROFILE_START,
-  getFaceDetails,
-  DefaultPlaneStr,
 } from 'clientSideScene/sceneEntities'
 import { Mesh, Object3D, Object3DEventMap } from 'three'
 import { AXIS_GROUP, X_AXIS } from 'clientSideScene/sceneInfra'
 import { PathToNodeMap } from 'lang/std/sketchcombos'
 import { err } from 'lib/trap'
-import { useModelingContext } from 'hooks/useModelingContext'
-import { ArtifactMapCommand } from 'lang/std/engineConnection'
 
 export const X_AXIS_UUID = 'ad792545-7fd3-482a-a602-a93924e3055b'
 export const Y_AXIS_UUID = '680fd157-266f-4b8a-984f-cdf46b8bdf01'
@@ -68,12 +62,8 @@ export async function getEventForSelectWithPoint(
     Models['OkModelingCmdResponse_type'],
     { type: 'select_with_point' }
   >,
-  state: ReturnType<typeof useModelingContext>['state']
+  { sketchEnginePathId }: { sketchEnginePathId?: string }
 ): Promise<ModelingMachineEvent | null> {
-  if (state.matches('Sketch no face'))
-    return handleSelectionInSketchNoFace(data?.entity_id || '')
-
-  // assumes XState state is `idle` from this point
   if (!data?.entity_id) {
     return {
       type: 'Set selection',
@@ -153,128 +143,6 @@ export async function getEventForSelectWithPoint(
   }
 }
 
-async function handleSelectionInSketchNoFace(
-  entity_id: string
-): Promise<ModelingMachineEvent | null> {
-  let _entity_id = entity_id
-  if (!_entity_id) return Promise.resolve(null)
-  if (
-    engineCommandManager.defaultPlanes?.xy === _entity_id ||
-    engineCommandManager.defaultPlanes?.xz === _entity_id ||
-    engineCommandManager.defaultPlanes?.yz === _entity_id ||
-    engineCommandManager.defaultPlanes?.negXy === _entity_id ||
-    engineCommandManager.defaultPlanes?.negXz === _entity_id ||
-    engineCommandManager.defaultPlanes?.negYz === _entity_id
-  ) {
-    const defaultPlaneStrMap: Record<string, DefaultPlaneStr> = {
-      [engineCommandManager.defaultPlanes.xy]: 'XY',
-      [engineCommandManager.defaultPlanes.xz]: 'XZ',
-      [engineCommandManager.defaultPlanes.yz]: 'YZ',
-      [engineCommandManager.defaultPlanes.negXy]: '-XY',
-      [engineCommandManager.defaultPlanes.negXz]: '-XZ',
-      [engineCommandManager.defaultPlanes.negYz]: '-YZ',
-    }
-    // TODO can we get this information from rust land when it creates the default planes?
-    // maybe returned from make_default_planes (src/wasm-lib/src/wasm.rs)
-    let zAxis: [number, number, number] = [0, 0, 1]
-    let yAxis: [number, number, number] = [0, 1, 0]
-
-    // get unit vector from camera position to target
-    const camVector = sceneInfra.camControls.camera.position
-      .clone()
-      .sub(sceneInfra.camControls.target)
-
-    if (engineCommandManager.defaultPlanes?.xy === _entity_id) {
-      zAxis = [0, 0, 1]
-      yAxis = [0, 1, 0]
-      if (camVector.z < 0) {
-        zAxis = [0, 0, -1]
-        _entity_id = engineCommandManager.defaultPlanes?.negXy || ''
-      }
-    } else if (engineCommandManager.defaultPlanes?.yz === _entity_id) {
-      zAxis = [1, 0, 0]
-      yAxis = [0, 0, 1]
-      if (camVector.x < 0) {
-        zAxis = [-1, 0, 0]
-        _entity_id = engineCommandManager.defaultPlanes?.negYz || ''
-      }
-    } else if (engineCommandManager.defaultPlanes?.xz === _entity_id) {
-      zAxis = [0, 1, 0]
-      yAxis = [0, 0, 1]
-      _entity_id = engineCommandManager.defaultPlanes?.negXz || ''
-      if (camVector.y < 0) {
-        zAxis = [0, -1, 0]
-        _entity_id = engineCommandManager.defaultPlanes?.xz || ''
-      }
-    }
-
-    return {
-      type: 'Select default plane',
-      data: {
-        type: 'defaultPlane',
-        planeId: _entity_id,
-        plane: defaultPlaneStrMap[_entity_id],
-        zAxis,
-        yAxis,
-      },
-    }
-  }
-  const artifact = engineCommandManager.artifactMap[_entity_id]
-  // If we clicked on an extrude wall, we climb up the parent Id
-  // to get the sketch profile's face ID. If we clicked on an endcap,
-  // we already have it.
-  const targetId =
-    'additionalData' in artifact && artifact.additionalData?.type === 'cap'
-      ? _entity_id
-      : artifact.parentId
-
-  // tsc cannot infer that target can have extrusions
-  // from the commandType (why?) so we need to cast it
-  const target = engineCommandManager.artifactMap?.[
-    targetId || ''
-  ] as ArtifactMapCommand & { extrusions?: string[] }
-
-  // TODO: We get the first extrusion command ID,
-  // which is fine while backend systems only support one extrusion.
-  // but we need to more robustly handle resolving to the correct extrusion
-  // if there are multiple.
-  const extrusions =
-    engineCommandManager.artifactMap?.[target?.extrusions?.[0] || '']
-
-  if (artifact?.commandType !== 'solid3d_get_extrusion_face_info') return null
-
-  const faceInfo = await getFaceDetails(_entity_id)
-  if (!faceInfo?.origin || !faceInfo?.z_axis || !faceInfo?.y_axis) return null
-  const { z_axis, y_axis, origin } = faceInfo
-  const sketchPathToNode = getNodePathFromSourceRange(
-    kclManager.ast,
-    artifact.range
-  )
-
-  const extrudePathToNode = extrusions?.range
-    ? getNodePathFromSourceRange(kclManager.ast, extrusions.range)
-    : []
-
-  return {
-    type: 'Select default plane',
-    data: {
-      type: 'extrudeFace',
-      zAxis: [z_axis.x, z_axis.y, z_axis.z],
-      yAxis: [y_axis.x, y_axis.y, y_axis.z],
-      position: [origin.x, origin.y, origin.z].map(
-        (num) => num / sceneInfra._baseUnitMultiplier
-      ) as [number, number, number],
-      sketchPathToNode,
-      extrudePathToNode,
-      cap:
-        artifact?.additionalData?.type === 'cap'
-          ? artifact.additionalData.info
-          : 'none',
-      faceId: _entity_id,
-    },
-  }
-}
-
 export function getEventForSegmentSelection(
   obj: Object3D<Object3DEventMap>
 ): ModelingMachineEvent | null {

src/lib/singletons.ts

@@ -9,6 +9,10 @@ export const codeManager = new CodeManager()
 
 export const engineCommandManager = new EngineCommandManager()
 
+// Accessible for tests mostly
+// @ts-ignore
+window.tearDown = engineCommandManager.tearDown
+
 // This needs to be after codeManager is created.
 export const kclManager = new KclManager(engineCommandManager)
 kclManager.isFirstRender = true