diff --git a/e2e/playwright/test-utils.ts b/e2e/playwright/test-utils.ts index 294367bc5..7b83ea0b1 100644 --- a/e2e/playwright/test-utils.ts +++ b/e2e/playwright/test-utils.ts @@ -1198,3 +1198,174 @@ export async function enableConsoleLogEverything({ console.log(`[Main] ${msg.type()}: ${msg.text()}`) }) } + +/** + * Simulate a pan touch gesture from the center of an element. + * + * Adapted from Playwright docs: https://playwright.dev/docs/touch-events + */ +export async function panFromCenter( + locator: Locator, + deltaX = 0, + deltaY = 0, + steps = 5 +) { + const { centerX, centerY } = await locator.evaluate((target: HTMLElement) => { + const bounds = target.getBoundingClientRect() + const centerX = bounds.left + bounds.width / 2 + const centerY = bounds.top + bounds.height / 2 + return { centerX, centerY } + }) + + // Providing only clientX and clientY as the app only cares about those. + const touches = [ + { + identifier: 0, + clientX: centerX, + clientY: centerY, + }, + ] + await locator.dispatchEvent('touchstart', { + touches, + changedTouches: touches, + targetTouches: touches, + }) + + for (let j = 1; j <= steps; j++) { + const touches = [ + { + identifier: 0, + clientX: centerX + (deltaX * j) / steps, + clientY: centerY + (deltaY * j) / steps, + }, + ] + await locator.dispatchEvent('touchmove', { + touches, + changedTouches: touches, + targetTouches: touches, + }) + } + + await locator.dispatchEvent('touchend') +} + +/** + * Simulate a 2-finger pan touch gesture from the center of an element. + * with {touchSpacing} pixels between. + * + * Adapted from Playwright docs: https://playwright.dev/docs/touch-events + */ +export async function panTwoFingerFromCenter( + locator: Locator, + deltaX = 0, + deltaY = 0, + steps = 5, + spacingX = 20 +) { + const { centerX, centerY } = await locator.evaluate((target: HTMLElement) => { + const bounds = target.getBoundingClientRect() + const centerX = bounds.left + bounds.width / 2 + const centerY = bounds.top + bounds.height / 2 + return { centerX, centerY } + }) + + // Providing only clientX and clientY as the app only cares about those. + const touches = [ + { + identifier: 0, + clientX: centerX, + clientY: centerY, + }, + { + identifier: 1, + clientX: centerX + spacingX, + clientY: centerY, + }, + ] + await locator.dispatchEvent('touchstart', { + touches, + changedTouches: touches, + targetTouches: touches, + }) + + for (let j = 1; j <= steps; j++) { + const touches = [ + { + identifier: 0, + clientX: centerX + (deltaX * j) / steps, + clientY: centerY + (deltaY * j) / steps, + }, + { + identifier: 1, + clientX: centerX + spacingX + (deltaX * j) / steps, + clientY: centerY + (deltaY * j) / steps, + }, + ] + await locator.dispatchEvent('touchmove', { + touches, + changedTouches: touches, + targetTouches: touches, + }) + } + + await locator.dispatchEvent('touchend') +} + +/** + * Simulate a pinch touch gesture from the center of an element. + * Touch points are set horizontally from each other, separated by {startDistance} pixels. + */ +export async function pinchFromCenter( + locator: Locator, + startDistance = 100, + delta = 0, + steps = 5 +) { + const { centerX, centerY } = await locator.evaluate((target: HTMLElement) => { + const bounds = target.getBoundingClientRect() + const centerX = bounds.left + bounds.width / 2 + const centerY = bounds.top + bounds.height / 2 + return { centerX, centerY } + }) + + // Providing only clientX and clientY as the app only cares about those. + const touches = [ + { + identifier: 0, + clientX: centerX - startDistance / 2, + clientY: centerY, + }, + { + identifier: 1, + clientX: centerX + startDistance / 2, + clientY: centerY, + }, + ] + await locator.dispatchEvent('touchstart', { + touches, + changedTouches: touches, + targetTouches: touches, + }) + + for (let i = 1; i <= steps; i++) { + const touches = [ + { + identifier: 0, + clientX: centerX - startDistance / 2 + (delta * i) / steps, + clientY: centerY, + }, + { + identifier: 1, + clientX: centerX + startDistance / 2 + (delta * i) / steps, + clientY: centerY, + }, + ] + await locator.dispatchEvent('touchmove', { + touches, + changedTouches: touches, + targetTouches: touches, + }) + } + + await locator.dispatchEvent('touchend') +} diff --git a/e2e/playwright/testing-camera-movement-touch.spec.ts b/e2e/playwright/testing-camera-movement-touch.spec.ts new file mode 100644 index 000000000..41da3b729 --- /dev/null +++ b/e2e/playwright/testing-camera-movement-touch.spec.ts @@ -0,0 +1,146 @@ +import { getUtils } from '@e2e/playwright/test-utils' +import { expect, test } from '@e2e/playwright/zoo-test' +import { type Page } from '@playwright/test' +import type { SceneFixture } from '@e2e/playwright/fixtures/sceneFixture' + +test.use({ + hasTouch: true, +}) +test.describe('Testing Camera Movement (Touch Only)', () => { + /** + * DUPLICATED FROM `testing-camera-movement.spec.ts`, might need to become a util. + * + * hack that we're implemented our own retry instead of using retries built into playwright. + * however each of these camera drags can be flaky, because of udp + * and so putting them together means only one needs to fail to make this test extra flaky. + * this way we can retry within the test + * We could break them out into separate tests, but the longest past of the test is waiting + * for the stream to start, so it can be good to bundle related things together. + */ + const _bakeInRetries = async ({ + mouseActions, + afterPosition, + beforePosition, + retryCount = 0, + page, + scene, + }: { + mouseActions: () => Promise + beforePosition: [number, number, number] + afterPosition: [number, number, number] + retryCount?: number + page: Page + scene: SceneFixture + }) => { + const acceptableCamError = 5 + const u = await getUtils(page) + + await test.step('Set up initial camera position', async () => + await scene.moveCameraTo({ + x: beforePosition[0], + y: beforePosition[1], + z: beforePosition[2], + })) + + await test.step('Do actions and watch for changes', async () => + u.doAndWaitForImageDiff(async () => { + await mouseActions() + + await u.openAndClearDebugPanel() + await u.closeDebugPanel() + await page.waitForTimeout(100) + }, 300)) + + await u.openAndClearDebugPanel() + await expect(page.getByTestId('cam-x-position')).toBeAttached() + + const vals = await Promise.all([ + page.getByTestId('cam-x-position').inputValue(), + page.getByTestId('cam-y-position').inputValue(), + page.getByTestId('cam-z-position').inputValue(), + ]) + const errors = vals.map((v, i) => Math.abs(Number(v) - afterPosition[i])) + let shouldRetry = false + + if (errors.some((e) => e > acceptableCamError)) { + if (retryCount > 2) { + console.log('xVal', vals[0], 'xError', errors[0]) + console.log('yVal', vals[1], 'yError', errors[1]) + console.log('zVal', vals[2], 'zError', errors[2]) + + throw new Error('Camera position not as expected', { + cause: { + vals, + errors, + }, + }) + } + shouldRetry = true + } + if (shouldRetry) { + await _bakeInRetries({ + mouseActions, + afterPosition: afterPosition, + beforePosition: beforePosition, + retryCount: retryCount + 1, + page, + scene, + }) + } + } + // test( + // 'Touch camera controls', + // { + // tag: '@web', + // }, + // async ({ page, homePage, scene, cmdBar }) => { + // const u = await getUtils(page) + // const camInitialPosition: [number, number, number] = [0, 85, 85] + // + // await homePage.goToModelingScene() + // await scene.settled(cmdBar) + // const stream = page.getByTestId('stream') + // + // await u.openAndClearDebugPanel() + // await u.closeKclCodePanel() + // + // await test.step('Orbit', async () => { + // await bakeInRetries({ + // mouseActions: async () => { + // await panFromCenter(stream, 200, 200) + // await page.waitForTimeout(200) + // }, + // afterPosition: [19, 85, 85], + // beforePosition: camInitialPosition, + // page, + // scene, + // }) + // }) + // + // await test.step('Pan', async () => { + // await bakeInRetries({ + // mouseActions: async () => { + // await panTwoFingerFromCenter(stream, 200, 200) + // await page.waitForTimeout(200) + // }, + // afterPosition: [19, 85, 85], + // beforePosition: camInitialPosition, + // page, + // scene, + // }) + // }) + // + // await test.step('Zoom', async () => { + // await bakeInRetries({ + // mouseActions: async () => { + // await pinchFromCenter(stream, 300, -100, 5) + // }, + // afterPosition: [0, 118, 118], + // beforePosition: camInitialPosition, + // page, + // scene, + // }) + // }) + // } + // ) +}) diff --git a/package-lock.json b/package-lock.json index 4f59445a6..f6d219b48 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "diff": "^7.0.0", "electron-updater": "^6.6.2", "fuse.js": "^7.1.0", + "hammerjs": "^2.0.8", "html2canvas-pro": "^1.5.8", "isomorphic-fetch": "^3.0.0", "json-rpc-2.0": "^1.6.0", @@ -93,6 +94,7 @@ "@testing-library/react": "^15.0.7", "@types/diff": "^7.0.2", "@types/electron": "^1.6.10", + "@types/hammerjs": "^2.0.46", "@types/isomorphic-fetch": "^0.0.39", "@types/jest": "^29.5.14", "@types/minimist": "^1.2.5", @@ -7491,6 +7493,13 @@ "@types/node": "*" } }, + "node_modules/@types/hammerjs": { + "version": "2.0.46", + "resolved": "https://registry.npmjs.org/@types/hammerjs/-/hammerjs-2.0.46.tgz", + "integrity": "sha512-ynRvcq6wvqexJ9brDMS4BnBLzmr0e14d6ZJTEShTBWKymQiHwlAyGu0ZPEFI2Fh1U53F7tN9ufClWM5KvqkKOw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/http-cache-semantics": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", @@ -15174,6 +15183,15 @@ "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } }, + "node_modules/hammerjs": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/hammerjs/-/hammerjs-2.0.8.tgz", + "integrity": "sha512-tSQXBXS/MWQOn/RKckawJ61vvsDpCom87JgxiYdGwHdOa0ht0vzUWDlfioofFCRU0L+6NGDt6XzbgoJvZkMeRQ==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/happy-dom": { "version": "17.4.4", "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-17.4.4.tgz", diff --git a/package.json b/package.json index 564938ac6..72bcdcfde 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "diff": "^7.0.0", "electron-updater": "^6.6.2", "fuse.js": "^7.1.0", + "hammerjs": "^2.0.8", "html2canvas-pro": "^1.5.8", "isomorphic-fetch": "^3.0.0", "json-rpc-2.0": "^1.6.0", @@ -168,6 +169,7 @@ "@testing-library/react": "^15.0.7", "@types/diff": "^7.0.2", "@types/electron": "^1.6.10", + "@types/hammerjs": "^2.0.46", "@types/isomorphic-fetch": "^0.0.39", "@types/jest": "^29.5.14", "@types/minimist": "^1.2.5", diff --git a/src/clientSideScene/CameraControls.ts b/src/clientSideScene/CameraControls.ts index add8cec63..a3923ff36 100644 --- a/src/clientSideScene/CameraControls.ts +++ b/src/clientSideScene/CameraControls.ts @@ -1,3 +1,4 @@ +import Hammer from 'hammerjs' import type { CameraDragInteractionType_type, CameraViewState_type, @@ -44,6 +45,7 @@ import { } from '@src/lib/utils' import { deg2Rad } from '@src/lib/utils2d' import type { SettingsType } from '@src/lib/settings/initialSettings' +import { degToRad } from 'three/src/math/MathUtils' const ORTHOGRAPHIC_CAMERA_SIZE = 20 const FRAMES_TO_ANIMATE_IN = 30 @@ -118,7 +120,7 @@ export class CameraControls { enableZoom = true moveSender: CameraRateLimiter = new CameraRateLimiter() zoomSender: CameraRateLimiter = new CameraRateLimiter() - lastPerspectiveFov: number = 45 + lastPerspectiveFov = 45 pendingZoom: number | null = null pendingRotation: Vector2 | null = null pendingPan: Vector2 | null = null @@ -213,7 +215,10 @@ export class CameraControls { } } - doMove = (interaction: any, coordinates: any) => { + doMove = ( + interaction: CameraDragInteractionType_type, + coordinates: [number, number] + ) => { // eslint-disable-next-line @typescript-eslint/no-floating-promises this.engineCommandManager.sendSceneCommand({ type: 'modeling_cmd_req', @@ -244,9 +249,9 @@ export class CameraControls { } constructor( - isOrtho = false, domElement: HTMLCanvasElement, - engineCommandManager: EngineCommandManager + engineCommandManager: EngineCommandManager, + isOrtho = false ) { this.engineCommandManager = engineCommandManager this.camera = isOrtho ? new OrthographicCamera() : new PerspectiveCamera() @@ -263,6 +268,7 @@ export class CameraControls { this.domElement.addEventListener('pointermove', this.onMouseMove) this.domElement.addEventListener('pointerup', this.onMouseUp) this.domElement.addEventListener('wheel', this.onMouseWheel) + this.setUpMultiTouch(this.domElement) window.addEventListener('resize', this.onWindowResize) this.onWindowResize() @@ -422,6 +428,10 @@ export class CameraControls { } onMouseMove = (event: PointerEvent) => { + if (event.pointerType === 'touch') { + return + } + if (this.isDragging) { this.mouseNewPosition.set(event.clientX, event.clientY) const deltaMove = this.mouseNewPosition @@ -429,8 +439,10 @@ export class CameraControls { .sub(this.mouseDownPosition) this.mouseDownPosition.copy(this.mouseNewPosition) - let interaction = this.getInteractionType(event) - if (interaction === 'none') return + const interaction = this.getInteractionType(event) + if (interaction === 'none') { + return + } // If there's a valid interaction and the mouse is moving, // our past (and current) interaction was a drag. @@ -500,6 +512,14 @@ export class CameraControls { } onMouseUp = (event: PointerEvent) => { + if (event.pointerType === 'touch') { + // We support momentum flick gestures so we have to do these things after that completes + return + } + this.onMouseUpInner(event) + } + + onMouseUpInner = (event: PointerEvent) => { this.domElement.releasePointerCapture(event.pointerId) this.isDragging = false this.handleEnd() @@ -1304,15 +1324,20 @@ export class CameraControls { Object.values(this._camChangeCallbacks).forEach((cb) => cb()) } getInteractionType = ( - event: MouseEvent + event: PointerEvent | WheelEvent | MouseEvent ): CameraDragInteractionType_type | 'none' => { - const initialInteractionType = _getInteractionType( - this.interactionGuards, - event, - this.enablePan, - this.enableRotate, - this.enableZoom - ) + // We just need to send any start value to the engine for touch. + // I chose "rotate" because it's the 1-finger gesture + const initialInteractionType = + 'pointerType' in event && event.pointerType === 'touch' + ? 'rotate' + : _getInteractionType( + this.interactionGuards, + event, + this.enablePan, + this.enableRotate, + this.enableZoom + ) if ( initialInteractionType === 'rotate' && this.getSettings?.().modeling.cameraOrbit.current === 'trackball' @@ -1321,6 +1346,171 @@ export class CameraControls { } return initialInteractionType } + + /** + * Set up HammerJS, a small library for multi-touch listeners, + * and use it for the camera controls if touch is available. + * + * Note: users cannot change touch controls; this implementation + * treats them as distinct from the mouse control scheme. This is because + * a device may both have mouse controls and touch available. + * + * TODO: Add support for sketch mode touch camera movements + */ + setUpMultiTouch = (domElement: HTMLCanvasElement) => { + /** Amount in px needed to pan before recognizer runs */ + const panDistanceThreshold = 3 + /** Amount in scale delta needed to pinch before recognizer runs */ + const zoomScaleThreshold = 0.01 + /** + * Max speed a pinch can be moving (not the pinch but its XY drift) before it's considered a pan. + * The closer to this value, the more we reduce the calculated zoom transformation to reduce jitter. + */ + const velocityLimit = 0.5 + /** Amount of pixel delta of calculated zoom transform needed before we send to the engine */ + const normalizedScaleThreshold = 5 + const velocityFlickDecay = 0.03 + /** Refresh rate for flick orbit decay timer */ + const decayRefreshRate = 16 + const decayIntervals: ReturnType[] = [] + const clearIntervals = () => { + for (let i = decayIntervals.length - 1; i >= 0; i--) { + clearInterval(decayIntervals[i]) + decayIntervals.pop() + } + } + + const hammertime = new Hammer(domElement, { + recognizers: [ + [Hammer.Pan, { pointers: 1, direction: Hammer.DIRECTION_ALL }], + [ + Hammer.Pan, + { + event: 'doublepan', + pointers: 2, + direction: Hammer.DIRECTION_ALL, + threshold: panDistanceThreshold, + }, + ], + [Hammer.Pinch, { enable: true, threshold: zoomScaleThreshold }], + ], + }) + + // TODO: get the engine to coalesce simultaneous zoom/pan/orbit events, + // then we won't have to worry about jitter at all. + // https://github.com/KittyCAD/engine/issues/3528 + hammertime.get('pinch').recognizeWith(hammertime.get('doublepan')) + + // Clear decay intervals on any interaction start + hammertime.on('panstart doublepanstart pinchstart', () => { + clearIntervals() + }) + + // Orbit gesture is a 1-finger "pan" + hammertime.on('pan', (ev) => { + if (this.syncDirection === 'engineToClient' && ev.maxPointers === 1) { + if (this.enableRotate) { + const orbitMode = + this.getSettings?.().modeling.cameraOrbit.current !== 'spherical' + ? 'rotatetrackball' + : 'rotate' + this.moveSender.send(() => { + this.doMove(orbitMode, [ev.center.x, ev.center.y]) + }) + } + } + }) + // Fake flicking by sending decaying orbit events in the last direction of orbit + hammertime.on('panend', (ev) => { + /** HammerJS's `event.velocity` gives you `Math.max(ev.velocityX, ev.velocityY`)`, not the actual velocity. */ + let velocity = Math.sqrt(ev.velocityY ** 2 + ev.velocityX ** 2) + const center = ev.center + const direction = ev.angle + + if ( + this.syncDirection === 'engineToClient' && + ev.maxPointers === 1 && + this.enableRotate && + velocity > 0 + ) { + const orbitMode = + this.getSettings?.().modeling.cameraOrbit.current !== 'spherical' + ? 'rotatetrackball' + : 'rotate' + + const decayInterval = setInterval(() => { + const decayedVelocity = velocity - velocityFlickDecay + if (decayedVelocity <= 0) { + if (ev.srcEvent instanceof PointerEvent) { + this.onMouseUpInner(ev.srcEvent) + } + clearInterval(decayInterval) + } else { + velocity = decayedVelocity + } + + // We have to multiply by the refresh rate, because `velocity` is in px/ms + // but we only call every `decayRefreshRate` ms. + center.x = + center.x + + Math.cos(degToRad(direction)) * velocity * decayRefreshRate + center.y = + center.y + + Math.sin(degToRad(direction)) * velocity * decayRefreshRate + + this.moveSender.send(() => { + this.doMove(orbitMode, [center.x, center.y]) + }) + }, decayRefreshRate) + decayIntervals.push(decayInterval) + } + }) + + // Pan gesture is a 2-finger gesture I named "doublepan" + hammertime.on('doublepan', (ev) => { + if (this.syncDirection === 'engineToClient' && this.enablePan) { + this.moveSender.send(() => { + this.doMove('pan', [ev.center.x, ev.center.y]) + }) + } + }) + + // Zoom is a pinch, which is very similar to a 2-finger pan + // and must therefore be heuristically determined. My heuristics is: + // A zoom should only occur if the gesture velocity is low and the scale delta is high + let lastScale = 1 + hammertime.on('pinchmove', (ev) => { + const scaleDelta = lastScale - ev.scale + const isUnderVelocityLimit = ev.velocity < velocityLimit + // The faster you move the less you zoom + const velocityFactor = + Math.abs( + velocityLimit - Math.min(velocityLimit, Math.abs(ev.velocity)) + ) * 0.5 + const normalizedScale = Math.ceil(scaleDelta * 2000 * velocityFactor) + const isOverNormalizedScaleLimit = + Math.abs(normalizedScale) > normalizedScaleThreshold + + if ( + this.syncDirection === 'engineToClient' && + this.enableZoom && + isUnderVelocityLimit && + isOverNormalizedScaleLimit + ) { + this.zoomSender.send(() => { + this.doZoom(normalizedScale) + }) + lastScale = ev.scale + } + }) + hammertime.on('pinchend pinchcancel', () => { + lastScale = 1 + }) + + hammertime.on('pinchend pinchcancel doublepanend', (event) => { + this.onMouseUpInner(event.srcEvent as PointerEvent) + }) + } } // Pure function helpers diff --git a/src/clientSideScene/sceneInfra.ts b/src/clientSideScene/sceneInfra.ts index 27cdece23..038c33e32 100644 --- a/src/clientSideScene/sceneInfra.ts +++ b/src/clientSideScene/sceneInfra.ts @@ -291,9 +291,9 @@ export class SceneInfra { window.addEventListener('resize', this.onWindowResize) this.camControls = new CameraControls( - false, this.renderer.domElement, - engineCommandManager + engineCommandManager, + false ) this.camControls.subscribeToCamChange(() => this.onCameraChange()) this.camControls.camera.layers.enable(SKETCH_LAYER)