Nadro/3079/screenshot improvements (#3917)

* chore: swapped screenshot to only use the video stream

* feat: video stream screenshot, native electron screenshot

* fix: auto tsc, fmt, xgen, lint

* fix: fixing tsc errors

* fix: removing debug console.log

* fix: renaming ScreenShot to Screenshot

* fix: deleting console log from debugging

* fix: bug with what source was referenced

* fix: using a productName

* fix: improving usage for native screenshots and detecthing support

* fix: fmt

* chore: updated rust test documentation

* fix: typo in readme

* fix: leaving package.json and yarn.lock the same as main??

* bump

* bump

* bump again

* bump again2

README.md

@@ -337,13 +337,47 @@ For individual testing:
 yarn test abstractSyntaxTree -t "unexpected closed curly brace" --silent=false
 ```
 
-Which will run our suite of [Vitest unit](https://vitest.dev/) and [React Testing Library E2E](https://testing-library.com/docs/react-testing-library/intro/) tests, in interactive mode by default.
+Which will run our suite of [Vitest unit](https://vitest.dev/) and [React Testing Library E2E](https://testing-library.com/docs/react-testing-library/intro) tests, in interactive mode by default.
 
 ### Rust tests
 
-```bash
+**Dependencies**
+
+- `KITTYCAD_API_TOKEN`
+- `cargo-nextest`
+- `just`
+
+#### Setting KITTYCAD_API_TOKEN
+Use the production zoo.dev token, set this environment variable before running the tests
+
+#### Installing cargonextest
+
+```
 cd src/wasm-lib
-KITTYCAD_API_TOKEN=XXX cargo test -- --test-threads=1
+cargo search cargo-nextest
+cargo install cargo-nextest
+```
+
+#### just
+install [`just`](https://github.com/casey/just?tab=readme-ov-file#pre-built-binaries)
+
+#### Running the tests
+
+```bash
+# With just
+# Make sure KITTYCAD_API_TOKEN=<prod zoo.dev token> is set
+# Make sure you installed cargo-nextest
+# Make sure you installed just
+cd src/wasm-lib
+just test
+```
+
+```bash
+# Without just
+# Make sure KITTYCAD_API_TOKEN=<prod zoo.dev token> is set
+# Make sure you installed cargo-nextest
+cd src/wasm-lib
+export RUST_BRACKTRACE="full" && cargo nextest run --workspace --test-threads=1
 ```
 
 Where `XXX` is an API token from the production engine (NOT the dev environment).

interface.d.ts

@@ -11,6 +11,13 @@ export interface IElectronAPI {
   open: typeof dialog.showOpenDialog
   save: typeof dialog.showSaveDialog
   openExternal: typeof shell.openExternal
+  takeElectronWindowScreenshot: ({
+    width,
+    height,
+  }: {
+    width: number
+    height: number
+  }) => Promise<string>
   showInFolder: typeof shell.showItemInFolder
   /** Require to be called first before {@link loginWithDeviceFlow} */
   startDeviceFlow: (host: string) => Promise<string>

src/lib/screenshot.ts

@@ -1,6 +1,24 @@
-import html2canvas from 'html2canvas-pro'
+function takeScreenshotOfVideoStreamCanvas() {
+  const canvas = document.querySelector('[data-engine]')
+  const video = document.getElementById('video-stream')
+  if (
+    canvas &&
+    video &&
+    canvas instanceof HTMLCanvasElement &&
+    video instanceof HTMLVideoElement
+  ) {
+    const videoCanvas = document.createElement('canvas')
+    videoCanvas.width = canvas.width
+    videoCanvas.height = canvas.height
+    const context = videoCanvas.getContext('2d')
+    context?.drawImage(video, 0, 0, videoCanvas.width, videoCanvas.height)
+    const url = videoCanvas.toDataURL('image/png')
+    return url
+  } else {
+    return ''
+  }
+}
 
-// Return a data URL (png format) of the screenshot of the current page.
 export default async function screenshot(): Promise<string> {
   if (typeof window === 'undefined') {
     return Promise.reject(
@@ -9,11 +27,17 @@ export default async function screenshot(): Promise<string> {
       )
     )
   }
-  return html2canvas(document.documentElement)
-    .then((canvas) => {
-      return canvas.toDataURL()
-    })
-    .catch((error) => {
-      return Promise.reject(error)
-    })
+
+  if (window.electron) {
+    const canvas = document.querySelector('[data-engine]')
+    if (canvas instanceof HTMLCanvasElement) {
+      const url = await window.electron.takeElectronWindowScreenshot({
+        width: canvas?.width || 500,
+        height: canvas?.height || 500,
+      })
+      return url !== '' ? url : takeScreenshotOfVideoStreamCanvas()
+    }
+  }
+
+  return takeScreenshotOfVideoStreamCanvas()
 }

src/main.ts

@@ -8,6 +8,8 @@ import {
   dialog,
   shell,
   nativeTheme,
+  desktopCapturer,
+  systemPreferences,
 } from 'electron'
 import path from 'path'
 import { Issuer } from 'openid-client'
@@ -21,6 +23,8 @@ import os from 'node:os'
 import { reportRejection } from 'lib/trap'
 import argvFromYargs from './commandLineArgs'
 
+import * as packageJSON from '../package.json'
+
 let mainWindow: BrowserWindow | null = null
 
 // Check the command line arguments for a project path
@@ -181,6 +185,44 @@ ipcMain.handle('shell.openExternal', (event, data) => {
   return shell.openExternal(data)
 })
 
+ipcMain.handle(
+  'take.screenshot',
+  async (event, data: { width: number; height: number }) => {
+    /**
+     * Operation system access to getting screen sources, even though we are only use application windows
+     * Linux: Yes!
+     * Mac OS: This user consent was not required on macOS 10.13 High Sierra so this method will always return granted. macOS 10.14 Mojave or higher requires consent for microphone and camera access. macOS 10.15 Catalina or higher requires consent for screen access.
+     * Windows 10: has a global setting controlling microphone and camera access for all win32 applications. It will always return granted for screen and for all media types on older versions of Windows.
+     */
+    let accessToScreenSources = true
+
+    // Can we check for access and if so, is it granted
+    // Linux does not even have access to the function getMediaAccessStatus, not going to polyfill
+    if (systemPreferences && systemPreferences.getMediaAccessStatus) {
+      const accessString = systemPreferences.getMediaAccessStatus('screen')
+      accessToScreenSources = accessString === 'granted' ? true : false
+    }
+
+    if (accessToScreenSources) {
+      const sources = await desktopCapturer.getSources({
+        types: ['window'],
+        thumbnailSize: { width: data.width, height: data.height },
+      })
+
+      for (const source of sources) {
+        // electron-builder uses the value of productName in package.json for the title of the application
+        if (source.name === packageJSON.productName) {
+          // @ts-ignore image/png is real.
+          return source.thumbnail.toDataURL('image/png') // The image to display the screenshot
+        }
+      }
+    }
+
+    // Cannot take a native desktop screenshot, unable to access screens
+    return ''
+  }
+)
+
 ipcMain.handle('argv.parser', (event, data) => {
   return argvFromYargs
 })

src/preload.ts

@@ -12,6 +12,13 @@ const resizeWindow = (width: number, height: number) =>
 const open = (args: any) => ipcRenderer.invoke('dialog.showOpenDialog', args)
 const save = (args: any) => ipcRenderer.invoke('dialog.showSaveDialog', args)
 const openExternal = (url: any) => ipcRenderer.invoke('shell.openExternal', url)
+const takeElectronWindowScreenshot = ({
+  width,
+  height,
+}: {
+  width: number
+  height: number
+}) => ipcRenderer.invoke('take.screenshot', { width, height })
 const showInFolder = (path: string) =>
   ipcRenderer.invoke('shell.showItemInFolder', path)
 const startDeviceFlow = (host: string): Promise<string> =>
@@ -160,6 +167,7 @@ contextBridge.exposeInMainWorld('electron', {
   version: process.version,
   join: path.join,
   sep: path.sep,
+  takeElectronWindowScreenshot,
   os: {
     isMac,
     isWindows,

src/wasm-lib/justfile

@@ -31,3 +31,5 @@ new-sim-test test_name render_to_png="true":
     {{cita}} -p kcl-lib -- simulation_tests::{{test_name}}::unparse
     TWENTY_TWENTY=overwrite {{cita}} -p kcl-lib -- tests::{{test_name}}::kcl_test_execute
 
+test:
+    export RUST_BRACKTRACE="full" && cargo nextest run --workspace --test-threads=1

src/wasm-lib/kcl/src/coredump/mod.rs

@@ -163,6 +163,9 @@ impl CoreDumpInfo {
 
 ![Screenshot]({screenshot_url})
 
+> _Note: If you are capturing from a browser there is limited support for screenshots, only captures the modeling scene.
+  If you are on MacOS native screenshots may be disabled by default. To enable native screenshots add Zoo Modeling App to System Settings -> Screen & SystemAudio Recording for native screenshots._
+
 <details>
 <summary><b>Core Dump</b></summary>