Reload projects when there are external changes on the file-system (#4077)

e2e/playwright/projects.spec.ts

@@ -12,11 +12,63 @@ import {
 import fsp from 'fs/promises'
 import fs from 'fs'
 import { join } from 'path'
+import { DEFAULT_PROJECT_KCL_FILE } from 'lib/constants'
 
 test.afterEach(async ({ page }, testInfo) => {
   await tearDown(page, testInfo)
 })
 
+test(
+  'projects reload if a new one is created, deleted, or renamed externally',
+  { tag: '@electron' },
+  async ({ browserName }, testInfo) => {
+    let externalCreatedProjectName = 'external-created-project'
+
+    let targetDir = ''
+
+    const { electronApp, page } = await setupElectron({
+      testInfo,
+      folderSetupFn: async (dir) => {
+        targetDir = dir
+        setTimeout(() => {
+          const myDir = join(dir, externalCreatedProjectName)
+          ;(async () => {
+            await fsp.mkdir(myDir)
+            await fsp.writeFile(
+              join(myDir, DEFAULT_PROJECT_KCL_FILE),
+              'sca ba be bop de day wawa skee'
+            )
+          })().catch(console.error)
+        }, 5000)
+      },
+    })
+
+    await page.setViewportSize({ width: 1200, height: 500 })
+
+    const projectLinks = page.getByTestId('project-link')
+
+    await projectLinks.first().waitFor()
+    await expect(projectLinks).toContainText(externalCreatedProjectName)
+
+    await fsp.rename(
+      join(targetDir, externalCreatedProjectName),
+      join(targetDir, externalCreatedProjectName + '1')
+    )
+
+    externalCreatedProjectName += '1'
+    await expect(projectLinks).toContainText(externalCreatedProjectName)
+
+    await fsp.rm(join(targetDir, externalCreatedProjectName), {
+      recursive: true,
+      force: true,
+    })
+
+    await expect(projectLinks).toHaveCount(0)
+
+    await electronApp.close()
+  }
+)
+
 test(
   'click help/keybindings from home page',
   { tag: '@electron' },

interface.d.ts

@@ -1,4 +1,5 @@
 import fs from 'node:fs/promises'
+import fsSync from 'node:fs'
 import path from 'path'
 import { dialog, shell } from 'electron'
 import { MachinesListing } from 'lib/machineManager'
@@ -17,6 +18,12 @@ export interface IElectronAPI {
   platform: typeof process.env.platform
   arch: typeof process.env.arch
   version: typeof process.env.version
+  watchFileOn: (
+    path: string,
+    callback: (eventType: string, path: string) => void
+  ) => void
+  watchFileOff: (path: string) => void
+  watchFileObliterate: () => void
   readFile: (path: string) => ReturnType<fs.readFile>
   writeFile: (
     path: string,

src/hooks/useFileSystemWatcher.tsx

@@ -0,0 +1,71 @@
+import { isDesktop } from 'lib/isDesktop'
+import { useEffect, useState, useRef } from 'react'
+
+type Path = string
+
+// Not having access to NodeJS functions has influenced the design a little.
+// There is some indirection going on because we can only pass data between
+// the NodeJS<->Browser boundary. The actual functions need to run on the
+// NodeJS side. Because EventEmitters come bundled with their listener
+// methods it complicates things because we can't just do
+// watcher.addListener(() => { ... }).
+
+export const useFileSystemWatcher = (
+  callback: (path: Path) => void,
+  dependencyArray: Path[]
+): void => {
+  // Track a ref to the callback. This is how we get the callback updated
+  // across the NodeJS<->Browser boundary.
+  const callbackRef = useRef<{ fn: (path: Path) => void }>({
+    fn: (_path) => {},
+  })
+
+  useEffect(() => {
+    callbackRef.current.fn = callback
+  }, [callback])
+
+  // Used to track if dependencyArrray changes.
+  const [dependencyArrayTracked, setDependencyArrayTracked] = useState<Path[]>(
+    []
+  )
+
+  // On component teardown obliterate all watchers.
+  useEffect(() => {
+    // The hook is useless on web.
+    if (!isDesktop()) return
+
+    return () => {
+      window.electron.watchFileObliterate()
+    }
+  }, [])
+
+  function difference<T>(l1: T[], l2: T[]): [T[], T[]] {
+    return [
+      l1.filter((x) => Boolean(!l2.find((x2) => x2 === x))),
+      l1.filter((x) => Boolean(l2.find((x2) => x2 === x))),
+    ]
+  }
+
+  // Removing 1 watcher at a time is only possible because in a filesystem,
+  // a path is unique (there can never be two paths with the same name).
+  // Otherwise we would have to obliterate() the whole list and reconstruct it.
+  useEffect(() => {
+    // The hook is useless on web.
+    if (!isDesktop()) return
+
+    const [pathsRemoved, pathsRemaining] = difference(
+      dependencyArrayTracked,
+      dependencyArray
+    )
+    for (let path of pathsRemoved) {
+      window.electron.watchFileOff(path)
+    }
+    const [pathsAdded] = difference(dependencyArray, dependencyArrayTracked)
+    for (let path of pathsAdded) {
+      window.electron.watchFileOn(path, (_eventType: string, path: Path) =>
+        callbackRef.current.fn(path)
+      )
+    }
+    setDependencyArrayTracked(pathsRemaining.concat(pathsAdded))
+  }, [difference(dependencyArray, dependencyArrayTracked)[0].length !== 0])
+}

src/hooks/useProjectsLoader.tsx

@@ -0,0 +1,41 @@
+import { trap } from 'lib/trap'
+import { useState, useEffect } from 'react'
+import { ensureProjectDirectoryExists, listProjects } from 'lib/desktop'
+import { loadAndValidateSettings } from 'lib/settings/settingsUtils'
+import { Project } from 'lib/project'
+import { isDesktop } from 'lib/isDesktop'
+
+// Hook uses [number] to give users familiarity. It is meant to mimic a
+// dependency array, but is intended to only ever be used with 1 value.
+export const useProjectsLoader = (deps?: [number]) => {
+  const [lastTs, setLastTs] = useState(-1)
+  const [projectPaths, setProjectPaths] = useState<Project[]>([])
+  const [projectsDir, setProjectsDir] = useState<string | undefined>(undefined)
+
+  useEffect(() => {
+    // Useless on web, until we get fake filesystems over there.
+    if (!isDesktop) return
+
+    if (deps && deps[0] === lastTs) return
+
+    if (deps) {
+      setLastTs(deps[0])
+    }
+
+    ;(async () => {
+      const { configuration } = await loadAndValidateSettings()
+      const _projectsDir = await ensureProjectDirectoryExists(configuration)
+      setProjectsDir(_projectsDir)
+
+      if (projectsDir) {
+        const _projectPaths = await listProjects(configuration)
+        setProjectPaths(_projectPaths)
+      }
+    })().catch(trap)
+  }, deps ?? [])
+
+  return {
+    projectPaths,
+    projectsDir,
+  }
+}

src/lib/commandBarConfigs/homeCommandConfig.ts

@@ -2,6 +2,7 @@ import { StateMachineCommandSetConfig } from 'lib/commandTypes'
 import { homeMachine } from 'machines/homeMachine'
 
 export type HomeCommandSchema = {
+  'Read projects': {}
   'Create project': {
     name: string
   }

src/lib/routeLoaders.ts

@@ -12,11 +12,7 @@ import { loadAndValidateSettings } from './settings/settingsUtils'
 import makeUrlPathRelative from './makeUrlPathRelative'
 import { codeManager } from 'lib/singletons'
 import { fileSystemManager } from 'lang/std/fileSystemManager'
-import {
-  getProjectInfo,
-  ensureProjectDirectoryExists,
-  listProjects,
-} from './desktop'
+import { getProjectInfo } from './desktop'
 import { createSettings } from './settings/initialSettings'
 
 // The root loader simply resolves the settings and any errors that
@@ -184,21 +180,7 @@ export const homeLoader: LoaderFunction = async (): Promise<
   if (!isDesktop()) {
     return redirect(PATHS.FILE + '/%2F' + BROWSER_PROJECT_NAME)
   }
-  const { configuration } = await loadAndValidateSettings()
-
-  const projectDir = await ensureProjectDirectoryExists(configuration)
-
-  if (projectDir) {
-    const projects = await listProjects(configuration)
-
-    return {
-      projects,
-    }
-  } else {
-    return {
-      projects: [],
-    }
-  }
+  return {}
 }
 
 const normalizeLineEndings = (str: string, normalized = '\n') => {

src/lib/types.ts

@@ -12,9 +12,7 @@ export type FileLoaderData = {
   file?: FileEntry
 }
 
-export type HomeLoaderData = {
-  projects: Project[]
-}
+export type HomeLoaderData = {}
 
 // From the very helpful @jcalz on StackOverflow: https://stackoverflow.com/a/58436959/22753272
 type Join<K, P> = K extends string | number

src/machines/homeMachine.ts

@@ -10,6 +10,7 @@ export const homeMachine = setup({
       defaultDirectory: string
     },
     events: {} as
+      | { type: 'Read projects'; data: {} }
       | { type: 'Open project'; data: HomeCommandSchema['Open project'] }
       | { type: 'Rename project'; data: HomeCommandSchema['Rename project'] }
       | { type: 'Create project'; data: HomeCommandSchema['Create project'] }
@@ -81,6 +82,9 @@ export const homeMachine = setup({
   states: {
     'Has no projects': {
       on: {
+        'Read projects': {
+          target: 'Reading projects',
+        },
         'Create project': {
           target: 'Creating project',
         },
@@ -89,6 +93,10 @@ export const homeMachine = setup({
 
     'Has projects': {
       on: {
+        'Read projects': {
+          target: 'Reading projects',
+        },
+
         'Rename project': {
           target: 'Renaming project',
         },

src/preload.ts

@@ -23,6 +23,36 @@ const isMac = os.platform() === 'darwin'
 const isWindows = os.platform() === 'win32'
 const isLinux = os.platform() === 'linux'
 
+let fsWatchListeners = new Map<
+  string,
+  {
+    watcher: fsSync.FSWatcher
+    callback: (eventType: string, path: string) => void
+  }
+>()
+
+const watchFileOn = (
+  path: string,
+  callback: (eventType: string, path: string) => void
+) => {
+  const watcher = fsSync.watch(path)
+  watcher.on('change', callback)
+  fsWatchListeners.set(path, { watcher, callback })
+}
+const watchFileOff = (path: string) => {
+  const entry = fsWatchListeners.get(path)
+  if (!entry) return
+  const { watcher, callback } = entry
+  watcher.off('change', callback)
+  watcher.close()
+  fsWatchListeners.delete(path)
+}
+const watchFileObliterate = () => {
+  for (let [pathAsKey] of fsWatchListeners) {
+    watchFileOff(pathAsKey)
+  }
+  fsWatchListeners = new Map()
+}
 const readFile = (path: string) => fs.readFile(path, 'utf-8')
 // It seems like from the node source code this does not actually block but also
 // don't trust me on that (jess).
@@ -71,6 +101,9 @@ contextBridge.exposeInMainWorld('electron', {
   // Passing fs directly is not recommended since it gives a lot of power
   // to the browser side / potential malicious code. We restrict what is
   // exported.
+  watchFileOn,
+  watchFileOff,
+  watchFileObliterate,
   readFile,
   writeFile,
   exists,

src/routes/Home.tsx

@@ -1,4 +1,4 @@
-import { FormEvent, useEffect, useRef } from 'react'
+import { FormEvent, useEffect, useRef, useState } from 'react'
 import {
   getNextProjectIndex,
   interpolateProjectNameWithIndex,
@@ -8,9 +8,8 @@ import { ActionButton } from 'components/ActionButton'
 import { toast } from 'react-hot-toast'
 import { AppHeader } from 'components/AppHeader'
 import ProjectCard from 'components/ProjectCard/ProjectCard'
-import { useLoaderData, useNavigate, useSearchParams } from 'react-router-dom'
+import { useNavigate, useSearchParams } from 'react-router-dom'
 import { Link } from 'react-router-dom'
-import { type HomeLoaderData } from 'lib/types'
 import Loading from 'components/Loading'
 import { useMachine } from '@xstate/react'
 import { homeMachine } from '../machines/homeMachine'
@@ -38,11 +37,17 @@ import {
 } from 'lib/desktop'
 import { ProjectSearchBar, useProjectSearch } from 'components/ProjectSearchBar'
 import { Project } from 'lib/project'
+import { useFileSystemWatcher } from 'hooks/useFileSystemWatcher'
+import { useProjectsLoader } from 'hooks/useProjectsLoader'
 
 // This route only opens in the desktop context for now,
 // as defined in Router.tsx, so we can use the desktop APIs and types.
 const Home = () => {
-  const { projects: loadedProjects } = useLoaderData() as HomeLoaderData
+  const [projectsLoaderTrigger, setProjectsLoaderTrigger] = useState(0)
+  const { projectPaths, projectsDir } = useProjectsLoader([
+    projectsLoaderTrigger,
+  ])
+
   useRefreshSettings(PATHS.HOME + 'SETTINGS')
   const { commandBarSend } = useCommandsContext()
   const navigate = useNavigate()
@@ -158,12 +163,25 @@ const Home = () => {
     }),
     {
       input: {
-        projects: loadedProjects,
+        projects: projectPaths,
         defaultProjectName: settings.projects.defaultProjectName.current,
         defaultDirectory: settings.app.projectDirectory.current,
       },
     }
   )
+
+  useEffect(() => {
+    send({ type: 'Read projects', data: {} })
+  }, [projectPaths])
+
+  // Re-read projects listing if the projectDir has any updates.
+  useFileSystemWatcher(
+    () => {
+      setProjectsLoaderTrigger(projectsLoaderTrigger + 1)
+    },
+    projectsDir ? [projectsDir] : []
+  )
+
   const { projects } = state.context
   const [searchParams, setSearchParams] = useSearchParams()
   const { searchResults, query, setQuery } = useProjectSearch(projects)