Make tests fail when there are console errors (#6015)

* Add a test to confirm console errors fail tests

* Check for console errors on all browsers

* Ignore error impacting lots of tests

* Add more detected errors to the allowlist for now

.env.development

@@ -1,5 +1,6 @@
 NODE_ENV=development
 DEV=true
+
 VITE_KC_API_WS_MODELING_URL=wss://api.dev.zoo.dev/ws/modeling/commands
 VITE_KC_API_BASE_URL=https://api.dev.zoo.dev
 VITE_KC_SITE_BASE_URL=https://dev.zoo.dev
@@ -8,3 +9,5 @@ VITE_KC_SKIP_AUTH=false
 VITE_KC_CONNECTION_TIMEOUT_MS=5000
 # ONLY add your token in .env.development.local if you want to skip auth, otherwise this token takes precedence!
 #VITE_KC_DEV_TOKEN="your token from dev.zoo.dev should go in .env.development.local"
+
+FAIL_ON_CONSOLE_ERRORS=true

e2e/playwright/lib/console-error-whitelist.ts

@@ -257,6 +257,46 @@ export const isErrorWhitelisted = (exception: Error) => {
       project: 'Google Chrome',
       foundInSpec: 'e2e/playwright/testing-settings.spec.ts',
     },
+    // TODO: fix this error in the code
+    {
+      name: 'TypeError',
+      message: "Cannot read properties of undefined (reading 'length')",
+      stack: '',
+      project: 'Google Chrome',
+      foundInSpec: '', // many tests are impacted by this error
+    },
+    // TODO: fix this error in the code
+    {
+      name: 'ReferenceError',
+      message: '_testUtils is not defined',
+      stack: '',
+      project: 'Google Chrome',
+      foundInSpec: 'e2e/playwright/snapshot-tests.spec.ts',
+    },
+    // TODO: fix this error in the code
+    {
+      name: 'TypeError',
+      message: 'Failed to fetch',
+      stack: '',
+      project: 'Google Chrome',
+      foundInSpec: 'e2e/playwright/snapshot-tests.spec.ts',
+    },
+    // TODO: fix this error in the code
+    {
+      name: 'ReferenceError',
+      message: 'originalCode is not defined',
+      stack: '',
+      project: 'Google Chrome',
+      foundInSpec: 'e2e/playwright/onboarding-tests.spec.ts',
+    },
+    // TODO: fix this error in the code
+    {
+      name: 'ReferenceError',
+      message: 'createNewVariableCheckbox is not defined',
+      stack: '',
+      project: 'Google Chrome',
+      foundInSpec: 'e2e/playwright/testing-constraints.spec.ts',
+    },
   ]
 
   const cleanString = (str: string) => str.replace(/[`"]/g, '')

e2e/playwright/projects.spec.ts

@@ -473,6 +473,9 @@ test.describe('Can export from electron app', () => {
         if (!tronApp) {
           fail()
         }
+        if (runningOnWindows()) {
+          test.fixme(orRunWhenFullSuiteEnabled())
+        }
 
         await context.folderSetupFn(async (dir) => {
           const bracketDir = path.join(dir, 'bracket')

e2e/playwright/regression-tests.spec.ts

@@ -778,6 +778,19 @@ plane002 = offsetPlane(XZ, offset = -2 * x)`
       await editor.expectEditor.not.toContain(`plane002`)
     })
   })
+
+  test.fail(
+    'Console errors cause tests to fail',
+    async ({ page, homePage }) => {
+      const u = await getUtils(page)
+      await homePage.goToModelingScene()
+      await u.openAndClearDebugPanel()
+
+      await page.getByTestId('custom-cmd-input').fill('foobar')
+      await page.getByTestId('custom-cmd-send-button').scrollIntoViewIfNeeded()
+      await page.getByTestId('custom-cmd-send-button').click()
+    }
+  )
 })
 
 async function clickExportButton(page: Page) {

e2e/playwright/test-utils.ts

@@ -935,47 +935,39 @@ export async function setup(
 }
 
 function failOnConsoleErrors(page: Page, testInfo?: TestInfo) {
-  // enabled for chrome for now
+  page.on('pageerror', (exception: any) => {
-  if (page.context().browser()?.browserType().name() === 'chromium') {
+    if (isErrorWhitelisted(exception)) {
-    // No idea wtf exception is
+      return
-    page.on('pageerror', (exception: any) => {
+    }
-      if (isErrorWhitelisted(exception)) {
+    // Only disable this environment variable if you want to collect console errors
-        return
+    if (process.env.FAIL_ON_CONSOLE_ERRORS !== 'false') {
-      }
+      // Use expect to prevent page from closing and not cleaning up
+      expect(`An error was detected in the console: \r\n message:${exception.message} \r\n name:${exception.name} \r\n stack:${exception.stack}
 
-      // only set this env var to false if you want to collect console errors
+        *Either fix the console error or add it to the whitelist defined in ./lib/console-error-whitelist.ts (if the error can be safely ignored)
-      // This can be configured in the GH workflow.  This should be set to true by default (we want tests to fail when
+        `).toEqual('Console error detected')
-      // unwhitelisted console errors are detected).
+    } else {
-      if (process.env.FAIL_ON_CONSOLE_ERRORS === 'true') {
+      // Add errors to `test-results/exceptions.txt` as a test artifact
-        // Fail when running on CI and FAIL_ON_CONSOLE_ERRORS is set
+      fsp
-        // use expect to prevent page from closing and not cleaning up
+        .appendFile(
-        expect(`An error was detected in the console: \r\n message:${exception.message} \r\n name:${exception.name} \r\n stack:${exception.stack}
+          './test-results/exceptions.txt',
-
+          [
-          *Either fix the console error or add it to the whitelist defined in ./lib/console-error-whitelist.ts (if the error can be safely ignored)
+            '~~~',
-          `).toEqual('Console error detected')
+            `triggered_by_test:${
-      } else {
+              testInfo?.file + ' ' + (testInfo?.title || ' ')
-        // the (test-results/exceptions.txt) file will be uploaded as part of an upload artifact in GH
+            }`,
-        fsp
+            `name:${exception.name}`,
-          .appendFile(
+            `message:${exception.message}`,
-            './test-results/exceptions.txt',
+            `stack:${exception.stack}`,
-            [
+            `project:${testInfo?.project.name}`,
-              '~~~',
+            '~~~',
-              `triggered_by_test:${
+          ].join('\n')
-                testInfo?.file + ' ' + (testInfo?.title || ' ')
+        )
-              }`,
+        .catch((err) => {
-              `name:${exception.name}`,
+          console.error(err)
-              `message:${exception.message}`,
+        })
-              `stack:${exception.stack}`,
+    }
-              `project:${testInfo?.project.name}`,
+  })
-              '~~~',
-            ].join('\n')
-          )
-          .catch((err) => {
-            console.error(err)
-          })
-      }
-    })
-  }
 }
 export async function isOutOfViewInScrollContainer(
   element: Locator,