prompt-to-edit API request snapshot testing infrastructure (#5514)

* POC write output to json

* move to cmd bar

* write files

* clean up

* A snapshot a day keeps the bugs away! 📷🐛 (OS: namespace-profile-ubuntu-8-cores)

* A snapshot a day keeps the bugs away! 📷🐛 (OS: namespace-profile-ubuntu-8-cores)

* A snapshot a day keeps the bugs away! 📷🐛 (OS: namespace-profile-ubuntu-8-cores)

* tweak

* tweak

* A snapshot a day keeps the bugs away! 📷🐛 (OS: namespace-profile-ubuntu-8-cores)

* update fmt ignore etc

* A snapshot a day keeps the bugs away! 📷🐛 (OS: namespace-profile-ubuntu-8-cores)

* A snapshot a day keeps the bugs away! 📷🐛 (OS: namespace-profile-ubuntu-8-cores)

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.eslintignore

@@ -1,3 +1,4 @@
 src/wasm-lib/*
 *.typegen.ts
 packages/codemirror-lsp-client/dist/*
+e2e/playwright/snapshots/prompt-to-edit/*

.github/workflows/e2e-tests.yml

@@ -167,7 +167,7 @@ jobs:
       shell: bash
       id: git-check
       run: |
-          git add e2e/playwright/snapshot-tests.spec.ts-snapshots
+          git add e2e/playwright/snapshot-tests.spec.ts-snapshots e2e/playwright/snapshots
           if git status | grep -q "Changes to be committed"
           then echo "modified=true" >> $GITHUB_OUTPUT
           else echo "modified=false" >> $GITHUB_OUTPUT
@@ -176,7 +176,7 @@ jobs:
       if: steps.git-check.outputs.modified == 'true'
       shell: bash
       run: |
-        git add e2e/playwright/snapshot-tests.spec.ts-snapshots
+        git add e2e/playwright/snapshot-tests.spec.ts-snapshots e2e/playwright/snapshots
         git config --local user.email "github-actions[bot]@users.noreply.github.com"
         git config --local user.name "github-actions[bot]"
         git remote set-url origin https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git

.prettierignore

@@ -10,6 +10,7 @@ target
 src/wasm-lib/pkg
 src/wasm-lib/kcl/bindings
 e2e/playwright/export-snapshots
+e2e/playwright/snapshots/prompt-to-edit
 
 
 # XState generated files

e2e/playwright/fixtures/cmdBarFixture.ts

@@ -1,5 +1,7 @@
-import type { Page, Locator } from '@playwright/test'
-import { expect } from '@playwright/test'
+import type { Page, Locator, Route, Request } from '@playwright/test'
+import { expect, TestInfo } from '@playwright/test'
+import * as fs from 'fs'
+import * as path from 'path'
 
 type CmdBarSerialised =
   | {
@@ -187,4 +189,71 @@ export class CmdBarFixture {
   selectOption = (options: Parameters<typeof this.page.getByRole>[1]) => {
     return this.page.getByRole('option', options)
   }
+
+  /**
+   * Captures a snapshot of the request sent to the text-to-cad API endpoint
+   * and saves it to a file named after the current test.
+   *
+   * The snapshot file will be saved in the specified directory with a filename
+   * derived from the test's full path (including describe blocks).
+   *
+   * @param testInfoInOrderToGetTestTitle The TestInfo object from the test context
+   * @param customOutputDir Optional custom directory for the output file
+   */
+  async captureTextToCadRequestSnapshot(
+    testInfoInOrderToGetTestTitle: TestInfo,
+    customOutputDir = 'e2e/playwright/snapshots/prompt-to-edit'
+  ) {
+    // First sanitize each title component individually
+    const sanitizedTitleComponents = [
+      ...testInfoInOrderToGetTestTitle.titlePath.slice(0, -1), // Get all parent titles
+      testInfoInOrderToGetTestTitle.title, // Add the test title
+    ].map(
+      (component) =>
+        component
+          .replace(/[^a-z0-9]/gi, '-') // Replace non-alphanumeric chars with hyphens
+          .toLowerCase()
+          .replace(/-+/g, '-') // Replace multiple consecutive hyphens with a single one
+          .replace(/^-|-$/g, '') // Remove leading/trailing hyphens
+    )
+
+    // Join the sanitized components with -- as a clear separator
+    const sanitizedTestName = sanitizedTitleComponents.join('--')
+
+    // Create the output path
+    const outputPath = path.join(
+      customOutputDir,
+      `${sanitizedTestName}.snap.json`
+    )
+
+    // Create a handler function that saves request bodies to a file
+    const requestHandler = (route: Route, request: Request) => {
+      try {
+        const requestBody = request.postDataJSON()
+
+        // Ensure directory exists
+        const dir = path.dirname(outputPath)
+        if (!fs.existsSync(dir)) {
+          fs.mkdirSync(dir, { recursive: true })
+        }
+
+        // Write the request body to the file
+        fs.writeFileSync(outputPath, JSON.stringify(requestBody, null, 2))
+
+        console.log(`Saved text-to-cad API request to: ${outputPath}`)
+      } catch (error) {
+        console.error('Error processing text-to-cad request:', error)
+      }
+
+      // Use void to explicitly mark the promise as ignored
+      void route.continue()
+    }
+
+    // Start monitoring requests
+    await this.page.route('**/ml/text-to-cad/iteration', requestHandler)
+
+    console.log(
+      `Monitoring text-to-cad API requests. Output will be saved to: ${outputPath}`
+    )
+  }
 }

e2e/playwright/prompt-to-edit-snapshot-tests.spec.ts

@@ -0,0 +1,98 @@
+import { test, expect } from './zoo-test'
+/* eslint-disable jest/no-conditional-expect */
+
+/**
+ * Snapshot Tests for Text-to-CAD API Requests
+ *
+ * These tests are primarily designed to capture the requests sent to the Text-to-CAD API
+ * rather than to verify application behavior. Unlike regular tests, these tests:
+ *
+ * 1. Don't assert much about the application's response or state changes
+ * 2. Focus on setting up specific scenarios and triggering API requests
+ * 3. Use the captureTextToCadRequestSnapshot() method to save request payloads to snapshot files
+ *
+ * The main purpose is to maintain a collection of real-world API request examples that can be:
+ * - Used for regression testing the (AI) API
+ * - Referenced when making changes to the Text-to-CAD integration, particularly the meta-prompts
+ *   the frontend adds to the user's prompt
+ *
+ * These tests intentionally don't wait for or verify responses, as we're primarily
+ * interested in capturing the outgoing requests for documentation and analysis.
+ *
+ */
+
+const file = `sketch001 = startSketchOn('XZ')
+profile001 = startProfileAt([57.81, 250.51], sketch001)
+  |> line(end = [121.13, 56.63], tag = $seg02)
+  |> line(end = [83.37, -34.61], tag = $seg01)
+  |> line(end = [19.66, -116.4])
+  |> line(end = [-221.8, -41.69])
+  |> line(endAbsolute = [profileStartX(%), profileStartY(%)])
+  |> close()
+extrude001 = extrude(profile001, length = 200)
+sketch002 = startSketchOn('XZ')
+  |> startProfileAt([-73.64, -42.89], %)
+  |> xLine(173.71, %)
+  |> line(end = [-22.12, -94.4])
+  |> xLine(-156.98, %)
+  |> line(endAbsolute = [profileStartX(%), profileStartY(%)])
+  |> close()
+extrude002 = extrude(sketch002, length = 50)
+sketch003 = startSketchOn('XY')
+  |> startProfileAt([52.92, 157.81], %)
+  |> angledLine([0, 176.4], %, $rectangleSegmentA001)
+  |> angledLine([
+       segAng(rectangleSegmentA001) - 90,
+       53.4
+     ], %, $rectangleSegmentB001)
+  |> angledLine([
+       segAng(rectangleSegmentA001),
+       -segLen(rectangleSegmentA001)
+     ], %, $rectangleSegmentC001)
+  |> line(endAbsolute = [profileStartX(%), profileStartY(%)])
+  |> close()
+extrude003 = extrude(sketch003, length = 20)
+`
+
+test(
+  `change colour`,
+  { tag: '@snapshot' },
+  async ({ context, homePage, cmdBar, editor, page, scene }) => {
+    await context.addInitScript((file) => {
+      localStorage.setItem('persistCode', file)
+    }, file)
+    await homePage.goToModelingScene()
+    await scene.waitForExecutionDone()
+
+    const body1CapCoords = { x: 571, y: 351 }
+    const [clickBody1Cap] = scene.makeMouseHelpers(
+      body1CapCoords.x,
+      body1CapCoords.y
+    )
+    const yellow: [number, number, number] = [179, 179, 131]
+    const submittingToast = page.getByText('Submitting to Text-to-CAD API...')
+
+    await test.step('wait for scene to load select body and check selection came through', async () => {
+      await scene.expectPixelColor([134, 134, 134], body1CapCoords, 15)
+      await clickBody1Cap()
+      await scene.expectPixelColor(yellow, body1CapCoords, 20)
+      await editor.expectState({
+        highlightedCode: '',
+        activeLines: ['|>startProfileAt([-73.64,-42.89],%)'],
+        diagnostics: [],
+      })
+    })
+
+    await test.step('fire off edit prompt', async () => {
+      await cmdBar.captureTextToCadRequestSnapshot(test.info())
+      await cmdBar.openCmdBar('promptToEdit')
+      // being specific about the color with a hex means asserting pixel color is more stable
+      await page
+        .getByTestId('cmd-bar-arg-value')
+        .fill('make this neon green please, use #39FF14')
+      await page.waitForTimeout(100)
+      await cmdBar.progressCmdBar()
+      await expect(submittingToast).toBeVisible()
+    })
+  }
+)

e2e/playwright/snapshots/prompt-to-edit/prompt-to-edit-snapshot-tests-spec-ts--change-colour.snap.json

@@ -0,0 +1,33 @@
+{
+  "original_source_code": "sketch001 = startSketchOn('XZ')\nprofile001 = startProfileAt([57.81, 250.51], sketch001)\n  |> line(end = [121.13, 56.63], tag = $seg02)\n  |> line(end = [83.37, -34.61], tag = $seg01)\n  |> line(end = [19.66, -116.4])\n  |> line(end = [-221.8, -41.69])\n  |> line(endAbsolute = [profileStartX(%), profileStartY(%)])\n  |> close()\nextrude001 = extrude(profile001, length = 200)\nsketch002 = startSketchOn('XZ')\n  |> startProfileAt([-73.64, -42.89], %)\n  |> xLine(173.71, %)\n  |> line(end = [-22.12, -94.4])\n  |> xLine(-156.98, %)\n  |> line(endAbsolute = [profileStartX(%), profileStartY(%)])\n  |> close()\nextrude002 = extrude(sketch002, length = 50)\nsketch003 = startSketchOn('XY')\n  |> startProfileAt([52.92, 157.81], %)\n  |> angledLine([0, 176.4], %, $rectangleSegmentA001)\n  |> angledLine([\n       segAng(rectangleSegmentA001) - 90,\n       53.4\n     ], %, $rectangleSegmentB001)\n  |> angledLine([\n       segAng(rectangleSegmentA001),\n       -segLen(rectangleSegmentA001)\n     ], %, $rectangleSegmentC001)\n  |> line(endAbsolute = [profileStartX(%), profileStartY(%)])\n  |> close()\nextrude003 = extrude(sketch003, length = 20)\n",
+  "prompt": "make this neon green please, use #39FF14",
+  "source_ranges": [
+    {
+      "prompt": "The users main selection is the end cap of a general-sweep (that is an extrusion, revolve, sweep or loft).\nThe source range most likely refers to \"startProfileAt\" simply because this is the start of the profile that was swept.\nIf you need to operate on this cap, for example for sketching on the face, you can use the special string END i.e. `startSketchOn(someSweepVariable, END)`\nWhen they made this selection they main have intended this surface directly or meant something more general like the sweep body.\nSee later source ranges for more context.",
+      "range": {
+        "start": {
+          "line": 11,
+          "column": 5
+        },
+        "end": {
+          "line": 11,
+          "column": 40
+        }
+      }
+    },
+    {
+      "prompt": "This is the sweep's source range from the user's main selection of the end cap.",
+      "range": {
+        "start": {
+          "line": 17,
+          "column": 13
+        },
+        "end": {
+          "line": 17,
+          "column": 44
+        }
+      }
+    }
+  ],
+  "kcl_version": "0.2.38"
+}