Update Rust tests to use internal KCL samples on CI (#7014)

* Update Rust tests to use internal KCL samples on CI

* Regenerate manifest with internal KCL samples

* try again

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* remove the needs

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* features

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* features

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* secret

Signed-off-by: Jess Frazelle <github@jessfraz.com>

---------

Signed-off-by: Jess Frazelle <github@jessfraz.com>
Co-authored-by: Jess Frazelle <github@jessfraz.com>

.github/workflows/cargo-test.yml

@@ -155,7 +155,7 @@ jobs:
         shell: bash
         run: |
           [ -e rust-toolchain.toml ] || cp rust/rust-toolchain.toml ./
-      - name: Install rust
+      - name: Install Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           cache: false # Configured below.
@@ -190,6 +190,54 @@ jobs:
           TAB_API_KEY: ${{ secrets.TAB_API_KEY }}
           CI_COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
           CI_PR_NUMBER: ${{ github.event.pull_request.number }}
+  run-internal-kcl-samples:
+    name: cargo test (internal-kcl-samples)
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - runner=32cpu-linux-x64
+      - extras=s3-cache
+    steps:
+      - uses: runs-on/action@v1
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ secrets.MODELING_APP_GH_APP_ID }}
+          private-key: ${{ secrets.MODELING_APP_GH_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+      - name: Use correct Rust toolchain
+        shell: bash
+        run: |
+          [ -e rust-toolchain.toml ] || cp rust/rust-toolchain.toml ./
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache: false # Configured below.
+      - name: Start Vector
+        run: .github/ci-cd-scripts/start-vector-ubuntu.sh
+        env:
+          GH_ACTIONS_AXIOM_TOKEN: ${{ secrets.GH_ACTIONS_AXIOM_TOKEN }}
+          OS_NAME: ${{ env.OS_NAME }}
+      - uses: taiki-e/install-action@nextest
+      - name: Download internal KCL samples
+        run: git clone --depth=1 https://x-access-token:${{ secrets.GH_PAT_KCL_SAMPLES_INTERNAL }}@github.com/KittyCAD/kcl-samples-internal public/kcl-samples/internal
+      - name: Run tests
+        shell: bash
+        run: |-
+          cd rust/kcl-lib
+          cargo nextest run \
+           --retries=10 --no-fail-fast --features artifact-graph --profile=ci \
+           internal \
+           2>&1 | tee /tmp/github-actions.log
+        env:
+          TWENTY_TWENTY: overwrite
+          INSTA_UPDATE: always
+          EXPECTORATE: overwrite
+          KITTYCAD_API_TOKEN: ${{secrets.KITTYCAD_API_TOKEN_DEV}}
+          ZOO_HOST: https://api.dev.zoo.dev
+          MODELING_APP_INTERNAL_SAMPLES_SECRET: ${{secrets.MODELING_APP_INTERNAL_SAMPLES_SECRET}}
   run-wasm-tests:
     name: Run wasm tests
     strategy:

.github/workflows/static-analysis.yml

@@ -127,54 +127,3 @@ jobs:
         uses: actions/checkout@v4
       - name: Run codespell
         uses: crate-ci/typos@v1.32.0
-
-  npm-unit-test-kcl-samples:
-    runs-on: ubuntu-latest
-    needs: npm-build-wasm
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version-file: '.nvmrc'
-          cache: 'npm'
-
-      - run: npm install
-      - uses: taiki-e/install-action@d4635f2de61c8b8104d59cd4aede2060638378cc
-        with:
-          tool: wasm-pack
-
-      - name: Download all artifacts
-        uses: actions/download-artifact@v4
-
-      - name: Copy prepared wasm
-        run: |
-          ls -R prepared-wasm
-          cp prepared-wasm/kcl_wasm_lib_bg.wasm public
-          mkdir rust/kcl-wasm-lib/pkg
-          cp prepared-wasm/kcl_wasm_lib* rust/kcl-wasm-lib/pkg
-
-      - name: Copy prepared ts-rs bindings
-        run: |
-          ls -R prepared-ts-rs-bindings
-          mkdir rust/kcl-lib/bindings
-          cp -r prepared-ts-rs-bindings/* rust/kcl-lib/bindings/
-
-      - run: npm run simpleserver:bg
-        if: ${{ github.event_name != 'release' && github.event_name != 'schedule' }}
-
-      - name: Install Chromium Browser
-        if: ${{ github.event_name != 'release' && github.event_name != 'schedule' }}
-        run: npm run playwright install chromium --with-deps
-
-      - name: Download internal KCL samples
-        run: git clone --depth=1 https://x-access-token:${{ secrets.GH_PAT_KCL_SAMPLES_INTERNAL }}@github.com/KittyCAD/kcl-samples-internal public/kcl-samples/internal
-
-      - name: Regenerate KCL samples manifest
-        run: cd rust/kcl-lib && EXPECTORATE=overwrite cargo test generate_manifest
-
-      - name: Check public and internal KCL samples
-        if: ${{ github.event_name != 'release' && github.event_name != 'schedule' }}
-        run: npm run test:unit:kcl-samples
-        env:
-          VITE_KC_DEV_TOKEN: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}

.gitignore

@@ -58,6 +58,8 @@ trace.zip
 /public/kcl-samples/.github
 /public/kcl-samples/screenshots/main.kcl
 /public/kcl-samples/step/main.kcl
+/public/kcl-samples/internal
+/rust/kcl-lib/tests/kcl_samples/internal
 /test-results/
 /playwright-report/
 /blob-report/

Makefile

@@ -114,7 +114,6 @@ test-unit: install ## Run the unit tests
 	npm run test:unit:components
 	@ curl -fs localhost:3000 >/dev/null || ( echo "Error: localhost:3000 not available, 'make run-web' first" && exit 1 )
 	npm run test:unit
-	npm run test:unit:kcl-samples
 
 .PHONY: test-e2e
 test-e2e: test-e2e-$(TARGET)

package.json

@@ -94,7 +94,6 @@
     "build:wasm:dev": "./scripts/build-wasm-dev.sh",
     "build:wasm:dev:windows": "powershell -ExecutionPolicy Bypass -File ./scripts/build-wasm-dev.ps1",
     "pretest": "npm run remove-importmeta",
-    "test:rust": "(cd rust && just test && just lint)",
     "simpleserver": "npm run pretest && http-server ./public --cors -p 3000",
     "simpleserver:ci": "npm run pretest && http-server ./public --cors -p 3000 &",
     "simpleserver:bg": "npm run pretest && http-server ./public --cors -p 3000 &",
@@ -130,15 +129,14 @@
     "tronb:package:prod": "npm run tronb:vite:prod && electron-builder --config electron-builder.yml --publish always",
     "test-setup": "npm install && npm run build:wasm",
     "test": "vitest --mode development",
+    "test:rust": "(cd rust && just test && just lint)",
     "test:snapshots": "PLATFORM=web NODE_ENV=development playwright test --config=playwright.config.ts --grep=@snapshot --trace=on --shard=1/1",
-    "test:unit": "vitest run --mode development --exclude **/kclSamples.test.ts --exclude **/jest-component-unit-tests/*",
+    "test:unit": "vitest run --mode development --exclude **/jest-component-unit-tests/*",
     "test:unit:components": "jest -c jest-component-unit-tests/jest.config.ts --rootDir jest-component-unit-tests/",
-    "test:unit:kcl-samples": "vitest run --mode development ./src/lang/kclSamples.test.ts",
     "test:playwright:electron": "playwright test --config=playwright.electron.config.ts --grep-invert=@snapshot",
     "test:playwright:electron:local": "npm run tronb:vite:dev && playwright test --config=playwright.electron.config.ts --grep-invert=@snapshot --grep-invert=\"$(curl --silent https://test-analysis-bot.hawk-dinosaur.ts.net/projects/KittyCAD/modeling-app/tests/disabled/regex)\"",
     "test:playwright:electron:local-engine": "npm run tronb:vite:dev && playwright test --config=playwright.electron.config.ts --grep-invert='@snapshot|@skipLocalEngine' --grep-invert=\"$(curl --silent https://test-analysis-bot.hawk-dinosaur.ts.net/projects/KittyCAD/modeling-app/tests/disabled/regex)\"",
-    "test:unit:local": "npm run simpleserver:bg && npm run test:unit; kill-port 3000",
+    "test:unit:local": "npm run simpleserver:bg && npm run test:unit; kill-port 3000"
-    "test:unit:kcl-samples:local": "npm run simpleserver:bg && npm run test:unit:kcl-samples; kill-port 3000"
   },
   "browserslist": {
     "production": [

rust/Cargo.lock

@@ -1847,6 +1847,7 @@ dependencies = [
 name = "kcl-directory-test-macro"
 version = "0.1.73"
 dependencies = [
+ "convert_case",
  "proc-macro2",
  "quote",
  "syn 2.0.100",

rust/kcl-directory-test-macro/Cargo.toml

@@ -11,6 +11,7 @@ proc-macro = true
 bench = false
 
 [dependencies]
+convert_case = "0.8.0"
 proc-macro2 = "1"
 quote = "1"
 syn = { version = "2.0.96", features = ["full"] }

rust/kcl-directory-test-macro/src/lib.rs

@@ -1,10 +1,13 @@
 use std::fs;
 
+use convert_case::Casing;
 use proc_macro::TokenStream;
 use quote::{format_ident, quote};
 use syn::{parse_macro_input, LitStr};
 
 /// A macro that generates test functions for each directory within a given path.
+/// To be included the test directory must have a main.kcl file.
+/// This will also recursively search for directories within the given path.
 ///
 /// # Example
 ///
@@ -45,7 +48,11 @@ pub fn test_all_dirs(attr: TokenStream, item: TokenStream) -> TokenStream {
 
     // Generate a test function for each directory
     let test_fns = dirs.iter().map(|(dir_name, dir_path)| {
-        let test_fn_name = format_ident!("{}_{}", fn_name, sanitize_dir_name(dir_name));
+        let relative_path = dir_path
+            .strip_prefix(&path.to_string_lossy().to_string())
+            .unwrap()
+            .trim();
+        let test_fn_name = format_ident!("{}_{}", fn_name, sanitize_dir_name(relative_path));
         let dir_name_str = dir_name.clone();
         let dir_path_str = dir_path.clone();
 
@@ -75,16 +82,26 @@ fn get_all_directories(path: &std::path::Path) -> Result<Vec<(String, String)>,
 
     for entry in fs::read_dir(path)? {
         let entry = entry?;
-        let path = entry.path();
+        let new_path = entry.path();
 
-        if path.is_dir() && !IGNORE_DIRS.contains(&path.file_name().and_then(|name| name.to_str()).unwrap_or("")) {
+        if new_path.is_dir()
-            let dir_name = path
+            && !IGNORE_DIRS.contains(&new_path.file_name().and_then(|name| name.to_str()).unwrap_or(""))
+        {
+            // Check if the directory contains a main.kcl file.
+            let main_kcl_path = new_path.join("main.kcl");
+            if !main_kcl_path.exists() {
+                // Recurse into the directory.
+                let sub_dirs = get_all_directories(&new_path)?;
+                dirs.extend(sub_dirs);
+                continue;
+            }
+            let dir_name = new_path
                 .file_name()
                 .and_then(|name| name.to_str())
                 .unwrap_or("unknown")
                 .to_string();
 
-            let dir_path = path.to_str().unwrap_or("unknown").to_string();
+            let dir_path = new_path.to_str().unwrap_or("unknown").to_string();
 
             dirs.push((dir_name, dir_path));
         }
@@ -95,10 +112,9 @@ fn get_all_directories(path: &std::path::Path) -> Result<Vec<(String, String)>,
 
 /// Sanitize directory name to create a valid Rust identifier
 fn sanitize_dir_name(name: &str) -> String {
-    let name = name.replace(|c: char| !c.is_ascii_alphanumeric() && c != '_', "_");
+    let binding = name
-    if name.chars().next().is_some_and(|c| c.is_numeric()) {
+        .replace(|c: char| !c.is_ascii_alphanumeric() && c != '_', "_")
-        format!("d_{}", name)
+        .replace("/", "_");
-    } else {
+    let name = binding.trim_start_matches('_').to_string();
-        name
+    name.to_case(convert_case::Case::Snake)
-    }
 }

rust/kcl-lib/src/simulation_tests.rs

@@ -1,4 +1,5 @@
 use std::{
+    fs::read_to_string,
     panic::{catch_unwind, AssertUnwindSafe},
     path::{Path, PathBuf},
 };
@@ -21,7 +22,7 @@ struct Test {
     name: String,
     /// The name of the KCL file that's the entry point, e.g. "main.kcl", in the
     /// `input_dir`.
-    entry_point: String,
+    entry_point: PathBuf,
     /// Input KCL files are in this directory.
     input_dir: PathBuf,
     /// Expected snapshot output files are in this directory.
@@ -34,7 +35,7 @@ impl Test {
     fn new(name: &str) -> Self {
         Self {
             name: name.to_owned(),
-            entry_point: "input.kcl".to_owned(),
+            entry_point: Path::new("tests").join(name).join("input.kcl"),
             input_dir: Path::new("tests").join(name),
             output_dir: Path::new("tests").join(name),
         }
@@ -66,19 +67,12 @@ where
     settings.bind(f);
 }
 
-fn read<P>(filename: &str, dir: P) -> String
-where
-    P: AsRef<Path>,
-{
-    std::fs::read_to_string(dir.as_ref().join(filename)).expect("Failed to read file: {filename}")
-}
-
 fn parse(test_name: &str) {
     parse_test(&Test::new(test_name));
 }
 
 fn parse_test(test: &Test) {
-    let input = read(&test.entry_point, &test.input_dir);
+    let input = read_to_string(&test.entry_point).unwrap();
     let tokens = crate::parsing::token::lex(&input, ModuleId::default()).unwrap();
 
     // Parse the tokens into an AST.
@@ -98,7 +92,7 @@ async fn unparse(test_name: &str) {
 
 async fn unparse_test(test: &Test) {
     // Parse into an AST
-    let input = read(&test.entry_point, &test.input_dir);
+    let input = read_to_string(&test.entry_point).unwrap();
     let tokens = crate::parsing::token::lex(&input, ModuleId::default()).unwrap();
     let ast = crate::parsing::parse_tokens(tokens).unwrap();
 
@@ -111,10 +105,9 @@ async fn unparse_test(test: &Test) {
     }));
 
     // Check all the rest of the files in the directory.
-    let entry_point = test.input_dir.join(&test.entry_point);
     let kcl_files = crate::unparser::walk_dir(&test.input_dir).await.unwrap();
     // Filter out the entry point file.
-    let kcl_files = kcl_files.into_iter().filter(|f| f != &entry_point);
+    let kcl_files = kcl_files.into_iter().filter(|f| f != &test.entry_point);
     let futures = kcl_files
         .into_iter()
         .filter(|file| file.extension().is_some_and(|ext| ext == "kcl")) // We only care about kcl
@@ -154,13 +147,11 @@ async fn execute(test_name: &str, render_to_png: bool) {
 }
 
 async fn execute_test(test: &Test, render_to_png: bool, export_step: bool) {
-    let input = read(&test.entry_point, &test.input_dir);
+    let input = read_to_string(&test.entry_point).unwrap();
     let ast = crate::Program::parse_no_errs(&input).unwrap();
 
     // Run the program.
-    let exec_res =
+    let exec_res = crate::test_server::execute_and_snapshot_ast(ast, Some(test.entry_point.clone()), export_step).await;
-        crate::test_server::execute_and_snapshot_ast(ast, Some(test.input_dir.join(&test.entry_point)), export_step)
-            .await;
     match exec_res {
         Ok((exec_state, env_ref, png, step)) => {
             let fail_path = test.output_dir.join("execution_error.snap");

rust/kcl-lib/src/simulation_tests/kcl_samples.rs

@@ -23,7 +23,7 @@ lazy_static::lazy_static! {
 
 #[kcl_directory_test_macro::test_all_dirs("../public/kcl-samples")]
 fn parse(dir_name: &str, dir_path: &Path) {
-    let t = test(dir_name, dir_path.join("main.kcl").to_str().unwrap().to_owned());
+    let t = test(dir_name, dir_path.join("main.kcl"));
     let write_new = matches!(
         std::env::var("INSTA_UPDATE").as_deref(),
         Ok("auto" | "always" | "new" | "unseen")
@@ -37,7 +37,7 @@ fn parse(dir_name: &str, dir_path: &Path) {
 
 #[kcl_directory_test_macro::test_all_dirs("../public/kcl-samples")]
 async fn unparse(dir_name: &str, dir_path: &Path) {
-    let t = test(dir_name, dir_path.join("main.kcl").to_str().unwrap().to_owned());
+    let t = test(dir_name, dir_path.join("main.kcl"));
     unparse_test(&t).await;
 }
 
@@ -71,7 +71,7 @@ async fn unparse_test(test: &Test) {
 
 #[kcl_directory_test_macro::test_all_dirs("../public/kcl-samples")]
 async fn kcl_test_execute(dir_name: &str, dir_path: &Path) {
-    let t = test(dir_name, dir_path.join("main.kcl").to_str().unwrap().to_owned());
+    let t = test(dir_name, dir_path.join("main.kcl"));
     super::execute_test(&t, true, true).await;
 }
 
@@ -129,12 +129,22 @@ fn test_after_engine_generate_manifest() {
     generate_kcl_manifest(&INPUTS_DIR).unwrap();
 }
 
-fn test(test_name: &str, entry_point: String) -> Test {
+fn test(test_name: &str, entry_point: std::path::PathBuf) -> Test {
+    let parent = std::fs::canonicalize(entry_point.parent().unwrap()).unwrap();
+    let inputs_dir = std::fs::canonicalize(INPUTS_DIR.as_path()).unwrap();
+    let relative_path = parent.strip_prefix(inputs_dir).unwrap();
+    let output_dir = std::fs::canonicalize(OUTPUTS_DIR.as_path()).unwrap();
+    let relative_output_dir = output_dir.join(relative_path);
+
+    // Ensure the output directory exists.
+    if !relative_output_dir.exists() {
+        std::fs::create_dir_all(&relative_output_dir).unwrap();
+    }
     Test {
         name: test_name.to_owned(),
-        entry_point,
+        entry_point: entry_point.clone(),
-        input_dir: INPUTS_DIR.join(test_name),
+        input_dir: parent.to_path_buf(),
-        output_dir: OUTPUTS_DIR.join(test_name),
+        output_dir: relative_output_dir,
     }
 }
 
@@ -173,8 +183,9 @@ fn kcl_samples_inputs() -> Vec<Test> {
         eprintln!("Found KCL sample: {:?}", dir_name.to_string_lossy());
         // Look for the entry point inside the directory.
         let sub_dir = INPUTS_DIR.join(dir_name);
-        let entry_point = if sub_dir.join("main.kcl").exists() {
+        let main_kcl_path = sub_dir.join("main.kcl");
-            "main.kcl".to_owned()
+        let entry_point = if main_kcl_path.exists() {
+            main_kcl_path
         } else {
             panic!("No main.kcl found in {:?}", sub_dir);
         };

src/lang/kclSamples.test.ts

@@ -1,71 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-
-import { assertParse } from '@src/lang/wasm'
-import { initPromise } from '@src/lang/wasmUtils'
-import { enginelessExecutor } from '@src/lib/testHelpers'
-
-// The purpose of these tests is to act as a first line of defense
-// if something gets real screwy with our KCL ecosystem.
-// THESE TESTS ONLY RUN UNDER A NODEJS ENVIRONMENT. They DO NOT
-// test under our application.
-
-const DIR_KCL_SAMPLES = 'public/kcl-samples'
-
-interface KclSampleFile {
-  file: string
-  pathFromProjectDirectoryToFirstFile: string
-  title: string
-  filename: string
-  description: string
-}
-
-// @ts-expect-error
-let files = await fs.readdir(DIR_KCL_SAMPLES)
-// @ts-expect-error
-const manifestJsonStr = await fs.readFile(
-  path.resolve(DIR_KCL_SAMPLES, 'manifest.json'),
-  'utf-8'
-)
-const manifest = JSON.parse(manifestJsonStr)
-
-process.chdir(DIR_KCL_SAMPLES)
-
-beforeAll(async () => {
-  await initPromise
-})
-
-afterAll(async () => {
-  try {
-    process.chdir('..')
-    await fs.rm(DIR_KCL_SAMPLES, { recursive: true })
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  } catch (e) {}
-})
-
-describe('Test KCL Samples from public Github repository', () => {
-  describe('when performing enginelessExecutor', () => {
-    manifest.forEach((file: KclSampleFile) => {
-      it(
-        `should execute ${file.title} (${file.file}) successfully`,
-        async () => {
-          // This one is being a little bitch but works in rust.
-          if (file.title === 'Helium Tank') {
-            return
-          }
-          const code = await fs.readFile(
-            file.pathFromProjectDirectoryToFirstFile,
-            'utf-8'
-          )
-          const ast = assertParse(code)
-          await enginelessExecutor(
-            ast,
-            false,
-            file.pathFromProjectDirectoryToFirstFile
-          )
-        },
-        files.length * 1000
-      )
-    })
-  })
-})