Update Rust tests to use internal KCL samples on CI (#7014)

* Update Rust tests to use internal KCL samples on CI * Regenerate manifest with internal KCL samples * try again Signed-off-by: Jess Frazelle <github@jessfraz.com> * remove the needs Signed-off-by: Jess Frazelle <github@jessfraz.com> * updates Signed-off-by: Jess Frazelle <github@jessfraz.com> * updates Signed-off-by: Jess Frazelle <github@jessfraz.com> * updates Signed-off-by: Jess Frazelle <github@jessfraz.com> * updates Signed-off-by: Jess Frazelle <github@jessfraz.com> * features Signed-off-by: Jess Frazelle <github@jessfraz.com> * features Signed-off-by: Jess Frazelle <github@jessfraz.com> * updates Signed-off-by: Jess Frazelle <github@jessfraz.com> * secret Signed-off-by: Jess Frazelle <github@jessfraz.com> --------- Signed-off-by: Jess Frazelle <github@jessfraz.com> Co-authored-by: Jess Frazelle <github@jessfraz.com>
2025-05-16 15:38:55 -04:00
parent e3e67b00d5
commit 620b7401aa
11 changed files with 111 additions and 166 deletions
--- a/.github/workflows/cargo-test.yml
+++ b/.github/workflows/cargo-test.yml
@ -155,7 +155,7 @@ jobs:
        shell: bash
        run: |
          [ -e rust-toolchain.toml ] || cp rust/rust-toolchain.toml ./
-      - name: Install rust
+      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache: false # Configured below.
@ -190,6 +190,54 @@ jobs:
          TAB_API_KEY: ${{ secrets.TAB_API_KEY }}
          CI_COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
          CI_PR_NUMBER: ${{ github.event.pull_request.number }}
+  run-internal-kcl-samples:
+    name: cargo test (internal-kcl-samples)
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - runner=32cpu-linux-x64
+      - extras=s3-cache
+    steps:
+      - uses: runs-on/action@v1
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ secrets.MODELING_APP_GH_APP_ID }}
+          private-key: ${{ secrets.MODELING_APP_GH_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+      - name: Use correct Rust toolchain
+        shell: bash
+        run: |
+          [ -e rust-toolchain.toml ] || cp rust/rust-toolchain.toml ./
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache: false # Configured below.
+      - name: Start Vector
+        run: .github/ci-cd-scripts/start-vector-ubuntu.sh
+        env:
+          GH_ACTIONS_AXIOM_TOKEN: ${{ secrets.GH_ACTIONS_AXIOM_TOKEN }}
+          OS_NAME: ${{ env.OS_NAME }}
+      - uses: taiki-e/install-action@nextest
+      - name: Download internal KCL samples
+        run: git clone --depth=1 https://x-access-token:${{ secrets.GH_PAT_KCL_SAMPLES_INTERNAL }}@github.com/KittyCAD/kcl-samples-internal public/kcl-samples/internal
+      - name: Run tests
+        shell: bash
+        run: |-
+          cd rust/kcl-lib
+          cargo nextest run \
+           --retries=10 --no-fail-fast --features artifact-graph --profile=ci \
+           internal \
+           2>&1 | tee /tmp/github-actions.log
+        env:
+          TWENTY_TWENTY: overwrite
+          INSTA_UPDATE: always
+          EXPECTORATE: overwrite
+          KITTYCAD_API_TOKEN: ${{secrets.KITTYCAD_API_TOKEN_DEV}}
+          ZOO_HOST: https://api.dev.zoo.dev
+          MODELING_APP_INTERNAL_SAMPLES_SECRET: ${{secrets.MODELING_APP_INTERNAL_SAMPLES_SECRET}}
  run-wasm-tests:
    name: Run wasm tests
    strategy:
--- a/.github/workflows/static-analysis.yml
+++ b/.github/workflows/static-analysis.yml
@ -127,54 +127,3 @@ jobs:
        uses: actions/checkout@v4
      - name: Run codespell
        uses: crate-ci/typos@v1.32.0
-
-  npm-unit-test-kcl-samples:
-    runs-on: ubuntu-latest
-    needs: npm-build-wasm
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version-file: '.nvmrc'
-          cache: 'npm'
-
-      - run: npm install
-      - uses: taiki-e/install-action@d4635f2de61c8b8104d59cd4aede2060638378cc
-        with:
-          tool: wasm-pack
-
-      - name: Download all artifacts
-        uses: actions/download-artifact@v4
-
-      - name: Copy prepared wasm
-        run: |
-          ls -R prepared-wasm
-          cp prepared-wasm/kcl_wasm_lib_bg.wasm public
-          mkdir rust/kcl-wasm-lib/pkg
-          cp prepared-wasm/kcl_wasm_lib* rust/kcl-wasm-lib/pkg
-
-      - name: Copy prepared ts-rs bindings
-        run: |
-          ls -R prepared-ts-rs-bindings
-          mkdir rust/kcl-lib/bindings
-          cp -r prepared-ts-rs-bindings/* rust/kcl-lib/bindings/
-
-      - run: npm run simpleserver:bg
-        if: ${{ github.event_name != 'release' && github.event_name != 'schedule' }}
-
-      - name: Install Chromium Browser
-        if: ${{ github.event_name != 'release' && github.event_name != 'schedule' }}
-        run: npm run playwright install chromium --with-deps
-
-      - name: Download internal KCL samples
-        run: git clone --depth=1 https://x-access-token:${{ secrets.GH_PAT_KCL_SAMPLES_INTERNAL }}@github.com/KittyCAD/kcl-samples-internal public/kcl-samples/internal
-
-      - name: Regenerate KCL samples manifest
-        run: cd rust/kcl-lib && EXPECTORATE=overwrite cargo test generate_manifest
-
-      - name: Check public and internal KCL samples
-        if: ${{ github.event_name != 'release' && github.event_name != 'schedule' }}
-        run: npm run test:unit:kcl-samples
-        env:
-          VITE_KC_DEV_TOKEN: ${{ secrets.KITTYCAD_API_TOKEN_DEV }}
--- a/.gitignore
+++ b/.gitignore
@ -58,6 +58,8 @@ trace.zip
 /public/kcl-samples/.github
 /public/kcl-samples/screenshots/main.kcl
 /public/kcl-samples/step/main.kcl
+/public/kcl-samples/internal
+/rust/kcl-lib/tests/kcl_samples/internal
 /test-results/
 /playwright-report/
 /blob-report/
--- a/1
+++ b/1
@ -114,7 +114,6 @@ test-unit: install ## Run the unit tests
 	npm run test:unit:components
 	@ curl -fs localhost:3000 >/dev/null || ( echo "Error: localhost:3000 not available, 'make run-web' first" && exit 1 )
 	npm run test:unit
-	npm run test:unit:kcl-samples

 .PHONY: test-e2e
 test-e2e: test-e2e-$(TARGET)
--- a/package.json
+++ b/package.json
@ -94,7 +94,6 @@
    "build:wasm:dev": "./scripts/build-wasm-dev.sh",
    "build:wasm:dev:windows": "powershell -ExecutionPolicy Bypass -File ./scripts/build-wasm-dev.ps1",
    "pretest": "npm run remove-importmeta",
-    "test:rust": "(cd rust && just test && just lint)",
    "simpleserver": "npm run pretest && http-server ./public --cors -p 3000",
    "simpleserver:ci": "npm run pretest && http-server ./public --cors -p 3000 &",
    "simpleserver:bg": "npm run pretest && http-server ./public --cors -p 3000 &",
@ -130,15 +129,14 @@
    "tronb:package:prod": "npm run tronb:vite:prod && electron-builder --config electron-builder.yml --publish always",
    "test-setup": "npm install && npm run build:wasm",
    "test": "vitest --mode development",
+    "test:rust": "(cd rust && just test && just lint)",
    "test:snapshots": "PLATFORM=web NODE_ENV=development playwright test --config=playwright.config.ts --grep=@snapshot --trace=on --shard=1/1",
-    "test:unit": "vitest run --mode development --exclude **/kclSamples.test.ts --exclude **/jest-component-unit-tests/*",
+    "test:unit": "vitest run --mode development --exclude **/jest-component-unit-tests/*",
    "test:unit:components": "jest -c jest-component-unit-tests/jest.config.ts --rootDir jest-component-unit-tests/",
-    "test:unit:kcl-samples": "vitest run --mode development ./src/lang/kclSamples.test.ts",
    "test:playwright:electron": "playwright test --config=playwright.electron.config.ts --grep-invert=@snapshot",
    "test:playwright:electron:local": "npm run tronb:vite:dev && playwright test --config=playwright.electron.config.ts --grep-invert=@snapshot --grep-invert=\"$(curl --silent https://test-analysis-bot.hawk-dinosaur.ts.net/projects/KittyCAD/modeling-app/tests/disabled/regex)\"",
    "test:playwright:electron:local-engine": "npm run tronb:vite:dev && playwright test --config=playwright.electron.config.ts --grep-invert='@snapshot|@skipLocalEngine' --grep-invert=\"$(curl --silent https://test-analysis-bot.hawk-dinosaur.ts.net/projects/KittyCAD/modeling-app/tests/disabled/regex)\"",
-    "test:unit:local": "npm run simpleserver:bg && npm run test:unit; kill-port 3000",
-    "test:unit:kcl-samples:local": "npm run simpleserver:bg && npm run test:unit:kcl-samples; kill-port 3000"
+    "test:unit:local": "npm run simpleserver:bg && npm run test:unit; kill-port 3000"
  },
  "browserslist": {
    "production": [
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@ -1847,6 +1847,7 @@ dependencies = [
 name = "kcl-directory-test-macro"
 version = "0.1.73"
 dependencies = [
+ "convert_case",
 "proc-macro2",
 "quote",
 "syn 2.0.100",
--- a/rust/kcl-directory-test-macro/Cargo.toml
+++ b/rust/kcl-directory-test-macro/Cargo.toml
@ -11,6 +11,7 @@ proc-macro = true
 bench = false

 [dependencies]
+convert_case = "0.8.0"
 proc-macro2 = "1"
 quote = "1"
 syn = { version = "2.0.96", features = ["full"] }
--- a/rust/kcl-directory-test-macro/src/lib.rs
+++ b/rust/kcl-directory-test-macro/src/lib.rs
@ -1,10 +1,13 @@
 use std::fs;

+use convert_case::Casing;
 use proc_macro::TokenStream;
 use quote::{format_ident, quote};
 use syn::{parse_macro_input, LitStr};

 /// A macro that generates test functions for each directory within a given path.
+/// To be included the test directory must have a main.kcl file.
+/// This will also recursively search for directories within the given path.
 ///
 /// # Example
 ///
@ -45,7 +48,11 @@ pub fn test_all_dirs(attr: TokenStream, item: TokenStream) -> TokenStream {

    // Generate a test function for each directory
    let test_fns = dirs.iter().map(|(dir_name, dir_path)| {
-        let test_fn_name = format_ident!("{}_{}", fn_name, sanitize_dir_name(dir_name));
+        let relative_path = dir_path
+            .strip_prefix(&path.to_string_lossy().to_string())
+            .unwrap()
+            .trim();
+        let test_fn_name = format_ident!("{}_{}", fn_name, sanitize_dir_name(relative_path));
        let dir_name_str = dir_name.clone();
        let dir_path_str = dir_path.clone();

@ -75,16 +82,26 @@ fn get_all_directories(path: &std::path::Path) -> Result<Vec<(String, String)>,

    for entry in fs::read_dir(path)? {
        let entry = entry?;
-        let path = entry.path();
+        let new_path = entry.path();

-        if path.is_dir() && !IGNORE_DIRS.contains(&path.file_name().and_then(|name| name.to_str()).unwrap_or("")) {
-            let dir_name = path
+        if new_path.is_dir()
+            && !IGNORE_DIRS.contains(&new_path.file_name().and_then(|name| name.to_str()).unwrap_or(""))
+        {
+            // Check if the directory contains a main.kcl file.
+            let main_kcl_path = new_path.join("main.kcl");
+            if !main_kcl_path.exists() {
+                // Recurse into the directory.
+                let sub_dirs = get_all_directories(&new_path)?;
+                dirs.extend(sub_dirs);
+                continue;
+            }
+            let dir_name = new_path
                .file_name()
                .and_then(|name| name.to_str())
                .unwrap_or("unknown")
                .to_string();

-            let dir_path = path.to_str().unwrap_or("unknown").to_string();
+            let dir_path = new_path.to_str().unwrap_or("unknown").to_string();

            dirs.push((dir_name, dir_path));
        }
@ -95,10 +112,9 @@ fn get_all_directories(path: &std::path::Path) -> Result<Vec<(String, String)>,

 /// Sanitize directory name to create a valid Rust identifier
 fn sanitize_dir_name(name: &str) -> String {
-    let name = name.replace(|c: char| !c.is_ascii_alphanumeric() && c != '_', "_");
-    if name.chars().next().is_some_and(|c| c.is_numeric()) {
-        format!("d_{}", name)
-    } else {
-        name
-    }
+    let binding = name
+        .replace(|c: char| !c.is_ascii_alphanumeric() && c != '_', "_")
+        .replace("/", "_");
+    let name = binding.trim_start_matches('_').to_string();
+    name.to_case(convert_case::Case::Snake)
 }
--- a/rust/kcl-lib/src/simulation_tests.rs
+++ b/rust/kcl-lib/src/simulation_tests.rs
@ -1,4 +1,5 @@
 use std::{
+    fs::read_to_string,
    panic::{catch_unwind, AssertUnwindSafe},
    path::{Path, PathBuf},
 };
@ -21,7 +22,7 @@ struct Test {
    name: String,
    /// The name of the KCL file that's the entry point, e.g. "main.kcl", in the
    /// `input_dir`.
-    entry_point: String,
+    entry_point: PathBuf,
    /// Input KCL files are in this directory.
    input_dir: PathBuf,
    /// Expected snapshot output files are in this directory.
@ -34,7 +35,7 @@ impl Test {
    fn new(name: &str) -> Self {
        Self {
            name: name.to_owned(),
-            entry_point: "input.kcl".to_owned(),
+            entry_point: Path::new("tests").join(name).join("input.kcl"),
            input_dir: Path::new("tests").join(name),
            output_dir: Path::new("tests").join(name),
        }
@ -66,19 +67,12 @@ where
    settings.bind(f);
 }

-fn read<P>(filename: &str, dir: P) -> String
-where
-    P: AsRef<Path>,
-{
-    std::fs::read_to_string(dir.as_ref().join(filename)).expect("Failed to read file: {filename}")
-}
-
 fn parse(test_name: &str) {
    parse_test(&Test::new(test_name));
 }

 fn parse_test(test: &Test) {
-    let input = read(&test.entry_point, &test.input_dir);
+    let input = read_to_string(&test.entry_point).unwrap();
    let tokens = crate::parsing::token::lex(&input, ModuleId::default()).unwrap();

    // Parse the tokens into an AST.
@ -98,7 +92,7 @@ async fn unparse(test_name: &str) {

 async fn unparse_test(test: &Test) {
    // Parse into an AST
-    let input = read(&test.entry_point, &test.input_dir);
+    let input = read_to_string(&test.entry_point).unwrap();
    let tokens = crate::parsing::token::lex(&input, ModuleId::default()).unwrap();
    let ast = crate::parsing::parse_tokens(tokens).unwrap();

@ -111,10 +105,9 @@ async fn unparse_test(test: &Test) {
    }));

    // Check all the rest of the files in the directory.
-    let entry_point = test.input_dir.join(&test.entry_point);
    let kcl_files = crate::unparser::walk_dir(&test.input_dir).await.unwrap();
    // Filter out the entry point file.
-    let kcl_files = kcl_files.into_iter().filter(|f| f != &entry_point);
+    let kcl_files = kcl_files.into_iter().filter(|f| f != &test.entry_point);
    let futures = kcl_files
        .into_iter()
        .filter(|file| file.extension().is_some_and(|ext| ext == "kcl")) // We only care about kcl
@ -154,13 +147,11 @@ async fn execute(test_name: &str, render_to_png: bool) {
 }

 async fn execute_test(test: &Test, render_to_png: bool, export_step: bool) {
-    let input = read(&test.entry_point, &test.input_dir);
+    let input = read_to_string(&test.entry_point).unwrap();
    let ast = crate::Program::parse_no_errs(&input).unwrap();

    // Run the program.
-    let exec_res =
-        crate::test_server::execute_and_snapshot_ast(ast, Some(test.input_dir.join(&test.entry_point)), export_step)
-            .await;
+    let exec_res = crate::test_server::execute_and_snapshot_ast(ast, Some(test.entry_point.clone()), export_step).await;
    match exec_res {
        Ok((exec_state, env_ref, png, step)) => {
            let fail_path = test.output_dir.join("execution_error.snap");
--- a/rust/kcl-lib/src/simulation_tests/kcl_samples.rs
+++ b/rust/kcl-lib/src/simulation_tests/kcl_samples.rs
@ -23,7 +23,7 @@ lazy_static::lazy_static! {

 #[kcl_directory_test_macro::test_all_dirs("../public/kcl-samples")]
 fn parse(dir_name: &str, dir_path: &Path) {
-    let t = test(dir_name, dir_path.join("main.kcl").to_str().unwrap().to_owned());
+    let t = test(dir_name, dir_path.join("main.kcl"));
    let write_new = matches!(
        std::env::var("INSTA_UPDATE").as_deref(),
        Ok("auto" | "always" | "new" | "unseen")
@ -37,7 +37,7 @@ fn parse(dir_name: &str, dir_path: &Path) {

 #[kcl_directory_test_macro::test_all_dirs("../public/kcl-samples")]
 async fn unparse(dir_name: &str, dir_path: &Path) {
-    let t = test(dir_name, dir_path.join("main.kcl").to_str().unwrap().to_owned());
+    let t = test(dir_name, dir_path.join("main.kcl"));
    unparse_test(&t).await;
 }

@ -71,7 +71,7 @@ async fn unparse_test(test: &Test) {

 #[kcl_directory_test_macro::test_all_dirs("../public/kcl-samples")]
 async fn kcl_test_execute(dir_name: &str, dir_path: &Path) {
-    let t = test(dir_name, dir_path.join("main.kcl").to_str().unwrap().to_owned());
+    let t = test(dir_name, dir_path.join("main.kcl"));
    super::execute_test(&t, true, true).await;
 }

@ -129,12 +129,22 @@ fn test_after_engine_generate_manifest() {
    generate_kcl_manifest(&INPUTS_DIR).unwrap();
 }

-fn test(test_name: &str, entry_point: String) -> Test {
+fn test(test_name: &str, entry_point: std::path::PathBuf) -> Test {
+    let parent = std::fs::canonicalize(entry_point.parent().unwrap()).unwrap();
+    let inputs_dir = std::fs::canonicalize(INPUTS_DIR.as_path()).unwrap();
+    let relative_path = parent.strip_prefix(inputs_dir).unwrap();
+    let output_dir = std::fs::canonicalize(OUTPUTS_DIR.as_path()).unwrap();
+    let relative_output_dir = output_dir.join(relative_path);
+
+    // Ensure the output directory exists.
+    if !relative_output_dir.exists() {
+        std::fs::create_dir_all(&relative_output_dir).unwrap();
+    }
    Test {
        name: test_name.to_owned(),
-        entry_point,
-        input_dir: INPUTS_DIR.join(test_name),
-        output_dir: OUTPUTS_DIR.join(test_name),
+        entry_point: entry_point.clone(),
+        input_dir: parent.to_path_buf(),
+        output_dir: relative_output_dir,
    }
 }

@ -173,8 +183,9 @@ fn kcl_samples_inputs() -> Vec<Test> {
        eprintln!("Found KCL sample: {:?}", dir_name.to_string_lossy());
        // Look for the entry point inside the directory.
        let sub_dir = INPUTS_DIR.join(dir_name);
-        let entry_point = if sub_dir.join("main.kcl").exists() {
-            "main.kcl".to_owned()
+        let main_kcl_path = sub_dir.join("main.kcl");
+        let entry_point = if main_kcl_path.exists() {
+            main_kcl_path
        } else {
            panic!("No main.kcl found in {:?}", sub_dir);
        };
--- a/src/lang/kclSamples.test.ts
+++ b/src/lang/kclSamples.test.ts
@ -1,71 +0,0 @@
-import fs from 'node:fs/promises'
-import path from 'node:path'
-
-import { assertParse } from '@src/lang/wasm'
-import { initPromise } from '@src/lang/wasmUtils'
-import { enginelessExecutor } from '@src/lib/testHelpers'
-
-// The purpose of these tests is to act as a first line of defense
-// if something gets real screwy with our KCL ecosystem.
-// THESE TESTS ONLY RUN UNDER A NODEJS ENVIRONMENT. They DO NOT
-// test under our application.
-
-const DIR_KCL_SAMPLES = 'public/kcl-samples'
-
-interface KclSampleFile {
-  file: string
-  pathFromProjectDirectoryToFirstFile: string
-  title: string
-  filename: string
-  description: string
-}
-
-// @ts-expect-error
-let files = await fs.readdir(DIR_KCL_SAMPLES)
-// @ts-expect-error
-const manifestJsonStr = await fs.readFile(
-  path.resolve(DIR_KCL_SAMPLES, 'manifest.json'),
-  'utf-8'
-)
-const manifest = JSON.parse(manifestJsonStr)
-
-process.chdir(DIR_KCL_SAMPLES)
-
-beforeAll(async () => {
-  await initPromise
-})
-
-afterAll(async () => {
-  try {
-    process.chdir('..')
-    await fs.rm(DIR_KCL_SAMPLES, { recursive: true })
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  } catch (e) {}
-})
-
-describe('Test KCL Samples from public Github repository', () => {
-  describe('when performing enginelessExecutor', () => {
-    manifest.forEach((file: KclSampleFile) => {
-      it(
-        `should execute ${file.title} (${file.file}) successfully`,
-        async () => {
-          // This one is being a little bitch but works in rust.
-          if (file.title === 'Helium Tank') {
-            return
-          }
-          const code = await fs.readFile(
-            file.pathFromProjectDirectoryToFirstFile,
-            'utf-8'
-          )
-          const ast = assertParse(code)
-          await enginelessExecutor(
-            ast,
-            false,
-            file.pathFromProjectDirectoryToFirstFile
-          )
-        },
-        files.length * 1000
-      )
-    })
-  })
-})