ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: Vendor kcl-samples and add simulation tests for them (#5460)

Browse Source 
* Change to unzip

* Download kcl-samples as zip to public dir

* Fix fetch:samples, e2e electron still not working

* Change error message to be clearer

* Refactor so that input and output directories of sim tests can be different

* Add kcl samples test implementation

* Update output since adding kcl_samples tests

* Update kcl-samples branch

* Fix git-ignore pattern to only apply to the root

* Fix yarn install and yarn fetch:samples to work the first time

* Remove unneeded exists check

* Change to use kcl-samples in public directory

* Add kcl-samples

* Update output since updating kcl-samples

* Update output files

* Change to not fetch samples during yarn install

* Update output after merge

* Ignore kcl-samples in codespell

* WIP: Don't run e2e if only kcl-samples changed

* Conditionally run cargo tests

* Fix to round floating point values in program memory arrays

* Update output since merge and rounding numbers in memory

* Fix memory redaction for floating point to find more values

* Fix float redaction pattern

* Update output since rounding floating point numbers

* Add center to floating point pattern

* Fix trigger to use picomatch syntax

* Update output since rounding center

* Remove kcl-samples github workflows

* Enable Rust backtrace

* Update output after re-running

* Update output after changing order of post-extrude commands

* Fix to have deterministic order of commands

* Update output after reverting ordering changes

* Update kcl-samples

* Update output after updating samples

* Fix error messages to show the names of all samples that failed

* Change cargo test command to match current one

* Update kcl-samples

* Update output since updating kcl-samples

* Add generate manifest workflow and yarn script

* Fix error check to actually work

* Change util function to be what we actually need

* Move new files after merge

* Fix paths since directory move

* Add dependabot updates for kcl-samples

* Add GitHub workflow to make PR to kcl-samples repo

* Add GitHub workflow to check kcl-samples header comments

* Fix worfklow to change to the right directory

* Add auto-commit simulation test output changes

* Add permissions to workflows

* Fix to run git commit step

* Install just if needed

* Fix directory of justfile

* Add installation of cargo-insta

* Fix to use underscore

* Fix to allow just command failure

* Change to always install CLI tools and cache them

* Trying to fix overwrite failing

* Combine commands

* Change reviewer

* Change to PR targeting the next branch

* Change git commands to not do unnecessary fetch

* Comment out trigger for creating a PR

* Update kcl-samples from next branch

* Update outputs after kcl-samples change

* Fix to use bash pipefail

* Add rust backtrace

* Print full env from sim tests

* Change command to use long option name

* Fix to use ci profile even when calling through just

* Add INSTA_UPDATE=always

* Fix git push by using an app token on checkout

* Add comments

* Fix to use bash options

* Change to echo when no changes are found

* Fix so that kcl-samples updates don't trigger full run

* Fix paths to reflect new crate location

* Fix path detection

* Fix e2e job to ignore kcl_samples simulation test output

* Fix the fetch logic for the KCL samples after vendoring (#5661)

Fixes the last 2 E2E tests for #5460.

---------

Co-authored-by: Pierre Jacquier <pierre@zoo.dev>
Co-authored-by: Pierre Jacquier <pierrejacquier39@gmail.com>
Co-authored-by: Frank Noirot <frank@zoo.dev>
This commit is contained in:
Jonathan Tran
2025-03-06 18:01:24 -05:00
committed by GitHub
parent 200a9af61f
commit 69553fded7
347 changed files with 794355 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
rust/kcl-lib/src/simulation_tests.rs
View File

@ -1,4 +1,4 @@
use std::path::Path;
use std::path::{Path, PathBuf};
use insta::rounded_redaction;
@ -10,6 +10,33 @@ use crate::{
ModuleId,
};
mod kcl_samples;
/// A simulation test.
#[derive(Debug, Clone)]
struct Test {
/// The name of the test.
name: String,
/// The name of the KCL file that's the entry point, e.g. "main.kcl", in the
/// `input_dir`.
entry_point: String,
/// Input KCL files are in this directory.
input_dir: PathBuf,
/// Expected snapshot output files are in this directory.
output_dir: PathBuf,
}
impl Test {
fn new(name: &str) -> Self {
Self {
name: name.to_owned(),
entry_point: "input.kcl".to_owned(),
input_dir: Path::new("tests").join(name),
output_dir: Path::new("tests").join(name),
}
}
}
/// Deserialize the data from a snapshot.
fn get<T: serde::de::DeserializeOwned>(snapshot: &str) -> T {
let mut parts = snapshot.split("---");
@ -21,16 +48,16 @@ fn get<T: serde::de::DeserializeOwned>(snapshot: &str) -> T {
.unwrap()
}
fn assert_snapshot<F, R>(test_name: &str, operation: &str, f: F)
fn assert_snapshot<F, R>(test: &Test, operation: &str, f: F)
where
F: FnOnce() -> R,
{
let mut settings = insta::Settings::clone_current();
// These make the snapshots more readable and match our dir structure.
settings.set_omit_expression(true);
settings.set_snapshot_path(format!("../tests/{test_name}"));
settings.set_snapshot_path(Path::new("..").join(&test.output_dir));
settings.set_prepend_module_to_snapshot(false);
settings.set_description(format!("{operation} {test_name}.kcl"));
settings.set_description(format!("{operation} {}.kcl", &test.name));
// Sorting maps makes them easier to diff.
settings.set_sort_maps(true);
// Replace UUIDs with the string "[uuid]", because otherwise the tests would constantly
@ -43,23 +70,34 @@ where
settings.bind(f);
}
fn read(filename: &'static str, test_name: &str) -> String {
std::fs::read_to_string(format!("tests/{test_name}/{filename}")).unwrap()
fn read<P>(filename: &str, dir: P) -> String
where
P: AsRef<Path>,
{
std::fs::read_to_string(dir.as_ref().join(filename)).unwrap()
}
fn parse(test_name: &str) {
let input = read("input.kcl", test_name);
parse_test(&Test::new(test_name));
}
fn parse_test(test: &Test) {
let input = read(&test.entry_point, &test.input_dir);
let tokens = crate::parsing::token::lex(&input, ModuleId::default()).unwrap();
// Parse the tokens into an AST.
let parse_res = Result::<_, KclError>::Ok(crate::parsing::parse_tokens(tokens).unwrap());
assert_snapshot(test_name, "Result of parsing", || {
assert_snapshot(test, "Result of parsing", || {
insta::assert_json_snapshot!("ast", parse_res);
});
}
fn unparse(test_name: &str) {
let input = read("ast.snap", test_name);
unparse_test(&Test::new(test_name));
}
fn unparse_test(test: &Test) {
let input = read("ast.snap", &test.output_dir);
let ast_res: Result<Program, KclError> = get(&input);
let Ok(ast) = ast_res else {
return;
@ -67,9 +105,9 @@ fn unparse(test_name: &str) {
// Check recasting the AST produces the original string.
let actual = ast.recast(&Default::default(), 0);
if matches!(std::env::var("EXPECTORATE").as_deref(), Ok("overwrite")) {
std::fs::write(format!("tests/{test_name}/input.kcl"), &actual).unwrap();
std::fs::write(test.input_dir.join(&test.entry_point), &actual).unwrap();
}
let expected = read("input.kcl", test_name);
let expected = read(&test.entry_point, &test.input_dir);
pretty_assertions::assert_eq!(
actual,
expected,
@ -78,42 +116,45 @@ fn unparse(test_name: &str) {
}
async fn execute(test_name: &str, render_to_png: bool) {
execute_test(&Test::new(test_name), render_to_png).await
}
async fn execute_test(test: &Test, render_to_png: bool) {
// Read the AST from disk.
let input = read("ast.snap", test_name);
let input = read("ast.snap", &test.output_dir);
let ast_res: Result<Node<Program>, KclError> = get(&input);
let Ok(ast) = ast_res else {
return;
};
let ast = crate::Program {
ast,
original_file_contents: read("input.kcl", test_name),
original_file_contents: read(&test.entry_point, &test.input_dir),
};
// Run the program.
let exec_res = crate::test_server::execute_and_snapshot_ast(
ast,
crate::settings::types::UnitLength::Mm,
Some(Path::new("tests").join(test_name).join("input.kcl").to_owned()),
Some(test.input_dir.join(&test.entry_point)),
)
.await;
match exec_res {
Ok((exec_state, env_ref, png)) => {
let fail_path_str = format!("tests/{test_name}/execution_error.snap");
let fail_path = Path::new(&fail_path_str);
if std::fs::exists(fail_path).unwrap() {
panic!("This test case is expected to fail, but it passed. If this is intended, and the test should actually be passing now, please delete kcl/{fail_path_str}")
let fail_path = test.output_dir.join("execution_error.snap");
if std::fs::exists(&fail_path).unwrap() {
panic!("This test case is expected to fail, but it passed. If this is intended, and the test should actually be passing now, please delete kcl-lib/{}", fail_path.to_string_lossy())
}
if render_to_png {
twenty_twenty::assert_image(format!("tests/{test_name}/rendered_model.png"), &png, 0.99);
twenty_twenty::assert_image(test.output_dir.join("rendered_model.png"), &png, 0.99);
}
let outcome = exec_state.to_wasm_outcome(env_ref);
assert_common_snapshots(
test_name,
test,
outcome.operations,
outcome.artifact_commands,
outcome.artifact_graph,
);
assert_snapshot(test_name, "Variables in memory after executing", || {
assert_snapshot(test, "Variables in memory after executing", || {
insta::assert_json_snapshot!("program_memory", outcome.variables, {
".**.value" => rounded_redaction(4),
".**[].value" => rounded_redaction(4),
@ -127,9 +168,8 @@ async fn execute(test_name: &str, render_to_png: bool) {
});
}
Err(e) => {
let ok_path_str = format!("tests/{test_name}/program_memory.snap");
let ok_path = Path::new(&ok_path_str);
let previously_passed = std::fs::exists(ok_path).unwrap();
let ok_path = test.output_dir.join("program_memory.snap");
let previously_passed = std::fs::exists(&ok_path).unwrap();
match e.error {
crate::errors::ExecError::Kcl(error) => {
// Snapshot the KCL error with a fancy graphical report.
@ -142,21 +182,16 @@ async fn execute(test_name: &str, render_to_png: bool) {
let report = error.clone().into_miette_report_with_outputs().unwrap();
let report = miette::Report::new(report);
if previously_passed {
eprintln!("This test case failed, but it previously passed. If this is intended, and the test should actually be failing now, please delete kcl/{ok_path_str} and other associated passing artifacts");
eprintln!("This test case failed, but it previously passed. If this is intended, and the test should actually be failing now, please delete kcl-lib/{} and other associated passing artifacts", ok_path.to_string_lossy());
panic!("{report:?}");
}
let report = format!("{:?}", report);
assert_snapshot(test_name, "Error from executing", || {
assert_snapshot(test, "Error from executing", || {
insta::assert_snapshot!("execution_error", report);
});
assert_common_snapshots(
test_name,
error.operations,
error.artifact_commands,
error.artifact_graph,
);
assert_common_snapshots(test, error.operations, error.artifact_commands, error.artifact_graph);
}
e => {
// These kinds of errors aren't expected to occur. We don't
@ -172,12 +207,12 @@ async fn execute(test_name: &str, render_to_png: bool) {
/// Assert snapshots that should happen both when KCL execution succeeds and
/// when it results in an error.
fn assert_common_snapshots(
test_name: &str,
test: &Test,
operations: Vec<Operation>,
artifact_commands: Vec<ArtifactCommand>,
artifact_graph: ArtifactGraph,
) {
assert_snapshot(test_name, "Operations executed", || {
assert_snapshot(test, "Operations executed", || {
insta::assert_json_snapshot!("ops", operations, {
"[].unlabeledArg.*.value.**[].from[]" => rounded_redaction(4),
"[].unlabeledArg.*.value.**[].to[]" => rounded_redaction(4),
@ -185,14 +220,14 @@ fn assert_common_snapshots(
"[].labeledArgs.*.value.**[].to[]" => rounded_redaction(4),
});
});
assert_snapshot(test_name, "Artifact commands", || {
assert_snapshot(test, "Artifact commands", || {
insta::assert_json_snapshot!("artifact_commands", artifact_commands, {
"[].command.segment.*.x" => rounded_redaction(4),
"[].command.segment.*.y" => rounded_redaction(4),
"[].command.segment.*.z" => rounded_redaction(4),
});
});
assert_snapshot(test_name, "Artifact graph flowchart", || {
assert_snapshot(test, "Artifact graph flowchart", || {
let flowchart = artifact_graph
.to_mermaid_flowchart()
.unwrap_or_else(|e| format!("Failed to convert artifact graph to flowchart: {e}"));

186
rust/kcl-lib/src/simulation_tests/kcl_samples.rs Normal file
View File

@ -0,0 +1,186 @@
//! Run all the KCL samples in the `kcl_samples` directory.
//!
//! Use the `KCL_SAMPLES_ONLY=gear` environment variable to run only a subset of
//! the samples, in this case, all those that start with "gear".
use std::{
collections::HashMap,
path::{Path, PathBuf},
};
use fnv::FnvHashSet;
use tokio::task::JoinSet;
use super::Test;
lazy_static::lazy_static! {
/// The directory containing the KCL samples source.
static ref INPUTS_DIR: PathBuf = Path::new("../../public/kcl-samples").to_path_buf();
/// The directory containing the expected output. We keep them isolated in
/// their own directory, separate from other simulation tests, so that we
/// know whether we've checked them all.
static ref OUTPUTS_DIR: PathBuf = Path::new("tests/kcl_samples").to_path_buf();
}
#[test]
fn parse() {
let write_new = matches!(
std::env::var("INSTA_UPDATE").as_deref(),
Ok("auto" | "always" | "new" | "unseen")
);
let filter = filter_from_env();
let tests = kcl_samples_inputs(filter.as_deref());
let expected_outputs = kcl_samples_outputs(filter.as_deref());
assert!(!tests.is_empty(), "No KCL samples found");
let input_names = FnvHashSet::from_iter(tests.iter().map(|t| t.name.clone()));
for test in tests {
if write_new {
// Ensure the directory exists for new tests.
std::fs::create_dir_all(test.output_dir.clone()).unwrap();
}
super::parse_test(&test);
}
// Ensure that inputs aren't missing.
let missing = expected_outputs
.into_iter()
.filter(|name| !input_names.contains(name))
.collect::<Vec<_>>();
assert!(missing.is_empty(), "Expected input kcl-samples for the following. If these are no longer tests, delete the expected output directories for them in {}: {missing:?}", OUTPUTS_DIR.to_string_lossy());
}
#[test]
fn unparse() {
// kcl-samples don't always use correct formatting. We don't ignore the
// test because we want to allow the just command to work. It's actually
// fine when no test runs.
}
#[tokio::test(flavor = "multi_thread")]
async fn kcl_test_execute() {
let filter = filter_from_env();
let tests = kcl_samples_inputs(filter.as_deref());
let expected_outputs = kcl_samples_outputs(filter.as_deref());
assert!(!tests.is_empty(), "No KCL samples found");
// Note: This is unordered.
let mut tasks = JoinSet::new();
// Mapping from task ID to test index.
let mut id_to_index = HashMap::new();
// Spawn a task for each test.
for (index, test) in tests.iter().cloned().enumerate() {
let handle = tasks.spawn(async move {
super::execute_test(&test, true).await;
});
id_to_index.insert(handle.id(), index);
}
// Join all the tasks and collect the failures. We cannot just join_all
// because insta's error messages don't clearly indicate which test failed.
let mut failed = vec![None; tests.len()];
while let Some(result) = tasks.join_next().await {
let Err(err) = result else {
continue;
};
// When there's an error, store the test name and error message.
let index = *id_to_index.get(&err.id()).unwrap();
failed[index] = Some(format!("{}: {err}", &tests[index].name));
}
let failed = failed.into_iter().flatten().collect::<Vec<_>>();
assert!(failed.is_empty(), "Failed tests: {}", failed.join("\n"));
// Ensure that inputs aren't missing.
let input_names = FnvHashSet::from_iter(tests.iter().map(|t| t.name.clone()));
let missing = expected_outputs
.into_iter()
.filter(|name| !input_names.contains(name))
.collect::<Vec<_>>();
assert!(missing.is_empty(), "Expected input kcl-samples for the following. If these are no longer tests, delete the expected output directories for them in {}: {missing:?}", OUTPUTS_DIR.to_string_lossy());
}
fn test(test_name: &str, entry_point: String) -> Test {
Test {
name: test_name.to_owned(),
entry_point,
input_dir: INPUTS_DIR.join(test_name),
output_dir: OUTPUTS_DIR.join(test_name),
}
}
fn filter_from_env() -> Option<String> {
std::env::var("KCL_SAMPLES_ONLY").ok().filter(|s| !s.is_empty())
}
fn kcl_samples_inputs(filter: Option<&str>) -> Vec<Test> {
let mut tests = Vec::new();
for entry in INPUTS_DIR.read_dir().unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if !path.is_dir() {
// We're looking for directories only.
continue;
}
let Some(dir_name) = path.file_name() else {
continue;
};
let dir_name_str = dir_name.to_string_lossy();
if dir_name_str.starts_with('.') {
// Skip hidden directories.
continue;
}
if matches!(dir_name_str.as_ref(), "step" | "screenshots") {
// Skip output directories.
continue;
}
if let Some(filter) = &filter {
if !dir_name_str.starts_with(filter) {
continue;
}
}
eprintln!("Found KCL sample: {:?}", dir_name.to_string_lossy());
// Look for the entry point inside the directory.
let sub_dir = INPUTS_DIR.join(dir_name);
let entry_point = if sub_dir.join("main.kcl").exists() {
"main.kcl".to_owned()
} else {
format!("{dir_name_str}.kcl")
};
tests.push(test(&dir_name_str, entry_point));
}
tests
}
fn kcl_samples_outputs(filter: Option<&str>) -> Vec<String> {
let mut outputs = Vec::new();
for entry in OUTPUTS_DIR.read_dir().unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if !path.is_dir() {
// We're looking for directories only.
continue;
}
let Some(dir_name) = path.file_name() else {
continue;
};
let dir_name_str = dir_name.to_string_lossy();
if dir_name_str.starts_with('.') {
// Skip hidden.
continue;
}
if let Some(filter) = &filter {
if !dir_name_str.starts_with(filter) {
continue;
}
}
eprintln!("Found expected KCL sample: {:?}", &dir_name_str);
outputs.push(dir_name_str.into_owned());
}
outputs
}