test: Vendor kcl-samples and add simulation tests for them (#5460)
* Change to unzip * Download kcl-samples as zip to public dir * Fix fetch:samples, e2e electron still not working * Change error message to be clearer * Refactor so that input and output directories of sim tests can be different * Add kcl samples test implementation * Update output since adding kcl_samples tests * Update kcl-samples branch * Fix git-ignore pattern to only apply to the root * Fix yarn install and yarn fetch:samples to work the first time * Remove unneeded exists check * Change to use kcl-samples in public directory * Add kcl-samples * Update output since updating kcl-samples * Update output files * Change to not fetch samples during yarn install * Update output after merge * Ignore kcl-samples in codespell * WIP: Don't run e2e if only kcl-samples changed * Conditionally run cargo tests * Fix to round floating point values in program memory arrays * Update output since merge and rounding numbers in memory * Fix memory redaction for floating point to find more values * Fix float redaction pattern * Update output since rounding floating point numbers * Add center to floating point pattern * Fix trigger to use picomatch syntax * Update output since rounding center * Remove kcl-samples github workflows * Enable Rust backtrace * Update output after re-running * Update output after changing order of post-extrude commands * Fix to have deterministic order of commands * Update output after reverting ordering changes * Update kcl-samples * Update output after updating samples * Fix error messages to show the names of all samples that failed * Change cargo test command to match current one * Update kcl-samples * Update output since updating kcl-samples * Add generate manifest workflow and yarn script * Fix error check to actually work * Change util function to be what we actually need * Move new files after merge * Fix paths since directory move * Add dependabot updates for kcl-samples * Add GitHub workflow to make PR to kcl-samples repo * Add GitHub workflow to check kcl-samples header comments * Fix worfklow to change to the right directory * Add auto-commit simulation test output changes * Add permissions to workflows * Fix to run git commit step * Install just if needed * Fix directory of justfile * Add installation of cargo-insta * Fix to use underscore * Fix to allow just command failure * Change to always install CLI tools and cache them * Trying to fix overwrite failing * Combine commands * Change reviewer * Change to PR targeting the next branch * Change git commands to not do unnecessary fetch * Comment out trigger for creating a PR * Update kcl-samples from next branch * Update outputs after kcl-samples change * Fix to use bash pipefail * Add rust backtrace * Print full env from sim tests * Change command to use long option name * Fix to use ci profile even when calling through just * Add INSTA_UPDATE=always * Fix git push by using an app token on checkout * Add comments * Fix to use bash options * Change to echo when no changes are found * Fix so that kcl-samples updates don't trigger full run * Fix paths to reflect new crate location * Fix path detection * Fix e2e job to ignore kcl_samples simulation test output * Fix the fetch logic for the KCL samples after vendoring (#5661) Fixes the last 2 E2E tests for #5460. --------- Co-authored-by: Pierre Jacquier <pierre@zoo.dev> Co-authored-by: Pierre Jacquier <pierrejacquier39@gmail.com> Co-authored-by: Frank Noirot <frank@zoo.dev>
This commit is contained in:
Jonathan Tran
GitHub
186
rust/kcl-lib/src/simulation_tests/kcl_samples.rs
Normal file
186
rust/kcl-lib/src/simulation_tests/kcl_samples.rs
Normal file
@ -0,0 +1,186 @@
|
||||
//! Run all the KCL samples in the `kcl_samples` directory.
|
||||
//!
|
||||
//! Use the `KCL_SAMPLES_ONLY=gear` environment variable to run only a subset of
|
||||
//! the samples, in this case, all those that start with "gear".
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use fnv::FnvHashSet;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
use super::Test;
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
/// The directory containing the KCL samples source.
|
||||
static ref INPUTS_DIR: PathBuf = Path::new("../../public/kcl-samples").to_path_buf();
|
||||
/// The directory containing the expected output. We keep them isolated in
|
||||
/// their own directory, separate from other simulation tests, so that we
|
||||
/// know whether we've checked them all.
|
||||
static ref OUTPUTS_DIR: PathBuf = Path::new("tests/kcl_samples").to_path_buf();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse() {
|
||||
let write_new = matches!(
|
||||
std::env::var("INSTA_UPDATE").as_deref(),
|
||||
Ok("auto" | "always" | "new" | "unseen")
|
||||
);
|
||||
let filter = filter_from_env();
|
||||
let tests = kcl_samples_inputs(filter.as_deref());
|
||||
let expected_outputs = kcl_samples_outputs(filter.as_deref());
|
||||
|
||||
assert!(!tests.is_empty(), "No KCL samples found");
|
||||
|
||||
let input_names = FnvHashSet::from_iter(tests.iter().map(|t| t.name.clone()));
|
||||
|
||||
for test in tests {
|
||||
if write_new {
|
||||
// Ensure the directory exists for new tests.
|
||||
std::fs::create_dir_all(test.output_dir.clone()).unwrap();
|
||||
}
|
||||
super::parse_test(&test);
|
||||
}
|
||||
|
||||
// Ensure that inputs aren't missing.
|
||||
let missing = expected_outputs
|
||||
.into_iter()
|
||||
.filter(|name| !input_names.contains(name))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(missing.is_empty(), "Expected input kcl-samples for the following. If these are no longer tests, delete the expected output directories for them in {}: {missing:?}", OUTPUTS_DIR.to_string_lossy());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unparse() {
|
||||
// kcl-samples don't always use correct formatting. We don't ignore the
|
||||
// test because we want to allow the just command to work. It's actually
|
||||
// fine when no test runs.
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn kcl_test_execute() {
|
||||
let filter = filter_from_env();
|
||||
let tests = kcl_samples_inputs(filter.as_deref());
|
||||
let expected_outputs = kcl_samples_outputs(filter.as_deref());
|
||||
|
||||
assert!(!tests.is_empty(), "No KCL samples found");
|
||||
|
||||
// Note: This is unordered.
|
||||
let mut tasks = JoinSet::new();
|
||||
// Mapping from task ID to test index.
|
||||
let mut id_to_index = HashMap::new();
|
||||
// Spawn a task for each test.
|
||||
for (index, test) in tests.iter().cloned().enumerate() {
|
||||
let handle = tasks.spawn(async move {
|
||||
super::execute_test(&test, true).await;
|
||||
});
|
||||
id_to_index.insert(handle.id(), index);
|
||||
}
|
||||
|
||||
// Join all the tasks and collect the failures. We cannot just join_all
|
||||
// because insta's error messages don't clearly indicate which test failed.
|
||||
let mut failed = vec![None; tests.len()];
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
let Err(err) = result else {
|
||||
continue;
|
||||
};
|
||||
// When there's an error, store the test name and error message.
|
||||
let index = *id_to_index.get(&err.id()).unwrap();
|
||||
failed[index] = Some(format!("{}: {err}", &tests[index].name));
|
||||
}
|
||||
let failed = failed.into_iter().flatten().collect::<Vec<_>>();
|
||||
assert!(failed.is_empty(), "Failed tests: {}", failed.join("\n"));
|
||||
|
||||
// Ensure that inputs aren't missing.
|
||||
let input_names = FnvHashSet::from_iter(tests.iter().map(|t| t.name.clone()));
|
||||
let missing = expected_outputs
|
||||
.into_iter()
|
||||
.filter(|name| !input_names.contains(name))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(missing.is_empty(), "Expected input kcl-samples for the following. If these are no longer tests, delete the expected output directories for them in {}: {missing:?}", OUTPUTS_DIR.to_string_lossy());
|
||||
}
|
||||
|
||||
fn test(test_name: &str, entry_point: String) -> Test {
|
||||
Test {
|
||||
name: test_name.to_owned(),
|
||||
entry_point,
|
||||
input_dir: INPUTS_DIR.join(test_name),
|
||||
output_dir: OUTPUTS_DIR.join(test_name),
|
||||
}
|
||||
}
|
||||
|
||||
fn filter_from_env() -> Option<String> {
|
||||
std::env::var("KCL_SAMPLES_ONLY").ok().filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
fn kcl_samples_inputs(filter: Option<&str>) -> Vec<Test> {
|
||||
let mut tests = Vec::new();
|
||||
for entry in INPUTS_DIR.read_dir().unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
let path = entry.path();
|
||||
if !path.is_dir() {
|
||||
// We're looking for directories only.
|
||||
continue;
|
||||
}
|
||||
let Some(dir_name) = path.file_name() else {
|
||||
continue;
|
||||
};
|
||||
let dir_name_str = dir_name.to_string_lossy();
|
||||
if dir_name_str.starts_with('.') {
|
||||
// Skip hidden directories.
|
||||
continue;
|
||||
}
|
||||
if matches!(dir_name_str.as_ref(), "step" | "screenshots") {
|
||||
// Skip output directories.
|
||||
continue;
|
||||
}
|
||||
if let Some(filter) = &filter {
|
||||
if !dir_name_str.starts_with(filter) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
eprintln!("Found KCL sample: {:?}", dir_name.to_string_lossy());
|
||||
// Look for the entry point inside the directory.
|
||||
let sub_dir = INPUTS_DIR.join(dir_name);
|
||||
let entry_point = if sub_dir.join("main.kcl").exists() {
|
||||
"main.kcl".to_owned()
|
||||
} else {
|
||||
format!("{dir_name_str}.kcl")
|
||||
};
|
||||
tests.push(test(&dir_name_str, entry_point));
|
||||
}
|
||||
|
||||
tests
|
||||
}
|
||||
|
||||
fn kcl_samples_outputs(filter: Option<&str>) -> Vec<String> {
|
||||
let mut outputs = Vec::new();
|
||||
|
||||
for entry in OUTPUTS_DIR.read_dir().unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
let path = entry.path();
|
||||
if !path.is_dir() {
|
||||
// We're looking for directories only.
|
||||
continue;
|
||||
}
|
||||
let Some(dir_name) = path.file_name() else {
|
||||
continue;
|
||||
};
|
||||
let dir_name_str = dir_name.to_string_lossy();
|
||||
if dir_name_str.starts_with('.') {
|
||||
// Skip hidden.
|
||||
continue;
|
||||
}
|
||||
if let Some(filter) = &filter {
|
||||
if !dir_name_str.starts_with(filter) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!("Found expected KCL sample: {:?}", &dir_name_str);
|
||||
outputs.push(dir_name_str.into_owned());
|
||||
}
|
||||
|
||||
outputs
|
||||
}
|
||||
Reference in New Issue
Block a user