Run std lib example tests one at a time (#7127)

Signed-off-by: Nick Cameron <nrc@ncameron.org>

rust/kcl-derive-docs/src/example_tests.rs

@@ -0,0 +1,139 @@
+use proc_macro2::Span;
+use quote::{quote, ToTokens};
+
+pub fn do_for_each_example_test(item: proc_macro2::TokenStream) -> proc_macro2::TokenStream {
+    let item: syn::ItemFn = syn::parse2(item.clone()).unwrap();
+    let mut result = proc_macro2::TokenStream::new();
+    for name in TEST_NAMES {
+        let mut item = item.clone();
+        item.sig.ident = syn::Ident::new(
+            &format!("{}_{}", item.sig.ident, name.replace('-', "_")),
+            Span::call_site(),
+        );
+        let name = name.to_owned();
+        let stmts = &item.block.stmts;
+        let block = quote! {
+            {
+                const NAME: &str = #name;
+                #(#stmts)*
+            }
+        };
+        item.block = Box::new(syn::parse2(block).unwrap());
+        result.extend(Some(item.into_token_stream()));
+    }
+
+    result
+}
+
+pub fn do_for_all_example_test(item: proc_macro2::TokenStream) -> proc_macro2::TokenStream {
+    let mut item: syn::ItemFn = syn::parse2(item).unwrap();
+    let len = TEST_NAMES.len();
+
+    let stmts = &item.block.stmts;
+    let test_names = TEST_NAMES.iter().map(|n| n.to_owned());
+    let block = quote! {
+        {
+            const TEST_NAMES: [&str; #len] = [#(#test_names,)*];
+            #(#stmts)*
+        }
+    };
+    item.block = Box::new(syn::parse2(block).unwrap());
+
+    item.into_token_stream()
+}
+
+pub const TEST_NAMES: [&str; 93] = [
+    "std-array-map-0",
+    "std-array-map-1",
+    "std-array-pop-0",
+    "std-array-push-0",
+    "std-array-reduce-0",
+    "std-array-reduce-1",
+    "std-array-reduce-2",
+    "std-clone-0",
+    "std-clone-1",
+    "std-clone-2",
+    "std-clone-3",
+    "std-clone-4",
+    "std-clone-5",
+    "std-clone-6",
+    "std-clone-7",
+    "std-clone-8",
+    "std-clone-9",
+    "std-helix-0",
+    "std-helix-1",
+    "std-helix-2",
+    "std-helix-3",
+    "std-math-abs-0",
+    "std-math-acos-0",
+    "std-math-asin-0",
+    "std-math-atan-0",
+    "std-math-atan2-0",
+    "std-math-ceil-0",
+    "std-math-cos-0",
+    "std-math-floor-0",
+    "std-math-ln-0",
+    "std-math-legLen-0",
+    "std-math-legAngX-0",
+    "std-math-legAngY-0",
+    "std-math-log-0",
+    "std-math-log10-0",
+    "std-math-log2-0",
+    "std-math-max-0",
+    "std-math-min-0",
+    "std-math-polar-0",
+    "std-math-pow-0",
+    "std-math-rem-0",
+    "std-math-round-0",
+    "std-math-sin-0",
+    "std-math-sqrt-0",
+    "std-math-tan-0",
+    "std-offsetPlane-0",
+    "std-offsetPlane-1",
+    "std-offsetPlane-2",
+    "std-offsetPlane-3",
+    "std-offsetPlane-4",
+    "std-sketch-circle-0",
+    "std-sketch-circle-1",
+    "std-sketch-patternTransform2d-0",
+    "std-sketch-revolve-0",
+    "std-sketch-revolve-1",
+    "std-sketch-revolve-10",
+    "std-sketch-revolve-11",
+    "std-sketch-revolve-12",
+    "std-sketch-revolve-2",
+    "std-sketch-revolve-3",
+    "std-sketch-revolve-4",
+    "std-sketch-revolve-5",
+    "std-sketch-revolve-6",
+    "std-sketch-revolve-7",
+    "std-sketch-revolve-8",
+    "std-sketch-revolve-9",
+    "std-solid-chamfer-0",
+    "std-solid-chamfer-1",
+    "std-solid-fillet-0",
+    "std-solid-fillet-1",
+    "std-solid-hollow-0",
+    "std-solid-hollow-1",
+    "std-solid-hollow-2",
+    "std-solid-patternTransform-0",
+    "std-solid-patternTransform-1",
+    "std-solid-patternTransform-2",
+    "std-solid-patternTransform-3",
+    "std-solid-patternTransform-4",
+    "std-solid-patternTransform-5",
+    "std-solid-shell-0",
+    "std-solid-shell-1",
+    "std-solid-shell-2",
+    "std-solid-shell-3",
+    "std-solid-shell-4",
+    "std-solid-shell-5",
+    "std-solid-shell-6",
+    "std-transform-mirror2d-0",
+    "std-transform-mirror2d-1",
+    "std-transform-mirror2d-2",
+    "std-transform-mirror2d-3",
+    "std-transform-mirror2d-4",
+    "std-units-toDegrees-0",
+    "std-units-toRadians-0",
+];

rust/kcl-derive-docs/src/lib.rs

@@ -2,16 +2,16 @@
 // automated enforcement.
 #![allow(clippy::style)]
 
+mod example_tests;
 #[cfg(test)]
 mod tests;
 mod unbox;
 
-use std::{collections::HashMap, fs};
+use std::collections::HashMap;
 
 use convert_case::Casing;
 use inflector::{cases::camelcase::to_camel_case, Inflector};
 use once_cell::sync::Lazy;
-use proc_macro2::Span;
 use quote::{format_ident, quote, quote_spanned, ToTokens};
 use regex::Regex;
 use serde::Deserialize;
@@ -28,8 +28,13 @@ pub fn stdlib(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> p
 }
 
 #[proc_macro_attribute]
-pub fn for_each_std_mod(_attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream {
+pub fn for_each_example_test(_attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream {
-    do_for_each_std_mod(item.into()).into()
+    example_tests::do_for_each_example_test(item.into()).into()
+}
+
+#[proc_macro_attribute]
+pub fn for_all_example_test(_attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    example_tests::do_for_all_example_test(item.into()).into()
 }
 
 /// Describes an argument of a stdlib function.
@@ -92,34 +97,6 @@ fn do_stdlib(
     do_stdlib_inner(metadata, attr, item)
 }
 
-fn do_for_each_std_mod(item: proc_macro2::TokenStream) -> proc_macro2::TokenStream {
-    let item: syn::ItemFn = syn::parse2(item.clone()).unwrap();
-    let mut result = proc_macro2::TokenStream::new();
-    for name in fs::read_dir("kcl-lib/std").unwrap().filter_map(|e| {
-        let e = e.unwrap();
-        let filename = e.file_name();
-        filename.to_str().unwrap().strip_suffix(".kcl").map(str::to_owned)
-    }) {
-        for i in 0..10_usize {
-            let mut item = item.clone();
-            item.sig.ident = syn::Ident::new(&format!("{}_{}_shard_{i}", item.sig.ident, name), Span::call_site());
-            let stmts = &item.block.stmts;
-            let block = quote! {
-                {
-                    const STD_MOD_NAME: &str = #name;
-                    const SHARD: usize = #i;
-                    const SHARD_COUNT: usize = 10;
-                    #(#stmts)*
-                }
-            };
-            item.block = Box::new(syn::parse2(block).unwrap());
-            result.extend(Some(item.into_token_stream()));
-        }
-    }
-
-    result
-}
-
 fn do_output(res: Result<(proc_macro2::TokenStream, Vec<Error>), Error>) -> proc_macro::TokenStream {
     match res {
         Err(err) => err.to_compile_error().into(),

rust/kcl-lib/src/docs/kcl_doc.rs

@@ -293,17 +293,6 @@ impl DocData {
         }
     }
 
-    #[cfg(test)]
-    fn examples(&self) -> impl Iterator<Item = &String> {
-        match self {
-            DocData::Fn(f) => f.examples.iter(),
-            DocData::Const(c) => c.examples.iter(),
-            DocData::Ty(t) => t.examples.iter(),
-            DocData::Mod(_) => unimplemented!(),
-        }
-        .filter_map(|(s, p)| (!p.norun).then_some(s))
-    }
-
     fn expect_mod(&self) -> &ModData {
         match self {
             DocData::Mod(m) => m,
@@ -1187,7 +1176,7 @@ impl ApplyMeta for ArgData {
 
 #[cfg(test)]
 mod test {
-    use kcl_derive_docs::for_each_std_mod;
+    use kcl_derive_docs::{for_all_example_test, for_each_example_test};
 
     use super::*;
 
@@ -1225,51 +1214,81 @@ mod test {
         );
     }
 
-    #[for_each_std_mod]
+    #[for_all_example_test]
+    #[tokio::test(flavor = "multi_thread")]
+    async fn missing_test_examples() {
+        fn check_mod(m: &ModData) {
+            for d in m.children.values() {
+                let DocData::Fn(f) = d else {
+                    continue;
+                };
+
+                for i in 0..f.examples.len() {
+                    let name = format!("{}-{i}", f.qual_name.replace("::", "-"));
+                    assert!(TEST_NAMES.contains(&&*name), "Missing test for example \"{name}\", maybe need to update kcl-derive-docs/src/example_tests.rs?")
+                }
+            }
+        }
+
+        let data = walk_prelude();
+
+        check_mod(&data);
+        for m in data.children.values() {
+            if let DocData::Mod(m) = m {
+                check_mod(m);
+            }
+        }
+    }
+
+    #[for_each_example_test]
     #[tokio::test(flavor = "multi_thread")]
     async fn kcl_test_examples() {
         let std = walk_prelude();
-        let mut errs = Vec::new();
 
-        let data = if STD_MOD_NAME == "prelude" {
+        let names = NAME.split('-');
+        let mut mods: Vec<_> = names.collect();
+        let number = mods.pop().unwrap();
+        let number: usize = number.parse().unwrap();
+        let name = mods.pop().unwrap();
+        let mut qualname = mods.join("::");
+        qualname.push_str("::");
+        qualname.push_str(name);
+
+        let data = if mods.len() == 1 {
             &std
         } else {
-            std.children
+            std.children.get(&format!("M:std::{}", mods[1])).unwrap().expect_mod()
-                .get(&format!("M:std::{STD_MOD_NAME}"))
-                .unwrap()
-                .expect_mod()
         };
 
-        let mut count = 0;
+        let Some(DocData::Fn(d)) = data.children.get(&format!("I:{qualname}")) else {
-        for d in data.children.values() {
+            panic!("Could not find data for {NAME} (missing a child entry for {qualname}), maybe need to update kcl-derive-docs/src/example_tests.rs?");
-            if let DocData::Mod(_) = d {
+        };
+
+        for (i, eg) in d.examples.iter().enumerate() {
+            if i != number {
                 continue;
             }
-
+            let result = match crate::test_server::execute_and_snapshot(&eg.0, None).await {
-            for (i, eg) in d.examples().enumerate() {
+                Err(crate::errors::ExecError::Kcl(e)) => {
-                count += 1;
+                    panic!("Error testing example {}{i}: {}", d.name, e.error.message());
-                if count % SHARD_COUNT != SHARD {
-                    continue;
                 }
-
+                Err(other_err) => panic!("{}", other_err),
-                let result = match crate::test_server::execute_and_snapshot(eg, None).await {
+                Ok(img) => img,
-                    Err(crate::errors::ExecError::Kcl(e)) => {
+            };
-                        errs.push(format!("Error testing example {}{i}: {}", d.name(), e.error.message()));
+            if eg.1.norun {
-                        continue;
+                return;
-                    }
-                    Err(other_err) => panic!("{}", other_err),
-                    Ok(img) => img,
-                };
-                twenty_twenty::assert_image(
-                    format!("tests/outputs/serial_test_example_{}{i}.png", d.example_name()),
-                    &result,
-                    0.99,
-                );
             }
+            twenty_twenty::assert_image(
+                format!(
+                    "tests/outputs/serial_test_example_fn_{}{i}.png",
+                    qualname.replace("::", "-")
+                ),
+                &result,
+                0.99,
+            );
+            return;
         }
 
-        if !errs.is_empty() {
+        panic!("Could not find data for {NAME} (no example {number}), maybe need to update kcl-derive-docs/src/example_tests.rs?");
-            panic!("{}", errs.join("\n\n"));
-        }
     }
 }