ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Foreign imports go thru parallelization too (#6488)

Browse Source 
* start of parallelizing forign imports

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* clippy

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* remove_printlns

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* remove shit that doesnt work

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* remove shit that doesnt work

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* put back

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* put back

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* put back

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* multiple

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* remove println

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* clone docs

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* add whole module import docs

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* add whole module import docs

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

---------

Signed-off-by: Jess Frazelle <github@jessfraz.com>
This commit is contained in:
Jess Frazelle
2025-04-24 20:25:02 -07:00
committed by GitHub
parent bff13f6bfe
commit 5c0ca52291
35 changed files with 17153 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
rust/kcl-lib/src/docs/kcl_doc.rs
View File

@ -1076,7 +1076,7 @@ mod test {
#[for_each_std_mod]
#[tokio::test(flavor = "multi_thread")]
async fn test_examples() {
async fn kcl_test_examples() {
let std = walk_prelude();
let mut errs = Vec::new();
for d in std {

21
rust/kcl-lib/src/execution/exec_ast.rs
View File

@ -459,7 +459,7 @@ impl ExecutorContext {
exec_state.add_path_to_source_id(resolved_path.clone(), id);
let format = super::import::format_from_annotations(attrs, path, source_range)?;
let geom = super::import::import_foreign(path, format, exec_state, self, source_range).await?;
exec_state.add_module(id, resolved_path, ModuleRepr::Foreign(geom));
exec_state.add_module(id, resolved_path, ModuleRepr::Foreign(geom, None));
Ok(id)
}
ImportPath::Std { .. } => {
@ -501,7 +501,7 @@ impl ExecutorContext {
*cache = Some((val, er, items.clone()));
(er, items)
}),
ModuleRepr::Foreign(geom) => Err(KclError::Semantic(KclErrorDetails {
ModuleRepr::Foreign(geom, _) => Err(KclError::Semantic(KclErrorDetails {
message: "Cannot import items from foreign modules".to_owned(),
source_ranges: vec![geom.source_range],
})),
@ -546,9 +546,20 @@ impl ExecutorContext {
Err(e) => Err(e),
}
}
ModuleRepr::Foreign(geom) => super::import::send_to_engine(geom.clone(), self)
.await
.map(|geom| Some(KclValue::ImportedGeometry(geom))),
ModuleRepr::Foreign(_, Some(imported)) => Ok(Some(imported.clone())),
ModuleRepr::Foreign(geom, cached) => {
let result = super::import::send_to_engine(geom.clone(), self)
.await
.map(|geom| Some(KclValue::ImportedGeometry(geom)));
match result {
Ok(val) => {
*cached = val.clone();
Ok(val)
}
Err(e) => Err(e),
}
}
ModuleRepr::Dummy => unreachable!(),
};

141
rust/kcl-lib/src/execution/mod.rs
View File

@ -752,26 +752,31 @@ impl ExecutorContext {
let mut universe = std::collections::HashMap::new();
let default_planes = self.engine.get_default_planes().read().await.clone();
crate::walk::import_universe(self, &program.ast, &mut universe, exec_state)
.await
.map_err(|err| {
let module_id_to_module_path: IndexMap<ModuleId, ModulePath> = exec_state
.global
.path_to_source_id
.iter()
.map(|(k, v)| ((*v), k.clone()))
.collect();
crate::walk::import_universe(
self,
&ModuleRepr::Kcl(program.ast.clone(), None),
&mut universe,
exec_state,
)
.await
.map_err(|err| {
let module_id_to_module_path: IndexMap<ModuleId, ModulePath> = exec_state
.global
.path_to_source_id
.iter()
.map(|(k, v)| ((*v), k.clone()))
.collect();
KclErrorWithOutputs::new(
err,
exec_state.global.operations.clone(),
exec_state.global.artifact_commands.clone(),
exec_state.global.artifact_graph.clone(),
module_id_to_module_path,
exec_state.global.id_to_source.clone(),
default_planes.clone(),
)
})?;
KclErrorWithOutputs::new(
err,
exec_state.global.operations.clone(),
exec_state.global.artifact_commands.clone(),
exec_state.global.artifact_graph.clone(),
module_id_to_module_path,
exec_state.global.id_to_source.clone(),
default_planes.clone(),
)
})?;
for modules in crate::walk::import_graph(&universe, self)
.map_err(|err| {
@ -799,16 +804,12 @@ impl ExecutorContext {
#[allow(clippy::type_complexity)]
let (results_tx, mut results_rx): (
tokio::sync::mpsc::Sender<(
ModuleId,
ModulePath,
Result<(Option<KclValue>, EnvironmentRef, Vec<String>), KclError>,
)>,
tokio::sync::mpsc::Sender<(ModuleId, ModulePath, Result<ModuleRepr, KclError>)>,
tokio::sync::mpsc::Receiver<_>,
) = tokio::sync::mpsc::channel(1);
for module in modules {
let Some((import_stmt, module_id, module_path, program)) = universe.get(&module) else {
let Some((import_stmt, module_id, module_path, repr)) = universe.get(&module) else {
return Err(KclErrorWithOutputs::no_outputs(KclError::Internal(KclErrorDetails {
message: format!("Module {module} not found in universe"),
source_ranges: Default::default(),
@ -816,12 +817,41 @@ impl ExecutorContext {
};
let module_id = *module_id;
let module_path = module_path.clone();
let program = program.clone();
let repr = repr.clone();
let exec_state = exec_state.clone();
let exec_ctxt = self.clone();
let results_tx = results_tx.clone();
let source_range = SourceRange::from(import_stmt);
let exec_module = async |exec_ctxt: &ExecutorContext,
repr: &ModuleRepr,
module_id: ModuleId,
module_path: &ModulePath,
exec_state: &mut ExecState,
source_range: SourceRange|
-> Result<ModuleRepr, KclError> {
match repr {
ModuleRepr::Kcl(program, _) => {
let result = exec_ctxt
.exec_module_from_ast(program, module_id, module_path, exec_state, source_range, false)
.await;
result.map(|val| ModuleRepr::Kcl(program.clone(), Some(val)))
}
ModuleRepr::Foreign(geom, _) => {
let result = crate::execution::import::send_to_engine(geom.clone(), exec_ctxt)
.await
.map(|geom| Some(KclValue::ImportedGeometry(geom)));
result.map(|val| ModuleRepr::Foreign(geom.clone(), val))
}
ModuleRepr::Dummy | ModuleRepr::Root => Err(KclError::Internal(KclErrorDetails {
message: format!("Module {module_path} not found in universe"),
source_ranges: vec![source_range],
})),
}
};
#[cfg(target_arch = "wasm32")]
{
wasm_bindgen_futures::spawn_local(async move {
@ -829,16 +859,15 @@ impl ExecutorContext {
let mut exec_state = exec_state;
let exec_ctxt = exec_ctxt;
let result = exec_ctxt
.exec_module_from_ast(
&program,
module_id,
&module_path,
&mut exec_state,
source_range,
false,
)
.await;
let result = exec_module(
&exec_ctxt,
&repr,
module_id,
&module_path,
&mut exec_state,
source_range,
)
.await;
results_tx
.send((module_id, module_path, result))
@ -852,16 +881,15 @@ impl ExecutorContext {
let mut exec_state = exec_state;
let exec_ctxt = exec_ctxt;
let result = exec_ctxt
.exec_module_from_ast(
&program,
module_id,
&module_path,
&mut exec_state,
source_range,
false,
)
.await;
let result = exec_module(
&exec_ctxt,
&repr,
module_id,
&module_path,
&mut exec_state,
source_range,
)
.await;
results_tx
.send((module_id, module_path, result))
@ -875,13 +903,24 @@ impl ExecutorContext {
while let Some((module_id, _, result)) = results_rx.recv().await {
match result {
Ok((val, session_data, variables)) => {
Ok(new_repr) => {
let mut repr = exec_state.global.module_infos[&module_id].take_repr();
let ModuleRepr::Kcl(_, cache) = &mut repr else {
continue;
};
*cache = Some((val, session_data, variables));
match &mut repr {
ModuleRepr::Kcl(_, cache) => {
let ModuleRepr::Kcl(_, session_data) = new_repr else {
unreachable!();
};
*cache = session_data;
}
ModuleRepr::Foreign(_, cache) => {
let ModuleRepr::Foreign(_, session_data) = new_repr else {
unreachable!();
};
*cache = session_data;
}
ModuleRepr::Dummy | ModuleRepr::Root => unreachable!(),
}
exec_state.global.module_infos[&module_id].restore_repr(repr);
}

2
rust/kcl-lib/src/modules.rs
View File

@ -124,7 +124,7 @@ pub enum ModuleRepr {
Root,
// AST, memory, exported names
Kcl(Node<Program>, Option<(Option<KclValue>, EnvironmentRef, Vec<String>)>),
Foreign(PreImportedGeometry),
Foreign(PreImportedGeometry, Option<KclValue>),
Dummy,
}

21
rust/kcl-lib/src/simulation_tests.rs
View File

@ -2579,3 +2579,24 @@ mod loop_tag {
super::execute(TEST_NAME, true).await
}
}
mod multiple_foreign_imports_all_render {
const TEST_NAME: &str = "multiple-foreign-imports-all-render";
/// Test parsing KCL.
#[test]
fn parse() {
super::parse(TEST_NAME)
}
/// Test that parsing and unparsing KCL produces the original KCL input.
#[tokio::test(flavor = "multi_thread")]
async fn unparse() {
super::unparse(TEST_NAME).await
}
/// Test that KCL is executed correctly.
#[tokio::test(flavor = "multi_thread")]
async fn kcl_test_execute() {
super::execute(TEST_NAME, true).await
}
}

81
rust/kcl-lib/src/walk/import_graph.rs
View File

@ -8,7 +8,7 @@ use anyhow::Result;
use crate::{
errors::KclErrorDetails,
modules::{ModulePath, ModuleRepr},
parsing::ast::types::{ImportPath, ImportStatement, Node as AstNode, NodeRef, Program},
parsing::ast::types::{ImportPath, ImportStatement, Node as AstNode},
walk::{Node, Visitable},
ExecState, ExecutorContext, KclError, ModuleId, SourceRange,
};
@ -20,7 +20,7 @@ type Dependency = (String, String);
type Graph = Vec<Dependency>;
type DependencyInfo = (AstNode<ImportStatement>, ModuleId, ModulePath, AstNode<Program>);
type DependencyInfo = (AstNode<ImportStatement>, ModuleId, ModulePath, ModuleRepr);
type Universe = HashMap<String, DependencyInfo>;
/// Process a number of programs, returning the graph of dependencies.
@ -32,9 +32,9 @@ type Universe = HashMap<String, DependencyInfo>;
pub fn import_graph(progs: &Universe, ctx: &ExecutorContext) -> Result<Vec<Vec<String>>, KclError> {
let mut graph = Graph::new();
for (name, (_, _, _, program)) in progs.iter() {
for (name, (_, _, _, repr)) in progs.iter() {
graph.extend(
import_dependencies(program, ctx)?
import_dependencies(repr, ctx)?
.into_iter()
.map(|(dependency, _, _)| (name.clone(), dependency))
.collect::<Vec<_>>(),
@ -118,28 +118,42 @@ fn topsort(all_modules: &[&str], graph: Graph) -> Result<Vec<Vec<String>>, KclEr
type ImportDependencies = Vec<(String, AstNode<ImportStatement>, ModulePath)>;
pub(crate) fn import_dependencies(
prog: NodeRef<Program>,
ctx: &ExecutorContext,
) -> Result<ImportDependencies, KclError> {
let ret = Arc::new(Mutex::new(vec![]));
pub(crate) fn import_dependencies(repr: &ModuleRepr, ctx: &ExecutorContext) -> Result<ImportDependencies, KclError> {
let ModuleRepr::Kcl(prog, _) = repr else {
// It has no dependencies, so return an empty list.
return Ok(vec![]);
};
let ret = Arc::new(Mutex::new(vec![]));
fn walk(ret: Arc<Mutex<ImportDependencies>>, node: Node<'_>, ctx: &ExecutorContext) -> Result<(), KclError> {
if let Node::ImportStatement(is) = node {
// We only care about Kcl imports for now.
if let ImportPath::Kcl { filename } = &is.path {
let resolved_path = ModulePath::from_import_path(&is.path, &ctx.settings.project_directory);
// We need to lock the mutex to push the dependency.
// This is a bit of a hack, but it works for now.
ret.lock()
.map_err(|err| {
KclError::Internal(KclErrorDetails {
message: format!("Failed to lock mutex: {}", err),
source_ranges: Default::default(),
})
})?
.push((filename.to_string(), is.clone(), resolved_path));
// We only care about Kcl and Foreign imports for now.
let resolved_path = ModulePath::from_import_path(&is.path, &ctx.settings.project_directory);
match &is.path {
ImportPath::Kcl { filename } => {
// We need to lock the mutex to push the dependency.
// This is a bit of a hack, but it works for now.
ret.lock()
.map_err(|err| {
KclError::Internal(KclErrorDetails {
message: format!("Failed to lock mutex: {}", err),
source_ranges: Default::default(),
})
})?
.push((filename.to_string(), is.clone(), resolved_path));
}
ImportPath::Foreign { path } => {
ret.lock()
.map_err(|err| {
KclError::Internal(KclErrorDetails {
message: format!("Failed to lock mutex: {}", err),
source_ranges: Default::default(),
})
})?
.push((path.to_string(), is.clone(), resolved_path));
}
ImportPath::Std { .. } => { // do nothing
}
}
}
@ -164,11 +178,11 @@ pub(crate) fn import_dependencies(
pub(crate) async fn import_universe(
ctx: &ExecutorContext,
prog: NodeRef<'_, Program>,
repr: &ModuleRepr,
out: &mut Universe,
exec_state: &mut ExecState,
) -> Result<(), KclError> {
let modules = import_dependencies(prog, ctx)?;
let modules = import_dependencies(repr, ctx)?;
for (filename, import_stmt, module_path) in modules {
if out.contains_key(&filename) {
continue;
@ -178,26 +192,21 @@ pub(crate) async fn import_universe(
.open_module(&import_stmt.path, &[], exec_state, Default::default())
.await?;
let program = {
let repr = {
let Some(module_info) = exec_state.get_module(module_id) else {
return Err(KclError::Internal(KclErrorDetails {
message: format!("Module {} not found", module_id),
source_ranges: vec![import_stmt.into()],
}));
};
let ModuleRepr::Kcl(program, _) = &module_info.repr else {
// if it's not a KCL module we can skip it since it has no
// dependencies.
continue;
};
program.clone()
module_info.repr.clone()
};
out.insert(
filename.clone(),
(import_stmt.clone(), module_id, module_path.clone(), program.clone()),
(import_stmt.clone(), module_id, module_path.clone(), repr.clone()),
);
Box::pin(import_universe(ctx, &program, out, exec_state)).await?;
Box::pin(import_universe(ctx, &repr, out, exec_state)).await?;
}
Ok(())
@ -206,7 +215,7 @@ pub(crate) async fn import_universe(
#[cfg(test)]
mod tests {
use super::*;
use crate::parsing::ast::types::ImportSelector;
use crate::parsing::ast::types::{ImportSelector, Program};
macro_rules! kcl {
( $kcl:expr ) => {{
@ -224,7 +233,7 @@ mod tests {
}),
ModuleId::default(),
ModulePath::Local { value: "".into() },
program,
ModuleRepr::Kcl(program, None),
)
}