From 10027b98b53b94e34dd5d3889f978204723bfbf6 Mon Sep 17 00:00:00 2001 From: Jess Frazelle Date: Wed, 6 Sep 2023 19:34:47 -0700 Subject: [PATCH] implement rename (#396) * updates Signed-off-by: Jess Frazelle * rename function Signed-off-by: Jess Frazelle * start of rename Signed-off-by: Jess Frazelle * updates Signed-off-by: Jess Frazelle * updates Signed-off-by: Jess Frazelle * updates Signed-off-by: Jess Frazelle * updates Signed-off-by: Jess Frazelle * cache rust Signed-off-by: Jess Frazelle * fix gnarly bug Signed-off-by: Jess Frazelle * updates Signed-off-by: Jess Frazelle * fixes Signed-off-by: Jess Frazelle * fucking tabs Signed-off-by: Jess Frazelle --------- Signed-off-by: Jess Frazelle Co-authored-by: Kurt Hutten --- .github/workflows/ci.yml | 36 ++- src/wasm-lib/Cargo.lock | 2 +- src/wasm-lib/kcl/Cargo.toml | 4 +- .../kcl/src/abstract_syntax_tree_types.rs | 299 ++++++++++++++++++ src/wasm-lib/kcl/src/parser.rs | 116 +++++-- src/wasm-lib/kcl/src/server/mod.rs | 38 +++ src/wasm-lib/kcl/src/tokeniser.rs | 37 +-- 7 files changed, 472 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 487a63a6b..4c97246e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI +name: CI on: pull_request: @@ -31,6 +31,10 @@ jobs: node-version-file: '.nvmrc' cache: 'yarn' - run: yarn install + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "./src/wasm-lib" + - run: yarn build:wasm - run: yarn tsc @@ -50,6 +54,10 @@ jobs: - run: yarn install + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "./src/wasm-lib" + - run: yarn build:wasm - run: yarn simpleserver:ci @@ -94,6 +102,10 @@ jobs: with: workspaces: './src-tauri -> target' + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "./src/wasm-lib" + - name: wasm prep shell: bash run: | @@ -117,12 +129,28 @@ jobs: - name: Fix format run: yarn fmt + # This will do intel silicon on mac - name: Build the app for the current platform (no upload) uses: tauri-apps/tauri-action@v0 env: TAURI_PRIVATE_KEY: ${{ secrets.TAURI_PRIVATE_KEY }} TAURI_KEY_PASSWORD: ${{ secrets.TAURI_KEY_PASSWORD }} + - name: install apple silicon target mac + if: matrix.os == 'macos-latest' + run: | + rustup target add aarch64-apple-darwin + + # this will do apple silicon on mac + - name: Build the app for the current platform (no upload) + uses: tauri-apps/tauri-action@v0 + if: matrix.os == 'macos-latest' + env: + TAURI_PRIVATE_KEY: ${{ secrets.TAURI_PRIVATE_KEY }} + TAURI_KEY_PASSWORD: ${{ secrets.TAURI_KEY_PASSWORD }} + with: + args: --target universal-apple-darwin + - uses: actions/upload-artifact@v3 with: path: src-tauri/target/release/bundle/*/* @@ -181,15 +209,15 @@ jobs: uses: google-github-actions/setup-gcloud@v1.1.1 with: project_id: kittycadapi - + - name: Upload release files to public bucket uses: google-github-actions/upload-cloud-storage@v1.0.3 with: path: artifact glob: '*/*itty*' parent: false - destination: dl.kittycad.io/releases/modeling-app/v${{ env.VERSION_NO_V }} - + destination: dl.kittycad.io/releases/modeling-app/v${{ env.VERSION_NO_V }} + - name: Upload update endpoint to public bucket uses: google-github-actions/upload-cloud-storage@v1.0.3 with: diff --git a/src/wasm-lib/Cargo.lock b/src/wasm-lib/Cargo.lock index 3059091cc..4d4e6fc90 100644 --- a/src/wasm-lib/Cargo.lock +++ b/src/wasm-lib/Cargo.lock @@ -1094,7 +1094,7 @@ dependencies = [ [[package]] name = "kcl-lib" -version = "0.1.21" +version = "0.1.24" dependencies = [ "anyhow", "bson", diff --git a/src/wasm-lib/kcl/Cargo.toml b/src/wasm-lib/kcl/Cargo.toml index 34d631e64..59c261dbd 100644 --- a/src/wasm-lib/kcl/Cargo.toml +++ b/src/wasm-lib/kcl/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "kcl-lib" description = "KittyCAD Language" -version = "0.1.21" +version = "0.1.24" edition = "2021" license = "MIT" @@ -11,7 +11,7 @@ license = "MIT" anyhow = { version = "1.0.75", features = ["backtrace"] } clap = { version = "4.4.2", features = ["cargo", "derive", "env", "unicode"] } dashmap = "5.5.3" -derive-docs = { version = "0.1.1" } +derive-docs = { version = "0.1.3" } #derive-docs = { path = "../derive-docs" } kittycad = { version = "0.2.23", default-features = false, features = ["js"] } lazy_static = "1.4.0" diff --git a/src/wasm-lib/kcl/src/abstract_syntax_tree_types.rs b/src/wasm-lib/kcl/src/abstract_syntax_tree_types.rs index a0a488ffc..4c617c385 100644 --- a/src/wasm-lib/kcl/src/abstract_syntax_tree_types.rs +++ b/src/wasm-lib/kcl/src/abstract_syntax_tree_types.rs @@ -117,6 +117,18 @@ impl Program { None } + /// Returns the body item that includes the given character position. + pub fn get_mut_body_item_for_position(&mut self, pos: usize) -> Option<&mut BodyItem> { + for item in &mut self.body { + let source_range: SourceRange = item.clone().into(); + if source_range.contains(pos) { + return Some(item); + } + } + + None + } + /// Returns a value that includes the given character position. /// This is a bit more recursive than `get_body_item_for_position`. pub fn get_value_for_position(&self, pos: usize) -> Option<&Value> { @@ -149,6 +161,82 @@ impl Program { symbols } + + /// Rename the variable declaration at the given position. + pub fn rename_symbol(&mut self, new_name: &str, pos: usize) { + // The position must be within the variable declaration. + let mut old_name = None; + for item in &mut self.body { + match item { + BodyItem::ExpressionStatement(_expression_statement) => { + continue; + } + BodyItem::VariableDeclaration(ref mut variable_declaration) => { + if let Some(var_old_name) = variable_declaration.rename_symbol(new_name, pos) { + old_name = Some(var_old_name); + break; + } + } + BodyItem::ReturnStatement(_return_statement) => continue, + } + } + + if let Some(old_name) = old_name { + // Now rename all the identifiers in the rest of the program. + self.rename_identifiers(&old_name, new_name); + } else { + // Okay so this was not a top level variable declaration. + // But it might be a variable declaration inside a function or function params. + // So we need to check that. + let Some(ref mut item) = self.get_mut_body_item_for_position(pos) else { + return; + }; + + // Recurse over the item. + let mut value = match item { + BodyItem::ExpressionStatement(ref mut expression_statement) => { + Some(&mut expression_statement.expression) + } + BodyItem::VariableDeclaration(ref mut variable_declaration) => { + variable_declaration.get_mut_value_for_position(pos) + } + BodyItem::ReturnStatement(ref mut return_statement) => Some(&mut return_statement.argument), + }; + + // Check if we have a function expression. + if let Some(Value::FunctionExpression(ref mut function_expression)) = &mut value { + // Check if the params to the function expression contain the position. + for param in &mut function_expression.params { + let param_source_range: SourceRange = param.clone().into(); + if param_source_range.contains(pos) { + let old_name = param.name.clone(); + // Rename the param. + param.rename(&old_name, new_name); + // Now rename all the identifiers in the rest of the program. + function_expression.body.rename_identifiers(&old_name, new_name); + return; + } + } + } + } + } + + /// Rename all identifiers that have the old name to the new given name. + fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + for item in &mut self.body { + match item { + BodyItem::ExpressionStatement(ref mut expression_statement) => { + expression_statement.expression.rename_identifiers(old_name, new_name); + } + BodyItem::VariableDeclaration(ref mut variable_declaration) => { + variable_declaration.rename_identifiers(old_name, new_name); + } + BodyItem::ReturnStatement(ref mut return_statement) => { + return_statement.argument.rename_identifiers(old_name, new_name); + } + } + } + } } pub trait ValueMeta { @@ -315,6 +403,29 @@ impl Value { Value::UnaryExpression(unary_expression) => unary_expression.get_hover_value_for_position(pos, code), } } + + /// Rename all identifiers that have the old name to the new given name. + fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + match self { + Value::Literal(_literal) => {} + Value::Identifier(ref mut identifier) => identifier.rename(old_name, new_name), + Value::BinaryExpression(ref mut binary_expression) => { + binary_expression.rename_identifiers(old_name, new_name) + } + Value::FunctionExpression(_function_identifier) => {} + Value::CallExpression(ref mut call_expression) => call_expression.rename_identifiers(old_name, new_name), + Value::PipeExpression(ref mut pipe_expression) => pipe_expression.rename_identifiers(old_name, new_name), + Value::PipeSubstitution(_) => {} + Value::ArrayExpression(ref mut array_expression) => array_expression.rename_identifiers(old_name, new_name), + Value::ObjectExpression(ref mut object_expression) => { + object_expression.rename_identifiers(old_name, new_name) + } + Value::MemberExpression(ref mut member_expression) => { + member_expression.rename_identifiers(old_name, new_name) + } + Value::UnaryExpression(ref mut unary_expression) => unary_expression.rename_identifiers(old_name, new_name), + } + } } impl From for crate::executor::SourceRange { @@ -420,6 +531,23 @@ impl BinaryPart { BinaryPart::UnaryExpression(unary_expression) => unary_expression.get_hover_value_for_position(pos, code), } } + + /// Rename all identifiers that have the old name to the new given name. + fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + match self { + BinaryPart::Literal(_literal) => {} + BinaryPart::Identifier(ref mut identifier) => identifier.rename(old_name, new_name), + BinaryPart::BinaryExpression(ref mut binary_expression) => { + binary_expression.rename_identifiers(old_name, new_name) + } + BinaryPart::CallExpression(ref mut call_expression) => { + call_expression.rename_identifiers(old_name, new_name) + } + BinaryPart::UnaryExpression(ref mut unary_expression) => { + unary_expression.rename_identifiers(old_name, new_name) + } + } + } } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ts_rs::TS, JsonSchema)] @@ -689,6 +817,15 @@ impl CallExpression { None } + + /// Rename all identifiers that have the old name to the new given name. + fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + self.callee.rename(old_name, new_name); + + for arg in &mut self.arguments { + arg.rename_identifiers(old_name, new_name); + } + } } /// A function declaration. @@ -741,6 +878,50 @@ impl VariableDeclaration { None } + /// Returns a value that includes the given character position. + pub fn get_mut_value_for_position(&mut self, pos: usize) -> Option<&mut Value> { + for declaration in &mut self.declarations { + let source_range: SourceRange = declaration.clone().into(); + if source_range.contains(pos) { + return Some(&mut declaration.init); + } + } + + None + } + + /// Rename the variable declaration at the given position. + /// This returns the old name of the variable, if it found one. + pub fn rename_symbol(&mut self, new_name: &str, pos: usize) -> Option { + // The position must be within the variable declaration. + let source_range: SourceRange = self.clone().into(); + if !source_range.contains(pos) { + return None; + } + + for declaration in &mut self.declarations { + let declaration_source_range: SourceRange = declaration.id.clone().into(); + if declaration_source_range.contains(pos) { + let old_name = declaration.id.name.clone(); + declaration.id.name = new_name.to_string(); + return Some(old_name); + } + } + + None + } + + pub fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + for declaration in &mut self.declarations { + // Skip the init for the variable with the new name since it is the one we are renaming. + if declaration.id.name == new_name { + continue; + } + + declaration.init.rename_identifiers(old_name, new_name); + } + } + pub fn get_lsp_symbols(&self, code: &str) -> Vec { let mut symbols = vec![]; @@ -857,7 +1038,9 @@ impl VariableKind { pub struct VariableDeclarator { pub start: usize, pub end: usize, + /// The identifier of the variable. pub id: Identifier, + /// The value of the variable. pub init: Value, } @@ -919,6 +1102,15 @@ pub struct Identifier { impl_value_meta!(Identifier); +impl Identifier { + /// Rename all identifiers that have the old name to the new given name. + fn rename(&mut self, old_name: &str, new_name: &str) { + if self.name == old_name { + self.name = new_name.to_string(); + } + } +} + #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ts_rs::TS, JsonSchema)] #[ts(export)] #[serde(tag = "type")] @@ -1045,6 +1237,13 @@ impl ArrayExpression { }], }) } + + /// Rename all identifiers that have the old name to the new given name. + fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + for element in &mut self.elements { + element.rename_identifiers(old_name, new_name); + } + } } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ts_rs::TS, JsonSchema)] @@ -1159,6 +1358,13 @@ impl ObjectExpression { }], }) } + + /// Rename all identifiers that have the old name to the new given name. + fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + for property in &mut self.properties { + property.value.rename_identifiers(old_name, new_name); + } + } } impl_value_meta!(ObjectExpression); @@ -1376,6 +1582,21 @@ impl MemberExpression { })) } } + + /// Rename all identifiers that have the old name to the new given name. + fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + match &mut self.object { + MemberObject::MemberExpression(ref mut member_expression) => { + member_expression.rename_identifiers(old_name, new_name) + } + MemberObject::Identifier(ref mut identifier) => identifier.rename(old_name, new_name), + } + + match &mut self.property { + LiteralIdentifier::Identifier(ref mut identifier) => identifier.rename(old_name, new_name), + LiteralIdentifier::Literal(_) => {} + } + } } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ts_rs::TS, JsonSchema)] @@ -1492,6 +1713,12 @@ impl BinaryExpression { }], }) } + + /// Rename all identifiers that have the old name to the new given name. + fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + self.left.rename_identifiers(old_name, new_name); + self.right.rename_identifiers(old_name, new_name); + } } pub fn parse_json_number_as_f64(j: &serde_json::Value, source_range: SourceRange) -> Result { @@ -1599,6 +1826,11 @@ impl UnaryExpression { None } + + /// Rename all identifiers that have the old name to the new given name. + fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + self.argument.rename_identifiers(old_name, new_name); + } } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ts_rs::TS, JsonSchema, FromStr, Display)] @@ -1675,6 +1907,13 @@ impl PipeExpression { pipe_info.index = 0; execute_pipe_body(memory, &self.body, pipe_info, self.into(), engine) } + + /// Rename all identifiers that have the old name to the new given name. + fn rename_identifiers(&mut self, old_name: &str, new_name: &str) { + for statement in &mut self.body { + statement.rename_identifiers(old_name, new_name); + } + } } fn execute_pipe_body( @@ -2158,4 +2397,64 @@ const part001 = startSketchAt([0, 0]) ); assert_eq!(recasted, some_program_string); } + + #[test] + fn test_recast_after_rename_std() { + let some_program_string = r#"const part001 = startSketchAt([0.0000000000, 5.0000000000]) + |> line([0.4900857016, -0.0240763666], %) + +const part002 = "part002" +const things = [part001, 0.0] +let blah = 1 +const foo = false +let baz = {a: 1, part001: "thing"} + +fn ghi = (part001) => { + return part001 +} + +show(part001)"#; + let tokens = crate::tokeniser::lexer(some_program_string); + let parser = crate::parser::Parser::new(tokens); + let mut program = parser.ast().unwrap(); + program.rename_symbol("mySuperCoolPart", 6); + + let recasted = program.recast(&Default::default(), 0); + assert_eq!( + recasted, + r#"const mySuperCoolPart = startSketchAt([0.0, 5.0]) + |> line([0.4900857016, -0.0240763666], %) + +const part002 = "part002" +const things = [mySuperCoolPart, 0.0] +let blah = 1 +const foo = false +let baz = { a: 1, part001: "thing" } + +fn ghi = (part001) => { + return part001 +} + +show(mySuperCoolPart)"# + ); + } + + #[test] + fn test_recast_after_rename_fn_args() { + let some_program_string = r#"fn ghi = (x, y, z) => { + return x +}"#; + let tokens = crate::tokeniser::lexer(some_program_string); + let parser = crate::parser::Parser::new(tokens); + let mut program = parser.ast().unwrap(); + program.rename_symbol("newName", 10); + + let recasted = program.recast(&Default::default(), 0); + assert_eq!( + recasted, + r#"fn ghi = (newName, y, z) => { + return newName +}"# + ); + } } diff --git a/src/wasm-lib/kcl/src/parser.rs b/src/wasm-lib/kcl/src/parser.rs index dc06dc2d6..e49b88671 100644 --- a/src/wasm-lib/kcl/src/parser.rs +++ b/src/wasm-lib/kcl/src/parser.rs @@ -170,6 +170,13 @@ impl Parser { })); } + if index >= self.tokens.len() { + return Err(KclError::Syntax(KclErrorDetails { + source_ranges: vec![self.tokens.last().unwrap().into()], + message: "unexpected end".to_string(), + })); + } + let Some(token) = self.tokens.get(index) else { return Err(KclError::Syntax(KclErrorDetails { source_ranges: vec![self.tokens.last().unwrap().into()], @@ -679,7 +686,11 @@ impl Parser { return Ok(index); } let next_right = self.next_meaningful_token(maybe_operator.index, None)?; - self.find_end_of_binary_expression(next_right.index) + if next_right.index != index { + self.find_end_of_binary_expression(next_right.index) + } else { + Ok(index) + } } else { Ok(index) } @@ -1105,42 +1116,42 @@ impl Parser { ) -> Result { let current_token = self.get_token(index)?; let assignment = self.next_meaningful_token(index, None)?; - if let Some(assignment_token) = assignment.token { - let contents_start_token = self.next_meaningful_token(assignment.index, None)?; - let pipe_start_index = if assignment_token.token_type == TokenType::Operator { - contents_start_token.index - } else { - assignment.index - }; - let next_pipe_operator = self.has_pipe_operator(pipe_start_index, None)?; - let init: Value; - let last_index = if next_pipe_operator.token.is_some() { - let pipe_expression_result = self.make_pipe_expression(assignment.index)?; - init = Value::PipeExpression(Box::new(pipe_expression_result.expression)); - pipe_expression_result.last_index - } else { - let value_result = self.make_value(contents_start_token.index)?; - init = value_result.value; - value_result.last_index - }; - let current_declarator = VariableDeclarator { - start: current_token.start, - end: self.get_token(last_index)?.end, - id: self.make_identifier(index)?, - init, - }; - let mut declarations = previous_declarators; - declarations.push(current_declarator); - Ok(VariableDeclaratorsReturn { - declarations, - last_index, - }) - } else { - Err(KclError::Unimplemented(KclErrorDetails { + let Some(assignment_token) = assignment.token else { + return Err(KclError::Unimplemented(KclErrorDetails { source_ranges: vec![current_token.clone().into()], message: format!("Unexpected token {} ", current_token.value), - })) - } + })); + }; + + let contents_start_token = self.next_meaningful_token(assignment.index, None)?; + let pipe_start_index = if assignment_token.token_type == TokenType::Operator { + contents_start_token.index + } else { + assignment.index + }; + let next_pipe_operator = self.has_pipe_operator(pipe_start_index, None)?; + let init: Value; + let last_index = if next_pipe_operator.token.is_some() { + let pipe_expression_result = self.make_pipe_expression(assignment.index)?; + init = Value::PipeExpression(Box::new(pipe_expression_result.expression)); + pipe_expression_result.last_index + } else { + let value_result = self.make_value(contents_start_token.index)?; + init = value_result.value; + value_result.last_index + }; + let current_declarator = VariableDeclarator { + start: current_token.start, + end: self.get_token(last_index)?.end, + id: self.make_identifier(index)?, + init, + }; + let mut declarations = previous_declarators; + declarations.push(current_declarator); + Ok(VariableDeclaratorsReturn { + declarations, + last_index, + }) } fn make_variable_declaration(&self, index: usize) -> Result { @@ -2716,4 +2727,39 @@ show(mySk1)"#; assert!(result.is_err()); assert!(result.err().unwrap().to_string().contains("file is empty")); } + + #[test] + fn test_parse_half_pipe_small() { + let tokens = crate::tokeniser::lexer( + "const secondExtrude = startSketchAt([0,0]) + |", + ); + let parser = Parser::new(tokens); + let result = parser.ast(); + assert!(result.is_err()); + assert!(result.err().unwrap().to_string().contains("Unexpected token")); + } + + #[test] + fn test_parse_half_pipe() { + let tokens = crate::tokeniser::lexer( + "const height = 10 + +const firstExtrude = startSketchAt([0,0]) + |> line([0, 8], %) + |> line([20, 0], %) + |> line([0, -8], %) + |> close(%) + |> extrude(2, %) + +show(firstExtrude) + +const secondExtrude = startSketchAt([0,0]) + |", + ); + let parser = Parser::new(tokens); + let result = parser.ast(); + assert!(result.is_err()); + assert!(result.err().unwrap().to_string().contains("Unexpected token")); + } } diff --git a/src/wasm-lib/kcl/src/server/mod.rs b/src/wasm-lib/kcl/src/server/mod.rs index d8fb388d1..3875764de 100644 --- a/src/wasm-lib/kcl/src/server/mod.rs +++ b/src/wasm-lib/kcl/src/server/mod.rs @@ -233,6 +233,7 @@ impl LanguageServer for Backend { document_symbol_provider: Some(OneOf::Left(true)), hover_provider: Some(HoverProviderCapability::Simple(true)), inlay_hint_provider: Some(OneOf::Left(true)), + rename_provider: Some(OneOf::Left(true)), semantic_tokens_provider: Some(SemanticTokensServerCapabilities::SemanticTokensRegistrationOptions( SemanticTokensRegistrationOptions { text_document_registration_options: { @@ -567,6 +568,43 @@ impl LanguageServer for Backend { range, }])) } + + async fn rename(&self, params: RenameParams) -> RpcResult> { + let filename = params.text_document_position.text_document.uri.to_string(); + + let Some(current_code) = self.current_code_map.get(&filename) else { + return Ok(None); + }; + + // Parse the ast. + // I don't know if we need to do this again since it should be updated in the context. + // But I figure better safe than sorry since this will write back out to the file. + let tokens = crate::tokeniser::lexer(¤t_code); + let parser = crate::parser::Parser::new(tokens); + let Ok(mut ast) = parser.ast() else { + return Ok(None); + }; + + // Let's convert the position to a character index. + let pos = position_to_char_index(params.text_document_position.position, ¤t_code); + // Now let's perform the rename on the ast. + ast.rename_symbol(¶ms.new_name, pos); + // Now recast it. + let recast = ast.recast(&Default::default(), 0); + let source_range = SourceRange([0, current_code.len() - 1]); + let range = source_range.to_lsp_range(¤t_code); + Ok(Some(WorkspaceEdit { + changes: Some(HashMap::from([( + params.text_document_position.text_document.uri, + vec![TextEdit { + new_text: recast, + range, + }], + )])), + document_changes: None, + change_annotations: None, + })) + } } /// Get completions from our stdlib. diff --git a/src/wasm-lib/kcl/src/tokeniser.rs b/src/wasm-lib/kcl/src/tokeniser.rs index 3918aedbf..fe02d8cb1 100644 --- a/src/wasm-lib/kcl/src/tokeniser.rs +++ b/src/wasm-lib/kcl/src/tokeniser.rs @@ -206,8 +206,8 @@ fn is_block_comment(character: &str) -> bool { BLOCKCOMMENT.is_match(character) } -fn match_first(str: &str, regex: &Regex) -> Option { - regex.find(str).map(|the_match| the_match.as_str().to_string()) +fn match_first(s: &str, regex: &Regex) -> Option { + regex.find(s).map(|the_match| the_match.as_str().to_string()) } fn make_token(token_type: TokenType, value: &str, start: usize) -> Token { @@ -219,8 +219,8 @@ fn make_token(token_type: TokenType, value: &str, start: usize) -> Token { } } -fn return_token_at_index(str: &str, start_index: usize) -> Option { - let str_from_index = &str[start_index..]; +fn return_token_at_index(s: &str, start_index: usize) -> Option { + let str_from_index = &s[start_index..]; if is_string(str_from_index) { return Some(make_token( TokenType::String, @@ -348,21 +348,22 @@ fn return_token_at_index(str: &str, start_index: usize) -> Option { None } -pub fn lexer(str: &str) -> Vec { - fn recursively_tokenise(str: &str, current_index: usize, previous_tokens: Vec) -> Vec { - if current_index >= str.len() { - return previous_tokens; - } - let token = return_token_at_index(str, current_index); - let Some(token) = token else { - return recursively_tokenise(str, current_index + 1, previous_tokens); - }; - let mut new_tokens = previous_tokens; - let token_length = token.value.len(); - new_tokens.push(token); - recursively_tokenise(str, current_index + token_length, new_tokens) +fn recursively_tokenise(s: &str, current_index: usize, previous_tokens: Vec) -> Vec { + if current_index >= s.len() { + return previous_tokens; } - recursively_tokenise(str, 0, Vec::new()) + let token = return_token_at_index(s, current_index); + let Some(token) = token else { + return recursively_tokenise(s, current_index + 1, previous_tokens); + }; + let mut new_tokens = previous_tokens; + let token_length = token.value.len(); + new_tokens.push(token); + recursively_tokenise(s, current_index + token_length, new_tokens) +} + +pub fn lexer(s: &str) -> Vec { + recursively_tokenise(s, 0, Vec::new()) } #[cfg(test)]