implement rename (#396)

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* rename function

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* start of rename

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* cache rust

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* fix gnarly bug

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* fucking tabs

Signed-off-by: Jess Frazelle <github@jessfraz.com>

---------

Signed-off-by: Jess Frazelle <github@jessfraz.com>
Co-authored-by: Kurt Hutten <k.hutten@protonmail.ch>

src/wasm-lib/Cargo.lock

@@ -1094,7 +1094,7 @@ dependencies = [
 
 [[package]]
 name = "kcl-lib"
-version = "0.1.21"
+version = "0.1.24"
 dependencies = [
  "anyhow",
  "bson",

src/wasm-lib/kcl/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "kcl-lib"
 description = "KittyCAD Language"
-version = "0.1.21"
+version = "0.1.24"
 edition = "2021"
 license = "MIT"
 
@@ -11,7 +11,7 @@ license = "MIT"
 anyhow = { version = "1.0.75", features = ["backtrace"] }
 clap = { version = "4.4.2", features = ["cargo", "derive", "env", "unicode"] }
 dashmap = "5.5.3"
-derive-docs = { version = "0.1.1" }
+derive-docs = { version = "0.1.3" }
 #derive-docs = { path = "../derive-docs" }
 kittycad = { version = "0.2.23", default-features = false, features = ["js"] }
 lazy_static = "1.4.0"

src/wasm-lib/kcl/src/abstract_syntax_tree_types.rs

@@ -117,6 +117,18 @@ impl Program {
         None
     }
 
+    /// Returns the body item that includes the given character position.
+    pub fn get_mut_body_item_for_position(&mut self, pos: usize) -> Option<&mut BodyItem> {
+        for item in &mut self.body {
+            let source_range: SourceRange = item.clone().into();
+            if source_range.contains(pos) {
+                return Some(item);
+            }
+        }
+
+        None
+    }
+
     /// Returns a value that includes the given character position.
     /// This is a bit more recursive than `get_body_item_for_position`.
     pub fn get_value_for_position(&self, pos: usize) -> Option<&Value> {
@@ -149,6 +161,82 @@ impl Program {
 
         symbols
     }
+
+    /// Rename the variable declaration at the given position.
+    pub fn rename_symbol(&mut self, new_name: &str, pos: usize) {
+        // The position must be within the variable declaration.
+        let mut old_name = None;
+        for item in &mut self.body {
+            match item {
+                BodyItem::ExpressionStatement(_expression_statement) => {
+                    continue;
+                }
+                BodyItem::VariableDeclaration(ref mut variable_declaration) => {
+                    if let Some(var_old_name) = variable_declaration.rename_symbol(new_name, pos) {
+                        old_name = Some(var_old_name);
+                        break;
+                    }
+                }
+                BodyItem::ReturnStatement(_return_statement) => continue,
+            }
+        }
+
+        if let Some(old_name) = old_name {
+            // Now rename all the identifiers in the rest of the program.
+            self.rename_identifiers(&old_name, new_name);
+        } else {
+            // Okay so this was not a top level variable declaration.
+            // But it might be a variable declaration inside a function or function params.
+            // So we need to check that.
+            let Some(ref mut item) = self.get_mut_body_item_for_position(pos) else {
+                return;
+            };
+
+            // Recurse over the item.
+            let mut value = match item {
+                BodyItem::ExpressionStatement(ref mut expression_statement) => {
+                    Some(&mut expression_statement.expression)
+                }
+                BodyItem::VariableDeclaration(ref mut variable_declaration) => {
+                    variable_declaration.get_mut_value_for_position(pos)
+                }
+                BodyItem::ReturnStatement(ref mut return_statement) => Some(&mut return_statement.argument),
+            };
+
+            // Check if we have a function expression.
+            if let Some(Value::FunctionExpression(ref mut function_expression)) = &mut value {
+                // Check if the params to the function expression contain the position.
+                for param in &mut function_expression.params {
+                    let param_source_range: SourceRange = param.clone().into();
+                    if param_source_range.contains(pos) {
+                        let old_name = param.name.clone();
+                        // Rename the param.
+                        param.rename(&old_name, new_name);
+                        // Now rename all the identifiers in the rest of the program.
+                        function_expression.body.rename_identifiers(&old_name, new_name);
+                        return;
+                    }
+                }
+            }
+        }
+    }
+
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        for item in &mut self.body {
+            match item {
+                BodyItem::ExpressionStatement(ref mut expression_statement) => {
+                    expression_statement.expression.rename_identifiers(old_name, new_name);
+                }
+                BodyItem::VariableDeclaration(ref mut variable_declaration) => {
+                    variable_declaration.rename_identifiers(old_name, new_name);
+                }
+                BodyItem::ReturnStatement(ref mut return_statement) => {
+                    return_statement.argument.rename_identifiers(old_name, new_name);
+                }
+            }
+        }
+    }
 }
 
 pub trait ValueMeta {
@@ -315,6 +403,29 @@ impl Value {
             Value::UnaryExpression(unary_expression) => unary_expression.get_hover_value_for_position(pos, code),
         }
     }
+
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        match self {
+            Value::Literal(_literal) => {}
+            Value::Identifier(ref mut identifier) => identifier.rename(old_name, new_name),
+            Value::BinaryExpression(ref mut binary_expression) => {
+                binary_expression.rename_identifiers(old_name, new_name)
+            }
+            Value::FunctionExpression(_function_identifier) => {}
+            Value::CallExpression(ref mut call_expression) => call_expression.rename_identifiers(old_name, new_name),
+            Value::PipeExpression(ref mut pipe_expression) => pipe_expression.rename_identifiers(old_name, new_name),
+            Value::PipeSubstitution(_) => {}
+            Value::ArrayExpression(ref mut array_expression) => array_expression.rename_identifiers(old_name, new_name),
+            Value::ObjectExpression(ref mut object_expression) => {
+                object_expression.rename_identifiers(old_name, new_name)
+            }
+            Value::MemberExpression(ref mut member_expression) => {
+                member_expression.rename_identifiers(old_name, new_name)
+            }
+            Value::UnaryExpression(ref mut unary_expression) => unary_expression.rename_identifiers(old_name, new_name),
+        }
+    }
 }
 
 impl From<Value> for crate::executor::SourceRange {
@@ -420,6 +531,23 @@ impl BinaryPart {
             BinaryPart::UnaryExpression(unary_expression) => unary_expression.get_hover_value_for_position(pos, code),
         }
     }
+
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        match self {
+            BinaryPart::Literal(_literal) => {}
+            BinaryPart::Identifier(ref mut identifier) => identifier.rename(old_name, new_name),
+            BinaryPart::BinaryExpression(ref mut binary_expression) => {
+                binary_expression.rename_identifiers(old_name, new_name)
+            }
+            BinaryPart::CallExpression(ref mut call_expression) => {
+                call_expression.rename_identifiers(old_name, new_name)
+            }
+            BinaryPart::UnaryExpression(ref mut unary_expression) => {
+                unary_expression.rename_identifiers(old_name, new_name)
+            }
+        }
+    }
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ts_rs::TS, JsonSchema)]
@@ -689,6 +817,15 @@ impl CallExpression {
 
         None
     }
+
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        self.callee.rename(old_name, new_name);
+
+        for arg in &mut self.arguments {
+            arg.rename_identifiers(old_name, new_name);
+        }
+    }
 }
 
 /// A function declaration.
@@ -741,6 +878,50 @@ impl VariableDeclaration {
         None
     }
 
+    /// Returns a value that includes the given character position.
+    pub fn get_mut_value_for_position(&mut self, pos: usize) -> Option<&mut Value> {
+        for declaration in &mut self.declarations {
+            let source_range: SourceRange = declaration.clone().into();
+            if source_range.contains(pos) {
+                return Some(&mut declaration.init);
+            }
+        }
+
+        None
+    }
+
+    /// Rename the variable declaration at the given position.
+    /// This returns the old name of the variable, if it found one.
+    pub fn rename_symbol(&mut self, new_name: &str, pos: usize) -> Option<String> {
+        // The position must be within the variable declaration.
+        let source_range: SourceRange = self.clone().into();
+        if !source_range.contains(pos) {
+            return None;
+        }
+
+        for declaration in &mut self.declarations {
+            let declaration_source_range: SourceRange = declaration.id.clone().into();
+            if declaration_source_range.contains(pos) {
+                let old_name = declaration.id.name.clone();
+                declaration.id.name = new_name.to_string();
+                return Some(old_name);
+            }
+        }
+
+        None
+    }
+
+    pub fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        for declaration in &mut self.declarations {
+            // Skip the init for the variable with the new name since it is the one we are renaming.
+            if declaration.id.name == new_name {
+                continue;
+            }
+
+            declaration.init.rename_identifiers(old_name, new_name);
+        }
+    }
+
     pub fn get_lsp_symbols(&self, code: &str) -> Vec<DocumentSymbol> {
         let mut symbols = vec![];
 
@@ -857,7 +1038,9 @@ impl VariableKind {
 pub struct VariableDeclarator {
     pub start: usize,
     pub end: usize,
+    /// The identifier of the variable.
     pub id: Identifier,
+    /// The value of the variable.
     pub init: Value,
 }
 
@@ -919,6 +1102,15 @@ pub struct Identifier {
 
 impl_value_meta!(Identifier);
 
+impl Identifier {
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename(&mut self, old_name: &str, new_name: &str) {
+        if self.name == old_name {
+            self.name = new_name.to_string();
+        }
+    }
+}
+
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ts_rs::TS, JsonSchema)]
 #[ts(export)]
 #[serde(tag = "type")]
@@ -1045,6 +1237,13 @@ impl ArrayExpression {
             }],
         })
     }
+
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        for element in &mut self.elements {
+            element.rename_identifiers(old_name, new_name);
+        }
+    }
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ts_rs::TS, JsonSchema)]
@@ -1159,6 +1358,13 @@ impl ObjectExpression {
             }],
         })
     }
+
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        for property in &mut self.properties {
+            property.value.rename_identifiers(old_name, new_name);
+        }
+    }
 }
 
 impl_value_meta!(ObjectExpression);
@@ -1376,6 +1582,21 @@ impl MemberExpression {
             }))
         }
     }
+
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        match &mut self.object {
+            MemberObject::MemberExpression(ref mut member_expression) => {
+                member_expression.rename_identifiers(old_name, new_name)
+            }
+            MemberObject::Identifier(ref mut identifier) => identifier.rename(old_name, new_name),
+        }
+
+        match &mut self.property {
+            LiteralIdentifier::Identifier(ref mut identifier) => identifier.rename(old_name, new_name),
+            LiteralIdentifier::Literal(_) => {}
+        }
+    }
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ts_rs::TS, JsonSchema)]
@@ -1492,6 +1713,12 @@ impl BinaryExpression {
             }],
         })
     }
+
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        self.left.rename_identifiers(old_name, new_name);
+        self.right.rename_identifiers(old_name, new_name);
+    }
 }
 
 pub fn parse_json_number_as_f64(j: &serde_json::Value, source_range: SourceRange) -> Result<f64, KclError> {
@@ -1599,6 +1826,11 @@ impl UnaryExpression {
 
         None
     }
+
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        self.argument.rename_identifiers(old_name, new_name);
+    }
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ts_rs::TS, JsonSchema, FromStr, Display)]
@@ -1675,6 +1907,13 @@ impl PipeExpression {
         pipe_info.index = 0;
         execute_pipe_body(memory, &self.body, pipe_info, self.into(), engine)
     }
+
+    /// Rename all identifiers that have the old name to the new given name.
+    fn rename_identifiers(&mut self, old_name: &str, new_name: &str) {
+        for statement in &mut self.body {
+            statement.rename_identifiers(old_name, new_name);
+        }
+    }
 }
 
 fn execute_pipe_body(
@@ -2158,4 +2397,64 @@ const part001 = startSketchAt([0, 0])
         );
         assert_eq!(recasted, some_program_string);
     }
+
+    #[test]
+    fn test_recast_after_rename_std() {
+        let some_program_string = r#"const part001 = startSketchAt([0.0000000000, 5.0000000000])
+    |> line([0.4900857016, -0.0240763666], %)
+
+const part002 = "part002"
+const things = [part001, 0.0]
+let blah = 1
+const foo = false
+let baz = {a: 1, part001: "thing"}
+
+fn ghi = (part001) => {
+  return part001
+}
+
+show(part001)"#;
+        let tokens = crate::tokeniser::lexer(some_program_string);
+        let parser = crate::parser::Parser::new(tokens);
+        let mut program = parser.ast().unwrap();
+        program.rename_symbol("mySuperCoolPart", 6);
+
+        let recasted = program.recast(&Default::default(), 0);
+        assert_eq!(
+            recasted,
+            r#"const mySuperCoolPart = startSketchAt([0.0, 5.0])
+  |> line([0.4900857016, -0.0240763666], %)
+
+const part002 = "part002"
+const things = [mySuperCoolPart, 0.0]
+let blah = 1
+const foo = false
+let baz = { a: 1, part001: "thing" }
+
+fn ghi = (part001) => {
+  return part001
+}
+
+show(mySuperCoolPart)"#
+        );
+    }
+
+    #[test]
+    fn test_recast_after_rename_fn_args() {
+        let some_program_string = r#"fn ghi = (x, y, z) => {
+  return x
+}"#;
+        let tokens = crate::tokeniser::lexer(some_program_string);
+        let parser = crate::parser::Parser::new(tokens);
+        let mut program = parser.ast().unwrap();
+        program.rename_symbol("newName", 10);
+
+        let recasted = program.recast(&Default::default(), 0);
+        assert_eq!(
+            recasted,
+            r#"fn ghi = (newName, y, z) => {
+  return newName
+}"#
+        );
+    }
 }

src/wasm-lib/kcl/src/parser.rs

@@ -170,6 +170,13 @@ impl Parser {
             }));
         }
 
+        if index >= self.tokens.len() {
+            return Err(KclError::Syntax(KclErrorDetails {
+                source_ranges: vec![self.tokens.last().unwrap().into()],
+                message: "unexpected end".to_string(),
+            }));
+        }
+
         let Some(token) = self.tokens.get(index) else {
             return Err(KclError::Syntax(KclErrorDetails {
                 source_ranges: vec![self.tokens.last().unwrap().into()],
@@ -679,7 +686,11 @@ impl Parser {
                 return Ok(index);
             }
             let next_right = self.next_meaningful_token(maybe_operator.index, None)?;
-            self.find_end_of_binary_expression(next_right.index)
+            if next_right.index != index {
+                self.find_end_of_binary_expression(next_right.index)
+            } else {
+                Ok(index)
+            }
         } else {
             Ok(index)
         }
@@ -1105,42 +1116,42 @@ impl Parser {
     ) -> Result<VariableDeclaratorsReturn, KclError> {
         let current_token = self.get_token(index)?;
         let assignment = self.next_meaningful_token(index, None)?;
-        if let Some(assignment_token) = assignment.token {
-            let contents_start_token = self.next_meaningful_token(assignment.index, None)?;
-            let pipe_start_index = if assignment_token.token_type == TokenType::Operator {
-                contents_start_token.index
-            } else {
-                assignment.index
-            };
-            let next_pipe_operator = self.has_pipe_operator(pipe_start_index, None)?;
-            let init: Value;
-            let last_index = if next_pipe_operator.token.is_some() {
-                let pipe_expression_result = self.make_pipe_expression(assignment.index)?;
-                init = Value::PipeExpression(Box::new(pipe_expression_result.expression));
-                pipe_expression_result.last_index
-            } else {
-                let value_result = self.make_value(contents_start_token.index)?;
-                init = value_result.value;
-                value_result.last_index
-            };
-            let current_declarator = VariableDeclarator {
-                start: current_token.start,
-                end: self.get_token(last_index)?.end,
-                id: self.make_identifier(index)?,
-                init,
-            };
-            let mut declarations = previous_declarators;
-            declarations.push(current_declarator);
-            Ok(VariableDeclaratorsReturn {
-                declarations,
-                last_index,
-            })
-        } else {
-            Err(KclError::Unimplemented(KclErrorDetails {
+        let Some(assignment_token) = assignment.token else {
+            return Err(KclError::Unimplemented(KclErrorDetails {
                 source_ranges: vec![current_token.clone().into()],
                 message: format!("Unexpected token {} ", current_token.value),
-            }))
-        }
+            }));
+        };
+
+        let contents_start_token = self.next_meaningful_token(assignment.index, None)?;
+        let pipe_start_index = if assignment_token.token_type == TokenType::Operator {
+            contents_start_token.index
+        } else {
+            assignment.index
+        };
+        let next_pipe_operator = self.has_pipe_operator(pipe_start_index, None)?;
+        let init: Value;
+        let last_index = if next_pipe_operator.token.is_some() {
+            let pipe_expression_result = self.make_pipe_expression(assignment.index)?;
+            init = Value::PipeExpression(Box::new(pipe_expression_result.expression));
+            pipe_expression_result.last_index
+        } else {
+            let value_result = self.make_value(contents_start_token.index)?;
+            init = value_result.value;
+            value_result.last_index
+        };
+        let current_declarator = VariableDeclarator {
+            start: current_token.start,
+            end: self.get_token(last_index)?.end,
+            id: self.make_identifier(index)?,
+            init,
+        };
+        let mut declarations = previous_declarators;
+        declarations.push(current_declarator);
+        Ok(VariableDeclaratorsReturn {
+            declarations,
+            last_index,
+        })
     }
 
     fn make_variable_declaration(&self, index: usize) -> Result<VariableDeclarationResult, KclError> {
@@ -2716,4 +2727,39 @@ show(mySk1)"#;
         assert!(result.is_err());
         assert!(result.err().unwrap().to_string().contains("file is empty"));
     }
+
+    #[test]
+    fn test_parse_half_pipe_small() {
+        let tokens = crate::tokeniser::lexer(
+            "const secondExtrude = startSketchAt([0,0])
+  |",
+        );
+        let parser = Parser::new(tokens);
+        let result = parser.ast();
+        assert!(result.is_err());
+        assert!(result.err().unwrap().to_string().contains("Unexpected token"));
+    }
+
+    #[test]
+    fn test_parse_half_pipe() {
+        let tokens = crate::tokeniser::lexer(
+            "const height = 10
+
+const firstExtrude = startSketchAt([0,0])
+  |> line([0, 8], %)
+  |> line([20, 0], %)
+  |> line([0, -8], %)
+  |> close(%)
+  |> extrude(2, %)
+
+show(firstExtrude)
+
+const secondExtrude = startSketchAt([0,0])
+  |",
+        );
+        let parser = Parser::new(tokens);
+        let result = parser.ast();
+        assert!(result.is_err());
+        assert!(result.err().unwrap().to_string().contains("Unexpected token"));
+    }
 }

src/wasm-lib/kcl/src/server/mod.rs

@@ -233,6 +233,7 @@ impl LanguageServer for Backend {
                 document_symbol_provider: Some(OneOf::Left(true)),
                 hover_provider: Some(HoverProviderCapability::Simple(true)),
                 inlay_hint_provider: Some(OneOf::Left(true)),
+                rename_provider: Some(OneOf::Left(true)),
                 semantic_tokens_provider: Some(SemanticTokensServerCapabilities::SemanticTokensRegistrationOptions(
                     SemanticTokensRegistrationOptions {
                         text_document_registration_options: {
@@ -567,6 +568,43 @@ impl LanguageServer for Backend {
             range,
         }]))
     }
+
+    async fn rename(&self, params: RenameParams) -> RpcResult<Option<WorkspaceEdit>> {
+        let filename = params.text_document_position.text_document.uri.to_string();
+
+        let Some(current_code) = self.current_code_map.get(&filename) else {
+            return Ok(None);
+        };
+
+        // Parse the ast.
+        // I don't know if we need to do this again since it should be updated in the context.
+        // But I figure better safe than sorry since this will write back out to the file.
+        let tokens = crate::tokeniser::lexer(&current_code);
+        let parser = crate::parser::Parser::new(tokens);
+        let Ok(mut ast) = parser.ast() else {
+            return Ok(None);
+        };
+
+        // Let's convert the position to a character index.
+        let pos = position_to_char_index(params.text_document_position.position, &current_code);
+        // Now let's perform the rename on the ast.
+        ast.rename_symbol(&params.new_name, pos);
+        // Now recast it.
+        let recast = ast.recast(&Default::default(), 0);
+        let source_range = SourceRange([0, current_code.len() - 1]);
+        let range = source_range.to_lsp_range(&current_code);
+        Ok(Some(WorkspaceEdit {
+            changes: Some(HashMap::from([(
+                params.text_document_position.text_document.uri,
+                vec![TextEdit {
+                    new_text: recast,
+                    range,
+                }],
+            )])),
+            document_changes: None,
+            change_annotations: None,
+        }))
+    }
 }
 
 /// Get completions from our stdlib.

src/wasm-lib/kcl/src/tokeniser.rs

@@ -206,8 +206,8 @@ fn is_block_comment(character: &str) -> bool {
     BLOCKCOMMENT.is_match(character)
 }
 
-fn match_first(str: &str, regex: &Regex) -> Option<String> {
-    regex.find(str).map(|the_match| the_match.as_str().to_string())
+fn match_first(s: &str, regex: &Regex) -> Option<String> {
+    regex.find(s).map(|the_match| the_match.as_str().to_string())
 }
 
 fn make_token(token_type: TokenType, value: &str, start: usize) -> Token {
@@ -219,8 +219,8 @@ fn make_token(token_type: TokenType, value: &str, start: usize) -> Token {
     }
 }
 
-fn return_token_at_index(str: &str, start_index: usize) -> Option<Token> {
-    let str_from_index = &str[start_index..];
+fn return_token_at_index(s: &str, start_index: usize) -> Option<Token> {
+    let str_from_index = &s[start_index..];
     if is_string(str_from_index) {
         return Some(make_token(
             TokenType::String,
@@ -348,21 +348,22 @@ fn return_token_at_index(str: &str, start_index: usize) -> Option<Token> {
     None
 }
 
-pub fn lexer(str: &str) -> Vec<Token> {
-    fn recursively_tokenise(str: &str, current_index: usize, previous_tokens: Vec<Token>) -> Vec<Token> {
-        if current_index >= str.len() {
-            return previous_tokens;
-        }
-        let token = return_token_at_index(str, current_index);
-        let Some(token) = token else {
-            return recursively_tokenise(str, current_index + 1, previous_tokens);
-        };
-        let mut new_tokens = previous_tokens;
-        let token_length = token.value.len();
-        new_tokens.push(token);
-        recursively_tokenise(str, current_index + token_length, new_tokens)
+fn recursively_tokenise(s: &str, current_index: usize, previous_tokens: Vec<Token>) -> Vec<Token> {
+    if current_index >= s.len() {
+        return previous_tokens;
     }
-    recursively_tokenise(str, 0, Vec::new())
+    let token = return_token_at_index(s, current_index);
+    let Some(token) = token else {
+        return recursively_tokenise(s, current_index + 1, previous_tokens);
+    };
+    let mut new_tokens = previous_tokens;
+    let token_length = token.value.len();
+    new_tokens.push(token);
+    recursively_tokenise(s, current_index + token_length, new_tokens)
+}
+
+pub fn lexer(s: &str) -> Vec<Token> {
+    recursively_tokenise(s, 0, Vec::new())
 }
 
 #[cfg(test)]