Restructure tokenizer module (#700)

* Remove duplicated tests

These tests already were copied to tokeniser2.rs, so removing them doesn't affect code coverage.

* Move tokeniser to its own module

Now there's a module for tokens, and the tokenizer/lexer implementation is private within the token module.
This commit is contained in:
Adam Chalmers
2023-09-24 20:01:17 -05:00
committed by GitHub
parent 2971b7752b
commit a03d09b41d
13 changed files with 145 additions and 248 deletions

View File

@ -32,7 +32,7 @@ async fn execute_and_snapshot(code: &str) -> Result<image::DynamicImage> {
// Create a temporary file to write the output to.
let output_file = std::env::temp_dir().join(format!("kcl_output_{}.png", uuid::Uuid::new_v4()));
let tokens = kcl_lib::tokeniser::lexer(code);
let tokens = kcl_lib::token::lexer(code);
let parser = kcl_lib::parser::Parser::new(tokens);
let program = parser.ast()?;
let mut mem: kcl_lib::executor::ProgramMemory = Default::default();