Restructure tokenizer module (#700)

* Remove duplicated tests

These tests already were copied to tokeniser2.rs, so removing them doesn't affect code coverage.

* Move tokeniser to its own module

Now there's a module for tokens, and the tokenizer/lexer implementation is private within the token module.
This commit is contained in:
Adam Chalmers
2023-09-24 20:01:17 -05:00
committed by GitHub
parent 2971b7752b
commit a03d09b41d
13 changed files with 145 additions and 248 deletions

View File

@ -33,7 +33,7 @@ async fn setup(code: &str, name: &str) -> Result<(EngineConnection, Program, uui
.commands_ws(None, None, None, None, Some(false))
.await?;
let tokens = kcl_lib::tokeniser::lexer(code);
let tokens = kcl_lib::token::lexer(code);
let parser = kcl_lib::parser::Parser::new(tokens);
let program = parser.ast()?;
let mut mem: kcl_lib::executor::ProgramMemory = Default::default();