parse the shebang and make it work with recast (#2289)

* parse the shebang and make it work with recast

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* fix playwright

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* fix playwright

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* fix playwright

Signed-off-by: Jess Frazelle <github@jessfraz.com>

---------

Signed-off-by: Jess Frazelle <github@jessfraz.com>

e2e/playwright/flow-tests.spec.ts

@@ -279,7 +279,7 @@ test('if you write invalid kcl you get inlined errors', async ({ page }) => {
     const bottomAng = 25
    */
   await page.click('.cm-content')
-  await page.keyboard.type('# error')
+  await page.keyboard.type('$ error')
 
   // press arrows to clear autocomplete
   await page.keyboard.press('ArrowLeft')
@@ -296,10 +296,10 @@ test('if you write invalid kcl you get inlined errors', async ({ page }) => {
 
   // error text on hover
   await page.hover('.cm-lint-marker-error')
-  await expect(page.getByText("found unknown token '#'")).toBeVisible()
+  await expect(page.getByText("found unknown token '$'")).toBeVisible()
 
   // select the line that's causing the error and delete it
-  await page.getByText('# error').click()
+  await page.getByText('$ error').click()
   await page.keyboard.press('End')
   await page.keyboard.down('Shift')
   await page.keyboard.press('Home')

src-tauri/Cargo.lock

@@ -416,9 +416,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
 [[package]]
 name = "base64"
-version = "0.22.0"
+version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "base64ct"
@@ -2426,7 +2426,7 @@ dependencies = [
  "approx",
  "async-recursion",
  "async-trait",
- "base64 0.22.0",
+ "base64 0.22.1",
  "bson",
  "chrono",
  "clap",
@@ -3886,7 +3886,7 @@ version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10"
 dependencies = [
- "base64 0.22.0",
+ "base64 0.22.1",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -4136,7 +4136,7 @@ version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d"
 dependencies = [
- "base64 0.22.0",
+ "base64 0.22.1",
  "rustls-pki-types",
 ]
 
@@ -4303,9 +4303,9 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.199"
+version = "1.0.200"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a"
+checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f"
 dependencies = [
  "serde_derive",
 ]
@@ -4321,9 +4321,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.199"
+version = "1.0.200"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc"
+checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5013,7 +5013,7 @@ version = "2.0.0-beta.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b383f341efb803852b0235a2f330ca90c4c113f422dd6d646b888685b372cace"
 dependencies = [
- "base64 0.22.0",
+ "base64 0.22.1",
  "brotli",
  "ico",
  "json-patch",
@@ -5207,7 +5207,7 @@ version = "2.0.0-beta.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f34be6851c7e84ca99b3bddd57e033d55d8bfca1dd153d6e8d18e9f1fb95469"
 dependencies = [
- "base64 0.22.0",
+ "base64 0.22.1",
  "dirs-next",
  "flate2",
  "futures-util",
@@ -6663,7 +6663,7 @@ version = "0.39.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e180ac2740d6cb4d5cec0abf63eacbea90f1b7e5e3803043b13c1c84c4b7884"
 dependencies = [
- "base64 0.22.0",
+ "base64 0.22.1",
  "block",
  "cocoa",
  "core-graphics",

src-tauri/src/main.rs

@@ -334,7 +334,7 @@ fn show_in_folder(path: &str) -> Result<(), InvokeError> {
     #[cfg(not(unix))]
     {
         Command::new("explorer")
-            .args(["/select,", &path]) // The comma after select is not a typo
+            .args(["/select,", path]) // The comma after select is not a typo
             .spawn()
             .map_err(|e| InvokeError::from_anyhow(e.into()))?;
     }
@@ -342,7 +342,7 @@ fn show_in_folder(path: &str) -> Result<(), InvokeError> {
     #[cfg(unix)]
     {
         Command::new("open")
-            .args(["-R", &path])
+            .args(["-R", path])
             .spawn()
             .map_err(|e| InvokeError::from_anyhow(e.into()))?;
     }

src/wasm-lib/kcl/src/ast/types.rs

@@ -821,6 +821,7 @@ impl NonCodeNode {
 
     pub fn value(&self) -> String {
         match &self.value {
+            NonCodeValue::Shebang { value } => value.clone(),
             NonCodeValue::InlineComment { value, style: _ } => value.clone(),
             NonCodeValue::BlockComment { value, style: _ } => value.clone(),
             NonCodeValue::NewLineBlockComment { value, style: _ } => value.clone(),
@@ -830,6 +831,7 @@ impl NonCodeNode {
 
     pub fn format(&self, indentation: &str) -> String {
         match &self.value {
+            NonCodeValue::Shebang { value } => format!("{}\n\n", value),
             NonCodeValue::InlineComment {
                 value,
                 style: CommentStyle::Line,
@@ -882,6 +884,15 @@ pub enum CommentStyle {
 #[ts(export)]
 #[serde(tag = "type", rename_all = "camelCase")]
 pub enum NonCodeValue {
+    /// A shebang.
+    /// This is a special type of comment that is at the top of the file.
+    /// It looks like this:
+    /// ```python,no_run
+    /// #!/usr/bin/env python
+    /// ```
+    Shebang {
+        value: String,
+    },
     /// An inline comment.
     /// Here are examples:
     /// `1 + 1 // This is an inline comment`.
@@ -3273,6 +3284,117 @@ fn ghi = (x) => {
         assert_eq!(recasted, r#""#);
     }
 
+    #[test]
+    fn test_recast_shebang_only() {
+        let some_program_string = r#"#!/usr/local/env zoo kcl"#;
+
+        let tokens = crate::token::lexer(some_program_string).unwrap();
+        let parser = crate::parser::Parser::new(tokens);
+        let result = parser.ast();
+
+        assert!(result.is_err());
+        assert_eq!(
+            result.unwrap_err().to_string(),
+            r#"syntax: KclErrorDetails { source_ranges: [SourceRange([21, 24])], message: "Unexpected end of file. The compiler expected a function body items (functions are made up of variable declarations, expressions, and return statements, each of those is a possible body item" }"#
+        );
+    }
+
+    #[test]
+    fn test_recast_shebang() {
+        let some_program_string = r#"#!/usr/local/env zoo kcl
+const part001 = startSketchOn('XY')
+  |> startProfileAt([-10, -10], %)
+  |> line([20, 0], %)
+  |> line([0, 20], %)
+  |> line([-20, 0], %)
+  |> close(%)
+"#;
+
+        let tokens = crate::token::lexer(some_program_string).unwrap();
+        let parser = crate::parser::Parser::new(tokens);
+        let program = parser.ast().unwrap();
+
+        let recasted = program.recast(&Default::default(), 0);
+        assert_eq!(
+            recasted,
+            r#"#!/usr/local/env zoo kcl
+
+const part001 = startSketchOn('XY')
+  |> startProfileAt([-10, -10], %)
+  |> line([20, 0], %)
+  |> line([0, 20], %)
+  |> line([-20, 0], %)
+  |> close(%)
+"#
+        );
+    }
+
+    #[test]
+    fn test_recast_shebang_new_lines() {
+        let some_program_string = r#"#!/usr/local/env zoo kcl
+        
+
+
+const part001 = startSketchOn('XY')
+  |> startProfileAt([-10, -10], %)
+  |> line([20, 0], %)
+  |> line([0, 20], %)
+  |> line([-20, 0], %)
+  |> close(%)
+"#;
+
+        let tokens = crate::token::lexer(some_program_string).unwrap();
+        let parser = crate::parser::Parser::new(tokens);
+        let program = parser.ast().unwrap();
+
+        let recasted = program.recast(&Default::default(), 0);
+        assert_eq!(
+            recasted,
+            r#"#!/usr/local/env zoo kcl
+
+const part001 = startSketchOn('XY')
+  |> startProfileAt([-10, -10], %)
+  |> line([20, 0], %)
+  |> line([0, 20], %)
+  |> line([-20, 0], %)
+  |> close(%)
+"#
+        );
+    }
+
+    #[test]
+    fn test_recast_shebang_with_comments() {
+        let some_program_string = r#"#!/usr/local/env zoo kcl
+        
+// Yo yo my comments.
+const part001 = startSketchOn('XY')
+  |> startProfileAt([-10, -10], %)
+  |> line([20, 0], %)
+  |> line([0, 20], %)
+  |> line([-20, 0], %)
+  |> close(%)
+"#;
+
+        let tokens = crate::token::lexer(some_program_string).unwrap();
+        let parser = crate::parser::Parser::new(tokens);
+        let program = parser.ast().unwrap();
+
+        let recasted = program.recast(&Default::default(), 0);
+        assert_eq!(
+            recasted,
+            r#"#!/usr/local/env zoo kcl
+
+// Yo yo my comments.
+const part001 = startSketchOn('XY')
+  |> startProfileAt([-10, -10], %)
+  |> line([20, 0], %)
+  |> line([0, 20], %)
+  |> line([-20, 0], %)
+  |> close(%)
+"#
+        );
+    }
+
     #[test]
     fn test_recast_large_file() {
         let some_program_string = r#"// define constants

src/wasm-lib/kcl/src/executor.rs

@@ -67,7 +67,7 @@ impl ProgramMemory {
 
     /// Add to the program memory.
     pub fn add(&mut self, key: &str, value: MemoryItem, source_range: SourceRange) -> Result<(), KclError> {
-        if self.root.get(key).is_some() {
+        if self.root.contains_key(key) {
             return Err(KclError::ValueAlreadyDefined(KclErrorDetails {
                 message: format!("Cannot redefine {}", key),
                 source_ranges: vec![source_range],

src/wasm-lib/kcl/src/lsp/tests.rs

@@ -742,7 +742,7 @@ async fn test_kcl_lsp_create_zip() {
 
     assert_eq!(files.len(), 11);
     let util_path = format!("{}/util.rs", string_path).replace("file://", "");
-    assert!(files.get(&util_path).is_some());
+    assert!(files.contains_key(&util_path));
     assert_eq!(files.get("/test.kcl"), Some(&4));
 }
 

src/wasm-lib/kcl/src/parser/parser_impl.rs

@@ -5,7 +5,7 @@ use winnow::{
     dispatch,
     error::{ErrMode, StrContext, StrContextValue},
     prelude::*,
-    token::{any, one_of},
+    token::{any, one_of, take_till},
 };
 
 use crate::{
@@ -39,7 +39,13 @@ fn expected(what: &'static str) -> StrContext {
 }
 
 fn program(i: TokenSlice) -> PResult<Program> {
+    let shebang = opt(shebang).parse_next(i)?;
     let mut out = function_body.parse_next(i)?;
+
+    // Add the shebang to the non-code meta.
+    if let Some(shebang) = shebang {
+        out.non_code_meta.start.insert(0, shebang);
+    }
     // Match original parser behaviour, for now.
     // Once this is merged and stable, consider changing this as I think it's more accurate
     // without the -1.
@@ -386,6 +392,39 @@ fn whitespace(i: TokenSlice) -> PResult<Vec<Token>> {
     .parse_next(i)
 }
 
+/// A shebang is a line at the start of a file that starts with `#!`.
+/// If the shebang is present it takes up the whole line.
+fn shebang(i: TokenSlice) -> PResult<NonCodeNode> {
+    // Parse the hash and the bang.
+    hash.parse_next(i)?;
+    bang.parse_next(i)?;
+    // Get the rest of the line.
+    // Parse everything until the next newline.
+    let tokens = take_till(0.., |token: Token| token.value.contains('\n')).parse_next(i)?;
+    let value = tokens.iter().map(|t| t.value.as_str()).collect::<String>();
+
+    if tokens.is_empty() {
+        return Err(ErrMode::Cut(
+            KclError::Syntax(KclErrorDetails {
+                source_ranges: vec![],
+                message: "expected a shebang value after #!".to_owned(),
+            })
+            .into(),
+        ));
+    }
+
+    // Strip all the whitespace after the shebang.
+    opt(whitespace).parse_next(i)?;
+
+    Ok(NonCodeNode {
+        start: 0,
+        end: tokens.last().unwrap().end,
+        value: NonCodeValue::Shebang {
+            value: format!("#!{}", value),
+        },
+    })
+}
+
 /// Parse the = operator.
 fn equals(i: TokenSlice) -> PResult<Token> {
     one_of((TokenType::Operator, "="))
@@ -601,6 +640,7 @@ fn noncode_just_after_code(i: TokenSlice) -> PResult<NonCodeNode> {
                 // There's an empty line between the body item and the comment,
                 // This means the comment is a NewLineBlockComment!
                 let value = match nc.value {
+                    NonCodeValue::Shebang { value } => NonCodeValue::Shebang { value },
                     // Change block comments to inline, as discussed above
                     NonCodeValue::BlockComment { value, style } => NonCodeValue::NewLineBlockComment { value, style },
                     // Other variants don't need to change.
@@ -620,6 +660,7 @@ fn noncode_just_after_code(i: TokenSlice) -> PResult<NonCodeNode> {
                 // There's no newline between the body item and comment,
                 // so if this is a comment, it must be inline with code.
                 let value = match nc.value {
+                    NonCodeValue::Shebang { value } => NonCodeValue::Shebang { value },
                     // Change block comments to inline, as discussed above
                     NonCodeValue::BlockComment { value, style } => NonCodeValue::InlineComment { value, style },
                     // Other variants don't need to change.
@@ -1204,6 +1245,16 @@ fn comma(i: TokenSlice) -> PResult<()> {
     Ok(())
 }
 
+fn hash(i: TokenSlice) -> PResult<()> {
+    TokenType::Hash.parse_from(i)?;
+    Ok(())
+}
+
+fn bang(i: TokenSlice) -> PResult<()> {
+    TokenType::Bang.parse_from(i)?;
+    Ok(())
+}
+
 fn period(i: TokenSlice) -> PResult<()> {
     TokenType::Period.parse_from(i)?;
     Ok(())
@@ -2331,7 +2382,7 @@ const secondExtrude = startSketchOn('XY')
         let err = parser.ast().unwrap_err();
         assert_eq!(
             err.to_string(),
-            r#"lexical: KclErrorDetails { source_ranges: [SourceRange([1, 2])], message: "found unknown token '!'" }"#
+            r#"syntax: KclErrorDetails { source_ranges: [SourceRange([0, 1])], message: "Unexpected token" }"#
         );
     }
 
@@ -2398,7 +2449,7 @@ z(-[["#,
         assert!(result.is_err());
         assert_eq!(
             result.err().unwrap().to_string(),
-            r#"lexical: KclErrorDetails { source_ranges: [SourceRange([6, 7])], message: "found unknown token '#'" }"#
+            r#"syntax: KclErrorDetails { source_ranges: [SourceRange([3, 4])], message: "Unexpected token" }"#
         );
     }
 
@@ -2410,7 +2461,7 @@ z(-[["#,
         assert!(result.is_err());
         assert_eq!(
             result.err().unwrap().to_string(),
-            r#"lexical: KclErrorDetails { source_ranges: [SourceRange([25, 26]), SourceRange([26, 27])], message: "found unknown tokens [#, #]" }"#
+            r#"syntax: KclErrorDetails { source_ranges: [SourceRange([2, 3])], message: "Unexpected token" }"#
         );
     }
 

src/wasm-lib/kcl/src/settings/types/mod.rs

@@ -31,7 +31,7 @@ impl Configuration {
 
         if let Some(project_directory) = &settings.settings.app.project_directory {
             if settings.settings.project.directory.to_string_lossy().is_empty() {
-                settings.settings.project.directory = project_directory.clone();
+                settings.settings.project.directory.clone_from(project_directory);
                 settings.settings.app.project_directory = None;
             }
         }

src/wasm-lib/kcl/src/settings/utils.rs

@@ -22,7 +22,10 @@ lazy_static::lazy_static! {
 
 /// Walk a directory recursively and return a list of all files.
 #[async_recursion::async_recursion]
-pub async fn walk_dir<P: AsRef<Path> + Send>(dir: P) -> Result<FileEntry> {
+pub async fn walk_dir<P>(dir: P) -> Result<FileEntry>
+where
+    P: AsRef<Path> + Send,
+{
     let mut entry = FileEntry {
         name: dir
             .as_ref()

src/wasm-lib/kcl/src/token.rs

@@ -31,6 +31,10 @@ pub enum TokenType {
     Type,
     /// A brace.
     Brace,
+    /// A hash.
+    Hash,
+    /// A bang.
+    Bang,
     /// Whitespace.
     Whitespace,
     /// A comma.
@@ -74,6 +78,8 @@ impl TryFrom<TokenType> for SemanticTokenType {
             | TokenType::Colon
             | TokenType::Period
             | TokenType::DoublePeriod
+            | TokenType::Hash
+            | TokenType::Bang
             | TokenType::Unknown => {
                 anyhow::bail!("unsupported token type: {:?}", token_type)
             }

src/wasm-lib/kcl/src/token/tokeniser.rs

@@ -25,6 +25,8 @@ pub fn token(i: &mut Located<&str>) -> PResult<Token> {
         '0'..='9' => number,
         ':' => colon,
         '.' => alt((number, double_period, period)),
+        '#' => hash,
+        '!' => bang,
         ' ' | '\t' | '\n' => whitespace,
         _ => alt((operator, keyword,type_, word))
     }
@@ -109,6 +111,16 @@ fn comma(i: &mut Located<&str>) -> PResult<Token> {
     Ok(Token::from_range(range, TokenType::Comma, value.to_string()))
 }
 
+fn hash(i: &mut Located<&str>) -> PResult<Token> {
+    let (value, range) = '#'.with_span().parse_next(i)?;
+    Ok(Token::from_range(range, TokenType::Hash, value.to_string()))
+}
+
+fn bang(i: &mut Located<&str>) -> PResult<Token> {
+    let (value, range) = '!'.with_span().parse_next(i)?;
+    Ok(Token::from_range(range, TokenType::Bang, value.to_string()))
+}
+
 fn question_mark(i: &mut Located<&str>) -> PResult<Token> {
     let (value, range) = '?'.with_span().parse_next(i)?;
     Ok(Token::from_range(range, TokenType::QuestionMark, value.to_string()))