Uses the grammar marijn made :) (#2967)

* Add a Lezer KCL grammar

* fmt

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* make tsc happy

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* turn off semantic tokens in favor of grammar

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* A snapshot a day keeps the bugs away! 📷🐛 (OS: ubuntu)

* fixups

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* A snapshot a day keeps the bugs away! 📷🐛 (OS: ubuntu)

* empty

---------

Signed-off-by: Jess Frazelle <github@jessfraz.com>
Co-authored-by: Marijn Haverbeke <marijn@haverbeke.berlin>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

e2e/playwright/flow-tests.spec.ts

@@ -560,6 +560,50 @@ test.describe('Testing Camera Movement', () => {
 })
 
 test.describe('Editor tests', () => {
+  test('can comment out code with ctrl+/', async ({ page }) => {
+    const u = await getUtils(page)
+    await page.setViewportSize({ width: 1000, height: 500 })
+
+    await u.waitForAuthSkipAppStart()
+    const CtrlKey = process.platform === 'darwin' ? 'Meta' : 'Control'
+
+    // check no error to begin with
+    await expect(page.locator('.cm-lint-marker-error')).not.toBeVisible()
+
+    await u.codeLocator.click()
+    await page.keyboard.type(`const sketch001 = startSketchOn('XY')
+  |> startProfileAt([-10, -10], %)
+  |> line([20, 0], %)
+  |> line([0, 20], %)
+  |> line([-20, 0], %)
+  |> close(%)`)
+
+    await page.keyboard.down(CtrlKey)
+    await page.keyboard.press('/')
+    await page.keyboard.up(CtrlKey)
+
+    await expect(page.locator('.cm-content'))
+      .toHaveText(`const sketch001 = startSketchOn('XY')
+    |> startProfileAt([-10, -10], %)
+    |> line([20, 0], %)
+    |> line([0, 20], %)
+    |> line([-20, 0], %)
+    // |> close(%)`)
+
+    // uncomment the code
+    await page.keyboard.down(CtrlKey)
+    await page.keyboard.press('/')
+    await page.keyboard.up(CtrlKey)
+
+    await expect(page.locator('.cm-content'))
+      .toHaveText(`const sketch001 = startSketchOn('XY')
+    |> startProfileAt([-10, -10], %)
+    |> line([20, 0], %)
+    |> line([0, 20], %)
+    |> line([-20, 0], %)
+    |> close(%)`)
+  })
+
   test('if you click the format button it formats your code', async ({
     page,
   }) => {

package.json

@@ -18,6 +18,8 @@
     "@headlessui/react": "^1.7.19",
     "@headlessui/tailwindcss": "^0.2.0",
     "@kittycad/lib": "^0.0.70",
+    "@lezer/highlight": "^1.2.0",
+    "@lezer/lr": "^1.4.1",
     "@react-hook/resize-observer": "^2.0.1",
     "@replit/codemirror-interact": "^6.3.1",
     "@tauri-apps/api": "^2.0.0-beta.14",
@@ -109,6 +111,7 @@
     "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
     "@babel/preset-env": "^7.24.3",
     "@iarna/toml": "^2.2.5",
+    "@lezer/generator": "^1.7.1",
     "@playwright/test": "^1.45.1",
     "@tauri-apps/cli": "==2.0.0-beta.13",
     "@testing-library/jest-dom": "^5.14.1",

packages/codemirror-lsp-client/src/plugin/lsp.ts

@@ -71,6 +71,9 @@ export interface LanguageServerOptions {
   ) => void
 
   changesDelay?: number
+
+  doSemanticTokens?: boolean
+  doFoldingRanges?: boolean
 }
 
 export class LanguageServerPlugin implements PluginValue {
@@ -87,6 +90,9 @@ export class LanguageServerPlugin implements PluginValue {
     notification: LSP.NotificationMessage
   ) => void
 
+  private doSemanticTokens: boolean = false
+  private doFoldingRanges: boolean = false
+
   private _defferer = deferExecution((code: string) => {
     try {
       // Update the state (not the editor) with the new code.
@@ -109,6 +115,9 @@ export class LanguageServerPlugin implements PluginValue {
     this.client = options.client
     this.documentVersion = 0
 
+    this.doSemanticTokens = options.doSemanticTokens ?? false
+    this.doFoldingRanges = options.doFoldingRanges ?? false
+
     if (options.changesDelay) {
       this.changesDelay = options.changesDelay
     }
@@ -220,6 +229,7 @@ export class LanguageServerPlugin implements PluginValue {
 
   async getFoldingRanges(): Promise<LSP.FoldingRange[] | null> {
     if (
+      !this.doFoldingRanges ||
       !this.client.ready ||
       !this.client.getServerCapabilities().foldingRangeProvider
     )
@@ -445,6 +455,7 @@ export class LanguageServerPlugin implements PluginValue {
 
   async requestSemanticTokens() {
     if (
+      !this.doSemanticTokens ||
       !this.client.ready ||
       !this.client.getServerCapabilities().semanticTokensProvider
     ) {

src/components/LspProvider.tsx

@@ -8,7 +8,7 @@ import {
   LanguageServerPlugin,
 } from '@kittycad/codemirror-lsp-client'
 import { TEST, VITE_KC_API_BASE_URL } from 'env'
-import KclLanguageSupport from 'editor/plugins/lsp/kcl/language'
+import { kcl } from 'editor/plugins/lsp/kcl/language'
 import { copilotPlugin } from 'editor/plugins/lsp/copilot'
 import { useSettingsAuthContext } from 'hooks/useSettingsAuthContext'
 import { Extension } from '@codemirror/state'
@@ -146,7 +146,7 @@ export const LspProvider = ({ children }: { children: React.ReactNode }) => {
     let plugin = null
     if (isKclLspReady && !TEST && kclLspClient) {
       // Set up the lsp plugin.
-      const lsp = new KclLanguageSupport({
+      const lsp = kcl({
         documentUri: `file:///${PROJECT_ENTRYPOINT}`,
         workspaceFolders: getWorkspaceFolders(),
         client: kclLspClient,

src/editor/plugins/lsp/kcl/highlight.ts

@@ -0,0 +1,26 @@
+import { styleTags, tags as t } from '@lezer/highlight'
+
+export const klcHighlight = styleTags({
+  'fn var let const': t.definitionKeyword,
+  return: t.controlKeyword,
+  'true false': t.bool,
+  nil: t.null,
+  'AddOp MultOp ExpOp': t.arithmeticOperator,
+  CompOp: t.logicOperator,
+  'Equals Arrow': t.definitionOperator,
+  PipeOperator: t.controlOperator,
+  String: t.string,
+  Number: t.number,
+  LineComment: t.lineComment,
+  BlockComment: t.blockComment,
+  Shebang: t.meta,
+  PipeSubstitution: t.atom,
+  VariableDefinition: t.definition(t.variableName),
+  VariableName: t.variableName,
+  PropertyName: t.propertyName,
+  TagDeclarator: t.tagName,
+  '( )': t.paren,
+  '{ }': t.brace,
+  '[ ]': t.bracket,
+  ', . : ? ..': t.punctuation,
+})

src/editor/plugins/lsp/kcl/kcl.grammar

@@ -0,0 +1,113 @@
+@precedence {
+  member
+  call
+  exp @left
+  mult @left
+  add @left
+  comp @left
+  pipe @left
+  range
+}
+
+@top Program {
+  Shebang?
+  statement*
+}
+
+statement[@isGroup=Statement] {
+  FunctionDeclaration { kw<"fn"> VariableDefinition Equals ParamList Arrow Body } |
+  VariableDeclaration { (kw<"var"> | kw<"let"> | kw<"const">) VariableDefinition Equals expression } |
+  ReturnStatement { kw<"return"> expression } |
+  ExpressionStatement { expression }
+}
+
+ParamList { "(" commaSep<Parameter { VariableDefinition "?"? (":" type)? }> ")" }
+
+Body { "{" statement* "}" }
+
+expression[@isGroup=Expression] {
+  String |
+  Number |
+  VariableName |
+  TagDeclarator |
+  kw<"true"> | kw<"false"> | kw<"nil"> |
+  PipeSubstitution |
+  BinaryExpression {
+    expression !add AddOp expression |
+    expression !mult MultOp expression |
+    expression !exp ExpOp expression |
+    expression !comp CompOp expression
+  } |
+  UnaryExpression { AddOp expression } |
+  ParenthesizedExpression { "(" expression ")" } |
+  CallExpression { expression !call ArgumentList } |
+  ArrayExpression { "[" commaSep<expression | IntegerRange { expression !range ".." expression }> "]" } |
+  ObjectExpression { "{" commaSep<ObjectProperty> "}" } |
+  MemberExpression { expression !member "." PropertyName } |
+  SubscriptExpression { expression !member "[" expression "]" } |
+  PipeExpression { expression (!pipe PipeOperator expression)+ }
+}
+
+ObjectProperty { PropertyName ":" expression }
+
+ArgumentList { "(" commaSep<expression> ")" }
+
+type[@isGroup=Type] {
+  @specialize[@name=PrimitiveType]<
+    identifier,
+    "string" | "number" | "bool" | "sketch_group" | "sketch_surface" | "extrude_group"
+  > |
+  ArrayType { type !member "[" "]" } |
+  ObjectType { "{" commaSep<ObjectProperty { PropertyName ":" type }> "}" }
+}
+
+VariableDefinition { identifier }
+
+VariableName { identifier }
+
+@skip { whitespace | LineComment | BlockComment }
+
+kw<term> { @specialize[@name={term}]<identifier, term> }
+
+commaSep<term> { (term ("," term)*)? ","? }
+
+@tokens {
+  String[isolate] { "'" ("\\" _ | !['\\])* "'" | '"' ("\\" _ | !["\\])* '"' }
+
+  Number { "." @digit+ | @digit+ ("." @digit*)? }
+  @precedence { Number, "." }
+
+  AddOp { "+" | "-" }
+  MultOp { "/" | "*" | "\\" }
+  ExpOp { "^" }
+  CompOp { $[<>] "="? | "!=" | "==" }
+  Equals { "=" }
+  Arrow { "=>" }
+  PipeOperator { "|>" }
+
+  PipeSubstitution { "%" }
+
+  identifier { (@asciiLetter | "_") (@asciiLetter | @digit | "_")* }
+  PropertyName { identifier }
+  TagDeclarator { "$" identifier }
+
+  whitespace { @whitespace+ }
+
+  LineComment[isolate] { "//" ![\n]* }
+  BlockComment[isolate] { "/*" blockCommentRest }
+  blockCommentRest { @eof | ![*] blockCommentRest | "*" blockCommentStar }
+  blockCommentStar { @eof | "/" | ![/] blockCommentRest | "*" blockCommentStar }
+
+  @precedence { LineComment, BlockComment, MultOp }
+
+  Shebang { "#!" ![\n]* }
+
+  "(" ")"
+  "{" "}"
+  "[" "]"
+  "," "?" ":" "." ".."
+}
+
+@external propSource klcHighlight from "./highlight"
+
+@detectDelim

src/editor/plugins/lsp/kcl/language.ts

@@ -1,9 +1,13 @@
 // Code mirror language implementation for kcl.
 
 import {
-  Language,
+  LRLanguage,
-  defineLanguageFacet,
   LanguageSupport,
+  indentNodeProp,
+  continuedIndent,
+  delimitedIndent,
+  foldNodeProp,
+  foldInside,
 } from '@codemirror/language'
 import {
   LanguageServerClient,
@@ -11,18 +15,8 @@ import {
 } from '@kittycad/codemirror-lsp-client'
 import { kclPlugin } from '.'
 import type * as LSP from 'vscode-languageserver-protocol'
-import KclParser from './parser'
+// @ts-ignore: No types available
-
+import { parser } from './kcl.grammar'
-const data = defineLanguageFacet({
-  // https://codemirror.net/docs/ref/#commands.CommentTokens
-  commentTokens: {
-    line: '//',
-    block: {
-      open: '/*',
-      close: '*/',
-    },
-  },
-})
 
 export interface LanguageOptions {
   workspaceFolders: LSP.WorkspaceFolder[]
@@ -34,26 +28,40 @@ export interface LanguageOptions {
   ) => void
 }
 
-class KclLanguage extends Language {
+export const KclLanguage = LRLanguage.define({
-  constructor(options: LanguageOptions) {
+  name: 'klc',
-    const plugin = kclPlugin({
+  parser: parser.configure({
+    props: [
+      indentNodeProp.add({
+        Body: delimitedIndent({ closing: '}' }),
+        BlockComment: () => null,
+        'Statement Property': continuedIndent({ except: /^{/ }),
+      }),
+      foldNodeProp.add({
+        'Body ArrayExpression ObjectExpression': foldInside,
+        BlockComment(tree) {
+          return { from: tree.from + 2, to: tree.to - 2 }
+        },
+        PipeExpression(tree) {
+          return { from: tree.firstChild!.to, to: tree.to }
+        },
+      }),
+    ],
+  }),
+  languageData: {
+    commentTokens: { line: '//', block: { open: '/*', close: '*/' } },
+  },
+})
+
+export function kcl(options: LanguageOptions) {
+  return new LanguageSupport(
+    KclLanguage,
+    kclPlugin({
       documentUri: options.documentUri,
       workspaceFolders: options.workspaceFolders,
       allowHTMLContent: true,
       client: options.client,
       processLspNotification: options.processLspNotification,
     })
-
+  )
-    const parser = new KclParser()
-
-    super(data, parser, [plugin], 'kcl')
-  }
-}
-
-export default class KclLanguageSupport extends LanguageSupport {
-  constructor(options: LanguageOptions) {
-    const lang = new KclLanguage(options)
-
-    super(lang)
-  }
 }

src/editor/plugins/lsp/kcl/parser.ts

@@ -1,47 +0,0 @@
-// Extends the codemirror Parser for kcl.
-// This is really just a no-op parser since we use semantic tokens from the LSP
-// server.
-
-import {
-  Parser,
-  Input,
-  TreeFragment,
-  PartialParse,
-  Tree,
-  NodeType,
-} from '@lezer/common'
-import { DocInput } from '@codemirror/language'
-
-export default class KclParser extends Parser {
-  createParse(
-    input: Input,
-    fragments: readonly TreeFragment[],
-    ranges: readonly { from: number; to: number }[]
-  ): PartialParse {
-    let parse: PartialParse = new Context(input)
-    return parse
-  }
-}
-
-class Context implements PartialParse {
-  private input: DocInput
-
-  stoppedAt: number = 0
-
-  constructor(input: Input) {
-    this.input = input as DocInput
-  }
-
-  get parsedPos(): number {
-    return 0
-  }
-
-  advance(): Tree | null {
-    this.stoppedAt = this.input.doc.length
-    return new Tree(NodeType.none, [], [], this.input.doc.length)
-  }
-
-  stopAt(pos: number) {
-    this.stoppedAt = pos
-  }
-}

tsconfig.json

@@ -12,7 +12,8 @@
       "@types/wicg-file-system-access",
       "node",
       "@wdio/globals/types",
-      "mocha"
+      "mocha",
+      "@lezer/generator"
     ],
     "target": "esnext",
     "lib": ["dom", "dom.iterable", "esnext"],
@@ -32,6 +33,6 @@
     "jsx": "react-jsx"
   },
   "include": ["src", "e2e", "packages", "./*.ts"],
-  "exclude": ["node_modules"],
+  "exclude": ["node_modules", "./*.grammar"],
   "references": [{ "path": "./tsconfig.node.json" }]
 }

vite.config.ts

@@ -3,6 +3,8 @@ import viteTsconfigPaths from 'vite-tsconfig-paths'
 import eslint from 'vite-plugin-eslint'
 import { defineConfig, configDefaults } from 'vitest/config'
 import version from 'vite-plugin-package-version'
+// @ts-ignore: No types available
+import { lezer } from '@lezer/generator/rollup'
 
 const config = defineConfig({
   server: {
@@ -58,7 +60,7 @@ const config = defineConfig({
       '@kittycad/codemirror-lsp-client': '/packages/codemirror-lsp-client/src',
     },
   },
-  plugins: [react(), viteTsconfigPaths(), eslint(), version()],
+  plugins: [react(), viteTsconfigPaths(), eslint(), version(), lezer()],
   worker: {
     plugins: () => [viteTsconfigPaths()],
   },

yarn.lock

@@ -1643,14 +1643,22 @@
   resolved "https://registry.yarnpkg.com/@lezer/common/-/common-1.2.1.tgz#198b278b7869668e1bebbe687586e12a42731049"
   integrity sha512-yemX0ZD2xS/73llMZIK6KplkjIjf2EvAHcinDi/TfJ9hS25G0388+ClHt6/3but0oOxinTcQHJLDXh6w1crzFQ==
 
-"@lezer/highlight@^1.0.0":
+"@lezer/generator@^1.7.1":
+  version "1.7.1"
+  resolved "https://registry.yarnpkg.com/@lezer/generator/-/generator-1.7.1.tgz#90c1a9de2fb4d5a714216fa659058c7859accaab"
+  integrity sha512-MgPJN9Si+ccxzXl3OAmCeZuUKw4XiPl4y664FX/hnnyG9CTqUPq65N3/VGPA2jD23D7QgMTtNqflta+cPN+5mQ==
+  dependencies:
+    "@lezer/common" "^1.1.0"
+    "@lezer/lr" "^1.3.0"
+
+"@lezer/highlight@^1.0.0", "@lezer/highlight@^1.2.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@lezer/highlight/-/highlight-1.2.0.tgz#e5898c3644208b4b589084089dceeea2966f7780"
   integrity sha512-WrS5Mw51sGrpqjlh3d4/fOwpEV2Hd3YOkp9DBt4k8XZQcoTHZFB7sx030A6OcahF4J1nDQAa3jXlTVVYH50IFA==
   dependencies:
     "@lezer/common" "^1.0.0"
 
-"@lezer/lr@^1.0.0":
+"@lezer/lr@^1.0.0", "@lezer/lr@^1.3.0", "@lezer/lr@^1.4.1":
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/@lezer/lr/-/lr-1.4.1.tgz#fe25f051880a754e820b28148d90aa2a96b8bdd2"
   integrity sha512-CHsKq8DMKBf9b3yXPDIU4DbH+ZJd/sJdYOW2llbW/HudP5u0VS6Bfq1hLYfgU7uAYGFIyGGQIsSOXGPEErZiJw==