pull lsp client out into a fake module (#2846)

* initial commit

Signed-off-by: Jess Frazelle <github@jessfraz.com>

tsc passing

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

working

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fixups

Signed-off-by: Jess Frazelle <github@jessfraz.com>

updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fmt

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* cleanups

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* udpates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* cleanup

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* cleanup

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

---------

Signed-off-by: Jess Frazelle <github@jessfraz.com>

packages/codemirror-lsp-client/src/client/codec/bytes.ts

@@ -0,0 +1,27 @@
+import { encoder, decoder } from '../codec'
+
+export default class Bytes {
+  static encode(input: string): Uint8Array {
+    return encoder.encode(input)
+  }
+
+  static decode(input: Uint8Array): string {
+    return decoder.decode(input)
+  }
+
+  static append<
+    T extends { length: number; set(arr: T, offset: number): void }
+  >(constructor: { new (length: number): T }, ...arrays: T[]) {
+    let totalLength = 0
+    for (const arr of arrays) {
+      totalLength += arr.length
+    }
+    const result = new constructor(totalLength)
+    let offset = 0
+    for (const arr of arrays) {
+      result.set(arr, offset)
+      offset += arr.length
+    }
+    return result
+  }
+}

packages/codemirror-lsp-client/src/client/codec/demuxer.ts

@@ -0,0 +1,109 @@
+import * as vsrpc from 'vscode-jsonrpc'
+
+import { Codec } from '.'
+import Bytes from './bytes'
+import Queue from './queue'
+import Tracer from './tracer'
+import PromiseMap from './map'
+
+export default class StreamDemuxer extends Queue<Uint8Array> {
+  readonly responses: PromiseMap<number | string, vsrpc.ResponseMessage> =
+    new PromiseMap()
+  readonly notifications: Queue<vsrpc.NotificationMessage> =
+    new Queue<vsrpc.NotificationMessage>()
+  readonly requests: Queue<vsrpc.RequestMessage> =
+    new Queue<vsrpc.RequestMessage>()
+
+  readonly #start: Promise<void>
+  private trace: boolean = false
+
+  constructor(trace?: boolean) {
+    super()
+    this.trace = trace || false
+
+    this.#start = this.start()
+  }
+
+  private async start(): Promise<void> {
+    let contentLength: null | number = null
+    let buffer = new Uint8Array()
+
+    for await (const bytes of this) {
+      buffer = Bytes.append(Uint8Array, buffer, bytes)
+      while (buffer.length > 0) {
+        // check if the content length is known
+        if (null == contentLength) {
+          // if not, try to match the prefixed headers
+          const match = Bytes.decode(buffer).match(
+            /^Content-Length:\s*(\d+)\s*/
+          )
+          if (null == match) continue
+
+          // try to parse the content-length from the headers
+          const length = parseInt(match[1])
+
+          if (isNaN(length))
+            return Promise.reject(new Error('invalid content length'))
+
+          // slice the headers since we now have the content length
+          buffer = buffer.slice(match[0].length)
+
+          // set the content length
+          contentLength = length
+        }
+
+        // if the buffer doesn't contain a full message; await another iteration
+        if (buffer.length < contentLength) continue
+
+        // Get just the slice of the buffer that is our content length.
+        const slice = buffer.slice(0, contentLength)
+
+        // decode buffer to a string
+        const delimited = Bytes.decode(slice)
+
+        // reset the buffer
+        buffer = buffer.slice(contentLength)
+        // reset the contentLength
+        contentLength = null
+
+        const message = JSON.parse(delimited) as vsrpc.Message
+
+        if (this.trace) {
+          Tracer.server(message)
+        }
+
+        // demux the message stream
+        if (vsrpc.Message.isResponse(message) && null != message.id) {
+          this.responses.set(message.id, message)
+          continue
+        }
+        if (vsrpc.Message.isNotification(message)) {
+          this.notifications.enqueue(message)
+          continue
+        }
+        if (vsrpc.Message.isRequest(message)) {
+          this.requests.enqueue(message)
+          continue
+        }
+      }
+    }
+  }
+
+  add(bytes: Uint8Array): void {
+    const message = Codec.decode(bytes) as vsrpc.Message
+    if (this.trace) {
+      Tracer.server(message)
+    }
+
+    // demux the message stream
+    if (vsrpc.Message.isResponse(message) && null != message.id) {
+      this.responses.set(message.id, message)
+    }
+    if (vsrpc.Message.isNotification(message)) {
+      this.notifications.enqueue(message)
+    }
+    if (vsrpc.Message.isRequest(message)) {
+      this.requests.enqueue(message)
+    }
+  }
+}

packages/codemirror-lsp-client/src/client/codec/headers.ts

@@ -0,0 +1,9 @@
+export default class Headers {
+  static add(message: string): string {
+    return `Content-Length: ${message.length}\r\n\r\n${message}`
+  }
+
+  static remove(delimited: string): string {
+    return delimited.replace(/^Content-Length:\s*\d+\s*/, '')
+  }
+}

packages/codemirror-lsp-client/src/client/codec/index.ts

@@ -0,0 +1,91 @@
+import * as jsrpc from 'json-rpc-2.0'
+import * as vsrpc from 'vscode-jsonrpc'
+
+import Bytes from './bytes'
+import StreamDemuxer from './demuxer'
+import Headers from './headers'
+import Queue from './queue'
+import Tracer from './tracer'
+
+export enum LspWorkerEventType {
+  Init = 'init',
+  Call = 'call',
+}
+
+export const encoder = new TextEncoder()
+export const decoder = new TextDecoder()
+
+export class Codec {
+  static encode(
+    json: jsrpc.JSONRPCRequest | jsrpc.JSONRPCResponse
+  ): Uint8Array {
+    const message = JSON.stringify(json)
+    const delimited = Headers.add(message)
+    return Bytes.encode(delimited)
+  }
+
+  static decode<T>(data: Uint8Array): T {
+    const delimited = Bytes.decode(data)
+    const message = Headers.remove(delimited)
+    return JSON.parse(message) as T
+  }
+}
+
+// FIXME: tracing efficiency
+export class IntoServer
+  extends Queue<Uint8Array>
+  implements AsyncGenerator<Uint8Array, never, void>
+{
+  private worker: Worker | null = null
+  private type_: String | null = null
+
+  private trace: boolean = false
+
+  constructor(type_?: String, worker?: Worker, trace?: boolean) {
+    super()
+    if (worker && type_) {
+      this.worker = worker
+      this.type_ = type_
+    }
+
+    this.trace = trace || false
+  }
+  enqueue(item: Uint8Array): void {
+    if (this.trace) {
+      Tracer.client(Headers.remove(decoder.decode(item)))
+    }
+
+    if (this.worker) {
+      this.worker.postMessage({
+        worker: this.type_,
+        eventType: LspWorkerEventType.Call,
+        eventData: item,
+      })
+    } else {
+      super.enqueue(item)
+    }
+  }
+}
+
+export interface FromServer extends WritableStream<Uint8Array> {
+  readonly responses: {
+    get(key: number | string): null | Promise<vsrpc.ResponseMessage>
+  }
+  readonly notifications: AsyncGenerator<vsrpc.NotificationMessage, never, void>
+  readonly requests: AsyncGenerator<vsrpc.RequestMessage, never, void>
+
+  add(item: Uint8Array): void
+}
+
+// eslint-disable-next-line @typescript-eslint/no-namespace
+export namespace FromServer {
+  export function create(): FromServer | Error {
+    // Calls private method .start() which can throw.
+    // This is an odd one of the bunch but try/catch seems most suitable here.
+    try {
+      return new StreamDemuxer(false)
+    } catch (e: any) {
+      return e
+    }
+  }
+}

packages/codemirror-lsp-client/src/client/codec/map.ts

@@ -0,0 +1,72 @@
+export default class PromiseMap<K, V extends { toString(): string }> {
+  #map: Map<K, PromiseMap.Entry<V>> = new Map()
+
+  get(key: K & { toString(): string }): null | Promise<V> {
+    let initialized: PromiseMap.Entry<V>
+    // if the entry doesn't exist, set it
+    if (!this.#map.has(key)) {
+      initialized = this.#set(key)
+    } else {
+      // otherwise return the entry
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      initialized = this.#map.get(key)!
+    }
+    // if the entry is a pending promise, return it
+    if (initialized.status === 'pending') {
+      return initialized.promise
+    } else {
+      // otherwise return null
+      return null
+    }
+  }
+
+  #set(key: K, value?: V): PromiseMap.Entry<V> {
+    if (this.#map.has(key)) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      return this.#map.get(key)!
+    }
+    // placeholder resolver for entry
+    let resolve = (item: V) => {
+      void item
+    }
+    // promise for entry (which assigns the resolver
+    const promise = new Promise<V>((resolver) => {
+      resolve = resolver
+    })
+    // the initialized entry
+    const initialized: PromiseMap.Entry<V> = {
+      status: 'pending',
+      resolve,
+      promise,
+    }
+    if (null != value) {
+      initialized.resolve(value)
+    }
+    // set the entry
+    this.#map.set(key, initialized)
+    return initialized
+  }
+
+  set(key: K & { toString(): string }, value: V): this {
+    const initialized = this.#set(key, value)
+    // if the promise is pending ...
+    if (initialized.status === 'pending') {
+      // ... set the entry status to resolved to free the promise
+      this.#map.set(key, { status: 'resolved' })
+      // ... and resolve the promise with the given value
+      initialized.resolve(value)
+    }
+    return this
+  }
+
+  get size(): number {
+    return this.#map.size
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/no-namespace
+export namespace PromiseMap {
+  export type Entry<V> =
+    | { status: 'pending'; resolve: (item: V) => void; promise: Promise<V> }
+    | { status: 'resolved' }
+}

packages/codemirror-lsp-client/src/client/codec/queue.ts

@@ -0,0 +1,113 @@
+export default class Queue<T>
+  implements WritableStream<T>, AsyncGenerator<T, never, void>
+{
+  readonly #promises: Promise<T>[] = []
+  readonly #resolvers: ((item: T) => void)[] = []
+  readonly #observers: ((item: T) => void)[] = []
+
+  #closed = false
+  #locked = false
+  readonly #stream: WritableStream<T>
+
+  static #__add<X>(
+    promises: Promise<X>[],
+    resolvers: ((item: X) => void)[]
+  ): void {
+    promises.push(
+      new Promise((resolve) => {
+        resolvers.push(resolve)
+      })
+    )
+  }
+
+  static #__enqueue<X>(
+    closed: boolean,
+    promises: Promise<X>[],
+    resolvers: ((item: X) => void)[],
+    item: X
+  ): void {
+    if (!closed) {
+      if (!resolvers.length) Queue.#__add(promises, resolvers)
+      const resolve = resolvers.shift()! // eslint-disable-line @typescript-eslint/no-non-null-assertion
+      resolve(item)
+    }
+  }
+
+  constructor() {
+    const closed = this.#closed
+    const promises = this.#promises
+    const resolvers = this.#resolvers
+    this.#stream = new WritableStream({
+      write(item: T): void {
+        Queue.#__enqueue(closed, promises, resolvers, item)
+      },
+    })
+  }
+
+  #add(): void {
+    return Queue.#__add(this.#promises, this.#resolvers)
+  }
+
+  enqueue(item: T): void {
+    return Queue.#__enqueue(this.#closed, this.#promises, this.#resolvers, item)
+  }
+
+  dequeue(): Promise<T> {
+    if (!this.#promises.length) this.#add()
+    const item = this.#promises.shift()! // eslint-disable-line @typescript-eslint/no-non-null-assertion
+    return item
+  }
+
+  isEmpty(): boolean {
+    return !this.#promises.length
+  }
+
+  isBlocked(): boolean {
+    return !!this.#resolvers.length
+  }
+
+  get length(): number {
+    return this.#promises.length - this.#resolvers.length
+  }
+
+  async next(): Promise<IteratorResult<T, never>> {
+    const done = false
+    const value = await this.dequeue()
+    for (const observer of this.#observers) {
+      observer(value)
+    }
+    return { done, value }
+  }
+
+  return(): Promise<IteratorResult<T, never>> {
+    return new Promise(() => {
+      // empty
+    })
+  }
+
+  throw(err: Error): Promise<IteratorResult<T, never>> {
+    return new Promise((_resolve, reject) => {
+      reject(err)
+    })
+  }
+
+  [Symbol.asyncIterator](): AsyncGenerator<T, never, void> {
+    return this
+  }
+
+  get locked(): boolean {
+    return this.#stream.locked
+  }
+
+  abort(reason?: Error): Promise<void> {
+    return this.#stream.abort(reason)
+  }
+
+  close(): Promise<void> {
+    return this.#stream.close()
+  }
+
+  getWriter(): WritableStreamDefaultWriter<T> {
+    return this.#stream.getWriter()
+  }
+}

packages/codemirror-lsp-client/src/client/codec/tracer.ts

@@ -0,0 +1,13 @@
+import { Message } from 'vscode-languageserver-protocol'
+
+export default class Tracer {
+  static client(message: string): void {
+    console.log('lsp client message', message)
+  }
+
+  static server(input: string | Message): void {
+    const message: string =
+      typeof input === 'string' ? input : JSON.stringify(input)
+    console.log('lsp server message', message)
+  }
+}

packages/codemirror-lsp-client/src/client/index.ts

@@ -0,0 +1,200 @@
+import type * as LSP from 'vscode-languageserver-protocol'
+
+import { FromServer, IntoServer } from './codec'
+import Client from './jsonrpc'
+import { LanguageServerPlugin } from '../plugin/lsp'
+
+// https://microsoft.github.io/language-server-protocol/specifications/specification-current/
+
+// Client to server then server to client
+interface LSPRequestMap {
+  initialize: [LSP.InitializeParams, LSP.InitializeResult]
+  'textDocument/hover': [LSP.HoverParams, LSP.Hover]
+  'textDocument/completion': [
+    LSP.CompletionParams,
+    LSP.CompletionItem[] | LSP.CompletionList | null
+  ]
+  'textDocument/semanticTokens/full': [
+    LSP.SemanticTokensParams,
+    LSP.SemanticTokens
+  ]
+  'textDocument/formatting': [
+    LSP.DocumentFormattingParams,
+    LSP.TextEdit[] | null
+  ]
+  'textDocument/foldingRange': [LSP.FoldingRangeParams, LSP.FoldingRange[]]
+}
+
+// Client to server
+interface LSPNotifyMap {
+  initialized: LSP.InitializedParams
+  'textDocument/didChange': LSP.DidChangeTextDocumentParams
+  'textDocument/didOpen': LSP.DidOpenTextDocumentParams
+  'textDocument/didClose': LSP.DidCloseTextDocumentParams
+  'workspace/didChangeWorkspaceFolders': LSP.DidChangeWorkspaceFoldersParams
+  'workspace/didCreateFiles': LSP.CreateFilesParams
+  'workspace/didRenameFiles': LSP.RenameFilesParams
+  'workspace/didDeleteFiles': LSP.DeleteFilesParams
+}
+
+export interface LanguageServerClientOptions {
+  name: string
+  fromServer: FromServer
+  intoServer: IntoServer
+  initializedCallback: () => void
+}
+
+export class LanguageServerClient {
+  private client: Client
+  readonly name: string
+
+  public ready: boolean
+
+  readonly plugins: LanguageServerPlugin[]
+
+  public initializePromise: Promise<void>
+
+  constructor(options: LanguageServerClientOptions) {
+    this.name = options.name
+    this.plugins = []
+
+    this.client = new Client(
+      options.fromServer,
+      options.intoServer,
+      options.initializedCallback
+    )
+
+    this.ready = false
+
+    this.initializePromise = this.initialize()
+  }
+
+  async initialize() {
+    // Start the client in the background.
+    this.client.setNotifyFn(this.processNotifications.bind(this))
+    this.client.start()
+
+    this.ready = true
+  }
+
+  getName(): string {
+    return this.name
+  }
+
+  getServerCapabilities(): LSP.ServerCapabilities<any> {
+    return this.client.getServerCapabilities()
+  }
+
+  close() {}
+
+  textDocumentDidOpen(params: LSP.DidOpenTextDocumentParams) {
+    this.notify('textDocument/didOpen', params)
+  }
+
+  textDocumentDidChange(params: LSP.DidChangeTextDocumentParams) {
+    this.notify('textDocument/didChange', params)
+  }
+
+  textDocumentDidClose(params: LSP.DidCloseTextDocumentParams) {
+    this.notify('textDocument/didClose', params)
+  }
+
+  workspaceDidChangeWorkspaceFolders(
+    added: LSP.WorkspaceFolder[],
+    removed: LSP.WorkspaceFolder[]
+  ) {
+    this.notify('workspace/didChangeWorkspaceFolders', {
+      event: { added, removed },
+    })
+  }
+
+  workspaceDidCreateFiles(params: LSP.CreateFilesParams) {
+    this.notify('workspace/didCreateFiles', params)
+  }
+
+  workspaceDidRenameFiles(params: LSP.RenameFilesParams) {
+    this.notify('workspace/didRenameFiles', params)
+  }
+
+  workspaceDidDeleteFiles(params: LSP.DeleteFilesParams) {
+    this.notify('workspace/didDeleteFiles', params)
+  }
+
+  async textDocumentSemanticTokensFull(params: LSP.SemanticTokensParams) {
+    const serverCapabilities = this.getServerCapabilities()
+    if (!serverCapabilities.semanticTokensProvider) {
+      return
+    }
+
+    return this.request('textDocument/semanticTokens/full', params)
+  }
+
+  async textDocumentHover(params: LSP.HoverParams) {
+    const serverCapabilities = this.getServerCapabilities()
+    if (!serverCapabilities.hoverProvider) {
+      return
+    }
+    return await this.request('textDocument/hover', params)
+  }
+
+  async textDocumentFormatting(params: LSP.DocumentFormattingParams) {
+    const serverCapabilities = this.getServerCapabilities()
+    if (!serverCapabilities.documentFormattingProvider) {
+      return
+    }
+    return await this.request('textDocument/formatting', params)
+  }
+
+  async textDocumentFoldingRange(params: LSP.FoldingRangeParams) {
+    const serverCapabilities = this.getServerCapabilities()
+    if (!serverCapabilities.foldingRangeProvider) {
+      return
+    }
+    return await this.request('textDocument/foldingRange', params)
+  }
+
+  async textDocumentCompletion(params: LSP.CompletionParams) {
+    const serverCapabilities = this.getServerCapabilities()
+    if (!serverCapabilities.completionProvider) {
+      return
+    }
+    const response = await this.request('textDocument/completion', params)
+    return response
+  }
+
+  attachPlugin(plugin: LanguageServerPlugin) {
+    this.plugins.push(plugin)
+  }
+
+  detachPlugin(plugin: LanguageServerPlugin) {
+    const i = this.plugins.indexOf(plugin)
+    if (i === -1) return
+    this.plugins.splice(i, 1)
+  }
+
+  private request<K extends keyof LSPRequestMap>(
+    method: K,
+    params: LSPRequestMap[K][0]
+  ): Promise<LSPRequestMap[K][1]> {
+    return this.client.request(method, params) as Promise<LSPRequestMap[K][1]>
+  }
+
+  requestCustom<P, R>(method: string, params: P): Promise<R> {
+    return this.client.request(method, params) as Promise<R>
+  }
+
+  private notify<K extends keyof LSPNotifyMap>(
+    method: K,
+    params: LSPNotifyMap[K]
+  ): void {
+    return this.client.notify(method, params)
+  }
+
+  notifyCustom<P>(method: string, params: P): void {
+    return this.client.notify(method, params)
+  }
+
+  private processNotifications(notification: LSP.NotificationMessage) {
+    for (const plugin of this.plugins) plugin.processNotification(notification)
+  }
+}

packages/codemirror-lsp-client/src/client/jsonrpc.ts

@@ -0,0 +1,208 @@
+import * as jsrpc from 'json-rpc-2.0'
+import * as LSP from 'vscode-languageserver-protocol'
+
+import {
+  registerServerCapability,
+  unregisterServerCapability,
+} from './server-capability-registration'
+import { Codec, FromServer, IntoServer } from './codec'
+
+const client_capabilities: LSP.ClientCapabilities = {
+  textDocument: {
+    hover: {
+      dynamicRegistration: true,
+      contentFormat: ['plaintext', 'markdown'],
+    },
+    moniker: {},
+    synchronization: {
+      dynamicRegistration: true,
+      willSave: false,
+      didSave: false,
+      willSaveWaitUntil: false,
+    },
+    completion: {
+      dynamicRegistration: true,
+      completionItem: {
+        snippetSupport: false,
+        commitCharactersSupport: true,
+        documentationFormat: ['plaintext', 'markdown'],
+        deprecatedSupport: false,
+        preselectSupport: false,
+      },
+      contextSupport: false,
+    },
+    signatureHelp: {
+      dynamicRegistration: true,
+      signatureInformation: {
+        documentationFormat: ['plaintext', 'markdown'],
+      },
+    },
+    declaration: {
+      dynamicRegistration: true,
+      linkSupport: true,
+    },
+    definition: {
+      dynamicRegistration: true,
+      linkSupport: true,
+    },
+    typeDefinition: {
+      dynamicRegistration: true,
+      linkSupport: true,
+    },
+    implementation: {
+      dynamicRegistration: true,
+      linkSupport: true,
+    },
+  },
+  workspace: {
+    didChangeConfiguration: {
+      dynamicRegistration: true,
+    },
+  },
+}
+
+export default class Client extends jsrpc.JSONRPCServerAndClient {
+  afterInitializedHooks: (() => Promise<void>)[] = []
+  #fromServer: FromServer
+  private serverCapabilities: LSP.ServerCapabilities<any> = {}
+  private notifyFn: ((message: LSP.NotificationMessage) => void) | null = null
+  private initializedCallback: () => void
+
+  constructor(
+    fromServer: FromServer,
+    intoServer: IntoServer,
+    initializedCallback: () => void
+  ) {
+    super(
+      new jsrpc.JSONRPCServer(),
+      new jsrpc.JSONRPCClient(async (json: jsrpc.JSONRPCRequest) => {
+        const encoded = Codec.encode(json)
+        intoServer.enqueue(encoded)
+        if (null != json.id) {
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          const response = await fromServer.responses.get(json.id)
+          this.client.receive(response as jsrpc.JSONRPCResponse)
+        }
+      })
+    )
+    this.#fromServer = fromServer
+    this.initializedCallback = initializedCallback
+  }
+
+  async start(): Promise<void> {
+    // process "window/logMessage": client <- server
+    this.addMethod(LSP.LogMessageNotification.type.method, (params) => {
+      const { type, message } = params as {
+        type: LSP.MessageType
+        message: string
+      }
+      let messageString = ''
+      switch (type) {
+        case LSP.MessageType.Error: {
+          messageString += '[error] '
+          break
+        }
+        case LSP.MessageType.Warning: {
+          messageString += ' [warn] '
+          break
+        }
+        case LSP.MessageType.Info: {
+          messageString += ' [info] '
+          break
+        }
+        case LSP.MessageType.Log: {
+          messageString += '  [log] '
+          break
+        }
+      }
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      messageString += message
+      return
+    })
+
+    // process "client/registerCapability": client <- server
+    this.addMethod(LSP.RegistrationRequest.type.method, (params) => {
+      // Register a server capability.
+      params.registrations.forEach(
+        (capabilityRegistration: LSP.Registration) => {
+          const caps = registerServerCapability(
+            this.serverCapabilities,
+            capabilityRegistration
+          )
+          if (caps instanceof Error) {
+            return (this.serverCapabilities = {})
+          }
+          this.serverCapabilities = caps
+        }
+      )
+    })
+
+    // process "client/unregisterCapability": client <- server
+    this.addMethod(LSP.UnregistrationRequest.type.method, (params) => {
+      // Unregister a server capability.
+      params.unregisterations.forEach(
+        (capabilityUnregistration: LSP.Unregistration) => {
+          const caps = unregisterServerCapability(
+            this.serverCapabilities,
+            capabilityUnregistration
+          )
+          if (caps instanceof Error) {
+            return (this.serverCapabilities = {})
+          }
+          this.serverCapabilities = caps
+        }
+      )
+    })
+
+    // request "initialize": client <-> server
+    const { capabilities } = await this.request(
+      LSP.InitializeRequest.type.method,
+      {
+        processId: null,
+        clientInfo: {
+          name: 'codemirror-lsp-client',
+        },
+        capabilities: client_capabilities,
+        rootUri: null,
+      } as LSP.InitializeParams
+    )
+
+    this.serverCapabilities = capabilities
+
+    // notify "initialized": client --> server
+    this.notify(LSP.InitializedNotification.type.method, {})
+
+    this.initializedCallback()
+
+    await Promise.all(
+      this.afterInitializedHooks.map((f: () => Promise<void>) => f())
+    )
+    await Promise.all([this.processNotifications(), this.processRequests()])
+  }
+
+  getServerCapabilities(): LSP.ServerCapabilities<any> {
+    return this.serverCapabilities
+  }
+
+  setNotifyFn(fn: (message: LSP.NotificationMessage) => void): void {
+    this.notifyFn = fn
+  }
+
+  async processNotifications(): Promise<void> {
+    for await (const notification of this.#fromServer.notifications) {
+      if (this.notifyFn) {
+        this.notifyFn(notification)
+      }
+    }
+  }
+
+  async processRequests(): Promise<void> {
+    for await (const request of this.#fromServer.requests) {
+      await this.receiveAndSend(request)
+    }
+  }
+
+  pushAfterInitializeHook(...hooks: (() => Promise<void>)[]): void {
+    this.afterInitializedHooks.push(...hooks)
+  }
+}

packages/codemirror-lsp-client/src/client/server-capability-registration.ts

@@ -0,0 +1,82 @@
+import {
+  Registration,
+  ServerCapabilities,
+  Unregistration,
+} from 'vscode-languageserver-protocol'
+
+interface IFlexibleServerCapabilities extends ServerCapabilities {
+  [key: string]: any
+}
+
+interface IMethodServerCapabilityProviderDictionary {
+  [key: string]: string
+}
+
+const ServerCapabilitiesProviders: IMethodServerCapabilityProviderDictionary = {
+  'textDocument/hover': 'hoverProvider',
+  'textDocument/completion': 'completionProvider',
+  'textDocument/signatureHelp': 'signatureHelpProvider',
+  'textDocument/definition': 'definitionProvider',
+  'textDocument/typeDefinition': 'typeDefinitionProvider',
+  'textDocument/implementation': 'implementationProvider',
+  'textDocument/references': 'referencesProvider',
+  'textDocument/documentHighlight': 'documentHighlightProvider',
+  'textDocument/documentSymbol': 'documentSymbolProvider',
+  'textDocument/workspaceSymbol': 'workspaceSymbolProvider',
+  'textDocument/codeAction': 'codeActionProvider',
+  'textDocument/codeLens': 'codeLensProvider',
+  'textDocument/documentFormatting': 'documentFormattingProvider',
+  'textDocument/documentRangeFormatting': 'documentRangeFormattingProvider',
+  'textDocument/documentOnTypeFormatting': 'documentOnTypeFormattingProvider',
+  'textDocument/rename': 'renameProvider',
+  'textDocument/documentLink': 'documentLinkProvider',
+  'textDocument/color': 'colorProvider',
+  'textDocument/foldingRange': 'foldingRangeProvider',
+  'textDocument/declaration': 'declarationProvider',
+  'textDocument/executeCommand': 'executeCommandProvider',
+  'textDocument/semanticTokens/full': 'semanticTokensProvider',
+  'textDocument/publishDiagnostics': 'diagnosticsProvider',
+}
+
+function registerServerCapability(
+  serverCapabilities: ServerCapabilities,
+  registration: Registration
+): ServerCapabilities | Error {
+  const serverCapabilitiesCopy = JSON.parse(
+    JSON.stringify(serverCapabilities)
+  ) as IFlexibleServerCapabilities
+  const { method, registerOptions } = registration
+  const providerName = ServerCapabilitiesProviders[method]
+
+  if (providerName) {
+    if (!registerOptions) {
+      serverCapabilitiesCopy[providerName] = true
+    } else {
+      serverCapabilitiesCopy[providerName] = Object.assign(
+        {},
+        JSON.parse(JSON.stringify(registerOptions))
+      )
+    }
+  } else {
+    return new Error('Could not register server capability.')
+  }
+
+  return serverCapabilitiesCopy
+}
+
+function unregisterServerCapability(
+  serverCapabilities: ServerCapabilities,
+  unregistration: Unregistration
+): ServerCapabilities {
+  const serverCapabilitiesCopy = JSON.parse(
+    JSON.stringify(serverCapabilities)
+  ) as IFlexibleServerCapabilities
+  const { method } = unregistration
+  const providerName = ServerCapabilitiesProviders[method]
+
+  delete serverCapabilitiesCopy[providerName]
+
+  return serverCapabilitiesCopy
+}
+
+export { registerServerCapability, unregisterServerCapability }

packages/codemirror-lsp-client/src/index.ts

@@ -0,0 +1,113 @@
+import { autocompletion } from '@codemirror/autocomplete'
+import { foldService, syntaxTree } from '@codemirror/language'
+import { Extension, EditorState } from '@codemirror/state'
+import { ViewPlugin } from '@codemirror/view'
+
+import { CompletionTriggerKind } from 'vscode-languageserver-protocol'
+
+import {
+  docPathFacet,
+  LanguageServerPlugin,
+  LanguageServerPluginSpec,
+  languageId,
+  workspaceFolders,
+  LanguageServerOptions,
+} from './plugin/lsp'
+import { offsetToPos } from './plugin/util'
+
+export type { LanguageServerClientOptions } from './client'
+export { LanguageServerClient } from './client'
+export {
+  Codec,
+  FromServer,
+  IntoServer,
+  LspWorkerEventType,
+} from './client/codec'
+export type { LanguageServerOptions } from './plugin/lsp'
+export type { TransactionInfo, RelevantUpdate } from './plugin/annotations'
+export { updateInfo, TransactionAnnotation } from './plugin/annotations'
+export {
+  LanguageServerPlugin,
+  LanguageServerPluginSpec,
+  docPathFacet,
+  languageId,
+  workspaceFolders,
+} from './plugin/lsp'
+export { posToOffset, offsetToPos } from './plugin/util'
+
+export function lspPlugin(options: LanguageServerOptions): Extension {
+  let plugin: LanguageServerPlugin | null = null
+  const viewPlugin = ViewPlugin.define(
+    (view) => (plugin = new LanguageServerPlugin(options, view)),
+    new LanguageServerPluginSpec()
+  )
+
+  let ext = [
+    docPathFacet.of(options.documentUri),
+    languageId.of('kcl'),
+    workspaceFolders.of(options.workspaceFolders),
+    viewPlugin,
+    foldService.of((state: EditorState, lineStart: number, lineEnd: number) => {
+      if (plugin == null) return null
+      // Get the folding ranges from the language server.
+      // Since this is async we directly need to update the folding ranges after.
+      return plugin?.foldingRange(lineStart, lineEnd)
+    }),
+  ]
+
+  if (options.client.getServerCapabilities().completionProvider) {
+    ext.push(
+      autocompletion({
+        defaultKeymap: false,
+        override: [
+          async (context) => {
+            if (plugin === null) {
+              return null
+            }
+
+            const { state, pos, explicit } = context
+
+            let nodeBefore = syntaxTree(state).resolveInner(pos, -1)
+            if (
+              nodeBefore.name === 'BlockComment' ||
+              nodeBefore.name === 'LineComment'
+            )
+              return null
+
+            const line = state.doc.lineAt(pos)
+            let trigKind: CompletionTriggerKind = CompletionTriggerKind.Invoked
+            let trigChar: string | undefined
+            if (
+              !explicit &&
+              plugin.client
+                .getServerCapabilities()
+                .completionProvider?.triggerCharacters?.includes(
+                  line.text[pos - line.from - 1]
+                )
+            ) {
+              trigKind = CompletionTriggerKind.TriggerCharacter
+              trigChar = line.text[pos - line.from - 1]
+            }
+            if (
+              trigKind === CompletionTriggerKind.Invoked &&
+              !context.matchBefore(/\w+$/)
+            ) {
+              return null
+            }
+
+            return await plugin.requestCompletion(
+              context,
+              offsetToPos(state.doc, pos),
+              {
+                triggerKind: trigKind,
+                triggerCharacter: trigChar,
+              }
+            )
+          },
+        ],
+      })
+    )
+  }
+
+  return ext
+}

packages/codemirror-lsp-client/src/plugin/annotations.ts

@@ -0,0 +1,131 @@
+import { hasNextSnippetField, pickedCompletion } from '@codemirror/autocomplete'
+import { Annotation, Transaction } from '@codemirror/state'
+import type { ViewUpdate } from '@codemirror/view'
+
+export enum LspAnnotation {
+  SemanticTokens = 'semantic-tokens',
+  FormatCode = 'format-code',
+  Diagnostics = 'diagnostics',
+}
+
+const lspEvent = Annotation.define<LspAnnotation>()
+export const lspSemanticTokensEvent = lspEvent.of(LspAnnotation.SemanticTokens)
+export const lspFormatCodeEvent = lspEvent.of(LspAnnotation.FormatCode)
+export const lspDiagnosticsEvent = lspEvent.of(LspAnnotation.Diagnostics)
+
+export enum TransactionAnnotation {
+  Remote = 'remote',
+  UserSelect = 'user.select',
+  UserInput = 'user.input',
+  UserMove = 'user.move',
+  UserDelete = 'user.delete',
+  UserUndo = 'user.undo',
+  UserRedo = 'user.redo',
+
+  SemanticTokens = 'SemanticTokens',
+  FormatCode = 'FormatCode',
+  Diagnostics = 'Diagnostics',
+
+  PickedCompletion = 'PickedCompletion',
+}
+
+export interface TransactionInfo {
+  annotations: TransactionAnnotation[]
+  time: number | null
+  docChanged: boolean
+  addToHistory: boolean
+  inSnippet: boolean
+  transaction: Transaction
+}
+
+export const updateInfo = (update: ViewUpdate): TransactionInfo[] => {
+  let transactionInfos: TransactionInfo[] = []
+
+  for (const tr of update.transactions) {
+    let annotations: TransactionAnnotation[] = []
+
+    if (tr.isUserEvent('select')) {
+      annotations.push(TransactionAnnotation.UserSelect)
+    }
+
+    if (tr.isUserEvent('input')) {
+      annotations.push(TransactionAnnotation.UserInput)
+    }
+    if (tr.isUserEvent('delete')) {
+      annotations.push(TransactionAnnotation.UserDelete)
+    }
+    if (tr.isUserEvent('undo')) {
+      annotations.push(TransactionAnnotation.UserUndo)
+    }
+    if (tr.isUserEvent('redo')) {
+      annotations.push(TransactionAnnotation.UserRedo)
+    }
+    if (tr.isUserEvent('move')) {
+      annotations.push(TransactionAnnotation.UserMove)
+    }
+
+    if (tr.annotation(pickedCompletion) !== undefined) {
+      annotations.push(TransactionAnnotation.PickedCompletion)
+    }
+
+    if (tr.annotation(lspSemanticTokensEvent.type) !== undefined) {
+      annotations.push(TransactionAnnotation.SemanticTokens)
+    }
+
+    if (tr.annotation(lspFormatCodeEvent.type) !== undefined) {
+      annotations.push(TransactionAnnotation.FormatCode)
+    }
+
+    if (tr.annotation(lspDiagnosticsEvent.type) !== undefined) {
+      annotations.push(TransactionAnnotation.Diagnostics)
+    }
+
+    if (tr.annotation(Transaction.remote) !== undefined) {
+      annotations.push(TransactionAnnotation.Remote)
+    }
+
+    transactionInfos.push({
+      annotations,
+      time: tr.annotation(Transaction.time) || null,
+      docChanged: tr.docChanged,
+      addToHistory: tr.annotation(Transaction.addToHistory) || false,
+      inSnippet: hasNextSnippetField(update.state),
+      transaction: tr,
+    })
+  }
+
+  return transactionInfos
+}
+
+export interface RelevantUpdate {
+  overall: boolean
+  userSelect: boolean
+  time: number | null
+}
+
+export const relevantUpdate = (update: ViewUpdate): RelevantUpdate => {
+  const infos = updateInfo(update)
+  // Make sure we are not in a snippet
+  if (infos.some((info) => info.inSnippet)) {
+    return {
+      overall: false,
+      userSelect: false,
+      time: null,
+    }
+  }
+  return {
+    overall: infos.some(
+      (info) =>
+        info.docChanged ||
+        info.annotations.includes(TransactionAnnotation.UserInput) ||
+        info.annotations.includes(TransactionAnnotation.UserDelete) ||
+        info.annotations.includes(TransactionAnnotation.UserUndo) ||
+        info.annotations.includes(TransactionAnnotation.UserRedo) ||
+        info.annotations.includes(TransactionAnnotation.UserMove)
+    ),
+    userSelect: infos.some((info) =>
+      info.annotations.includes(TransactionAnnotation.UserSelect)
+    ),
+    time: infos.length ? infos[0].time : null,
+  }
+}

packages/codemirror-lsp-client/src/plugin/autocomplete.ts

@@ -0,0 +1,51 @@
+import {
+  acceptCompletion,
+  clearSnippet,
+  closeCompletion,
+  hasNextSnippetField,
+  moveCompletionSelection,
+  nextSnippetField,
+  prevSnippetField,
+  startCompletion,
+} from '@codemirror/autocomplete'
+import { Prec } from '@codemirror/state'
+import { EditorView, keymap, KeyBinding } from '@codemirror/view'
+
+import { CompletionItemKind } from 'vscode-languageserver-protocol'
+
+export const CompletionItemKindMap = Object.fromEntries(
+  Object.entries(CompletionItemKind).map(([key, value]) => [value, key])
+) as Record<CompletionItemKind, string>
+
+const lspAutocompleteKeymap: readonly KeyBinding[] = [
+  { key: 'Ctrl-Space', run: startCompletion },
+  {
+    key: 'Escape',
+    run: (view: EditorView): boolean => {
+      if (clearSnippet(view)) return true
+
+      return closeCompletion(view)
+    },
+  },
+  { key: 'ArrowDown', run: moveCompletionSelection(true) },
+  { key: 'ArrowUp', run: moveCompletionSelection(false) },
+  { key: 'PageDown', run: moveCompletionSelection(true, 'page') },
+  { key: 'PageUp', run: moveCompletionSelection(false, 'page') },
+  { key: 'Enter', run: acceptCompletion },
+  {
+    key: 'Tab',
+    run: (view: EditorView): boolean => {
+      if (hasNextSnippetField(view.state)) {
+        const result = nextSnippetField(view)
+        return result
+      }
+
+      return acceptCompletion(view)
+    },
+    shift: prevSnippetField,
+  },
+]
+
+export const lspAutocompleteKeymapExt = Prec.highest(
+  keymap.computeN([], () => [lspAutocompleteKeymap])
+)

packages/codemirror-lsp-client/src/plugin/format.ts

@@ -0,0 +1,27 @@
+import { Extension, Prec } from '@codemirror/state'
+import { EditorView, keymap, KeyBinding, ViewPlugin } from '@codemirror/view'
+
+import { LanguageServerPlugin } from './lsp'
+
+export default function lspFormatExt(
+  plugin: ViewPlugin<LanguageServerPlugin>
+): Extension {
+  const formatKeymap: readonly KeyBinding[] = [
+    {
+      key: 'Alt-Shift-f',
+      run: (view: EditorView) => {
+        let value = view.plugin(plugin)
+        if (!value) return false
+        value.requestFormatting()
+        return true
+      },
+    },
+  ]
+
+  // Create an extension for the key mappings.
+  const formatKeymapExt = Prec.highest(
+    keymap.computeN([], () => [formatKeymap])
+  )
+
+  return formatKeymapExt
+}

packages/codemirror-lsp-client/src/plugin/hover.ts

@@ -0,0 +1,22 @@
+import { Extension } from '@codemirror/state'
+import { hoverTooltip, tooltips, ViewPlugin } from '@codemirror/view'
+
+import { LanguageServerPlugin } from './lsp'
+import { offsetToPos } from './util'
+
+export default function lspHoverExt(
+  plugin: ViewPlugin<LanguageServerPlugin>
+): Extension {
+  return [
+    hoverTooltip((view, pos) => {
+      const value = view.plugin(plugin)
+      return (
+        value?.requestHoverTooltip(view, offsetToPos(view.state.doc, pos)) ??
+        null
+      )
+    }),
+    tooltips({
+      position: 'absolute',
+    }),
+  ]
+}

packages/codemirror-lsp-client/src/plugin/indent.ts

@@ -0,0 +1,21 @@
+import { indentService } from '@codemirror/language'
+import { Extension } from '@codemirror/state'
+
+export default function lspIndentExt(): Extension {
+  // Match the indentation of the previous line (if present).
+  return indentService.of((context, pos) => {
+    try {
+      const previousLine = context.lineAt(pos, -1)
+      const previousLineText = previousLine.text.replaceAll(
+        '\t',
+        ' '.repeat(context.state.tabSize)
+      )
+      const match = previousLineText.match(/^(\s)*/)
+      if (match === null || match.length <= 0) return null
+      return match[0].length
+    } catch (err) {
+      console.error('Error in codemirror indentService', err)
+    }
+    return null
+  })
+}

packages/codemirror-lsp-client/src/plugin/lint.ts

@@ -0,0 +1,14 @@
+import { Extension } from '@codemirror/state'
+import { linter, forEachDiagnostic, Diagnostic } from '@codemirror/lint'
+
+import { LanguageServerPlugin } from './lsp'
+
+export default function lspLintExt(): Extension {
+  return linter((view) => {
+    let diagnostics: Diagnostic[] = []
+    forEachDiagnostic(view.state, (d: Diagnostic, from: number, to: number) => {
+      diagnostics.push(d)
+    })
+    return diagnostics
+  })
+}

packages/codemirror-lsp-client/src/plugin/lsp.ts

@@ -0,0 +1,565 @@
+import type {
+  Completion,
+  CompletionContext,
+  CompletionResult,
+} from '@codemirror/autocomplete'
+import { completeFromList, snippetCompletion } from '@codemirror/autocomplete'
+import { Facet, StateEffect, Extension, Transaction } from '@codemirror/state'
+import type {
+  ViewUpdate,
+  PluginValue,
+  PluginSpec,
+  ViewPlugin,
+} from '@codemirror/view'
+import { EditorView, Tooltip } from '@codemirror/view'
+import { setDiagnosticsEffect } from '@codemirror/lint'
+
+import type { PublishDiagnosticsParams } from 'vscode-languageserver-protocol'
+import type * as LSP from 'vscode-languageserver-protocol'
+import {
+  DiagnosticSeverity,
+  CompletionTriggerKind,
+} from 'vscode-languageserver-protocol'
+import { URI } from 'vscode-uri'
+
+import { LanguageServerClient } from '../client'
+import {
+  lspSemanticTokensEvent,
+  lspFormatCodeEvent,
+  lspDiagnosticsEvent,
+  relevantUpdate,
+} from './annotations'
+import { CompletionItemKindMap } from './autocomplete'
+import { addToken, SemanticToken } from './semantic-tokens'
+import { deferExecution, posToOffset, formatMarkdownContents } from './util'
+import { lspAutocompleteKeymapExt } from './autocomplete'
+import lspHoverExt from './hover'
+import lspFormatExt from './format'
+import lspIndentExt from './indent'
+import lspLintExt from './lint'
+import lspSemanticTokensExt from './semantic-tokens'
+
+const useLast = (values: readonly any[]) => values.reduce((_, v) => v, '')
+export const docPathFacet = Facet.define<string, string>({
+  combine: useLast,
+})
+export const languageId = Facet.define<string, string>({ combine: useLast })
+export const workspaceFolders = Facet.define<
+  LSP.WorkspaceFolder[],
+  LSP.WorkspaceFolder[]
+>({ combine: useLast })
+
+export interface LanguageServerOptions {
+  // We assume this is the main project directory, we are currently working in.
+  workspaceFolders: LSP.WorkspaceFolder[]
+  documentUri: string
+  allowHTMLContent: boolean
+  client: LanguageServerClient
+  processLspNotification?: (
+    plugin: LanguageServerPlugin,
+    notification: LSP.NotificationMessage
+  ) => void
+
+  changesDelay?: number
+}
+
+export class LanguageServerPlugin implements PluginValue {
+  public client: LanguageServerClient
+  private documentVersion: number
+  private foldingRanges: LSP.FoldingRange[] | null = null
+
+  private previousSemanticTokens: SemanticToken[] = []
+
+  private allowHTMLContent: boolean = true
+  private changesDelay: number = 600
+  private processLspNotification?: (
+    plugin: LanguageServerPlugin,
+    notification: LSP.NotificationMessage
+  ) => void
+
+  private _defferer = deferExecution((code: string) => {
+    try {
+      // Update the state (not the editor) with the new code.
+      this.client.textDocumentDidChange({
+        textDocument: {
+          uri: this.getDocUri(),
+          version: this.documentVersion++,
+        },
+        contentChanges: [{ text: code }],
+      })
+
+      this.requestSemanticTokens()
+      this.updateFoldingRanges()
+    } catch (e) {
+      console.error(e)
+    }
+  }, this.changesDelay)
+
+  constructor(options: LanguageServerOptions, private view: EditorView) {
+    this.client = options.client
+    this.documentVersion = 0
+
+    if (options.changesDelay) {
+      this.changesDelay = options.changesDelay
+    }
+
+    if (options.allowHTMLContent !== undefined) {
+      this.allowHTMLContent = options.allowHTMLContent
+    }
+
+    this.client.attachPlugin(this)
+
+    this.processLspNotification = options.processLspNotification
+
+    this.initialize({
+      documentText: this.getDocText(),
+    })
+  }
+
+  private getDocPath(view = this.view) {
+    return view.state.facet(docPathFacet)
+  }
+
+  private getDocText(view = this.view) {
+    return view.state.doc.toString()
+  }
+
+  private getDocUri(view = this.view) {
+    return URI.file(this.getDocPath(view)).toString()
+  }
+
+  private getLanguageId(view = this.view) {
+    return view.state.facet(languageId)
+  }
+
+  update(viewUpdate: ViewUpdate) {
+    const isRelevant = relevantUpdate(viewUpdate)
+    if (!isRelevant.overall) {
+      return
+    }
+
+    // If the doc didn't change we can return early.
+    if (!viewUpdate.docChanged) {
+      return
+    }
+
+    this.sendChange({
+      documentText: viewUpdate.state.doc.toString(),
+    })
+  }
+
+  destroy() {
+    this.client.detachPlugin(this)
+  }
+
+  async initialize({ documentText }: { documentText: string }) {
+    if (this.client.initializePromise) {
+      await this.client.initializePromise
+    }
+
+    this.client.textDocumentDidOpen({
+      textDocument: {
+        uri: this.getDocUri(),
+        languageId: this.getLanguageId(),
+        text: documentText,
+        version: this.documentVersion,
+      },
+    })
+
+    this.requestSemanticTokens()
+    this.updateFoldingRanges()
+  }
+
+  async sendChange({ documentText }: { documentText: string }) {
+    if (!this.client.ready) return
+
+    this._defferer(documentText)
+  }
+
+  requestDiagnostics() {
+    this.sendChange({ documentText: this.getDocText() })
+  }
+
+  async requestHoverTooltip(
+    view: EditorView,
+    { line, character }: { line: number; character: number }
+  ): Promise<Tooltip | null> {
+    if (
+      !this.client.ready ||
+      !this.client.getServerCapabilities().hoverProvider
+    )
+      return null
+
+    this.sendChange({ documentText: this.getDocText() })
+    const result = await this.client.textDocumentHover({
+      textDocument: { uri: this.getDocUri() },
+      position: { line, character },
+    })
+    if (!result) return null
+    const { contents, range } = result
+    let pos = posToOffset(view.state.doc, { line, character })!
+    let end: number | undefined
+    if (range) {
+      pos = posToOffset(view.state.doc, range.start)!
+      end = posToOffset(view.state.doc, range.end)
+    }
+    if (pos === null) return null
+    const dom = document.createElement('div')
+    dom.classList.add('documentation')
+    dom.classList.add('hover-tooltip')
+    dom.style.zIndex = '99999999'
+    if (this.allowHTMLContent) dom.innerHTML = formatMarkdownContents(contents)
+    else dom.textContent = formatMarkdownContents(contents)
+    return { pos, end, create: (view) => ({ dom }), above: true }
+  }
+
+  async getFoldingRanges(): Promise<LSP.FoldingRange[] | null> {
+    if (
+      !this.client.ready ||
+      !this.client.getServerCapabilities().foldingRangeProvider
+    )
+      return null
+
+    const result = await this.client.textDocumentFoldingRange({
+      textDocument: { uri: this.getDocUri() },
+    })
+
+    return result || null
+  }
+
+  async updateFoldingRanges() {
+    const foldingRanges = await this.getFoldingRanges()
+    if (foldingRanges === null) return
+    // Update the folding ranges.
+    this.foldingRanges = foldingRanges
+  }
+
+  // In the future if codemirrors foldService accepts async folding ranges
+  // then we will not have to store these and we can call getFoldingRanges
+  // here.
+  foldingRange(
+    lineStart: number,
+    lineEnd: number
+  ): { from: number; to: number } | null {
+    if (this.foldingRanges === null) {
+      return null
+    }
+
+    for (let i = 0; i < this.foldingRanges.length; i++) {
+      const { startLine, endLine } = this.foldingRanges[i]
+      if (startLine === lineEnd) {
+        const range = {
+          // Set the fold start to the end of the first line
+          // With this, the fold will not include the first line
+          from: startLine,
+          to: endLine,
+        }
+
+        return range
+      }
+    }
+
+    return null
+  }
+
+  async requestFormatting() {
+    if (
+      !this.client.ready ||
+      !this.client.getServerCapabilities().documentFormattingProvider
+    )
+      return null
+
+    this.client.textDocumentDidChange({
+      textDocument: {
+        uri: this.getDocUri(),
+        version: this.documentVersion++,
+      },
+      contentChanges: [{ text: this.getDocText() }],
+    })
+
+    const result = await this.client.textDocumentFormatting({
+      textDocument: { uri: this.getDocUri() },
+      options: {
+        tabSize: 2,
+        insertSpaces: true,
+        insertFinalNewline: true,
+      },
+    })
+
+    if (!result) return null
+
+    for (let i = 0; i < result.length; i++) {
+      const { range, newText } = result[i]
+      this.view.dispatch({
+        changes: {
+          from: posToOffset(this.view.state.doc, range.start)!,
+          to: posToOffset(this.view.state.doc, range.end)!,
+          insert: newText,
+        },
+        annotations: [lspFormatCodeEvent, Transaction.addToHistory.of(true)],
+      })
+    }
+  }
+
+  async requestCompletion(
+    context: CompletionContext,
+    { line, character }: { line: number; character: number },
+    {
+      triggerKind,
+      triggerCharacter,
+    }: {
+      triggerKind: CompletionTriggerKind
+      triggerCharacter: string | undefined
+    }
+  ): Promise<CompletionResult | null> {
+    if (
+      !this.client.ready ||
+      !this.client.getServerCapabilities().completionProvider
+    )
+      return null
+
+    this.sendChange({
+      documentText: context.state.doc.toString(),
+    })
+
+    const result = await this.client.textDocumentCompletion({
+      textDocument: { uri: this.getDocUri() },
+      position: { line, character },
+      context: {
+        triggerKind,
+        triggerCharacter,
+      },
+    })
+
+    if (!result) return null
+
+    const items = 'items' in result ? result.items : result
+
+    let options = items.map(
+      ({
+        detail,
+        label,
+        labelDetails,
+        kind,
+        textEdit,
+        documentation,
+        deprecated,
+        insertText,
+        insertTextFormat,
+        sortText,
+        filterText,
+      }) => {
+        const completion: Completion & {
+          filterText: string
+          sortText?: string
+          apply: string
+        } = {
+          label,
+          detail: labelDetails ? labelDetails.detail : detail,
+          apply: label,
+          type: kind && CompletionItemKindMap[kind].toLowerCase(),
+          sortText: sortText ?? label,
+          filterText: filterText ?? label,
+        }
+        if (documentation) {
+          completion.info = () => {
+            const htmlString = formatMarkdownContents(documentation)
+            const htmlNode = document.createElement('div')
+            htmlNode.style.display = 'contents'
+            htmlNode.innerHTML = htmlString
+            return { dom: htmlNode }
+          }
+        }
+
+        if (insertText && insertTextFormat === 2) {
+          return snippetCompletion(insertText, completion)
+        }
+
+        return completion
+      }
+    )
+
+    return completeFromList(options)(context)
+  }
+
+  parseSemanticTokens(view: EditorView, data: number[]) {
+    // decode the lsp semantic token types
+    const tokens = []
+    for (let i = 0; i < data.length; i += 5) {
+      tokens.push({
+        deltaLine: data[i],
+        startChar: data[i + 1],
+        length: data[i + 2],
+        tokenType: data[i + 3],
+        modifiers: data[i + 4],
+      })
+    }
+
+    // convert the tokens into an array of {to, from, type} objects
+    const tokenTypes =
+      this.client.getServerCapabilities().semanticTokensProvider!.legend
+        .tokenTypes
+    const tokenModifiers =
+      this.client.getServerCapabilities().semanticTokensProvider!.legend
+        .tokenModifiers
+    const tokenRanges: any = []
+    let curLine = 0
+    let prevStart = 0
+    for (let i = 0; i < tokens.length; i++) {
+      const token = tokens[i]
+      const tokenType = tokenTypes[token.tokenType]
+      // get a list of modifiers
+      const tokenModifier = []
+      for (let j = 0; j < tokenModifiers.length; j++) {
+        if (token.modifiers & (1 << j)) {
+          tokenModifier.push(tokenModifiers[j])
+        }
+      }
+
+      if (token.deltaLine !== 0) prevStart = 0
+
+      const tokenRange = {
+        from: posToOffset(view.state.doc, {
+          line: curLine + token.deltaLine,
+          character: prevStart + token.startChar,
+        })!,
+        to: posToOffset(view.state.doc, {
+          line: curLine + token.deltaLine,
+          character: prevStart + token.startChar + token.length,
+        })!,
+        type: tokenType,
+        modifiers: tokenModifier,
+      }
+      tokenRanges.push(tokenRange)
+
+      curLine += token.deltaLine
+      prevStart += token.startChar
+    }
+
+    // sort by from
+    tokenRanges.sort((a: any, b: any) => a.from - b.from)
+    return tokenRanges
+  }
+
+  async requestSemanticTokens() {
+    if (
+      !this.client.ready ||
+      !this.client.getServerCapabilities().semanticTokensProvider
+    ) {
+      return null
+    }
+
+    const result = await this.client.textDocumentSemanticTokensFull({
+      textDocument: { uri: this.getDocUri() },
+    })
+    if (!result) return null
+
+    const { data } = result
+    this.previousSemanticTokens = this.parseSemanticTokens(this.view, data)
+
+    const effects: StateEffect<SemanticToken | Extension>[] =
+      this.previousSemanticTokens.map((tokenRange: any) =>
+        addToken.of(tokenRange)
+      )
+
+    this.view.dispatch({
+      effects,
+
+      annotations: [lspSemanticTokensEvent, Transaction.addToHistory.of(false)],
+    })
+  }
+
+  async processNotification(notification: LSP.NotificationMessage) {
+    try {
+      switch (notification.method) {
+        case 'textDocument/publishDiagnostics':
+          if (notification === undefined) break
+          if (notification.params === undefined) break
+          if (!notification.params) break
+          const params = notification.params as PublishDiagnosticsParams
+          if (!params) break
+          console.log(
+            '[lsp] [window/publishDiagnostics]',
+            this.client.getName(),
+            params
+          )
+          // this is sometimes slower than our actual typing.
+          this.processDiagnostics(params)
+          break
+        case 'window/logMessage':
+          console.log(
+            '[lsp] [window/logMessage]',
+            this.client.getName(),
+            notification.params
+          )
+          break
+        case 'window/showMessage':
+          console.log(
+            '[lsp] [window/showMessage]',
+            this.client.getName(),
+            notification.params
+          )
+          break
+      }
+    } catch (error) {
+      console.error(error)
+    }
+
+    // Send it to the plugin
+    this.processLspNotification?.(this, notification)
+  }
+
+  processDiagnostics(params: PublishDiagnosticsParams) {
+    if (params.uri !== this.getDocUri()) return
+
+    const diagnostics = params.diagnostics
+      .map(({ range, message, severity }) => ({
+        from: posToOffset(this.view.state.doc, range.start)!,
+        to: posToOffset(this.view.state.doc, range.end)!,
+        severity: (
+          {
+            [DiagnosticSeverity.Error]: 'error',
+            [DiagnosticSeverity.Warning]: 'warning',
+            [DiagnosticSeverity.Information]: 'info',
+            [DiagnosticSeverity.Hint]: 'info',
+          } as const
+        )[severity!],
+        message,
+      }))
+      .filter(
+        ({ from, to }) =>
+          from !== null && to !== null && from !== undefined && to !== undefined
+      )
+      .sort((a, b) => {
+        switch (true) {
+          case a.from < b.from:
+            return -1
+          case a.from > b.from:
+            return 1
+        }
+        return 0
+      })
+
+    /* This creates infighting with the others.
+     * TODO: turn it back on when we have a better way to handle it.
+     * this.view.dispatch({
+      effects: [setDiagnosticsEffect.of(diagnostics)],
+      annotations: [lspDiagnosticsEvent, Transaction.addToHistory.of(false)],
+    })*/
+  }
+}
+
+export class LanguageServerPluginSpec
+  implements PluginSpec<LanguageServerPlugin>
+{
+  provide(plugin: ViewPlugin<LanguageServerPlugin>): Extension {
+    return [
+      lspAutocompleteKeymapExt,
+      lspFormatExt(plugin),
+      lspHoverExt(plugin),
+      lspIndentExt(),
+      lspLintExt(),
+      lspSemanticTokensExt(),
+    ]
+  }
+}

packages/codemirror-lsp-client/src/plugin/semantic-tokens.ts

@@ -0,0 +1,175 @@
+import { highlightingFor } from '@codemirror/language'
+import { StateEffect, StateField, Extension } from '@codemirror/state'
+import { EditorView, Decoration, DecorationSet } from '@codemirror/view'
+
+import { Tag, tags } from '@lezer/highlight'
+
+import { lspSemanticTokensEvent } from './annotations'
+
+export interface SemanticToken {
+  from: number
+  to: number
+  type: string
+  modifiers: string[]
+}
+
+export const addToken = StateEffect.define<SemanticToken>({
+  map: (token: SemanticToken, change) => ({
+    ...token,
+    from: change.mapPos(token.from),
+    to: change.mapPos(token.to),
+  }),
+})
+
+export default function lspSemanticTokenExt(): Extension {
+  return StateField.define<DecorationSet>({
+    create() {
+      return Decoration.none
+    },
+    update(highlights, tr) {
+      // Nothing can come before this line, this is very important!
+      // It makes sure the highlights are updated correctly for the changes.
+      highlights = highlights.map(tr.changes)
+
+      const isSemanticTokensEvent = tr.annotation(lspSemanticTokensEvent.type)
+      if (!isSemanticTokensEvent) {
+        return highlights
+      }
+
+      // Check if any of the changes are addToken
+      const hasAddToken = tr.effects.some((e) => e.is(addToken))
+      if (hasAddToken) {
+        highlights = highlights.update({
+          filter: (from, to) => false,
+        })
+      }
+
+      for (const e of tr.effects)
+        if (e.is(addToken)) {
+          const tag = getTag(e.value)
+          const className = tag
+            ? highlightingFor(tr.startState, [tag])
+            : undefined
+
+          if (e.value.from < e.value.to && tag) {
+            if (className) {
+              highlights = highlights.update({
+                add: [
+                  Decoration.mark({ class: className }).range(
+                    e.value.from,
+                    e.value.to
+                  ),
+                ],
+              })
+            }
+          }
+        }
+      return highlights
+    },
+    provide: (f) => EditorView.decorations.from(f),
+  })
+}
+
+export function getTag(semanticToken: SemanticToken): Tag | null {
+  let tokenType = convertSemanticTokenTypeToCodeMirrorTag(semanticToken.type)
+
+  if (
+    semanticToken.modifiers === undefined ||
+    semanticToken.modifiers === null ||
+    semanticToken.modifiers.length === 0
+  ) {
+    return tokenType
+  }
+
+  for (let modifier of semanticToken.modifiers) {
+    tokenType = convertSemanticTokenToCodeMirrorTag(
+      '',
+      modifier,
+      tokenType || undefined
+    )
+  }
+
+  return tokenType
+}
+
+export function getTagName(semanticToken: SemanticToken): string {
+  let tokenType = semanticToken.type
+
+  if (
+    semanticToken.modifiers === undefined ||
+    semanticToken.modifiers === null ||
+    semanticToken.modifiers.length === 0
+  ) {
+    return tokenType
+  }
+
+  for (let modifier of semanticToken.modifiers) {
+    tokenType = `${tokenType}.${modifier}`
+  }
+
+  return tokenType
+}
+
+function convertSemanticTokenTypeToCodeMirrorTag(
+  tokenType: string
+): Tag | null {
+  switch (tokenType) {
+    case 'keyword':
+      return tags.keyword
+    case 'variable':
+      return tags.variableName
+    case 'string':
+      return tags.string
+    case 'number':
+      return tags.number
+    case 'comment':
+      return tags.comment
+    case 'operator':
+      return tags.operator
+    case 'function':
+      return tags.function(tags.name)
+    case 'type':
+      return tags.typeName
+    case 'property':
+      return tags.propertyName
+    case 'parameter':
+      return tags.local(tags.name)
+    default:
+      console.error('Unknown token type:', tokenType)
+      return null
+  }
+}
+
+function convertSemanticTokenToCodeMirrorTag(
+  tokenType: string,
+  tokenModifier: string,
+  givenTag?: Tag
+): Tag | null {
+  let tag = givenTag
+    ? givenTag
+    : convertSemanticTokenTypeToCodeMirrorTag(tokenType)
+
+  if (!tag) {
+    return null
+  }
+
+  if (tokenModifier) {
+    switch (tokenModifier) {
+      case 'definition':
+        return tags.definition(tag)
+      case 'declaration':
+        return tags.definition(tag)
+      case 'readonly':
+        return tags.constant(tag)
+      case 'static':
+        return tags.constant(tag)
+      case 'defaultLibrary':
+        return tags.standard(tag)
+      default:
+        console.error('Unknown token modifier:', tokenModifier)
+        return tag
+    }
+  }
+
+  return tag
+}

packages/codemirror-lsp-client/src/plugin/util.ts

@@ -0,0 +1,55 @@
+import { Text } from '@codemirror/state'
+import { Marked } from '@ts-stack/markdown'
+
+import type * as LSP from 'vscode-languageserver-protocol'
+
+// takes a function and executes it after the wait time, if the function is called again before the wait time is up, the timer is reset
+export function deferExecution<T>(func: (args: T) => any, wait: number) {
+  let timeout: ReturnType<typeof setTimeout> | null
+  let latestArgs: T
+
+  function later() {
+    timeout = null
+    func(latestArgs)
+  }
+
+  function deferred(args: T) {
+    latestArgs = args
+    if (timeout) {
+      clearTimeout(timeout)
+    }
+    timeout = setTimeout(later, wait)
+  }
+
+  return deferred
+}
+
+export function posToOffset(
+  doc: Text,
+  pos: { line: number; character: number }
+): number | undefined {
+  if (pos.line >= doc.lines) return
+  const offset = doc.line(pos.line + 1).from + pos.character
+  if (offset > doc.length) return
+  return offset
+}
+
+export function offsetToPos(doc: Text, offset: number) {
+  const line = doc.lineAt(offset)
+  return {
+    line: line.number - 1,
+    character: offset - line.from,
+  }
+}
+
+export function formatMarkdownContents(
+  contents: LSP.MarkupContent | LSP.MarkedString | LSP.MarkedString[]
+): string {
+  if (Array.isArray(contents)) {
+    return contents.map((c) => formatMarkdownContents(c) + '\n\n').join('')
+  } else if (typeof contents === 'string') {
+    return Marked.parse(contents)
+  } else {
+    return Marked.parse(contents.value)
+  }
+}