pull lsp client out into a fake module (#2846)

* initial commit

Signed-off-by: Jess Frazelle <github@jessfraz.com>

tsc passing

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

working

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fixups

Signed-off-by: Jess Frazelle <github@jessfraz.com>

updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

fmt

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* cleanups

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* udpates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* cleanup

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* cleanup

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* fixes

Signed-off-by: Jess Frazelle <github@jessfraz.com>

* updates

Signed-off-by: Jess Frazelle <github@jessfraz.com>

---------

Signed-off-by: Jess Frazelle <github@jessfraz.com>

packages/codemirror-lsp-client/src/plugin/annotations.ts

@@ -0,0 +1,131 @@
+import { hasNextSnippetField, pickedCompletion } from '@codemirror/autocomplete'
+import { Annotation, Transaction } from '@codemirror/state'
+import type { ViewUpdate } from '@codemirror/view'
+
+export enum LspAnnotation {
+  SemanticTokens = 'semantic-tokens',
+  FormatCode = 'format-code',
+  Diagnostics = 'diagnostics',
+}
+
+const lspEvent = Annotation.define<LspAnnotation>()
+export const lspSemanticTokensEvent = lspEvent.of(LspAnnotation.SemanticTokens)
+export const lspFormatCodeEvent = lspEvent.of(LspAnnotation.FormatCode)
+export const lspDiagnosticsEvent = lspEvent.of(LspAnnotation.Diagnostics)
+
+export enum TransactionAnnotation {
+  Remote = 'remote',
+  UserSelect = 'user.select',
+  UserInput = 'user.input',
+  UserMove = 'user.move',
+  UserDelete = 'user.delete',
+  UserUndo = 'user.undo',
+  UserRedo = 'user.redo',
+
+  SemanticTokens = 'SemanticTokens',
+  FormatCode = 'FormatCode',
+  Diagnostics = 'Diagnostics',
+
+  PickedCompletion = 'PickedCompletion',
+}
+
+export interface TransactionInfo {
+  annotations: TransactionAnnotation[]
+  time: number | null
+  docChanged: boolean
+  addToHistory: boolean
+  inSnippet: boolean
+  transaction: Transaction
+}
+
+export const updateInfo = (update: ViewUpdate): TransactionInfo[] => {
+  let transactionInfos: TransactionInfo[] = []
+
+  for (const tr of update.transactions) {
+    let annotations: TransactionAnnotation[] = []
+
+    if (tr.isUserEvent('select')) {
+      annotations.push(TransactionAnnotation.UserSelect)
+    }
+
+    if (tr.isUserEvent('input')) {
+      annotations.push(TransactionAnnotation.UserInput)
+    }
+    if (tr.isUserEvent('delete')) {
+      annotations.push(TransactionAnnotation.UserDelete)
+    }
+    if (tr.isUserEvent('undo')) {
+      annotations.push(TransactionAnnotation.UserUndo)
+    }
+    if (tr.isUserEvent('redo')) {
+      annotations.push(TransactionAnnotation.UserRedo)
+    }
+    if (tr.isUserEvent('move')) {
+      annotations.push(TransactionAnnotation.UserMove)
+    }
+
+    if (tr.annotation(pickedCompletion) !== undefined) {
+      annotations.push(TransactionAnnotation.PickedCompletion)
+    }
+
+    if (tr.annotation(lspSemanticTokensEvent.type) !== undefined) {
+      annotations.push(TransactionAnnotation.SemanticTokens)
+    }
+
+    if (tr.annotation(lspFormatCodeEvent.type) !== undefined) {
+      annotations.push(TransactionAnnotation.FormatCode)
+    }
+
+    if (tr.annotation(lspDiagnosticsEvent.type) !== undefined) {
+      annotations.push(TransactionAnnotation.Diagnostics)
+    }
+
+    if (tr.annotation(Transaction.remote) !== undefined) {
+      annotations.push(TransactionAnnotation.Remote)
+    }
+
+    transactionInfos.push({
+      annotations,
+      time: tr.annotation(Transaction.time) || null,
+      docChanged: tr.docChanged,
+      addToHistory: tr.annotation(Transaction.addToHistory) || false,
+      inSnippet: hasNextSnippetField(update.state),
+      transaction: tr,
+    })
+  }
+
+  return transactionInfos
+}
+
+export interface RelevantUpdate {
+  overall: boolean
+  userSelect: boolean
+  time: number | null
+}
+
+export const relevantUpdate = (update: ViewUpdate): RelevantUpdate => {
+  const infos = updateInfo(update)
+  // Make sure we are not in a snippet
+  if (infos.some((info) => info.inSnippet)) {
+    return {
+      overall: false,
+      userSelect: false,
+      time: null,
+    }
+  }
+  return {
+    overall: infos.some(
+      (info) =>
+        info.docChanged ||
+        info.annotations.includes(TransactionAnnotation.UserInput) ||
+        info.annotations.includes(TransactionAnnotation.UserDelete) ||
+        info.annotations.includes(TransactionAnnotation.UserUndo) ||
+        info.annotations.includes(TransactionAnnotation.UserRedo) ||
+        info.annotations.includes(TransactionAnnotation.UserMove)
+    ),
+    userSelect: infos.some((info) =>
+      info.annotations.includes(TransactionAnnotation.UserSelect)
+    ),
+    time: infos.length ? infos[0].time : null,
+  }
+}

packages/codemirror-lsp-client/src/plugin/autocomplete.ts

@@ -0,0 +1,51 @@
+import {
+  acceptCompletion,
+  clearSnippet,
+  closeCompletion,
+  hasNextSnippetField,
+  moveCompletionSelection,
+  nextSnippetField,
+  prevSnippetField,
+  startCompletion,
+} from '@codemirror/autocomplete'
+import { Prec } from '@codemirror/state'
+import { EditorView, keymap, KeyBinding } from '@codemirror/view'
+
+import { CompletionItemKind } from 'vscode-languageserver-protocol'
+
+export const CompletionItemKindMap = Object.fromEntries(
+  Object.entries(CompletionItemKind).map(([key, value]) => [value, key])
+) as Record<CompletionItemKind, string>
+
+const lspAutocompleteKeymap: readonly KeyBinding[] = [
+  { key: 'Ctrl-Space', run: startCompletion },
+  {
+    key: 'Escape',
+    run: (view: EditorView): boolean => {
+      if (clearSnippet(view)) return true
+
+      return closeCompletion(view)
+    },
+  },
+  { key: 'ArrowDown', run: moveCompletionSelection(true) },
+  { key: 'ArrowUp', run: moveCompletionSelection(false) },
+  { key: 'PageDown', run: moveCompletionSelection(true, 'page') },
+  { key: 'PageUp', run: moveCompletionSelection(false, 'page') },
+  { key: 'Enter', run: acceptCompletion },
+  {
+    key: 'Tab',
+    run: (view: EditorView): boolean => {
+      if (hasNextSnippetField(view.state)) {
+        const result = nextSnippetField(view)
+        return result
+      }
+
+      return acceptCompletion(view)
+    },
+    shift: prevSnippetField,
+  },
+]
+
+export const lspAutocompleteKeymapExt = Prec.highest(
+  keymap.computeN([], () => [lspAutocompleteKeymap])
+)

packages/codemirror-lsp-client/src/plugin/format.ts

@@ -0,0 +1,27 @@
+import { Extension, Prec } from '@codemirror/state'
+import { EditorView, keymap, KeyBinding, ViewPlugin } from '@codemirror/view'
+
+import { LanguageServerPlugin } from './lsp'
+
+export default function lspFormatExt(
+  plugin: ViewPlugin<LanguageServerPlugin>
+): Extension {
+  const formatKeymap: readonly KeyBinding[] = [
+    {
+      key: 'Alt-Shift-f',
+      run: (view: EditorView) => {
+        let value = view.plugin(plugin)
+        if (!value) return false
+        value.requestFormatting()
+        return true
+      },
+    },
+  ]
+
+  // Create an extension for the key mappings.
+  const formatKeymapExt = Prec.highest(
+    keymap.computeN([], () => [formatKeymap])
+  )
+
+  return formatKeymapExt
+}

packages/codemirror-lsp-client/src/plugin/hover.ts

@@ -0,0 +1,22 @@
+import { Extension } from '@codemirror/state'
+import { hoverTooltip, tooltips, ViewPlugin } from '@codemirror/view'
+
+import { LanguageServerPlugin } from './lsp'
+import { offsetToPos } from './util'
+
+export default function lspHoverExt(
+  plugin: ViewPlugin<LanguageServerPlugin>
+): Extension {
+  return [
+    hoverTooltip((view, pos) => {
+      const value = view.plugin(plugin)
+      return (
+        value?.requestHoverTooltip(view, offsetToPos(view.state.doc, pos)) ??
+        null
+      )
+    }),
+    tooltips({
+      position: 'absolute',
+    }),
+  ]
+}

packages/codemirror-lsp-client/src/plugin/indent.ts

@@ -0,0 +1,21 @@
+import { indentService } from '@codemirror/language'
+import { Extension } from '@codemirror/state'
+
+export default function lspIndentExt(): Extension {
+  // Match the indentation of the previous line (if present).
+  return indentService.of((context, pos) => {
+    try {
+      const previousLine = context.lineAt(pos, -1)
+      const previousLineText = previousLine.text.replaceAll(
+        '\t',
+        ' '.repeat(context.state.tabSize)
+      )
+      const match = previousLineText.match(/^(\s)*/)
+      if (match === null || match.length <= 0) return null
+      return match[0].length
+    } catch (err) {
+      console.error('Error in codemirror indentService', err)
+    }
+    return null
+  })
+}

packages/codemirror-lsp-client/src/plugin/lint.ts

@@ -0,0 +1,14 @@
+import { Extension } from '@codemirror/state'
+import { linter, forEachDiagnostic, Diagnostic } from '@codemirror/lint'
+
+import { LanguageServerPlugin } from './lsp'
+
+export default function lspLintExt(): Extension {
+  return linter((view) => {
+    let diagnostics: Diagnostic[] = []
+    forEachDiagnostic(view.state, (d: Diagnostic, from: number, to: number) => {
+      diagnostics.push(d)
+    })
+    return diagnostics
+  })
+}

packages/codemirror-lsp-client/src/plugin/lsp.ts

@@ -0,0 +1,565 @@
+import type {
+  Completion,
+  CompletionContext,
+  CompletionResult,
+} from '@codemirror/autocomplete'
+import { completeFromList, snippetCompletion } from '@codemirror/autocomplete'
+import { Facet, StateEffect, Extension, Transaction } from '@codemirror/state'
+import type {
+  ViewUpdate,
+  PluginValue,
+  PluginSpec,
+  ViewPlugin,
+} from '@codemirror/view'
+import { EditorView, Tooltip } from '@codemirror/view'
+import { setDiagnosticsEffect } from '@codemirror/lint'
+
+import type { PublishDiagnosticsParams } from 'vscode-languageserver-protocol'
+import type * as LSP from 'vscode-languageserver-protocol'
+import {
+  DiagnosticSeverity,
+  CompletionTriggerKind,
+} from 'vscode-languageserver-protocol'
+import { URI } from 'vscode-uri'
+
+import { LanguageServerClient } from '../client'
+import {
+  lspSemanticTokensEvent,
+  lspFormatCodeEvent,
+  lspDiagnosticsEvent,
+  relevantUpdate,
+} from './annotations'
+import { CompletionItemKindMap } from './autocomplete'
+import { addToken, SemanticToken } from './semantic-tokens'
+import { deferExecution, posToOffset, formatMarkdownContents } from './util'
+import { lspAutocompleteKeymapExt } from './autocomplete'
+import lspHoverExt from './hover'
+import lspFormatExt from './format'
+import lspIndentExt from './indent'
+import lspLintExt from './lint'
+import lspSemanticTokensExt from './semantic-tokens'
+
+const useLast = (values: readonly any[]) => values.reduce((_, v) => v, '')
+export const docPathFacet = Facet.define<string, string>({
+  combine: useLast,
+})
+export const languageId = Facet.define<string, string>({ combine: useLast })
+export const workspaceFolders = Facet.define<
+  LSP.WorkspaceFolder[],
+  LSP.WorkspaceFolder[]
+>({ combine: useLast })
+
+export interface LanguageServerOptions {
+  // We assume this is the main project directory, we are currently working in.
+  workspaceFolders: LSP.WorkspaceFolder[]
+  documentUri: string
+  allowHTMLContent: boolean
+  client: LanguageServerClient
+  processLspNotification?: (
+    plugin: LanguageServerPlugin,
+    notification: LSP.NotificationMessage
+  ) => void
+
+  changesDelay?: number
+}
+
+export class LanguageServerPlugin implements PluginValue {
+  public client: LanguageServerClient
+  private documentVersion: number
+  private foldingRanges: LSP.FoldingRange[] | null = null
+
+  private previousSemanticTokens: SemanticToken[] = []
+
+  private allowHTMLContent: boolean = true
+  private changesDelay: number = 600
+  private processLspNotification?: (
+    plugin: LanguageServerPlugin,
+    notification: LSP.NotificationMessage
+  ) => void
+
+  private _defferer = deferExecution((code: string) => {
+    try {
+      // Update the state (not the editor) with the new code.
+      this.client.textDocumentDidChange({
+        textDocument: {
+          uri: this.getDocUri(),
+          version: this.documentVersion++,
+        },
+        contentChanges: [{ text: code }],
+      })
+
+      this.requestSemanticTokens()
+      this.updateFoldingRanges()
+    } catch (e) {
+      console.error(e)
+    }
+  }, this.changesDelay)
+
+  constructor(options: LanguageServerOptions, private view: EditorView) {
+    this.client = options.client
+    this.documentVersion = 0
+
+    if (options.changesDelay) {
+      this.changesDelay = options.changesDelay
+    }
+
+    if (options.allowHTMLContent !== undefined) {
+      this.allowHTMLContent = options.allowHTMLContent
+    }
+
+    this.client.attachPlugin(this)
+
+    this.processLspNotification = options.processLspNotification
+
+    this.initialize({
+      documentText: this.getDocText(),
+    })
+  }
+
+  private getDocPath(view = this.view) {
+    return view.state.facet(docPathFacet)
+  }
+
+  private getDocText(view = this.view) {
+    return view.state.doc.toString()
+  }
+
+  private getDocUri(view = this.view) {
+    return URI.file(this.getDocPath(view)).toString()
+  }
+
+  private getLanguageId(view = this.view) {
+    return view.state.facet(languageId)
+  }
+
+  update(viewUpdate: ViewUpdate) {
+    const isRelevant = relevantUpdate(viewUpdate)
+    if (!isRelevant.overall) {
+      return
+    }
+
+    // If the doc didn't change we can return early.
+    if (!viewUpdate.docChanged) {
+      return
+    }
+
+    this.sendChange({
+      documentText: viewUpdate.state.doc.toString(),
+    })
+  }
+
+  destroy() {
+    this.client.detachPlugin(this)
+  }
+
+  async initialize({ documentText }: { documentText: string }) {
+    if (this.client.initializePromise) {
+      await this.client.initializePromise
+    }
+
+    this.client.textDocumentDidOpen({
+      textDocument: {
+        uri: this.getDocUri(),
+        languageId: this.getLanguageId(),
+        text: documentText,
+        version: this.documentVersion,
+      },
+    })
+
+    this.requestSemanticTokens()
+    this.updateFoldingRanges()
+  }
+
+  async sendChange({ documentText }: { documentText: string }) {
+    if (!this.client.ready) return
+
+    this._defferer(documentText)
+  }
+
+  requestDiagnostics() {
+    this.sendChange({ documentText: this.getDocText() })
+  }
+
+  async requestHoverTooltip(
+    view: EditorView,
+    { line, character }: { line: number; character: number }
+  ): Promise<Tooltip | null> {
+    if (
+      !this.client.ready ||
+      !this.client.getServerCapabilities().hoverProvider
+    )
+      return null
+
+    this.sendChange({ documentText: this.getDocText() })
+    const result = await this.client.textDocumentHover({
+      textDocument: { uri: this.getDocUri() },
+      position: { line, character },
+    })
+    if (!result) return null
+    const { contents, range } = result
+    let pos = posToOffset(view.state.doc, { line, character })!
+    let end: number | undefined
+    if (range) {
+      pos = posToOffset(view.state.doc, range.start)!
+      end = posToOffset(view.state.doc, range.end)
+    }
+    if (pos === null) return null
+    const dom = document.createElement('div')
+    dom.classList.add('documentation')
+    dom.classList.add('hover-tooltip')
+    dom.style.zIndex = '99999999'
+    if (this.allowHTMLContent) dom.innerHTML = formatMarkdownContents(contents)
+    else dom.textContent = formatMarkdownContents(contents)
+    return { pos, end, create: (view) => ({ dom }), above: true }
+  }
+
+  async getFoldingRanges(): Promise<LSP.FoldingRange[] | null> {
+    if (
+      !this.client.ready ||
+      !this.client.getServerCapabilities().foldingRangeProvider
+    )
+      return null
+
+    const result = await this.client.textDocumentFoldingRange({
+      textDocument: { uri: this.getDocUri() },
+    })
+
+    return result || null
+  }
+
+  async updateFoldingRanges() {
+    const foldingRanges = await this.getFoldingRanges()
+    if (foldingRanges === null) return
+    // Update the folding ranges.
+    this.foldingRanges = foldingRanges
+  }
+
+  // In the future if codemirrors foldService accepts async folding ranges
+  // then we will not have to store these and we can call getFoldingRanges
+  // here.
+  foldingRange(
+    lineStart: number,
+    lineEnd: number
+  ): { from: number; to: number } | null {
+    if (this.foldingRanges === null) {
+      return null
+    }
+
+    for (let i = 0; i < this.foldingRanges.length; i++) {
+      const { startLine, endLine } = this.foldingRanges[i]
+      if (startLine === lineEnd) {
+        const range = {
+          // Set the fold start to the end of the first line
+          // With this, the fold will not include the first line
+          from: startLine,
+          to: endLine,
+        }
+
+        return range
+      }
+    }
+
+    return null
+  }
+
+  async requestFormatting() {
+    if (
+      !this.client.ready ||
+      !this.client.getServerCapabilities().documentFormattingProvider
+    )
+      return null
+
+    this.client.textDocumentDidChange({
+      textDocument: {
+        uri: this.getDocUri(),
+        version: this.documentVersion++,
+      },
+      contentChanges: [{ text: this.getDocText() }],
+    })
+
+    const result = await this.client.textDocumentFormatting({
+      textDocument: { uri: this.getDocUri() },
+      options: {
+        tabSize: 2,
+        insertSpaces: true,
+        insertFinalNewline: true,
+      },
+    })
+
+    if (!result) return null
+
+    for (let i = 0; i < result.length; i++) {
+      const { range, newText } = result[i]
+      this.view.dispatch({
+        changes: {
+          from: posToOffset(this.view.state.doc, range.start)!,
+          to: posToOffset(this.view.state.doc, range.end)!,
+          insert: newText,
+        },
+        annotations: [lspFormatCodeEvent, Transaction.addToHistory.of(true)],
+      })
+    }
+  }
+
+  async requestCompletion(
+    context: CompletionContext,
+    { line, character }: { line: number; character: number },
+    {
+      triggerKind,
+      triggerCharacter,
+    }: {
+      triggerKind: CompletionTriggerKind
+      triggerCharacter: string | undefined
+    }
+  ): Promise<CompletionResult | null> {
+    if (
+      !this.client.ready ||
+      !this.client.getServerCapabilities().completionProvider
+    )
+      return null
+
+    this.sendChange({
+      documentText: context.state.doc.toString(),
+    })
+
+    const result = await this.client.textDocumentCompletion({
+      textDocument: { uri: this.getDocUri() },
+      position: { line, character },
+      context: {
+        triggerKind,
+        triggerCharacter,
+      },
+    })
+
+    if (!result) return null
+
+    const items = 'items' in result ? result.items : result
+
+    let options = items.map(
+      ({
+        detail,
+        label,
+        labelDetails,
+        kind,
+        textEdit,
+        documentation,
+        deprecated,
+        insertText,
+        insertTextFormat,
+        sortText,
+        filterText,
+      }) => {
+        const completion: Completion & {
+          filterText: string
+          sortText?: string
+          apply: string
+        } = {
+          label,
+          detail: labelDetails ? labelDetails.detail : detail,
+          apply: label,
+          type: kind && CompletionItemKindMap[kind].toLowerCase(),
+          sortText: sortText ?? label,
+          filterText: filterText ?? label,
+        }
+        if (documentation) {
+          completion.info = () => {
+            const htmlString = formatMarkdownContents(documentation)
+            const htmlNode = document.createElement('div')
+            htmlNode.style.display = 'contents'
+            htmlNode.innerHTML = htmlString
+            return { dom: htmlNode }
+          }
+        }
+
+        if (insertText && insertTextFormat === 2) {
+          return snippetCompletion(insertText, completion)
+        }
+
+        return completion
+      }
+    )
+
+    return completeFromList(options)(context)
+  }
+
+  parseSemanticTokens(view: EditorView, data: number[]) {
+    // decode the lsp semantic token types
+    const tokens = []
+    for (let i = 0; i < data.length; i += 5) {
+      tokens.push({
+        deltaLine: data[i],
+        startChar: data[i + 1],
+        length: data[i + 2],
+        tokenType: data[i + 3],
+        modifiers: data[i + 4],
+      })
+    }
+
+    // convert the tokens into an array of {to, from, type} objects
+    const tokenTypes =
+      this.client.getServerCapabilities().semanticTokensProvider!.legend
+        .tokenTypes
+    const tokenModifiers =
+      this.client.getServerCapabilities().semanticTokensProvider!.legend
+        .tokenModifiers
+    const tokenRanges: any = []
+    let curLine = 0
+    let prevStart = 0
+    for (let i = 0; i < tokens.length; i++) {
+      const token = tokens[i]
+      const tokenType = tokenTypes[token.tokenType]
+      // get a list of modifiers
+      const tokenModifier = []
+      for (let j = 0; j < tokenModifiers.length; j++) {
+        if (token.modifiers & (1 << j)) {
+          tokenModifier.push(tokenModifiers[j])
+        }
+      }
+
+      if (token.deltaLine !== 0) prevStart = 0
+
+      const tokenRange = {
+        from: posToOffset(view.state.doc, {
+          line: curLine + token.deltaLine,
+          character: prevStart + token.startChar,
+        })!,
+        to: posToOffset(view.state.doc, {
+          line: curLine + token.deltaLine,
+          character: prevStart + token.startChar + token.length,
+        })!,
+        type: tokenType,
+        modifiers: tokenModifier,
+      }
+      tokenRanges.push(tokenRange)
+
+      curLine += token.deltaLine
+      prevStart += token.startChar
+    }
+
+    // sort by from
+    tokenRanges.sort((a: any, b: any) => a.from - b.from)
+    return tokenRanges
+  }
+
+  async requestSemanticTokens() {
+    if (
+      !this.client.ready ||
+      !this.client.getServerCapabilities().semanticTokensProvider
+    ) {
+      return null
+    }
+
+    const result = await this.client.textDocumentSemanticTokensFull({
+      textDocument: { uri: this.getDocUri() },
+    })
+    if (!result) return null
+
+    const { data } = result
+    this.previousSemanticTokens = this.parseSemanticTokens(this.view, data)
+
+    const effects: StateEffect<SemanticToken | Extension>[] =
+      this.previousSemanticTokens.map((tokenRange: any) =>
+        addToken.of(tokenRange)
+      )
+
+    this.view.dispatch({
+      effects,
+
+      annotations: [lspSemanticTokensEvent, Transaction.addToHistory.of(false)],
+    })
+  }
+
+  async processNotification(notification: LSP.NotificationMessage) {
+    try {
+      switch (notification.method) {
+        case 'textDocument/publishDiagnostics':
+          if (notification === undefined) break
+          if (notification.params === undefined) break
+          if (!notification.params) break
+          const params = notification.params as PublishDiagnosticsParams
+          if (!params) break
+          console.log(
+            '[lsp] [window/publishDiagnostics]',
+            this.client.getName(),
+            params
+          )
+          // this is sometimes slower than our actual typing.
+          this.processDiagnostics(params)
+          break
+        case 'window/logMessage':
+          console.log(
+            '[lsp] [window/logMessage]',
+            this.client.getName(),
+            notification.params
+          )
+          break
+        case 'window/showMessage':
+          console.log(
+            '[lsp] [window/showMessage]',
+            this.client.getName(),
+            notification.params
+          )
+          break
+      }
+    } catch (error) {
+      console.error(error)
+    }
+
+    // Send it to the plugin
+    this.processLspNotification?.(this, notification)
+  }
+
+  processDiagnostics(params: PublishDiagnosticsParams) {
+    if (params.uri !== this.getDocUri()) return
+
+    const diagnostics = params.diagnostics
+      .map(({ range, message, severity }) => ({
+        from: posToOffset(this.view.state.doc, range.start)!,
+        to: posToOffset(this.view.state.doc, range.end)!,
+        severity: (
+          {
+            [DiagnosticSeverity.Error]: 'error',
+            [DiagnosticSeverity.Warning]: 'warning',
+            [DiagnosticSeverity.Information]: 'info',
+            [DiagnosticSeverity.Hint]: 'info',
+          } as const
+        )[severity!],
+        message,
+      }))
+      .filter(
+        ({ from, to }) =>
+          from !== null && to !== null && from !== undefined && to !== undefined
+      )
+      .sort((a, b) => {
+        switch (true) {
+          case a.from < b.from:
+            return -1
+          case a.from > b.from:
+            return 1
+        }
+        return 0
+      })
+
+    /* This creates infighting with the others.
+     * TODO: turn it back on when we have a better way to handle it.
+     * this.view.dispatch({
+      effects: [setDiagnosticsEffect.of(diagnostics)],
+      annotations: [lspDiagnosticsEvent, Transaction.addToHistory.of(false)],
+    })*/
+  }
+}
+
+export class LanguageServerPluginSpec
+  implements PluginSpec<LanguageServerPlugin>
+{
+  provide(plugin: ViewPlugin<LanguageServerPlugin>): Extension {
+    return [
+      lspAutocompleteKeymapExt,
+      lspFormatExt(plugin),
+      lspHoverExt(plugin),
+      lspIndentExt(),
+      lspLintExt(),
+      lspSemanticTokensExt(),
+    ]
+  }
+}

packages/codemirror-lsp-client/src/plugin/semantic-tokens.ts

@@ -0,0 +1,175 @@
+import { highlightingFor } from '@codemirror/language'
+import { StateEffect, StateField, Extension } from '@codemirror/state'
+import { EditorView, Decoration, DecorationSet } from '@codemirror/view'
+
+import { Tag, tags } from '@lezer/highlight'
+
+import { lspSemanticTokensEvent } from './annotations'
+
+export interface SemanticToken {
+  from: number
+  to: number
+  type: string
+  modifiers: string[]
+}
+
+export const addToken = StateEffect.define<SemanticToken>({
+  map: (token: SemanticToken, change) => ({
+    ...token,
+    from: change.mapPos(token.from),
+    to: change.mapPos(token.to),
+  }),
+})
+
+export default function lspSemanticTokenExt(): Extension {
+  return StateField.define<DecorationSet>({
+    create() {
+      return Decoration.none
+    },
+    update(highlights, tr) {
+      // Nothing can come before this line, this is very important!
+      // It makes sure the highlights are updated correctly for the changes.
+      highlights = highlights.map(tr.changes)
+
+      const isSemanticTokensEvent = tr.annotation(lspSemanticTokensEvent.type)
+      if (!isSemanticTokensEvent) {
+        return highlights
+      }
+
+      // Check if any of the changes are addToken
+      const hasAddToken = tr.effects.some((e) => e.is(addToken))
+      if (hasAddToken) {
+        highlights = highlights.update({
+          filter: (from, to) => false,
+        })
+      }
+
+      for (const e of tr.effects)
+        if (e.is(addToken)) {
+          const tag = getTag(e.value)
+          const className = tag
+            ? highlightingFor(tr.startState, [tag])
+            : undefined
+
+          if (e.value.from < e.value.to && tag) {
+            if (className) {
+              highlights = highlights.update({
+                add: [
+                  Decoration.mark({ class: className }).range(
+                    e.value.from,
+                    e.value.to
+                  ),
+                ],
+              })
+            }
+          }
+        }
+      return highlights
+    },
+    provide: (f) => EditorView.decorations.from(f),
+  })
+}
+
+export function getTag(semanticToken: SemanticToken): Tag | null {
+  let tokenType = convertSemanticTokenTypeToCodeMirrorTag(semanticToken.type)
+
+  if (
+    semanticToken.modifiers === undefined ||
+    semanticToken.modifiers === null ||
+    semanticToken.modifiers.length === 0
+  ) {
+    return tokenType
+  }
+
+  for (let modifier of semanticToken.modifiers) {
+    tokenType = convertSemanticTokenToCodeMirrorTag(
+      '',
+      modifier,
+      tokenType || undefined
+    )
+  }
+
+  return tokenType
+}
+
+export function getTagName(semanticToken: SemanticToken): string {
+  let tokenType = semanticToken.type
+
+  if (
+    semanticToken.modifiers === undefined ||
+    semanticToken.modifiers === null ||
+    semanticToken.modifiers.length === 0
+  ) {
+    return tokenType
+  }
+
+  for (let modifier of semanticToken.modifiers) {
+    tokenType = `${tokenType}.${modifier}`
+  }
+
+  return tokenType
+}
+
+function convertSemanticTokenTypeToCodeMirrorTag(
+  tokenType: string
+): Tag | null {
+  switch (tokenType) {
+    case 'keyword':
+      return tags.keyword
+    case 'variable':
+      return tags.variableName
+    case 'string':
+      return tags.string
+    case 'number':
+      return tags.number
+    case 'comment':
+      return tags.comment
+    case 'operator':
+      return tags.operator
+    case 'function':
+      return tags.function(tags.name)
+    case 'type':
+      return tags.typeName
+    case 'property':
+      return tags.propertyName
+    case 'parameter':
+      return tags.local(tags.name)
+    default:
+      console.error('Unknown token type:', tokenType)
+      return null
+  }
+}
+
+function convertSemanticTokenToCodeMirrorTag(
+  tokenType: string,
+  tokenModifier: string,
+  givenTag?: Tag
+): Tag | null {
+  let tag = givenTag
+    ? givenTag
+    : convertSemanticTokenTypeToCodeMirrorTag(tokenType)
+
+  if (!tag) {
+    return null
+  }
+
+  if (tokenModifier) {
+    switch (tokenModifier) {
+      case 'definition':
+        return tags.definition(tag)
+      case 'declaration':
+        return tags.definition(tag)
+      case 'readonly':
+        return tags.constant(tag)
+      case 'static':
+        return tags.constant(tag)
+      case 'defaultLibrary':
+        return tags.standard(tag)
+      default:
+        console.error('Unknown token modifier:', tokenModifier)
+        return tag
+    }
+  }
+
+  return tag
+}

packages/codemirror-lsp-client/src/plugin/util.ts

@@ -0,0 +1,55 @@
+import { Text } from '@codemirror/state'
+import { Marked } from '@ts-stack/markdown'
+
+import type * as LSP from 'vscode-languageserver-protocol'
+
+// takes a function and executes it after the wait time, if the function is called again before the wait time is up, the timer is reset
+export function deferExecution<T>(func: (args: T) => any, wait: number) {
+  let timeout: ReturnType<typeof setTimeout> | null
+  let latestArgs: T
+
+  function later() {
+    timeout = null
+    func(latestArgs)
+  }
+
+  function deferred(args: T) {
+    latestArgs = args
+    if (timeout) {
+      clearTimeout(timeout)
+    }
+    timeout = setTimeout(later, wait)
+  }
+
+  return deferred
+}
+
+export function posToOffset(
+  doc: Text,
+  pos: { line: number; character: number }
+): number | undefined {
+  if (pos.line >= doc.lines) return
+  const offset = doc.line(pos.line + 1).from + pos.character
+  if (offset > doc.length) return
+  return offset
+}
+
+export function offsetToPos(doc: Text, offset: number) {
+  const line = doc.lineAt(offset)
+  return {
+    line: line.number - 1,
+    character: offset - line.from,
+  }
+}
+
+export function formatMarkdownContents(
+  contents: LSP.MarkupContent | LSP.MarkedString | LSP.MarkedString[]
+): string {
+  if (Array.isArray(contents)) {
+    return contents.map((c) => formatMarkdownContents(c) + '\n\n').join('')
+  } else if (typeof contents === 'string') {
+    return Marked.parse(contents)
+  } else {
+    return Marked.parse(contents.value)
+  }
+}