add pipe operator to tokeniser and ast

Executor and recast TODO

src/lang/abstractSyntaxTree.test.ts

@@ -1,4 +1,8 @@
-import { abstractSyntaxTree, findClosingBrace } from './abstractSyntaxTree'
+import {
+  abstractSyntaxTree,
+  findClosingBrace,
+  hasPipeOperator,
+} from './abstractSyntaxTree'
 import { lexer } from './tokeniser'
 
 describe('findClosingBrace', () => {
@@ -644,3 +648,229 @@ describe('structures specific to this lang', () => {
     ])
   })
 })
+
+
+describe('testing hasPipeOperator', () => {
+  test('hasPipeOperator is true', () => {
+    let code = `sketch mySketch {
+  lineTo(2, 3)
+} |> rx(45, %)
+`
+
+    const tokens = lexer(code)
+    expect(hasPipeOperator(tokens, 0)).toEqual({
+      index: 16,
+      token: { end: 37, start: 35, type: 'operator', value: '|>' },
+    })
+  })
+  test('matches the first pipe', () => {
+    let code = `sketch mySketch {
+  lineTo(2, 3)
+} |> rx(45, %) |> rx(45, %)
+`
+    const tokens = lexer(code)
+    expect(hasPipeOperator(tokens, 0)).toEqual({
+      index: 16,
+      token: { end: 37, start: 35, type: 'operator', value: '|>' },
+    })
+  })
+  test('hasPipeOperator is false when the pipe operator is after a new variable declaration', () => {
+    let code = `sketch mySketch {
+  lineTo(2, 3)
+}
+const yo = myFunc(9()
+  |> rx(45, %)
+`
+    const tokens = lexer(code)
+    expect(hasPipeOperator(tokens, 0)).toEqual(false)
+  })
+})
+
+describe('testing pipe operator', () => {
+  test('pipe operator with sketch', () => {
+    let code = `sketch mySketch {
+  lineTo(2, 3)
+} |> rx(45, %)
+`
+    const tokens = lexer(code)
+    const { body } = abstractSyntaxTree(tokens)
+    expect(body).toEqual([
+      {
+        "type": "VariableDeclaration",
+        "start": 0,
+        "end": 47,
+        "kind": "sketch",
+        "declarations": [
+          {
+            "type": "VariableDeclarator",
+            "start": 7,
+            "end": 47,
+            "id": {
+              "type": "Identifier",
+              "start": 7,
+              "end": 15,
+              "name": "mySketch"
+            },
+            "init": {
+              "type": "PipeExpression",
+              "start": 16,
+              "end": 47,
+              "body": [
+                {
+                  "type": "SketchExpression",
+                  "start": 16,
+                  "end": 34,
+                  "body": {
+                    "type": "BlockStatement",
+                    "start": 16,
+                    "end": 34,
+                    "body": [
+                      {
+                        "type": "ExpressionStatement",
+                        "start": 20,
+                        "end": 32,
+                        "expression": {
+                          "type": "CallExpression",
+                          "start": 20,
+                          "end": 32,
+                          "callee": {
+                            "type": "Identifier",
+                            "start": 20,
+                            "end": 26,
+                            "name": "lineTo"
+                          },
+                          "arguments": [
+                            {
+                              "type": "Literal",
+                              "start": 27,
+                              "end": 28,
+                              "value": 2,
+                              "raw": "2"
+                            },
+                            {
+                              "type": "Literal",
+                              "start": 30,
+                              "end": 31,
+                              "value": 3,
+                              "raw": "3"
+                            }
+                          ],
+                          "optional": false
+                        }
+                      }
+                    ]
+                  }
+                },
+                {
+                  "type": "CallExpression",
+                  "start": 38,
+                  "end": 47,
+                  "callee": {
+                    "type": "Identifier",
+                    "start": 38,
+                    "end": 40,
+                    "name": "rx"
+                  },
+                  "arguments": [
+                    {
+                      "type": "Literal",
+                      "start": 41,
+                      "end": 43,
+                      "value": 45,
+                      "raw": "45"
+                    },
+                    {
+                      "type": "PipeSubstitution",
+                      "start": 45,
+                      "end": 46
+                    }
+                  ],
+                  "optional": false
+                }
+              ]
+            }
+          }
+        ]
+      }
+    ])
+  })
+  test('pipe operator with binary expression', () => {
+    let code = `const myVar = 5 + 6 |> myFunc(45, %)`
+    const tokens = lexer(code)
+    const { body } = abstractSyntaxTree(tokens)
+    expect(body).toEqual([
+      {
+        "type": "VariableDeclaration",
+        "start": 0,
+        "end": 36,
+        "kind": "const",
+        "declarations": [
+          {
+            "type": "VariableDeclarator",
+            "start": 6,
+            "end": 36,
+            "id": {
+              "type": "Identifier",
+              "start": 6,
+              "end": 11,
+              "name": "myVar"
+            },
+            "init": {
+              "type": "PipeExpression",
+              "start": 12,
+              "end": 36,
+              "body": [
+                {
+                  "type": "BinaryExpression",
+                  "start": 14,
+                  "end": 19,
+                  "left": {
+                    "type": "Literal",
+                    "start": 14,
+                    "end": 15,
+                    "value": 5,
+                    "raw": "5"
+                  },
+                  "operator": "+",
+                  "right": {
+                    "type": "Literal",
+                    "start": 18,
+                    "end": 19,
+                    "value": 6,
+                    "raw": "6"
+                  }
+                },
+                {
+                  "type": "CallExpression",
+                  "start": 23,
+                  "end": 36,
+                  "callee": {
+                    "type": "Identifier",
+                    "start": 23,
+                    "end": 29,
+                    "name": "myFunc"
+                  },
+                  "arguments": [
+                    {
+                      "type": "Literal",
+                      "start": 30,
+                      "end": 32,
+                      "value": 45,
+                      "raw": "45"
+                    },
+                    {
+                      "type": "PipeSubstitution",
+                      "start": 34,
+                      "end": 35
+                    }
+                  ],
+                  "optional": false
+                }
+              ]
+            }
+          }
+        ]
+      }
+    ])
+  })
+})

src/lang/abstractSyntaxTree.ts

@@ -42,6 +42,8 @@ type syntaxType =
   // | "ArrowFunctionExpression"
   | 'FunctionExpression'
   | 'SketchExpression'
+  | 'PipeExpression'
+  | 'PipeSubstitution'
   | 'YieldExpression'
   | 'AwaitExpression'
   | 'ImportDeclaration'
@@ -183,6 +185,20 @@ function makeArguments(
     const { expression, lastIndex } = makeBinaryExpression(tokens, index)
     return makeArguments(tokens, lastIndex, [...previousArgs, expression])
   }
+  if (
+    argumentToken.token.type === 'operator' &&
+    argumentToken.token.value === '%'
+  ) {
+    const value: PipeSubstitution = {
+      type: 'PipeSubstitution',
+      start: argumentToken.token.start,
+      end: argumentToken.token.end,
+    }
+    return makeArguments(tokens, nextBraceOrCommaToken.index, [
+      ...previousArgs,
+      value,
+    ])
+  }
   if (argumentToken.token.type === 'word') {
     const identifier = makeIdentifier(tokens, argumentToken.index)
     return makeArguments(tokens, nextBraceOrCommaToken.index, [
@@ -204,7 +220,7 @@ function makeArguments(
   ) {
     return makeArguments(tokens, argumentToken.index, previousArgs)
   }
-  throw new Error('Expected a previous if statement to match')
+  throw new Error('Expected a previous Argument if statement to match')
 }
 
 interface VariableDeclaration extends GeneralStatement {
@@ -251,6 +267,8 @@ export type Value =
   | FunctionExpression
   | CallExpression
   | SketchExpression
+  | PipeExpression
+  | PipeSubstitution
 
 function makeValue(
   tokens: Token[],
@@ -265,7 +283,7 @@ function makeValue(
       lastIndex,
     }
   }
-  if (currentToken.type === 'word' && nextToken.type === 'operator') {
+  if ((currentToken.type === 'word' || currentToken.type === 'number') && nextToken.type === 'operator') {
     const { expression, lastIndex } = makeBinaryExpression(tokens, index)
     return {
       value: expression,
@@ -286,7 +304,7 @@ function makeValue(
       lastIndex: index,
     }
   }
-  throw new Error('Expected a previous if statement to match')
+  throw new Error('Expected a previous Value if statement to match')
 }
 
 interface VariableDeclarator extends GeneralStatement {
@@ -303,6 +321,7 @@ function makeVariableDeclarators(
   declarations: VariableDeclarator[]
   lastIndex: number
 } {
+  const nextPipeOperator = hasPipeOperator(tokens, 0)
   const currentToken = tokens[index]
   const assignmentToken = nextMeaningfulToken(tokens, index)
   const declarationToken = previousMeaningfulToken(tokens, index)
@@ -310,7 +329,14 @@ function makeVariableDeclarators(
   const nextAfterInit = nextMeaningfulToken(tokens, contentsStartToken.index)
   let init: Value
   let lastIndex = contentsStartToken.index
-  if (
+  if (nextPipeOperator) {
+    const { expression, lastIndex: pipeLastIndex } = makePipeExpression(
+      tokens,
+      assignmentToken.index
+    )
+    init = expression
+    lastIndex = pipeLastIndex
+  } else if (
     contentsStartToken.token.type === 'brace' &&
     contentsStartToken.token.value === '('
   ) {
@@ -392,6 +418,10 @@ function makeIdentifier(token: Token[], index: number): Identifier {
   }
 }
 
+interface PipeSubstitution extends GeneralStatement {
+  type: 'PipeSubstitution'
+}
+
 function makeLiteral(tokens: Token[], index: number): Literal {
   const token = tokens[index]
   const value =
@@ -431,7 +461,7 @@ function makeBinaryPart(
       lastIndex: index,
     }
   }
-  throw new Error('Expected a previous if statement to match')
+  throw new Error('Expected a previous BinaryPart if statement to match')
 }
 
 function makeBinaryExpression(
@@ -483,6 +513,61 @@ function makeSketchExpression(
   }
 }
 
+export interface PipeExpression extends GeneralStatement {
+  type: 'PipeExpression'
+  body: Value[]
+}
+
+function makePipeExpression(
+  tokens: Token[],
+  index: number
+): { expression: PipeExpression; lastIndex: number } {
+  const currentToken = tokens[index]
+  const { body, lastIndex: bodyLastIndex } = makePipeBody(tokens, index)
+  const endToken = tokens[bodyLastIndex]
+  return {
+    expression: {
+      type: 'PipeExpression',
+      start: currentToken.start,
+      end: endToken.end,
+      body,
+    },
+    lastIndex: bodyLastIndex,
+  }
+}
+
+function makePipeBody(
+  tokens: Token[],
+  index: number,
+  previousValues: Value[] = []
+): { body: Value[]; lastIndex: number } {
+  const currentToken = tokens[index]
+  const expressionStart = nextMeaningfulToken(tokens, index)
+  let value: Value
+  let lastIndex: number
+  if (currentToken.type === 'operator') {
+    const beep = makeValue(tokens, expressionStart.index)
+    value = beep.value
+    lastIndex = beep.lastIndex
+  } else if (currentToken.type === 'brace' && currentToken.value === '{') {
+    const sketch = makeSketchExpression(tokens, index)
+    value = sketch.expression
+    lastIndex = sketch.lastIndex
+  } else {
+    throw new Error('Expected a previous PipeValue if statement to match')
+  }
+
+  const nextPipeToken = hasPipeOperator(tokens, index)
+  if (!nextPipeToken) {
+    return {
+      body: [...previousValues, value],
+      lastIndex,
+    }
+  }
+  // const nextToken = nextMeaningfulToken(tokens, nextPipeToken.index + 1)
+  return makePipeBody(tokens, nextPipeToken.index, [...previousValues, value])
+}
+
 export interface FunctionExpression extends GeneralStatement {
   type: 'FunctionExpression'
   id: Identifier | null
@@ -699,7 +784,6 @@ function makeBody(
     // return startTree(tokens, tokenIndex, [...previousBody, makeExpressionStatement(tokens, tokenIndex)]);
     return { body: [...previousBody, expression], lastIndex }
   }
-  console.log('should throw', tokens.slice(tokenIndex))
   throw new Error('Unexpected token')
 }
 export const abstractSyntaxTree = (tokens: Token[]): Program => {
@@ -713,6 +797,46 @@ export const abstractSyntaxTree = (tokens: Token[]): Program => {
   return program
 }
 
+export function findNextDeclarationKeyword(
+  tokens: Token[],
+  index: number
+): { token: Token | null; index: number } {
+  const nextToken = nextMeaningfulToken(tokens, index)
+  if (nextToken.index >= tokens.length) {
+    return { token: null, index: tokens.length - 1 }
+  }
+  if (
+    nextToken.token.type === 'word' &&
+    (nextToken.token.value === 'const' ||
+      nextToken.token.value === 'fn' ||
+      nextToken.token.value === 'sketch' ||
+      nextToken.token.value === 'path')
+  ) {
+    return nextToken
+  }
+  return findNextDeclarationKeyword(tokens, nextToken.index)
+}
+
+export function hasPipeOperator(
+  tokens: Token[],
+  index: number,
+  _limitIndex = -1
+): { token: Token; index: number } | false {
+  let limitIndex = _limitIndex
+  if (limitIndex === -1) {
+    const nextDeclaration = findNextDeclarationKeyword(tokens, index)
+    limitIndex = nextDeclaration.index
+  }
+  const nextToken = nextMeaningfulToken(tokens, index)
+  if (nextToken.index >= limitIndex) {
+    return false
+  }
+  if (nextToken.token.type === 'operator' && nextToken.token.value === '|>') {
+    return nextToken
+  }
+  return hasPipeOperator(tokens, nextToken.index, limitIndex)
+}
+
 export function findClosingBrace(
   tokens: Token[],
   index: number,

src/lang/tokeniser.test.ts

@@ -299,6 +299,39 @@ describe('testing lexer', () => {
       "whitespace   ' '        from 6   to 7",
       "number       '2.5'      from 7   to 10",
     ])
+    
+  })
+  it('testing piping operator', () => {
+    const result = stringSummaryLexer(`sketch mySketch {
+      lineTo(2, 3)
+    } |> rx(45, %)`)
+    expect(result).toEqual([
+      "word         'sketch'   from 0   to 6",
+      "whitespace   ' '        from 6   to 7",
+      "word         'mySketch' from 7   to 15",
+      "whitespace   ' '        from 15  to 16",
+      "brace        '{'        from 16  to 17",
+      "whitespace   '\n      '  from 17  to 24",
+      "word         'lineTo'   from 24  to 30",
+      "brace        '('        from 30  to 31",
+      "number       '2'        from 31  to 32",
+      "comma        ','        from 32  to 33",
+      "whitespace   ' '        from 33  to 34",
+      "number       '3'        from 34  to 35",
+      "brace        ')'        from 35  to 36",
+      "whitespace   '\n    '    from 36  to 41",
+      "brace        '}'        from 41  to 42",
+      "whitespace   ' '        from 42  to 43",
+      "operator     '|>'       from 43  to 45",
+      "whitespace   ' '        from 45  to 46",
+      "word         'rx'       from 46  to 48",
+      "brace        '('        from 48  to 49",
+      "number       '45'       from 49  to 51",
+      "comma        ','        from 51  to 52",
+      "whitespace   ' '        from 52  to 53",
+      "operator     '%'        from 53  to 54",
+      "brace        ')'        from 54  to 55"
+    ])
   })
 })
 

src/lang/tokeniser.ts

@@ -6,7 +6,7 @@ const WORD = /^[a-zA-Z_][a-zA-Z0-9_]*/
 // regex that captures everything between two non escaped quotes and the quotes aren't captured in the match
 const STRING = /^(["'])(?:(?=(\\?))\2.)*?\1/
 // verbose regex for finding operators, multiple character operators need to be first
-const OPERATOR = /^(>=|<=|==|=>|!=|\*|\+|-|\/|%|=|<|>|\||\^)/
+const OPERATOR = /^(>=|<=|==|=>|!= |\|>|\*|\+|-|\/|%|=|<|>|\||\^)/
 
 const BLOCK_START = /^\{/
 const BLOCK_END = /^\}/