diff --git a/src/lang/abstractSyntaxTree.test.ts b/src/lang/abstractSyntaxTree.test.ts index fe485bca1..d397d4ff0 100644 --- a/src/lang/abstractSyntaxTree.test.ts +++ b/src/lang/abstractSyntaxTree.test.ts @@ -1,4 +1,8 @@ -import { abstractSyntaxTree, findClosingBrace } from './abstractSyntaxTree' +import { + abstractSyntaxTree, + findClosingBrace, + hasPipeOperator, +} from './abstractSyntaxTree' import { lexer } from './tokeniser' describe('findClosingBrace', () => { @@ -644,3 +648,229 @@ describe('structures specific to this lang', () => { ]) }) }) + + +describe('testing hasPipeOperator', () => { + test('hasPipeOperator is true', () => { + let code = `sketch mySketch { + lineTo(2, 3) +} |> rx(45, %) +` + + const tokens = lexer(code) + expect(hasPipeOperator(tokens, 0)).toEqual({ + index: 16, + token: { end: 37, start: 35, type: 'operator', value: '|>' }, + }) + }) + test('matches the first pipe', () => { + let code = `sketch mySketch { + lineTo(2, 3) +} |> rx(45, %) |> rx(45, %) +` + const tokens = lexer(code) + expect(hasPipeOperator(tokens, 0)).toEqual({ + index: 16, + token: { end: 37, start: 35, type: 'operator', value: '|>' }, + }) + }) + test('hasPipeOperator is false when the pipe operator is after a new variable declaration', () => { + let code = `sketch mySketch { + lineTo(2, 3) +} +const yo = myFunc(9() + |> rx(45, %) +` + const tokens = lexer(code) + expect(hasPipeOperator(tokens, 0)).toEqual(false) + }) +}) + +describe('testing pipe operator', () => { + test('pipe operator with sketch', () => { + let code = `sketch mySketch { + lineTo(2, 3) +} |> rx(45, %) +` + const tokens = lexer(code) + const { body } = abstractSyntaxTree(tokens) + expect(body).toEqual([ + { + "type": "VariableDeclaration", + "start": 0, + "end": 47, + "kind": "sketch", + "declarations": [ + { + "type": "VariableDeclarator", + "start": 7, + "end": 47, + "id": { + "type": "Identifier", + "start": 7, + "end": 15, + "name": "mySketch" + }, + "init": { + "type": "PipeExpression", + "start": 16, + "end": 47, + "body": [ + { + "type": "SketchExpression", + "start": 16, + "end": 34, + "body": { + "type": "BlockStatement", + "start": 16, + "end": 34, + "body": [ + { + "type": "ExpressionStatement", + "start": 20, + "end": 32, + "expression": { + "type": "CallExpression", + "start": 20, + "end": 32, + "callee": { + "type": "Identifier", + "start": 20, + "end": 26, + "name": "lineTo" + }, + "arguments": [ + { + "type": "Literal", + "start": 27, + "end": 28, + "value": 2, + "raw": "2" + }, + { + "type": "Literal", + "start": 30, + "end": 31, + "value": 3, + "raw": "3" + } + ], + "optional": false + } + } + ] + } + }, + { + "type": "CallExpression", + "start": 38, + "end": 47, + "callee": { + "type": "Identifier", + "start": 38, + "end": 40, + "name": "rx" + }, + "arguments": [ + { + "type": "Literal", + "start": 41, + "end": 43, + "value": 45, + "raw": "45" + }, + { + "type": "PipeSubstitution", + "start": 45, + "end": 46 + } + ], + "optional": false + } + ] + } + } + ] + } + ]) + }) + test('pipe operator with binary expression', () => { + let code = `const myVar = 5 + 6 |> myFunc(45, %)` + const tokens = lexer(code) + const { body } = abstractSyntaxTree(tokens) + expect(body).toEqual([ + { + "type": "VariableDeclaration", + "start": 0, + "end": 36, + "kind": "const", + "declarations": [ + { + "type": "VariableDeclarator", + "start": 6, + "end": 36, + "id": { + "type": "Identifier", + "start": 6, + "end": 11, + "name": "myVar" + }, + "init": { + "type": "PipeExpression", + "start": 12, + "end": 36, + "body": [ + { + "type": "BinaryExpression", + "start": 14, + "end": 19, + "left": { + "type": "Literal", + "start": 14, + "end": 15, + "value": 5, + "raw": "5" + }, + "operator": "+", + "right": { + "type": "Literal", + "start": 18, + "end": 19, + "value": 6, + "raw": "6" + } + }, + { + "type": "CallExpression", + "start": 23, + "end": 36, + "callee": { + "type": "Identifier", + "start": 23, + "end": 29, + "name": "myFunc" + }, + "arguments": [ + { + "type": "Literal", + "start": 30, + "end": 32, + "value": 45, + "raw": "45" + }, + { + "type": "PipeSubstitution", + "start": 34, + "end": 35 + } + ], + "optional": false + } + ] + } + } + ] + } + ]) + }) +}) \ No newline at end of file diff --git a/src/lang/abstractSyntaxTree.ts b/src/lang/abstractSyntaxTree.ts index f250d8377..37b884468 100644 --- a/src/lang/abstractSyntaxTree.ts +++ b/src/lang/abstractSyntaxTree.ts @@ -42,6 +42,8 @@ type syntaxType = // | "ArrowFunctionExpression" | 'FunctionExpression' | 'SketchExpression' + | 'PipeExpression' + | 'PipeSubstitution' | 'YieldExpression' | 'AwaitExpression' | 'ImportDeclaration' @@ -183,6 +185,20 @@ function makeArguments( const { expression, lastIndex } = makeBinaryExpression(tokens, index) return makeArguments(tokens, lastIndex, [...previousArgs, expression]) } + if ( + argumentToken.token.type === 'operator' && + argumentToken.token.value === '%' + ) { + const value: PipeSubstitution = { + type: 'PipeSubstitution', + start: argumentToken.token.start, + end: argumentToken.token.end, + } + return makeArguments(tokens, nextBraceOrCommaToken.index, [ + ...previousArgs, + value, + ]) + } if (argumentToken.token.type === 'word') { const identifier = makeIdentifier(tokens, argumentToken.index) return makeArguments(tokens, nextBraceOrCommaToken.index, [ @@ -204,7 +220,7 @@ function makeArguments( ) { return makeArguments(tokens, argumentToken.index, previousArgs) } - throw new Error('Expected a previous if statement to match') + throw new Error('Expected a previous Argument if statement to match') } interface VariableDeclaration extends GeneralStatement { @@ -251,6 +267,8 @@ export type Value = | FunctionExpression | CallExpression | SketchExpression + | PipeExpression + | PipeSubstitution function makeValue( tokens: Token[], @@ -265,7 +283,7 @@ function makeValue( lastIndex, } } - if (currentToken.type === 'word' && nextToken.type === 'operator') { + if ((currentToken.type === 'word' || currentToken.type === 'number') && nextToken.type === 'operator') { const { expression, lastIndex } = makeBinaryExpression(tokens, index) return { value: expression, @@ -286,7 +304,7 @@ function makeValue( lastIndex: index, } } - throw new Error('Expected a previous if statement to match') + throw new Error('Expected a previous Value if statement to match') } interface VariableDeclarator extends GeneralStatement { @@ -303,6 +321,7 @@ function makeVariableDeclarators( declarations: VariableDeclarator[] lastIndex: number } { + const nextPipeOperator = hasPipeOperator(tokens, 0) const currentToken = tokens[index] const assignmentToken = nextMeaningfulToken(tokens, index) const declarationToken = previousMeaningfulToken(tokens, index) @@ -310,7 +329,14 @@ function makeVariableDeclarators( const nextAfterInit = nextMeaningfulToken(tokens, contentsStartToken.index) let init: Value let lastIndex = contentsStartToken.index - if ( + if (nextPipeOperator) { + const { expression, lastIndex: pipeLastIndex } = makePipeExpression( + tokens, + assignmentToken.index + ) + init = expression + lastIndex = pipeLastIndex + } else if ( contentsStartToken.token.type === 'brace' && contentsStartToken.token.value === '(' ) { @@ -392,6 +418,10 @@ function makeIdentifier(token: Token[], index: number): Identifier { } } +interface PipeSubstitution extends GeneralStatement { + type: 'PipeSubstitution' +} + function makeLiteral(tokens: Token[], index: number): Literal { const token = tokens[index] const value = @@ -431,7 +461,7 @@ function makeBinaryPart( lastIndex: index, } } - throw new Error('Expected a previous if statement to match') + throw new Error('Expected a previous BinaryPart if statement to match') } function makeBinaryExpression( @@ -483,6 +513,61 @@ function makeSketchExpression( } } +export interface PipeExpression extends GeneralStatement { + type: 'PipeExpression' + body: Value[] +} + +function makePipeExpression( + tokens: Token[], + index: number +): { expression: PipeExpression; lastIndex: number } { + const currentToken = tokens[index] + const { body, lastIndex: bodyLastIndex } = makePipeBody(tokens, index) + const endToken = tokens[bodyLastIndex] + return { + expression: { + type: 'PipeExpression', + start: currentToken.start, + end: endToken.end, + body, + }, + lastIndex: bodyLastIndex, + } +} + +function makePipeBody( + tokens: Token[], + index: number, + previousValues: Value[] = [] +): { body: Value[]; lastIndex: number } { + const currentToken = tokens[index] + const expressionStart = nextMeaningfulToken(tokens, index) + let value: Value + let lastIndex: number + if (currentToken.type === 'operator') { + const beep = makeValue(tokens, expressionStart.index) + value = beep.value + lastIndex = beep.lastIndex + } else if (currentToken.type === 'brace' && currentToken.value === '{') { + const sketch = makeSketchExpression(tokens, index) + value = sketch.expression + lastIndex = sketch.lastIndex + } else { + throw new Error('Expected a previous PipeValue if statement to match') + } + + const nextPipeToken = hasPipeOperator(tokens, index) + if (!nextPipeToken) { + return { + body: [...previousValues, value], + lastIndex, + } + } + // const nextToken = nextMeaningfulToken(tokens, nextPipeToken.index + 1) + return makePipeBody(tokens, nextPipeToken.index, [...previousValues, value]) +} + export interface FunctionExpression extends GeneralStatement { type: 'FunctionExpression' id: Identifier | null @@ -699,7 +784,6 @@ function makeBody( // return startTree(tokens, tokenIndex, [...previousBody, makeExpressionStatement(tokens, tokenIndex)]); return { body: [...previousBody, expression], lastIndex } } - console.log('should throw', tokens.slice(tokenIndex)) throw new Error('Unexpected token') } export const abstractSyntaxTree = (tokens: Token[]): Program => { @@ -713,6 +797,46 @@ export const abstractSyntaxTree = (tokens: Token[]): Program => { return program } +export function findNextDeclarationKeyword( + tokens: Token[], + index: number +): { token: Token | null; index: number } { + const nextToken = nextMeaningfulToken(tokens, index) + if (nextToken.index >= tokens.length) { + return { token: null, index: tokens.length - 1 } + } + if ( + nextToken.token.type === 'word' && + (nextToken.token.value === 'const' || + nextToken.token.value === 'fn' || + nextToken.token.value === 'sketch' || + nextToken.token.value === 'path') + ) { + return nextToken + } + return findNextDeclarationKeyword(tokens, nextToken.index) +} + +export function hasPipeOperator( + tokens: Token[], + index: number, + _limitIndex = -1 +): { token: Token; index: number } | false { + let limitIndex = _limitIndex + if (limitIndex === -1) { + const nextDeclaration = findNextDeclarationKeyword(tokens, index) + limitIndex = nextDeclaration.index + } + const nextToken = nextMeaningfulToken(tokens, index) + if (nextToken.index >= limitIndex) { + return false + } + if (nextToken.token.type === 'operator' && nextToken.token.value === '|>') { + return nextToken + } + return hasPipeOperator(tokens, nextToken.index, limitIndex) +} + export function findClosingBrace( tokens: Token[], index: number, diff --git a/src/lang/tokeniser.test.ts b/src/lang/tokeniser.test.ts index fba09b5bf..03b8a380f 100644 --- a/src/lang/tokeniser.test.ts +++ b/src/lang/tokeniser.test.ts @@ -299,6 +299,39 @@ describe('testing lexer', () => { "whitespace ' ' from 6 to 7", "number '2.5' from 7 to 10", ]) + + }) + it('testing piping operator', () => { + const result = stringSummaryLexer(`sketch mySketch { + lineTo(2, 3) + } |> rx(45, %)`) + expect(result).toEqual([ + "word 'sketch' from 0 to 6", + "whitespace ' ' from 6 to 7", + "word 'mySketch' from 7 to 15", + "whitespace ' ' from 15 to 16", + "brace '{' from 16 to 17", + "whitespace '\n ' from 17 to 24", + "word 'lineTo' from 24 to 30", + "brace '(' from 30 to 31", + "number '2' from 31 to 32", + "comma ',' from 32 to 33", + "whitespace ' ' from 33 to 34", + "number '3' from 34 to 35", + "brace ')' from 35 to 36", + "whitespace '\n ' from 36 to 41", + "brace '}' from 41 to 42", + "whitespace ' ' from 42 to 43", + "operator '|>' from 43 to 45", + "whitespace ' ' from 45 to 46", + "word 'rx' from 46 to 48", + "brace '(' from 48 to 49", + "number '45' from 49 to 51", + "comma ',' from 51 to 52", + "whitespace ' ' from 52 to 53", + "operator '%' from 53 to 54", + "brace ')' from 54 to 55" + ]) }) }) diff --git a/src/lang/tokeniser.ts b/src/lang/tokeniser.ts index 7cbcaac6c..955eb2c9c 100644 --- a/src/lang/tokeniser.ts +++ b/src/lang/tokeniser.ts @@ -6,7 +6,7 @@ const WORD = /^[a-zA-Z_][a-zA-Z0-9_]*/ // regex that captures everything between two non escaped quotes and the quotes aren't captured in the match const STRING = /^(["'])(?:(?=(\\?))\2.)*?\1/ // verbose regex for finding operators, multiple character operators need to be first -const OPERATOR = /^(>=|<=|==|=>|!=|\*|\+|-|\/|%|=|<|>|\||\^)/ +const OPERATOR = /^(>=|<=|==|=>|!= |\|>|\*|\+|-|\/|%|=|<|>|\||\^)/ const BLOCK_START = /^\{/ const BLOCK_END = /^\}/