Improved math expressions (#6)
* Improved math expressions Things are in a better state, + - / * work now for basic const var = 5 <operator> 1 Though the current method I'm using to make the ast isn't really going to work for dealing with precedence rules so some refactoring is needed going forward * get complex math expressions working with precedence including parans Node that identifiers are working, call expressions are not, that's a TODO / * % + - are working both other things like exponent and logical operators are also not working. Recasting is the most important thing to implement next * get recasting working for nested binary expressions * clean up
This commit is contained in:
Kurt Hutten
GitHub
@ -2,6 +2,7 @@ import {
|
||||
abstractSyntaxTree,
|
||||
findClosingBrace,
|
||||
hasPipeOperator,
|
||||
findEndOfBinaryExpression,
|
||||
} from './abstractSyntaxTree'
|
||||
import { lexer } from './tokeniser'
|
||||
|
||||
@ -1521,3 +1522,334 @@ describe('testing pipe operator special', () => {
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('nests binary expressions correctly', () => {
|
||||
it('it works with the simple case', () => {
|
||||
const code = `const yo = 1 + 2`
|
||||
const { body } = abstractSyntaxTree(lexer(code))
|
||||
expect(body[0]).toEqual({
|
||||
type: 'VariableDeclaration',
|
||||
start: 0,
|
||||
end: 16,
|
||||
kind: 'const',
|
||||
declarations: [
|
||||
{
|
||||
type: 'VariableDeclarator',
|
||||
start: 6,
|
||||
end: 16,
|
||||
id: {
|
||||
type: 'Identifier',
|
||||
start: 6,
|
||||
end: 8,
|
||||
name: 'yo',
|
||||
},
|
||||
init: {
|
||||
type: 'BinaryExpression',
|
||||
start: 11,
|
||||
end: 16,
|
||||
left: {
|
||||
type: 'Literal',
|
||||
start: 11,
|
||||
end: 12,
|
||||
value: 1,
|
||||
raw: '1',
|
||||
},
|
||||
operator: '+',
|
||||
right: {
|
||||
type: 'Literal',
|
||||
start: 15,
|
||||
end: 16,
|
||||
value: 2,
|
||||
raw: '2',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
it('it should nest according to precedence with multiply first', () => {
|
||||
// should be binExp { binExp { lit-1 * lit-2 } + lit}
|
||||
const code = `const yo = 1 * 2 + 3`
|
||||
const { body } = abstractSyntaxTree(lexer(code))
|
||||
expect(body[0]).toEqual({
|
||||
type: 'VariableDeclaration',
|
||||
start: 0,
|
||||
end: 20,
|
||||
kind: 'const',
|
||||
declarations: [
|
||||
{
|
||||
type: 'VariableDeclarator',
|
||||
start: 6,
|
||||
end: 20,
|
||||
id: {
|
||||
type: 'Identifier',
|
||||
start: 6,
|
||||
end: 8,
|
||||
name: 'yo',
|
||||
},
|
||||
init: {
|
||||
type: 'BinaryExpression',
|
||||
start: 11,
|
||||
end: 20,
|
||||
left: {
|
||||
type: 'BinaryExpression',
|
||||
start: 11,
|
||||
end: 16,
|
||||
left: {
|
||||
type: 'Literal',
|
||||
start: 11,
|
||||
end: 12,
|
||||
value: 1,
|
||||
raw: '1',
|
||||
},
|
||||
operator: '*',
|
||||
right: {
|
||||
type: 'Literal',
|
||||
start: 15,
|
||||
end: 16,
|
||||
value: 2,
|
||||
raw: '2',
|
||||
},
|
||||
},
|
||||
operator: '+',
|
||||
right: {
|
||||
type: 'Literal',
|
||||
start: 19,
|
||||
end: 20,
|
||||
value: 3,
|
||||
raw: '3',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
it('it should nest according to precedence with sum first', () => {
|
||||
// should be binExp { lit-1 + binExp { lit-2 * lit-3 } }
|
||||
const code = `const yo = 1 + 2 * 3`
|
||||
const { body } = abstractSyntaxTree(lexer(code))
|
||||
expect(body[0]).toEqual({
|
||||
type: 'VariableDeclaration',
|
||||
start: 0,
|
||||
end: 20,
|
||||
kind: 'const',
|
||||
declarations: [
|
||||
{
|
||||
type: 'VariableDeclarator',
|
||||
start: 6,
|
||||
end: 20,
|
||||
id: {
|
||||
type: 'Identifier',
|
||||
start: 6,
|
||||
end: 8,
|
||||
name: 'yo',
|
||||
},
|
||||
init: {
|
||||
type: 'BinaryExpression',
|
||||
start: 11,
|
||||
end: 20,
|
||||
left: {
|
||||
type: 'Literal',
|
||||
start: 11,
|
||||
end: 12,
|
||||
value: 1,
|
||||
raw: '1',
|
||||
},
|
||||
operator: '+',
|
||||
right: {
|
||||
type: 'BinaryExpression',
|
||||
start: 15,
|
||||
end: 20,
|
||||
left: {
|
||||
type: 'Literal',
|
||||
start: 15,
|
||||
end: 16,
|
||||
value: 2,
|
||||
raw: '2',
|
||||
},
|
||||
operator: '*',
|
||||
right: {
|
||||
type: 'Literal',
|
||||
start: 19,
|
||||
end: 20,
|
||||
value: 3,
|
||||
raw: '3',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
it('it should nest properly with two opperators of equal precedence', () => {
|
||||
const code = `const yo = 1 + 2 - 3`
|
||||
const { body } = abstractSyntaxTree(lexer(code))
|
||||
expect((body[0] as any).declarations[0].init).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
start: 11,
|
||||
end: 20,
|
||||
left: {
|
||||
type: 'BinaryExpression',
|
||||
start: 11,
|
||||
end: 16,
|
||||
left: {
|
||||
type: 'Literal',
|
||||
start: 11,
|
||||
end: 12,
|
||||
value: 1,
|
||||
raw: '1',
|
||||
},
|
||||
operator: '+',
|
||||
right: {
|
||||
type: 'Literal',
|
||||
start: 15,
|
||||
end: 16,
|
||||
value: 2,
|
||||
raw: '2',
|
||||
},
|
||||
},
|
||||
operator: '-',
|
||||
right: {
|
||||
type: 'Literal',
|
||||
start: 19,
|
||||
end: 20,
|
||||
value: 3,
|
||||
raw: '3',
|
||||
},
|
||||
})
|
||||
})
|
||||
it('it should nest properly with two opperators of equal (but higher) precedence', () => {
|
||||
const code = `const yo = 1 * 2 / 3`
|
||||
const { body } = abstractSyntaxTree(lexer(code))
|
||||
expect((body[0] as any).declarations[0].init).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
start: 11,
|
||||
end: 20,
|
||||
left: {
|
||||
type: 'BinaryExpression',
|
||||
start: 11,
|
||||
end: 16,
|
||||
left: {
|
||||
type: 'Literal',
|
||||
start: 11,
|
||||
end: 12,
|
||||
value: 1,
|
||||
raw: '1',
|
||||
},
|
||||
operator: '*',
|
||||
right: {
|
||||
type: 'Literal',
|
||||
start: 15,
|
||||
end: 16,
|
||||
value: 2,
|
||||
raw: '2',
|
||||
},
|
||||
},
|
||||
operator: '/',
|
||||
right: {
|
||||
type: 'Literal',
|
||||
start: 19,
|
||||
end: 20,
|
||||
value: 3,
|
||||
raw: '3',
|
||||
},
|
||||
})
|
||||
})
|
||||
it('it should nest properly with longer example', () => {
|
||||
const code = `const yo = 1 + 2 * (3 - 4) / 5 + 6`
|
||||
const { body } = abstractSyntaxTree(lexer(code))
|
||||
const init = (body[0] as any).declarations[0].init
|
||||
expect(init).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 11,
|
||||
end: 34,
|
||||
left: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 11,
|
||||
end: 30,
|
||||
left: { type: 'Literal', value: 1, raw: '1', start: 11, end: 12 },
|
||||
right: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '/',
|
||||
start: 15,
|
||||
end: 30,
|
||||
left: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '*',
|
||||
start: 15,
|
||||
end: 26,
|
||||
left: { type: 'Literal', value: 2, raw: '2', start: 15, end: 16 },
|
||||
right: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '-',
|
||||
start: 20,
|
||||
end: 25,
|
||||
left: { type: 'Literal', value: 3, raw: '3', start: 20, end: 21 },
|
||||
right: {
|
||||
type: 'Literal',
|
||||
value: 4,
|
||||
raw: '4',
|
||||
start: 24,
|
||||
end: 25,
|
||||
},
|
||||
},
|
||||
},
|
||||
right: { type: 'Literal', value: 5, raw: '5', start: 29, end: 30 },
|
||||
},
|
||||
},
|
||||
right: { type: 'Literal', value: 6, raw: '6', start: 33, end: 34 },
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('testing findEndofBinaryExpression', () => {
|
||||
it('1 + 2 * 3', () => {
|
||||
const code = `1 + 2 * 3\nconst yo = 5`
|
||||
const tokens = lexer(code)
|
||||
const end = findEndOfBinaryExpression(tokens, 0)
|
||||
expect(end).toBe(8)
|
||||
})
|
||||
it('(1 + 2) / 5 - 3', () => {
|
||||
const code = `(1 + 25) / 5 - 3\nconst yo = 5`
|
||||
const tokens = lexer(code)
|
||||
const end = findEndOfBinaryExpression(tokens, 0)
|
||||
expect(end).toBe(14)
|
||||
|
||||
// expect to have the same end if started later in the string at a legitimate place
|
||||
const indexOf5 = code.indexOf('5')
|
||||
const endStartingAtThe5 = findEndOfBinaryExpression(tokens, indexOf5)
|
||||
expect(endStartingAtThe5).toBe(end)
|
||||
})
|
||||
it('whole thing wraped: ((1 + 2) / 5 - 3)', () => {
|
||||
const code = '((1 + 2) / 5 - 3)\nconst yo = 5'
|
||||
const tokens = lexer(code)
|
||||
const end = findEndOfBinaryExpression(tokens, 0)
|
||||
expect(end).toBe(code.indexOf('3)') + 1)
|
||||
})
|
||||
it('whole thing wraped but given index after the first brace: ((1 + 2) / 5 - 3)', () => {
|
||||
const code = '((1 + 2) / 5 - 3)\nconst yo = 5'
|
||||
const tokens = lexer(code)
|
||||
const end = findEndOfBinaryExpression(tokens, 1)
|
||||
expect(end).toBe(code.indexOf('3'))
|
||||
})
|
||||
it('given the index of a small wrapped section i.e. `1 + 2` in ((1 + 2) / 5 - 3)', () => {
|
||||
const code = '((1 + 2) / 5 - 3)\nconst yo = 5'
|
||||
const tokens = lexer(code)
|
||||
const end = findEndOfBinaryExpression(tokens, 2)
|
||||
expect(end).toBe(code.indexOf('2'))
|
||||
})
|
||||
it('lots of silly nesting: (1 + 2) / (5 - (3))', () => {
|
||||
const code = '(1 + 2) / (5 - (3))\nconst yo = 5'
|
||||
const tokens = lexer(code)
|
||||
const end = findEndOfBinaryExpression(tokens, 0)
|
||||
expect(end).toBe(code.indexOf('))') + 1)
|
||||
})
|
||||
it('with pipe operator at the end', () => {
|
||||
const code = '(1 + 2) / (5 - (3))\n |> fn(%)'
|
||||
const tokens = lexer(code)
|
||||
const end = findEndOfBinaryExpression(tokens, 0)
|
||||
expect(end).toBe(code.indexOf('))') + 1)
|
||||
})
|
||||
})
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { PathToNode } from './executor'
|
||||
import { Token } from './tokeniser'
|
||||
import { parseExpression } from './astMathExpressions'
|
||||
|
||||
type syntaxType =
|
||||
| 'Program'
|
||||
@ -473,8 +474,7 @@ function makeVariableDeclarators(
|
||||
}
|
||||
}
|
||||
|
||||
export type BinaryPart = Literal | Identifier
|
||||
// | BinaryExpression
|
||||
export type BinaryPart = Literal | Identifier | BinaryExpression
|
||||
// | CallExpression
|
||||
// | MemberExpression
|
||||
// | ArrayExpression
|
||||
@ -764,7 +764,7 @@ export interface BinaryExpression extends GeneralStatement {
|
||||
function makeBinaryPart(
|
||||
tokens: Token[],
|
||||
index: number
|
||||
): { part: BinaryPart; lastIndex: number } {
|
||||
): { part: Literal | Identifier; lastIndex: number } {
|
||||
const currentToken = tokens[index]
|
||||
if (currentToken.type === 'word') {
|
||||
const identifier = makeIdentifier(tokens, index)
|
||||
@ -783,28 +783,43 @@ function makeBinaryPart(
|
||||
throw new Error('Expected a previous BinaryPart if statement to match')
|
||||
}
|
||||
|
||||
export function findEndOfBinaryExpression(
|
||||
tokens: Token[],
|
||||
index: number
|
||||
): number {
|
||||
const currentToken = tokens[index]
|
||||
if (currentToken.type === 'brace' && currentToken.value === '(') {
|
||||
const closingParenthesis = findClosingBrace(tokens, index)
|
||||
const maybeAnotherOperator = nextMeaningfulToken(tokens, closingParenthesis)
|
||||
if (
|
||||
maybeAnotherOperator?.token?.type !== 'operator' ||
|
||||
maybeAnotherOperator?.token?.value === '|>'
|
||||
) {
|
||||
return closingParenthesis
|
||||
}
|
||||
const nextRight = nextMeaningfulToken(tokens, maybeAnotherOperator.index)
|
||||
return findEndOfBinaryExpression(tokens, nextRight.index)
|
||||
}
|
||||
const maybeOperator = nextMeaningfulToken(tokens, index)
|
||||
if (
|
||||
maybeOperator?.token?.type !== 'operator' ||
|
||||
maybeOperator?.token?.value === '|>'
|
||||
) {
|
||||
return index
|
||||
}
|
||||
const nextRight = nextMeaningfulToken(tokens, maybeOperator.index)
|
||||
return findEndOfBinaryExpression(tokens, nextRight.index)
|
||||
}
|
||||
|
||||
function makeBinaryExpression(
|
||||
tokens: Token[],
|
||||
index: number
|
||||
): { expression: BinaryExpression; lastIndex: number } {
|
||||
const currentToken = tokens[index]
|
||||
const { part: left } = makeBinaryPart(tokens, index)
|
||||
const { token: operatorToken, index: operatorIndex } = nextMeaningfulToken(
|
||||
tokens,
|
||||
index
|
||||
)
|
||||
const rightToken = nextMeaningfulToken(tokens, operatorIndex)
|
||||
const { part: right } = makeBinaryPart(tokens, rightToken.index)
|
||||
const endIndex = findEndOfBinaryExpression(tokens, index)
|
||||
const expression = parseExpression(tokens.slice(index, endIndex + 1))
|
||||
return {
|
||||
expression: {
|
||||
type: 'BinaryExpression',
|
||||
start: currentToken.start,
|
||||
end: right.end,
|
||||
left,
|
||||
operator: operatorToken.value,
|
||||
right,
|
||||
},
|
||||
lastIndex: rightToken.index,
|
||||
expression,
|
||||
lastIndex: endIndex,
|
||||
}
|
||||
}
|
||||
|
||||
@ -1296,6 +1311,61 @@ export function findClosingBrace(
|
||||
return findClosingBrace(tokens, index + 1, _braceCount, searchOpeningBrace)
|
||||
}
|
||||
|
||||
// function findOpeningBrace(
|
||||
// tokens: Token[],
|
||||
// index: number,
|
||||
// _braceCount: number = 0,
|
||||
// _searchClosingBrace: string = ''
|
||||
// ): number {
|
||||
// // should be called with the index of the opening brace
|
||||
// const closingBraceMap: { [key: string]: string } = {
|
||||
// ')': '(',
|
||||
// '}': '{',
|
||||
// ']': '[',
|
||||
// }
|
||||
// const currentToken = tokens[index]
|
||||
// let searchClosingBrace = _searchClosingBrace
|
||||
|
||||
// const isFirstCall = !searchClosingBrace && _braceCount === 0
|
||||
// if (isFirstCall) {
|
||||
// searchClosingBrace = currentToken.value
|
||||
// if (![')', '}', ']'].includes(searchClosingBrace)) {
|
||||
// throw new Error(
|
||||
// `expected to be started on a opening brace ( { [, instead found '${searchClosingBrace}'`
|
||||
// )
|
||||
// }
|
||||
// }
|
||||
|
||||
// const foundOpeningBrace =
|
||||
// _braceCount === 1 &&
|
||||
// currentToken.value === closingBraceMap[searchClosingBrace]
|
||||
// const foundAnotherClosingBrace = currentToken.value === searchClosingBrace
|
||||
// const foundAnotherOpeningBrace =
|
||||
// currentToken.value === closingBraceMap[searchClosingBrace]
|
||||
|
||||
// if (foundOpeningBrace) {
|
||||
// return index
|
||||
// }
|
||||
// if (foundAnotherClosingBrace) {
|
||||
// return findOpeningBrace(
|
||||
// tokens,
|
||||
// index - 1,
|
||||
// _braceCount + 1,
|
||||
// searchClosingBrace
|
||||
// )
|
||||
// }
|
||||
// if (foundAnotherOpeningBrace) {
|
||||
// return findOpeningBrace(
|
||||
// tokens,
|
||||
// index - 1,
|
||||
// _braceCount - 1,
|
||||
// searchClosingBrace
|
||||
// )
|
||||
// }
|
||||
// // non-brace token, increment and continue
|
||||
// return findOpeningBrace(tokens, index - 1, _braceCount, searchClosingBrace)
|
||||
// }
|
||||
|
||||
function isCallExpression(tokens: Token[], index: number): number {
|
||||
const currentToken = tokens[index]
|
||||
const veryNextToken = tokens[index + 1] // i.e. no whitespace
|
||||
|
||||
204
src/lang/astMathExpressions.test.ts
Normal file
204
src/lang/astMathExpressions.test.ts
Normal file
@ -0,0 +1,204 @@
|
||||
import { parseExpression, reversePolishNotation } from './astMathExpressions'
|
||||
import { lexer } from './tokeniser'
|
||||
|
||||
describe('parseExpression', () => {
|
||||
it('parses a simple expression', () => {
|
||||
const result = parseExpression(lexer('1 + 2'))
|
||||
expect(result).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 0,
|
||||
end: 5,
|
||||
left: { type: 'Literal', value: 1, raw: '1', start: 0, end: 1 },
|
||||
right: { type: 'Literal', value: 2, raw: '2', start: 4, end: 5 },
|
||||
})
|
||||
})
|
||||
it('parses a more complex expression + followed by *', () => {
|
||||
const tokens = lexer('1 + 2 * 3')
|
||||
const result = parseExpression(tokens)
|
||||
expect(result).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 0,
|
||||
end: 9,
|
||||
left: { type: 'Literal', value: 1, raw: '1', start: 0, end: 1 },
|
||||
right: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '*',
|
||||
start: 4,
|
||||
end: 9,
|
||||
left: { type: 'Literal', value: 2, raw: '2', start: 4, end: 5 },
|
||||
right: { type: 'Literal', value: 3, raw: '3', start: 8, end: 9 },
|
||||
},
|
||||
})
|
||||
})
|
||||
it('parses a more complex expression with parentheses: 1 * ( 2 + 3 )', () => {
|
||||
const result = parseExpression(lexer('1 * ( 2 + 3 )'))
|
||||
expect(result).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
operator: '*',
|
||||
start: 0,
|
||||
end: 13,
|
||||
left: { type: 'Literal', value: 1, raw: '1', start: 0, end: 1 },
|
||||
right: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 6,
|
||||
end: 11,
|
||||
left: { type: 'Literal', value: 2, raw: '2', start: 6, end: 7 },
|
||||
right: { type: 'Literal', value: 3, raw: '3', start: 10, end: 11 },
|
||||
},
|
||||
})
|
||||
})
|
||||
it('parses a more complex expression with parentheses with more ', () => {
|
||||
const result = parseExpression(lexer('1 * ( 2 + 3 ) / 4'))
|
||||
expect(result).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
operator: '/',
|
||||
start: 0,
|
||||
end: 17,
|
||||
left: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '*',
|
||||
start: 0,
|
||||
end: 13,
|
||||
left: { type: 'Literal', value: 1, raw: '1', start: 0, end: 1 },
|
||||
right: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 6,
|
||||
end: 11,
|
||||
left: { type: 'Literal', value: 2, raw: '2', start: 6, end: 7 },
|
||||
right: { type: 'Literal', value: 3, raw: '3', start: 10, end: 11 },
|
||||
},
|
||||
},
|
||||
right: { type: 'Literal', value: 4, raw: '4', start: 16, end: 17 },
|
||||
})
|
||||
})
|
||||
it('same as last one but with a 1 + at the start ', () => {
|
||||
const result = parseExpression(lexer('1 + ( 2 + 3 ) / 4'))
|
||||
expect(result).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 0,
|
||||
end: 17,
|
||||
left: { type: 'Literal', value: 1, raw: '1', start: 0, end: 1 },
|
||||
right: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '/',
|
||||
start: 4,
|
||||
end: 17,
|
||||
left: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 6,
|
||||
end: 11,
|
||||
left: { type: 'Literal', value: 2, raw: '2', start: 6, end: 7 },
|
||||
right: { type: 'Literal', value: 3, raw: '3', start: 10, end: 11 },
|
||||
},
|
||||
right: { type: 'Literal', value: 4, raw: '4', start: 16, end: 17 },
|
||||
},
|
||||
})
|
||||
})
|
||||
it('nested braces ', () => {
|
||||
const result = parseExpression(lexer('1 * (( 2 + 3 ) / 4 + 5 )'))
|
||||
expect(result).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
operator: '*',
|
||||
start: 0,
|
||||
end: 24,
|
||||
left: { type: 'Literal', value: 1, raw: '1', start: 0, end: 1 },
|
||||
right: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 5,
|
||||
end: 22,
|
||||
left: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '/',
|
||||
start: 5,
|
||||
end: 18,
|
||||
left: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 7,
|
||||
end: 12,
|
||||
left: { type: 'Literal', value: 2, raw: '2', start: 7, end: 8 },
|
||||
right: {
|
||||
type: 'Literal',
|
||||
value: 3,
|
||||
raw: '3',
|
||||
start: 11,
|
||||
end: 12,
|
||||
},
|
||||
},
|
||||
right: { type: 'Literal', value: 4, raw: '4', start: 17, end: 18 },
|
||||
},
|
||||
right: { type: 'Literal', value: 5, raw: '5', start: 21, end: 22 },
|
||||
},
|
||||
})
|
||||
})
|
||||
it('multiple braces around the same thing ', () => {
|
||||
const result = parseExpression(lexer('1 * ((( 2 + 3 )))'))
|
||||
expect(result).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
operator: '*',
|
||||
start: 0,
|
||||
end: 17,
|
||||
left: { type: 'Literal', value: 1, raw: '1', start: 0, end: 1 },
|
||||
right: {
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 8,
|
||||
end: 13,
|
||||
left: { type: 'Literal', value: 2, raw: '2', start: 8, end: 9 },
|
||||
right: { type: 'Literal', value: 3, raw: '3', start: 12, end: 13 },
|
||||
},
|
||||
})
|
||||
})
|
||||
it('multiple braces around a sing literal', () => {
|
||||
const code = '2 + (((3)))'
|
||||
const result = parseExpression(lexer(code))
|
||||
expect(result).toEqual({
|
||||
type: 'BinaryExpression',
|
||||
operator: '+',
|
||||
start: 0,
|
||||
end: code.indexOf(')))') + 3,
|
||||
left: { type: 'Literal', value: 2, raw: '2', start: 0, end: 1 },
|
||||
right: { type: 'Literal', value: 3, raw: '3', start: 7, end: 8 },
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('reversePolishNotation', () => {
|
||||
it('converts a simple expression', () => {
|
||||
const result = reversePolishNotation(lexer('1 + 2'))
|
||||
expect(result).toEqual([
|
||||
{ type: 'number', value: '1', start: 0, end: 1 },
|
||||
{ type: 'number', value: '2', start: 4, end: 5 },
|
||||
{ type: 'operator', value: '+', start: 2, end: 3 },
|
||||
])
|
||||
})
|
||||
it('converts a more complex expression', () => {
|
||||
const result = reversePolishNotation(lexer('1 + 2 * 3'))
|
||||
expect(result).toEqual([
|
||||
{ type: 'number', value: '1', start: 0, end: 1 },
|
||||
{ type: 'number', value: '2', start: 4, end: 5 },
|
||||
{ type: 'number', value: '3', start: 8, end: 9 },
|
||||
{ type: 'operator', value: '*', start: 6, end: 7 },
|
||||
{ type: 'operator', value: '+', start: 2, end: 3 },
|
||||
])
|
||||
})
|
||||
it('converts a more complex expression with parentheses', () => {
|
||||
const result = reversePolishNotation(lexer('1 * ( 2 + 3 )'))
|
||||
expect(result).toEqual([
|
||||
{ type: 'number', value: '1', start: 0, end: 1 },
|
||||
{ type: 'brace', value: '(', start: 4, end: 5 },
|
||||
{ type: 'number', value: '2', start: 6, end: 7 },
|
||||
{ type: 'number', value: '3', start: 10, end: 11 },
|
||||
{ type: 'operator', value: '+', start: 8, end: 9 },
|
||||
{ type: 'brace', value: ')', start: 12, end: 13 },
|
||||
{ type: 'operator', value: '*', start: 2, end: 3 },
|
||||
])
|
||||
})
|
||||
})
|
||||
218
src/lang/astMathExpressions.ts
Normal file
218
src/lang/astMathExpressions.ts
Normal file
@ -0,0 +1,218 @@
|
||||
import { BinaryExpression, Literal, Identifier } from './abstractSyntaxTree'
|
||||
import { Token } from './tokeniser'
|
||||
|
||||
interface Tree {
|
||||
value: string
|
||||
left?: Tree
|
||||
right?: Tree
|
||||
}
|
||||
|
||||
export function reversePolishNotation(
|
||||
tokens: Token[],
|
||||
previousPostfix: Token[] = [],
|
||||
operators: Token[] = []
|
||||
): Token[] {
|
||||
if (tokens.length === 0) {
|
||||
return [...previousPostfix, ...operators.slice().reverse()] // reverse mutates, so clone is needed
|
||||
}
|
||||
const currentToken = tokens[0]
|
||||
if (
|
||||
currentToken.type === 'number' ||
|
||||
currentToken.type === 'word' ||
|
||||
currentToken.type === 'string'
|
||||
) {
|
||||
return reversePolishNotation(
|
||||
tokens.slice(1),
|
||||
[...previousPostfix, currentToken],
|
||||
operators
|
||||
)
|
||||
} else if (['+', '-', '*', '/', '%'].includes(currentToken.value)) {
|
||||
if (
|
||||
operators.length > 0 &&
|
||||
_precedence(operators[operators.length - 1]) >= _precedence(currentToken)
|
||||
) {
|
||||
return reversePolishNotation(
|
||||
tokens,
|
||||
[...previousPostfix, operators[operators.length - 1]],
|
||||
operators.slice(0, -1)
|
||||
)
|
||||
}
|
||||
return reversePolishNotation(tokens.slice(1), previousPostfix, [
|
||||
...operators,
|
||||
currentToken,
|
||||
])
|
||||
} else if (currentToken.value === '(') {
|
||||
// push current token to both stacks as it is a legitimate operator
|
||||
// but later we'll need to pop other operators off the stack until we find the matching ')'
|
||||
return reversePolishNotation(
|
||||
tokens.slice(1),
|
||||
[...previousPostfix, currentToken],
|
||||
[...operators, currentToken]
|
||||
)
|
||||
} else if (currentToken.value === ')') {
|
||||
if (operators[operators.length - 1]?.value !== '(') {
|
||||
// pop operators off the stack and pust them to postFix until we find the matching '('
|
||||
return reversePolishNotation(
|
||||
tokens,
|
||||
[...previousPostfix, operators[operators.length - 1]],
|
||||
operators.slice(0, -1)
|
||||
)
|
||||
}
|
||||
return reversePolishNotation(
|
||||
tokens.slice(1),
|
||||
[...previousPostfix, currentToken],
|
||||
operators.slice(0, -1)
|
||||
)
|
||||
}
|
||||
if (currentToken.type === 'whitespace') {
|
||||
return reversePolishNotation(tokens.slice(1), previousPostfix, operators)
|
||||
}
|
||||
throw new Error('Unknown token')
|
||||
}
|
||||
|
||||
interface ParenthesisToken {
|
||||
type: 'parenthesis'
|
||||
value: '(' | ')'
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
interface ExtendedBinaryExpression extends BinaryExpression {
|
||||
startExtended?: number
|
||||
endExtended?: number
|
||||
}
|
||||
|
||||
const buildTree = (
|
||||
reversePolishNotationTokens: Token[],
|
||||
stack: (
|
||||
| ExtendedBinaryExpression
|
||||
| Literal
|
||||
| Identifier
|
||||
| ParenthesisToken
|
||||
)[] = []
|
||||
): BinaryExpression => {
|
||||
if (reversePolishNotationTokens.length === 0) {
|
||||
return stack[0] as BinaryExpression
|
||||
}
|
||||
const currentToken = reversePolishNotationTokens[0]
|
||||
if (currentToken.type === 'number' || currentToken.type === 'string') {
|
||||
return buildTree(reversePolishNotationTokens.slice(1), [
|
||||
...stack,
|
||||
{
|
||||
type: 'Literal',
|
||||
value:
|
||||
currentToken.type === 'number'
|
||||
? Number(currentToken.value)
|
||||
: currentToken.value.slice(1, -1),
|
||||
raw: currentToken.value,
|
||||
start: currentToken.start,
|
||||
end: currentToken.end,
|
||||
},
|
||||
])
|
||||
} else if (currentToken.type === 'word') {
|
||||
return buildTree(reversePolishNotationTokens.slice(1), [
|
||||
...stack,
|
||||
{
|
||||
type: 'Identifier',
|
||||
name: currentToken.value,
|
||||
start: currentToken.start,
|
||||
end: currentToken.end,
|
||||
},
|
||||
])
|
||||
} else if (currentToken.type === 'brace' && currentToken.value === '(') {
|
||||
const paranToken: ParenthesisToken = {
|
||||
type: 'parenthesis',
|
||||
value: '(',
|
||||
start: currentToken.start,
|
||||
end: currentToken.end,
|
||||
}
|
||||
return buildTree(reversePolishNotationTokens.slice(1), [
|
||||
...stack,
|
||||
paranToken,
|
||||
])
|
||||
} else if (currentToken.type === 'brace' && currentToken.value === ')') {
|
||||
const innerNode = stack[stack.length - 1]
|
||||
|
||||
const paran = stack[stack.length - 2]
|
||||
|
||||
const binExp: ExtendedBinaryExpression = {
|
||||
...innerNode,
|
||||
startExtended: paran.start,
|
||||
endExtended: currentToken.end,
|
||||
} as ExtendedBinaryExpression
|
||||
|
||||
return buildTree(reversePolishNotationTokens.slice(1), [
|
||||
...stack.slice(0, -2),
|
||||
binExp,
|
||||
])
|
||||
}
|
||||
|
||||
const left = { ...stack[stack.length - 2] }
|
||||
let start = left.start
|
||||
if (left.type === 'BinaryExpression') {
|
||||
start = left?.startExtended || left.start
|
||||
delete left.startExtended
|
||||
delete left.endExtended
|
||||
}
|
||||
|
||||
const right = { ...stack[stack.length - 1] }
|
||||
let end = right.end
|
||||
if (right.type === 'BinaryExpression') {
|
||||
end = right?.endExtended || right.end
|
||||
delete right.startExtended
|
||||
delete right.endExtended
|
||||
}
|
||||
|
||||
const binExp: BinaryExpression = {
|
||||
type: 'BinaryExpression',
|
||||
operator: currentToken.value,
|
||||
start,
|
||||
end,
|
||||
left: left as any,
|
||||
right: right as any,
|
||||
}
|
||||
return buildTree(reversePolishNotationTokens.slice(1), [
|
||||
...stack.slice(0, -2),
|
||||
binExp,
|
||||
])
|
||||
}
|
||||
|
||||
export function parseExpression(tokens: Token[]): BinaryExpression {
|
||||
const treeWithMabyeBadTopLevelStartEnd = buildTree(
|
||||
reversePolishNotation(tokens)
|
||||
)
|
||||
const left = treeWithMabyeBadTopLevelStartEnd?.left as any
|
||||
const start = left?.startExtended || treeWithMabyeBadTopLevelStartEnd?.start
|
||||
delete left.startExtended
|
||||
delete left.endExtended
|
||||
|
||||
const right = treeWithMabyeBadTopLevelStartEnd?.right as any
|
||||
const end = right?.endExtended || treeWithMabyeBadTopLevelStartEnd?.end
|
||||
delete right.startExtended
|
||||
delete right.endExtended
|
||||
|
||||
const tree: BinaryExpression = {
|
||||
...treeWithMabyeBadTopLevelStartEnd,
|
||||
start,
|
||||
end,
|
||||
left,
|
||||
right,
|
||||
}
|
||||
return tree
|
||||
}
|
||||
|
||||
function _precedence(operator: Token): number {
|
||||
return precedence(operator.value)
|
||||
}
|
||||
|
||||
export function precedence(operator: string): number {
|
||||
// might be useful for refenecne to make it match
|
||||
// another commonly used lang https://www.w3schools.com/js/js_precedence.asp
|
||||
if (['+', '-'].includes(operator)) {
|
||||
return 11
|
||||
} else if (['*', '/', '%'].includes(operator)) {
|
||||
return 12
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
@ -296,6 +296,63 @@ show(mySketch)
|
||||
})
|
||||
})
|
||||
|
||||
describe('testing math operators', () => {
|
||||
it('it can sum', () => {
|
||||
const code = ['const myVar = 1 + 2'].join('\n')
|
||||
const { root } = exe(code)
|
||||
expect(root.myVar.value).toBe(3)
|
||||
})
|
||||
it('it can subtract', () => {
|
||||
const code = ['const myVar = 1 - 2'].join('\n')
|
||||
const { root } = exe(code)
|
||||
expect(root.myVar.value).toBe(-1)
|
||||
})
|
||||
it('it can multiply', () => {
|
||||
const code = ['const myVar = 1 * 2'].join('\n')
|
||||
const { root } = exe(code)
|
||||
expect(root.myVar.value).toBe(2)
|
||||
})
|
||||
it('it can divide', () => {
|
||||
const code = ['const myVar = 1 / 2'].join('\n')
|
||||
const { root } = exe(code)
|
||||
expect(root.myVar.value).toBe(0.5)
|
||||
})
|
||||
it('it can modulus', () => {
|
||||
const code = ['const myVar = 5 % 2'].join('\n')
|
||||
const { root } = exe(code)
|
||||
expect(root.myVar.value).toBe(1)
|
||||
})
|
||||
it('it can do multiple operations', () => {
|
||||
const code = ['const myVar = 1 + 2 * 3'].join('\n')
|
||||
const { root } = exe(code)
|
||||
expect(root.myVar.value).toBe(7)
|
||||
})
|
||||
it('big example with parans', () => {
|
||||
const code = ['const myVar = 1 + 2 * (3 - 4) / -5 + 6'].join('\n')
|
||||
const { root } = exe(code)
|
||||
expect(root.myVar.value).toBe(7.4)
|
||||
})
|
||||
it('with identifier', () => {
|
||||
const code = ['const yo = 6', 'const myVar = yo / 2'].join('\n')
|
||||
const { root } = exe(code)
|
||||
expect(root.myVar.value).toBe(3)
|
||||
})
|
||||
it('with identifier', () => {
|
||||
const code = ['const myVar = 2 * ((2 + 3 ) / 4 + 5)'].join('\n')
|
||||
const { root } = exe(code)
|
||||
expect(root.myVar.value).toBe(12.5)
|
||||
})
|
||||
// TODO
|
||||
// it('with callExpression', () => {
|
||||
// const code = [
|
||||
// 'const yo = (a) => a * 2',
|
||||
// 'const myVar = yo(2) + 2'
|
||||
// ].join('\n')
|
||||
// const { root } = exe(code)
|
||||
// expect(root.myVar.value).toBe(6)
|
||||
// })
|
||||
})
|
||||
|
||||
// helpers
|
||||
|
||||
function exe(
|
||||
|
||||
@ -407,16 +407,22 @@ function getBinaryExpressionResult(
|
||||
expression: BinaryExpression,
|
||||
programMemory: ProgramMemory
|
||||
) {
|
||||
const getVal = (part: BinaryPart) => {
|
||||
const getVal = (part: BinaryPart): any => {
|
||||
if (part.type === 'Literal') {
|
||||
return part.value
|
||||
} else if (part.type === 'Identifier') {
|
||||
return programMemory.root[part.name].value
|
||||
} else if (part.type === 'BinaryExpression') {
|
||||
return getBinaryExpressionResult(part, programMemory)
|
||||
}
|
||||
}
|
||||
const left = getVal(expression.left)
|
||||
const right = getVal(expression.right)
|
||||
return left + right
|
||||
if (expression.operator === '+') return left + right
|
||||
if (expression.operator === '-') return left - right
|
||||
if (expression.operator === '*') return left * right
|
||||
if (expression.operator === '/') return left / right
|
||||
if (expression.operator === '%') return left % right
|
||||
}
|
||||
|
||||
function getPipeExpressionResult(
|
||||
|
||||
@ -99,6 +99,36 @@ show(mySketch)
|
||||
const recasted = recast(ast)
|
||||
expect(recasted).toBe(code.trim())
|
||||
})
|
||||
it('recast nested binary expression', () => {
|
||||
const code = ['const myVar = 1 + 2 * 5'].join('\n')
|
||||
const { ast } = code2ast(code)
|
||||
const recasted = recast(ast)
|
||||
expect(recasted).toBe(code.trim())
|
||||
})
|
||||
it('recast nested binary expression with parans', () => {
|
||||
const code = ['const myVar = 1 + (1 + 2) * 5'].join('\n')
|
||||
const { ast } = code2ast(code)
|
||||
const recasted = recast(ast)
|
||||
expect(recasted).toBe(code.trim())
|
||||
})
|
||||
it('unnecessary paran wrap will be remove', () => {
|
||||
const code = ['const myVar = 1 + (2 * 5)'].join('\n')
|
||||
const { ast } = code2ast(code)
|
||||
const recasted = recast(ast)
|
||||
expect(recasted).toBe(code.replace('(', '').replace(')', ''))
|
||||
})
|
||||
it('complex nested binary expression', () => {
|
||||
const code = ['1 * ((2 + 3) / 4 + 5)'].join('\n')
|
||||
const { ast } = code2ast(code)
|
||||
const recasted = recast(ast)
|
||||
expect(recasted).toBe(code.trim())
|
||||
})
|
||||
it('multiplied paren expressions', () => {
|
||||
const code = ['3 + (1 + 2) * (3 + 4)'].join('\n')
|
||||
const { ast } = code2ast(code)
|
||||
const recasted = recast(ast)
|
||||
expect(recasted).toBe(code.trim())
|
||||
})
|
||||
it('recast array declaration', () => {
|
||||
const code = ['const three = 3', "const yo = [1, '2', three, 4 + 5]"].join(
|
||||
'\n'
|
||||
|
||||
@ -11,6 +11,7 @@ import {
|
||||
ObjectExpression,
|
||||
MemberExpression,
|
||||
} from './abstractSyntaxTree'
|
||||
import { precedence } from './astMathExpressions'
|
||||
|
||||
export function recast(
|
||||
ast: Program,
|
||||
@ -54,9 +55,18 @@ export function recast(
|
||||
}
|
||||
|
||||
function recastBinaryExpression(expression: BinaryExpression): string {
|
||||
return `${recastBinaryPart(expression.left)} ${
|
||||
const maybeWrapIt = (a: string, doit: boolean) => (doit ? `(${a})` : a)
|
||||
|
||||
const shouldWrapRight =
|
||||
expression.right.type === 'BinaryExpression' &&
|
||||
precedence(expression.operator) > precedence(expression.right.operator)
|
||||
const shouldWrapLeft =
|
||||
expression.left.type === 'BinaryExpression' &&
|
||||
precedence(expression.operator) > precedence(expression.left.operator)
|
||||
|
||||
return `${maybeWrapIt(recastBinaryPart(expression.left), shouldWrapLeft)} ${
|
||||
expression.operator
|
||||
} ${recastBinaryPart(expression.right)}`
|
||||
} ${maybeWrapIt(recastBinaryPart(expression.right), shouldWrapRight)}`
|
||||
}
|
||||
|
||||
function recastArrayExpression(
|
||||
@ -102,8 +112,11 @@ function recastBinaryPart(part: BinaryPart): string {
|
||||
return recastLiteral(part)
|
||||
} else if (part.type === 'Identifier') {
|
||||
return part.name
|
||||
} else if (part.type === 'BinaryExpression') {
|
||||
return recastBinaryExpression(part)
|
||||
}
|
||||
throw new Error(`Cannot recast BinaryPart ${part}`)
|
||||
return ''
|
||||
// throw new Error(`Cannot recast BinaryPart ${part}`)
|
||||
}
|
||||
|
||||
function recastLiteral(literal: Literal): string {
|
||||
|
||||
Reference in New Issue
Block a user