diff --git a/src/App.tsx b/src/App.tsx index 51426b40a..01d348a66 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -42,7 +42,6 @@ function App() { setError, errorState, setProgramMemory, - tokens, } = useStore((s) => ({ editorView: s.editorView, setEditorView: s.setEditorView, @@ -61,7 +60,6 @@ function App() { setError: s.setError, errorState: s.errorState, setProgramMemory: s.setProgramMemory, - tokens: s.tokens, })) // const onChange = React.useCallback((value: string, viewUpdate: ViewUpdate) => { const onChange = (value: string, viewUpdate: ViewUpdate) => { @@ -91,7 +89,7 @@ function App() { } const tokens = lexer(code) const _ast = abstractSyntaxTree(tokens) - setAst(_ast, tokens) + setAst(_ast) const programMemory = executor(_ast, { root: { log: { @@ -144,7 +142,7 @@ function App() { }, [code]) const shouldFormat = useMemo(() => { if (!ast) return false - const recastedCode = recast(ast, tokens) + const recastedCode = recast(ast) return recastedCode !== code }, [code, ast]) return ( diff --git a/src/components/BasePlanes.tsx b/src/components/BasePlanes.tsx index 832d9e3b2..b659ac5b0 100644 --- a/src/components/BasePlanes.tsx +++ b/src/components/BasePlanes.tsx @@ -52,6 +52,7 @@ export const BasePlanes = () => { start: 0, end: 0, body: [], + nonCodeMeta: {}, } const axis = axisIndex === 0 ? 'xy' : axisIndex === 1 ? 'xz' : 'yz' const quaternion = new Quaternion() diff --git a/src/components/SketchPlane.tsx b/src/components/SketchPlane.tsx index 00705b4d8..9b7751f86 100644 --- a/src/components/SketchPlane.tsx +++ b/src/components/SketchPlane.tsx @@ -62,6 +62,7 @@ export const SketchPlane = () => { start: 0, end: 0, body: [], + nonCodeMeta: {}, } const addLinePoint: [number, number] = [point.x, point.y] const { modifiedAst } = addLine( diff --git a/src/lang/abstractSyntaxTree.test.ts b/src/lang/abstractSyntaxTree.test.ts index 1b5a7fab7..ebccefd08 100644 --- a/src/lang/abstractSyntaxTree.test.ts +++ b/src/lang/abstractSyntaxTree.test.ts @@ -29,6 +29,7 @@ describe('testing AST', () => { test('test 5 + 6', () => { const tokens = lexer('5 +6') const result = abstractSyntaxTree(tokens) + delete (result as any).nonCodeMeta expect(result).toEqual({ type: 'Program', start: 0, @@ -219,6 +220,7 @@ describe('testing function declaration', () => { test('fn funcN = () => {}', () => { const tokens = lexer('fn funcN = () => {}') const { body } = abstractSyntaxTree(tokens) + delete (body[0] as any).declarations[0].init.body.nonCodeMeta expect(body).toEqual([ { type: 'VariableDeclaration', @@ -259,6 +261,7 @@ describe('testing function declaration', () => { ['fn funcN = (a, b) => {', ' return a + b', '}'].join('\n') ) const { body } = abstractSyntaxTree(tokens) + delete (body[0] as any).declarations[0].init.body.nonCodeMeta expect(body).toEqual([ { type: 'VariableDeclaration', @@ -337,6 +340,7 @@ describe('testing function declaration', () => { const myVar = funcN(1, 2)` ) const { body } = abstractSyntaxTree(tokens) + delete (body[0] as any).declarations[0].init.body.nonCodeMeta expect(body).toEqual([ { type: 'VariableDeclaration', @@ -469,6 +473,7 @@ describe('structures specific to this lang', () => { ` const tokens = lexer(code) const { body } = abstractSyntaxTree(tokens) + delete (body[0] as any).declarations[0].init.body.nonCodeMeta expect(body).toEqual([ { type: 'VariableDeclaration', @@ -657,7 +662,9 @@ describe('testing hasPipeOperator', () => { ` const tokens = lexer(code) - expect(hasPipeOperator(tokens, 0)).toEqual({ + const result = hasPipeOperator(tokens, 0) + delete (result as any).bonusNonCodeNode + expect(result).toEqual({ index: 16, token: { end: 37, start: 35, type: 'operator', value: '|>' }, }) @@ -669,6 +676,7 @@ describe('testing hasPipeOperator', () => { ` const tokens = lexer(code) const result = hasPipeOperator(tokens, 0) + delete (result as any).bonusNonCodeNode expect(result).toEqual({ index: 16, token: { end: 37, start: 35, type: 'operator', value: '|>' }, @@ -690,6 +698,7 @@ const yo = myFunc(9() let code = `const myVar2 = 5 + 1 |> myFn(%)` const tokens = lexer(code) const result = hasPipeOperator(tokens, 1) + delete (result as any).bonusNonCodeNode expect(result).toEqual({ index: 12, token: { end: 23, start: 21, type: 'operator', value: '|>' }, @@ -718,6 +727,7 @@ const yo = myFunc(9() const braceTokenIndex = tokens.findIndex(({ value }) => value === '{') const result2 = hasPipeOperator(tokens, braceTokenIndex) + delete (result2 as any).bonusNonCodeNode expect(result2).toEqual({ index: 36, token: { end: 76, start: 74, type: 'operator', value: '|>' }, @@ -737,6 +747,8 @@ describe('testing pipe operator special', () => { ` const tokens = lexer(code) const { body } = abstractSyntaxTree(tokens) + delete (body[0] as any).declarations[0].init.nonCodeMeta + delete (body[0] as any).declarations[0].init.body[0].body.nonCodeMeta expect(body).toEqual([ { type: 'VariableDeclaration', @@ -921,6 +933,7 @@ describe('testing pipe operator special', () => { let code = `const myVar = 5 + 6 |> myFunc(45, %)` const tokens = lexer(code) const { body } = abstractSyntaxTree(tokens) + delete (body as any)[0].declarations[0].init.nonCodeMeta expect(body).toEqual([ { type: 'VariableDeclaration', @@ -1804,6 +1817,76 @@ describe('nests binary expressions correctly', () => { }) }) +describe('check nonCodeMeta data is attached to the AST correctly', () => { + it('comments between expressions', () => { + const code = ` +const yo = { a: { b: { c: '123' } } } +// this is a comment +const key = 'c'` + const nonCodeMetaInstance = { + type: 'NoneCodeNode', + start: code.indexOf('\n// this is a comment'), + end: code.indexOf('const key'), + value: '\n// this is a comment\n', + } + const { nonCodeMeta } = abstractSyntaxTree(lexer(code)) + expect(nonCodeMeta[0]).toEqual(nonCodeMetaInstance) + + // extra whitespace won't change it's position (0) or value (NB the start end would have changed though) + const codeWithExtraStartWhitespace = '\n\n\n' + code + const { nonCodeMeta: nonCodeMeta2 } = abstractSyntaxTree( + lexer(codeWithExtraStartWhitespace) + ) + expect(nonCodeMeta2[0].value).toBe(nonCodeMetaInstance.value) + expect(nonCodeMeta2[0].start).not.toBe(nonCodeMetaInstance.start) + }) + it('comments nested within a block statement', () => { + const code = `sketch mySketch { + path myPath = lineTo(0,1) + lineTo(1,1) /* this is + a comment + spanning a few lines */ + path rightPath = lineTo(1,0) + close() + } + ` + + const { body } = abstractSyntaxTree(lexer(code)) + const indexOfSecondLineToExpression = 1 // 0 index so `path myPath = lineTo(0,1)` is 0 + const sketchNonCodeMeta = (body as any)[0].declarations[0].init.body + .nonCodeMeta + expect(sketchNonCodeMeta[indexOfSecondLineToExpression]).toEqual({ + type: 'NoneCodeNode', + start: 67, + end: 133, + value: + ' /* this is \n a comment \n spanning a few lines */\n ', + }) + }) + it('comments in a pipe expression', () => { + const code = [ + 'sketch mySk1 {', + ' lineTo(1, 1)', + ' path myPath = lineTo(0, 1)', + ' lineTo(1, 1)', + '}', + '// a comment', + ' |> rx(90, %)', + ].join('\n') + + const { body } = abstractSyntaxTree(lexer(code)) + const bing = abstractSyntaxTree(lexer(code)) + const sketchNonCodeMeta = (body[0] as any).declarations[0].init.nonCodeMeta + expect(1).toBe(1) + expect(sketchNonCodeMeta[0]).toEqual({ + type: 'NoneCodeNode', + start: 75, + end: 91, + value: '\n// a comment\n ', + }) + }) +}) + describe('testing findEndofBinaryExpression', () => { it('1 + 2 * 3', () => { const code = `1 + 2 * 3\nconst yo = 5` @@ -1853,91 +1936,3 @@ describe('testing findEndofBinaryExpression', () => { expect(end).toBe(code.indexOf('))') + 1) }) }) - -describe('testing code with comments', () => { - it('should ignore line comments', () => { - const comment = '// this is a comment' - const codeWithComment = `const yo = 5 -${comment} -const yo2 = 6` - // filling with extra whitespace to make the source start end numbers match - const codeWithoutComment = `const yo = 5 -${comment - .split('') - .map(() => ' ') - .join('')} -const yo2 = 6` - const { body } = abstractSyntaxTree(lexer(codeWithComment)) - const { body: bodyWithoutComment } = abstractSyntaxTree( - lexer(codeWithoutComment) - ) - expect(body).toEqual(bodyWithoutComment) - }) - it('should ignore block comments', () => { - const comment = `/* this is a -multi line -comment */` - const codeWithComment = `const yo = 5${comment} -const yo2 = 6` - // filling with extra whitespace to make the source start end numbers match - const codeWithoutComment = `const yo = 5${comment - .split('') - .map(() => ' ') - .join('')} -const yo2 = 6` - const { body } = abstractSyntaxTree(lexer(codeWithComment)) - const { body: bodyWithoutComment } = abstractSyntaxTree( - lexer(codeWithoutComment) - ) - expect(body).toEqual(bodyWithoutComment) - }) - it('comment in function declaration', () => { - const code = `const yo=(a)=>{ - // this is a comment - return a -}` - const { body } = abstractSyntaxTree(lexer(code)) - const yo = [ - { - type: 'VariableDeclaration', - start: 0, - end: 51, - kind: 'const', - declarations: [ - { - type: 'VariableDeclarator', - start: 6, - end: 51, - id: { type: 'Identifier', start: 6, end: 8, name: 'yo' }, - init: { - type: 'FunctionExpression', - start: 9, - end: 51, - id: null, - params: [{ type: 'Identifier', start: 10, end: 11, name: 'a' }], - body: { - type: 'BlockStatement', - start: 14, - end: 51, - body: [ - { - type: 'ReturnStatement', - start: 41, - end: 49, - argument: { - type: 'Identifier', - start: 48, - end: 49, - name: 'a', - }, - }, - ], - }, - }, - }, - ], - }, - ] - expect(body).toEqual(yo) - }) -}) diff --git a/src/lang/abstractSyntaxTree.ts b/src/lang/abstractSyntaxTree.ts index fe7c1ffd6..9ea6cc684 100644 --- a/src/lang/abstractSyntaxTree.ts +++ b/src/lang/abstractSyntaxTree.ts @@ -21,6 +21,7 @@ type syntaxType = | 'PipeExpression' | 'PipeSubstitution' | 'Literal' + | 'NoneCodeNode' // | 'NumberLiteral' // | 'StringLiteral' // | 'IfStatement' @@ -84,6 +85,7 @@ export interface Program { start: number end: number body: BodyItem[] + nonCodeMeta: NoneCodeMeta } interface GeneralStatement { type: syntaxType @@ -91,6 +93,44 @@ interface GeneralStatement { end: number } +interface NoneCodeNode extends GeneralStatement { + type: 'NoneCodeNode' + value: string +} + +interface NoneCodeMeta { + // Stores the whitespace/comments that go after the statement who's index we're using here + [statementIndex: number]: NoneCodeNode + // Which is why we also need `start` for and whitespace at the start of the file/block + start?: NoneCodeNode +} + +function makeNoneCodeNode( + tokens: Token[], + index: number +): { node?: NoneCodeNode; lastIndex: number } { + const currentToken = tokens[index] + const endIndex = findEndOfNonCodeNode(tokens, index) + const nonCodeTokens = tokens.slice(index, endIndex) + let value = nonCodeTokens.map((t) => t.value).join('') + + const node: NoneCodeNode = { + type: 'NoneCodeNode', + start: currentToken.start, + end: tokens[endIndex - 1].end, + value, + } + return { node, lastIndex: endIndex - 1 } +} + +export function findEndOfNonCodeNode(tokens: Token[], index: number): number { + const currentToken = tokens[index] + if (isNotCodeToken(currentToken)) { + return findEndOfNonCodeNode(tokens, index + 1) + } + return index +} + export interface ExpressionStatement extends GeneralStatement { type: 'ExpressionStatement' expression: Value @@ -828,6 +868,7 @@ function makeSketchExpression( export interface PipeExpression extends GeneralStatement { type: 'PipeExpression' body: Value[] + nonCodeMeta: NoneCodeMeta } function makePipeExpression( @@ -835,7 +876,11 @@ function makePipeExpression( index: number ): { expression: PipeExpression; lastIndex: number } { const currentToken = tokens[index] - const { body, lastIndex: bodyLastIndex } = makePipeBody(tokens, index) + const { + body, + lastIndex: bodyLastIndex, + nonCodeMeta, + } = makePipeBody(tokens, index) const endToken = tokens[bodyLastIndex] return { expression: { @@ -843,6 +888,7 @@ function makePipeExpression( start: currentToken.start, end: endToken.end, body, + nonCodeMeta, }, lastIndex: bodyLastIndex, } @@ -851,8 +897,10 @@ function makePipeExpression( function makePipeBody( tokens: Token[], index: number, - previousValues: Value[] = [] -): { body: Value[]; lastIndex: number } { + previousValues: Value[] = [], + previousNonCodeMeta: NoneCodeMeta = {} +): { body: Value[]; lastIndex: number; nonCodeMeta: NoneCodeMeta } { + const nonCodeMeta = { ...previousNonCodeMeta } const currentToken = tokens[index] const expressionStart = nextMeaningfulToken(tokens, index) let value: Value @@ -874,10 +922,18 @@ function makePipeBody( return { body: [...previousValues, value], lastIndex, + nonCodeMeta, } } - // const nextToken = nextMeaningfulToken(tokens, nextPipeToken.index + 1) - return makePipeBody(tokens, nextPipeToken.index, [...previousValues, value]) + if (nextPipeToken.bonusNonCodeNode) { + nonCodeMeta[previousValues.length] = nextPipeToken.bonusNonCodeNode + } + return makePipeBody( + tokens, + nextPipeToken.index, + [...previousValues, value], + nonCodeMeta + ) } export interface FunctionExpression extends GeneralStatement { @@ -938,6 +994,7 @@ function makeParams( export interface BlockStatement extends GeneralStatement { type: 'BlockStatement' body: BodyItem[] + nonCodeMeta: NoneCodeMeta } function makeBlockStatement( @@ -945,10 +1002,10 @@ function makeBlockStatement( index: number ): { block: BlockStatement; lastIndex: number } { const openingCurly = tokens[index] - const nextToken = nextMeaningfulToken(tokens, index) - const { body, lastIndex } = + const nextToken = { token: tokens[index + 1], index: index + 1 } + const { body, lastIndex, nonCodeMeta } = nextToken.token.value === '}' - ? { body: [], lastIndex: nextToken.index } + ? { body: [], lastIndex: nextToken.index, nonCodeMeta: {} } : makeBody({ tokens, tokenIndex: nextToken.index }) return { block: { @@ -956,6 +1013,7 @@ function makeBlockStatement( start: openingCurly.start, end: tokens[lastIndex]?.end || 0, body, + nonCodeMeta, }, lastIndex, } @@ -986,18 +1044,24 @@ function makeReturnStatement( export type All = Program | ExpressionStatement[] | BinaryExpression | Literal -function nextMeaningfulToken( +export function nextMeaningfulToken( tokens: Token[], index: number, offset: number = 1 -): { token: Token; index: number } { +): { token: Token; index: number; bonusNonCodeNode?: NoneCodeNode } { const newIndex = index + offset const token = tokens[newIndex] if (!token) { return { token, index: tokens.length } } if (isNotCodeToken(token)) { - return nextMeaningfulToken(tokens, index, offset + 1) + const nonCodeNode = makeNoneCodeNode(tokens, newIndex) + const newnewIndex = nonCodeNode.lastIndex + 1 + return { + token: tokens[newnewIndex], + index: newnewIndex, + bonusNonCodeNode: nonCodeNode?.node?.value ? nonCodeNode.node : undefined, + } } return { token, index: newIndex } } @@ -1018,10 +1082,7 @@ function previousMeaningfulToken( return { token, index: newIndex } } -export type BodyItem = - | ExpressionStatement - | VariableDeclaration - | ReturnStatement +type BodyItem = ExpressionStatement | VariableDeclaration | ReturnStatement function makeBody( { @@ -1031,23 +1092,37 @@ function makeBody( tokens: Token[] tokenIndex?: number }, - previousBody: BodyItem[] = [] -): { body: BodyItem[]; lastIndex: number } { + previousBody: BodyItem[] = [], + previousNonCodeMeta: NoneCodeMeta = {} +): { body: BodyItem[]; lastIndex: number; nonCodeMeta: NoneCodeMeta } { + const nonCodeMeta = { ...previousNonCodeMeta } if (tokenIndex >= tokens.length) { - return { body: previousBody, lastIndex: tokenIndex } + return { body: previousBody, lastIndex: tokenIndex, nonCodeMeta } } const token = tokens[tokenIndex] if (token.type === 'brace' && token.value === '}') { - return { body: previousBody, lastIndex: tokenIndex } - } - if (typeof token === 'undefined') { - console.log('probably should throw') + return { body: previousBody, lastIndex: tokenIndex, nonCodeMeta } } if (isNotCodeToken(token)) { - return makeBody({ tokens, tokenIndex: tokenIndex + 1 }, previousBody) + const nextToken = nextMeaningfulToken(tokens, tokenIndex, 0) + if (nextToken.bonusNonCodeNode) { + if (previousBody.length === 0) { + nonCodeMeta.start = nextToken.bonusNonCodeNode + } else { + nonCodeMeta[previousBody.length] = nextToken.bonusNonCodeNode + } + } + return makeBody( + { tokens, tokenIndex: nextToken.index }, + previousBody, + nonCodeMeta + ) } const nextToken = nextMeaningfulToken(tokens, tokenIndex) + nextToken.bonusNonCodeNode && + (nonCodeMeta[previousBody.length] = nextToken.bonusNonCodeNode) + if ( token.type === 'word' && (token.value === 'const' || @@ -1060,18 +1135,26 @@ function makeBody( tokenIndex ) const nextThing = nextMeaningfulToken(tokens, lastIndex) - return makeBody({ tokens, tokenIndex: nextThing.index }, [ - ...previousBody, - declaration, - ]) + nextThing.bonusNonCodeNode && + (nonCodeMeta[previousBody.length] = nextThing.bonusNonCodeNode) + + return makeBody( + { tokens, tokenIndex: nextThing.index }, + [...previousBody, declaration], + nonCodeMeta + ) } if (token.type === 'word' && token.value === 'return') { const { statement, lastIndex } = makeReturnStatement(tokens, tokenIndex) const nextThing = nextMeaningfulToken(tokens, lastIndex) - return makeBody({ tokens, tokenIndex: nextThing.index }, [ - ...previousBody, - statement, - ]) + nextThing.bonusNonCodeNode && + (nonCodeMeta[previousBody.length] = nextThing.bonusNonCodeNode) + + return makeBody( + { tokens, tokenIndex: nextThing.index }, + [...previousBody, statement], + nonCodeMeta + ) } if ( token.type === 'word' && @@ -1083,31 +1166,44 @@ function makeBody( tokenIndex ) const nextThing = nextMeaningfulToken(tokens, lastIndex) - return makeBody({ tokens, tokenIndex: nextThing.index }, [ - ...previousBody, - expression, - ]) + if (nextThing.bonusNonCodeNode) { + nonCodeMeta[previousBody.length] = nextThing.bonusNonCodeNode + } + + return makeBody( + { tokens, tokenIndex: nextThing.index }, + [...previousBody, expression], + nonCodeMeta + ) } + const nextThing = nextMeaningfulToken(tokens, tokenIndex) if ( (token.type === 'number' || token.type === 'word') && - nextMeaningfulToken(tokens, tokenIndex).token.type === 'operator' + nextThing.token.type === 'operator' ) { + if (nextThing.bonusNonCodeNode) { + nonCodeMeta[previousBody.length] = nextThing.bonusNonCodeNode + } const { expression, lastIndex } = makeExpressionStatement( tokens, tokenIndex ) - // return startTree(tokens, tokenIndex, [...previousBody, makeExpressionStatement(tokens, tokenIndex)]); - return { body: [...previousBody, expression], lastIndex } + return { + body: [...previousBody, expression], + nonCodeMeta: nonCodeMeta, + lastIndex, + } } throw new Error('Unexpected token') } export const abstractSyntaxTree = (tokens: Token[]): Program => { - const { body } = makeBody({ tokens }) + const { body, nonCodeMeta } = makeBody({ tokens }) const program: Program = { type: 'Program', start: 0, end: body[body.length - 1].end, body: body, + nonCodeMeta, } return program } @@ -1138,7 +1234,6 @@ export function findNextDeclarationKeyword( ) { return nextToken } - // return findNextDeclarationKeyword(tokens, nextToken.index) // probably should do something else here // throw new Error('Unexpected token') } @@ -1190,7 +1285,7 @@ export function hasPipeOperator( tokens: Token[], index: number, _limitIndex = -1 -): { token: Token; index: number } | false { +): ReturnType | false { // this probably still needs some work // should be called on expression statuments (i.e "lineTo" for lineTo(10, 10)) or "{" for sketch declarations let limitIndex = _limitIndex @@ -1538,8 +1633,8 @@ export function getNodePathFromSourceRange( export function isNotCodeToken(token: Token): boolean { return ( - token.type === 'whitespace' || - token.type === 'linecomment' || - token.type === 'blockcomment' + token?.type === 'whitespace' || + token?.type === 'linecomment' || + token?.type === 'blockcomment' ) } diff --git a/src/lang/modifyAst.ts b/src/lang/modifyAst.ts index 71ad72427..83edcd1c2 100644 --- a/src/lang/modifyAst.ts +++ b/src/lang/modifyAst.ts @@ -24,6 +24,7 @@ export function addSketchTo( type: 'BlockStatement', ...dumbyStartend, body: [], + nonCodeMeta: {}, } const sketch: SketchExpression = { type: 'SketchExpression', @@ -56,6 +57,7 @@ export function addSketchTo( const pipChain: PipeExpression = { type: 'PipeExpression', + nonCodeMeta: {}, ...dumbyStartend, body: [sketch, rotate], } @@ -344,11 +346,13 @@ export function extrudeSketch( const pipeChain: PipeExpression = isInPipeExpression ? { type: 'PipeExpression', + nonCodeMeta: {}, ...dumbyStartend, body: [...pipeExpression.body, extrudeCall], } : { type: 'PipeExpression', + nonCodeMeta: {}, ...dumbyStartend, body: [sketchExpression, extrudeCall], } @@ -460,6 +464,7 @@ export function sketchOnExtrudedFace( // create pipe expression with a sketch block piped into a transform function const sketchPipe: PipeExpression = { type: 'PipeExpression', + nonCodeMeta: {}, ...dumbyStartend, body: [ { @@ -469,6 +474,7 @@ export function sketchOnExtrudedFace( type: 'BlockStatement', ...dumbyStartend, body: [], + nonCodeMeta: {}, }, }, { diff --git a/src/lang/nonAstTokenHelpers.test.ts b/src/lang/nonAstTokenHelpers.test.ts deleted file mode 100644 index 68f99dfc7..000000000 --- a/src/lang/nonAstTokenHelpers.test.ts +++ /dev/null @@ -1,210 +0,0 @@ -import { findTokensBetweenStatements } from './nonAstTokenHelpers' -import { Token } from './tokeniser' -import { BodyItem } from './abstractSyntaxTree' - -describe('verify code', () => { - it('should find tokens between statements', () => { - const statement1 = { - type: 'yoyo', - start: 105, - end: 111, - } - - const statement2 = { - type: 'yoyo', - start: 150, - end: 156, - } - - const tokens: Token[] = [ - { - type: 'word', - value: 'yoyo', - start: 100, - end: 104, - }, - { - type: 'whitespace', - value: ' ', - start: 111, - end: 115, - }, - { - type: 'linecomment', - value: '// this is a comment', - start: 115, - end: 119, - }, - { - type: 'whitespace', - value: ' ', - start: 157, - end: 161, - }, - ] - const result = findTokensBetweenStatements(statement1, statement2, tokens) - // should grab the middle two tokens an the start and end tokens are less than the first statement - // and greater than the second statement respectively - expect(result).toEqual([ - { type: 'whitespace', value: ' ', start: 111, end: 115 }, - { - type: 'linecomment', - value: '// this is a comment', - start: 115, - end: 119, - }, - ]) - }) - it('propert test with our types', () => { - const tokens: Token[] = [ - { - type: 'whitespace', - value: '\n', - start: 37, - end: 38, - }, - { - type: 'linecomment', - value: '// this is a comment', - start: 38, - end: 58, - }, - { - type: 'whitespace', - value: '\n', - start: 58, - end: 59, - }, - ] - - const statement1: BodyItem = { - type: 'VariableDeclaration', - start: 0, - end: 37, - kind: 'const', - declarations: [ - { - type: 'VariableDeclarator', - start: 6, - end: 37, - id: { - type: 'Identifier', - start: 6, - end: 8, - name: 'yo', - }, - init: { - type: 'ObjectExpression', - start: 11, - end: 37, - properties: [ - { - type: 'ObjectProperty', - start: 13, - end: 35, - key: { - type: 'Identifier', - start: 13, - end: 14, - name: 'a', - }, - value: { - type: 'ObjectExpression', - start: 16, - end: 35, - properties: [ - { - type: 'ObjectProperty', - start: 18, - end: 33, - key: { - type: 'Identifier', - start: 18, - end: 19, - name: 'b', - }, - value: { - type: 'ObjectExpression', - start: 21, - end: 33, - properties: [ - { - type: 'ObjectProperty', - start: 23, - end: 31, - key: { - type: 'Identifier', - start: 23, - end: 24, - name: 'c', - }, - value: { - type: 'Literal', - start: 26, - end: 31, - value: '123', - raw: "'123'", - }, - }, - ], - }, - }, - ], - }, - }, - ], - }, - }, - ], - } - - const statement2: BodyItem = { - type: 'VariableDeclaration', - start: 59, - end: 74, - kind: 'const', - declarations: [ - { - type: 'VariableDeclarator', - start: 65, - end: 74, - id: { - type: 'Identifier', - start: 65, - end: 68, - name: 'key', - }, - init: { - type: 'Literal', - start: 71, - end: 74, - value: 'c', - raw: "'c'", - }, - }, - ], - } - - const result = findTokensBetweenStatements(statement1, statement2, tokens) - expect(result).toEqual([ - { - type: 'whitespace', - value: '\n', - start: 37, - end: 38, - }, - { - type: 'linecomment', - value: '// this is a comment', - start: 38, - end: 58, - }, - { - type: 'whitespace', - value: '\n', - start: 58, - end: 59, - }, - ]) - }) -}) diff --git a/src/lang/nonAstTokenHelpers.ts b/src/lang/nonAstTokenHelpers.ts deleted file mode 100644 index b218ccfc8..000000000 --- a/src/lang/nonAstTokenHelpers.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { Token } from './tokeniser' -import { Program, BodyItem } from './abstractSyntaxTree' - -export function findTokensBetweenStatements( - statement1: { start: number; end: number }, - statement2: { start: number; end: number }, - tokens: Token[] -): Token[] { - // Find the start index of the range using binary search - let startIndex = firstGreaterThanBinarySearch(tokens, statement1.end, 'start') - if (startIndex < 0) { - startIndex = ~startIndex - } - - // Find the end index of the range using binary search - let endIndex = firstGreaterThanBinarySearch(tokens, statement2.end, 'start') - if (endIndex < 0) { - endIndex = ~endIndex - } - - // Return the tokens between the start and end index - return tokens.slice(startIndex, endIndex) -} - -function firstGreaterThanBinarySearch( - tokens: { start: number; end: number }[], - target: number, - property: 'start' | 'end' -): number { - let left = 0 - - // has trouble with including tokens at the end of the range - const paddedTokens = [ - { - type: 'whitespace', - value: '', - start: 0, - end: 0, - }, - ...tokens, - { - type: 'whitespace', - value: '', - start: tokens[tokens.length - 1]?.end + 1000, - end: tokens[tokens.length - 1]?.end + 1001, - }, - ] - - let right = paddedTokens.length - 1 - - while (left <= right) { - const middle = left + Math.floor((right - left) / 2) - if (paddedTokens[middle]?.[property] >= target) { - if (middle === 1 || paddedTokens[middle - 1]?.[property] < target) { - // minus 1 because of the padding - return middle - 1 - } - right = middle - 1 - } else { - left = middle + 1 - } - } - return -1 -} - -export function getNonCodeString( - body: Program['body'], - index: number, - tokens: Token[] -): string { - let tokensToIntegrate: Token[] = [] - const currentStatement = body[index] - const nextStatement = body[index + 1] - if (nextStatement && nextStatement.start && currentStatement.end) { - tokensToIntegrate = findTokensBetweenStatements( - currentStatement, - nextStatement, - tokens - ) - } else if (index === body.length - 1) { - const tokensAfter = firstGreaterThanBinarySearch( - tokens, - currentStatement?.end, - 'start' - ) - if (tokensAfter > 0) { - tokensToIntegrate = tokens.slice(tokensAfter) - } - } - - if (tokensToIntegrate.length > 0) { - const nonCodeString = tokensToIntegrate.map((token) => token.value).join('') - // check it extra ends with a line break followed by spaces (only spaces not new lines) - const hasWhitespaceOnEnd = nonCodeString.match(/(\n *)$/) - if (hasWhitespaceOnEnd) { - // we always put each statement on a new line, so this prevents it adding an extra line - // however if the user puts more than one line break between statements, we'll respect it since - // we're only removing the last one - return nonCodeString.slice(0, -hasWhitespaceOnEnd[0].length) - } - - return nonCodeString - } - return '' -} - -export function getStartNonCodeString( - firstStatement: BodyItem, - tokens: Token[] -): string { - if (!firstStatement) return '' - const tokensBeforeIndex = tokens.length - ? firstGreaterThanBinarySearch(tokens, firstStatement.start, 'end') - : 0 - let nonCodeString = '' - if (tokensBeforeIndex > 0) { - nonCodeString = tokens - .slice(0, tokensBeforeIndex) - .map((token) => token.value) - .join('') - } - return nonCodeString.trim() ? nonCodeString.trim() + '\n' : '' -} diff --git a/src/lang/recast.test.ts b/src/lang/recast.test.ts index 9ef1cbdd6..5f487b840 100644 --- a/src/lang/recast.test.ts +++ b/src/lang/recast.test.ts @@ -1,4 +1,4 @@ -import { recast, processTokens } from './recast' +import { recast } from './recast' import { Program, abstractSyntaxTree } from './abstractSyntaxTree' import { lexer, Token } from './tokeniser' import fs from 'node:fs' @@ -47,7 +47,7 @@ const myVar = "hello" log(5, myVar)` const { ast } = code2ast(code) const recasted = recast(ast) - expect(recasted).toBe(code.trim()) + expect(recasted).toBe(code) }) it('function declaration with call', () => { const code = [ @@ -59,7 +59,7 @@ log(5, myVar)` ].join('\n') const { ast } = code2ast(code) const recasted = recast(ast) - expect(recasted).toBe(code.trim()) + expect(recasted).toBe(code) }) it('sketch declaration', () => { let code = `sketch mySketch { @@ -97,7 +97,7 @@ show(mySketch) ].join('\n') const { ast } = code2ast(code) const recasted = recast(ast) - expect(recasted).toBe(code.trim()) + expect(recasted).toBe(code) }) it('recast nested binary expression', () => { const code = ['const myVar = 1 + 2 * 5'].join('\n') @@ -180,106 +180,114 @@ const myVar2 = yo['a'][key2].c` const recasted = recast(ast) expect(recasted).toBe(code.trim()) }) +}) + +describe('testing recasting with comments and whitespace', () => { it('code with comments', () => { const code = ` const yo = { a: { b: { c: '123' } } } // this is a comment const key = 'c'` - const { ast, tokens } = code2ast(code) - const processedTokens = processTokens(tokens) - const recasted = recast(ast, processedTokens) + const { ast } = code2ast(code) + const recasted = recast(ast) - expect(recasted).toBe(code.trim()) + expect(recasted).toBe(code) }) - it('code with extra whitespace should be respected when recasted', () => { - const withExtraEmptylLineBetween = ` -const yo = { a: { b: { c: '123' } } } + it('code with comment and extra lines', () => { + const code = ` +const yo = 'c' /* this is +a +comment */ -const key = 'c'` - - const { ast, tokens } = code2ast(withExtraEmptylLineBetween) - const processedTokens = processTokens(tokens) - const recasted = recast(ast, processedTokens) - - expect(recasted).toBe(withExtraEmptylLineBetween.trim()) +const yo = 'bing'` + const { ast } = code2ast(code) + const recasted = recast(ast) + expect(recasted).toBe(code) }) - it('code with block comment in between', () => { - const withExtraEmptylLineBetween = ` -const yo = { a: { b: { c: '123' } } } -/* hi there -yo yo yo -*/ -const key = 'c'` + it('comments at the start and end', () => { + const code = ` +// this is a comment - const { ast, tokens } = code2ast(withExtraEmptylLineBetween) - const processedTokens = processTokens(tokens) - const recasted = recast(ast, processedTokens) - - expect(recasted).toBe(withExtraEmptylLineBetween.trim()) - }) - it('code with block comment line comment and empty line', () => { - const withExtraEmptylLineBetween = ` -const yo = { a: { b: { c: '123' } } } -/* hi there -yo yo yo -*/ - -// empty line above and line comment here -const key = 'c'` - - const { ast, tokens } = code2ast(withExtraEmptylLineBetween) - const processedTokens = processTokens(tokens) - const recasted = recast(ast, processedTokens) - - expect(recasted).toBe(withExtraEmptylLineBetween.trim()) - }) - it('code comment at the start and end', () => { - const withExtraEmptylLineBetween = ` -// comment at the start const yo = { a: { b: { c: '123' } } } const key = 'c' -// comment at the end` - const { ast, tokens } = code2ast(withExtraEmptylLineBetween) - const processedTokens = processTokens(tokens) - const recasted = recast(ast, processedTokens) - - expect(recasted).toBe(withExtraEmptylLineBetween.trim()) +// this is also a comment` + const { ast } = code2ast(code) + const recasted = recast(ast) + expect(recasted).toBe(code) }) - it('comments and random new lines between statements within function declarations are fine', () => { - const withExtraEmptylLineBetween = ` -const fn = (a) => { - const yo = 5 + it('comments in a fn block', () => { + const code = ` +const myFn = () => { + // this is a comment + const yo = { a: { b: { c: '123' } } } /* block + comment */ - // a comment + const key = 'c' + // this is also a comment +}` + const { ast } = code2ast(code) + const recasted = recast(ast) + expect(recasted).toBe(code) + }) + it('comments in a sketch block', () => { + const code = ` +sketch mySketch { /* comment at start */ + // comment at start more + path myPath = lineTo(0, 1) /* comment here with + some whitespace below */ - return a + yo + lineTo(1, 1) + /* comment before declaration*/path rightPath = lineTo(1, 0) + close() + // comment at end }` - - const { ast, tokens } = code2ast(withExtraEmptylLineBetween) - const processedTokens = processTokens(tokens) - const recasted = recast(ast, processedTokens) - expect(recasted).toBe(withExtraEmptylLineBetween.trim()) + const { ast } = code2ast(code) + const recasted = recast(ast) + expect(recasted).toBe(code) }) - it('Comment with sketch', () => { - const withExtraEmptylLineBetween = `sketch part001 { - lineTo(5.98, -0.04) - // yo + it('comments in a pipe expression', () => { + const code = [ + 'sketch mySk1 {', + ' lineTo(1, 1)', + ' path myPath = lineTo(0, 1)', + ' lineTo(1, 1)', + '}', + ' // a comment', + ' |> rx(90, %)', + ].join('\n') + const { ast } = code2ast(code) + const recasted = recast(ast) + expect(recasted).toBe(code) + }) + it('comments sprinkled in all over the place', () => { + const code = ` +/* comment at start */ - lineTo(0.18, 0.03) +sketch mySk1 { + lineTo(1, 1) + // comment here + path myPath = lineTo(0, 1) + lineTo(1, 1) /* and + here + */ } + // a comment between pipe expression statements |> rx(90, %) - |> extrude(9.6, %) + // and another with just white space between others below + |> ry(45, %) -show(part001)` - const { ast, tokens } = code2ast(withExtraEmptylLineBetween) - const processedTokens = processTokens(tokens) - const recasted = recast(ast, processedTokens) - expect(recasted).toBe(withExtraEmptylLineBetween.trim()) + |> rx(45, %) + /* + one more for good measure + */` + const { ast } = code2ast(code) + const recasted = recast(ast) + expect(recasted).toBe(code) }) }) diff --git a/src/lang/recast.ts b/src/lang/recast.ts index b5ddcec20..bf790e5ff 100644 --- a/src/lang/recast.ts +++ b/src/lang/recast.ts @@ -1,3 +1,4 @@ +import { start } from 'repl' import { Program, BinaryExpression, @@ -10,72 +11,80 @@ import { ArrayExpression, ObjectExpression, MemberExpression, + PipeExpression, } from './abstractSyntaxTree' import { precedence } from './astMathExpressions' -import { Token } from './tokeniser' -import { getNonCodeString, getStartNonCodeString } from './nonAstTokenHelpers' - -export const processTokens = (tokens: Token[]): Token[] => { - return tokens.filter((token) => { - if (token.type === 'linecomment' || token.type === 'blockcomment') - return true - if (token.type === 'whitespace') { - if (token.value.includes('\n')) return true - } - return false - }) -} export function recast( ast: Program, - tokens: Token[] = [], previousWrittenCode = '', - indentation = '' + indentation = '', + isWithBlock = false ): string { - let startComments = getStartNonCodeString(ast?.body?.[0], tokens) - return ( - startComments + - ast.body - .map((statement) => { - if (statement.type === 'ExpressionStatement') { - if (statement.expression.type === 'BinaryExpression') { - return recastBinaryExpression(statement.expression) - } else if (statement.expression.type === 'ArrayExpression') { - return recastArrayExpression(statement.expression) - } else if (statement.expression.type === 'ObjectExpression') { - return recastObjectExpression(statement.expression) - } else if (statement.expression.type === 'CallExpression') { - return recastCallExpression(statement.expression, tokens) - } - } else if (statement.type === 'VariableDeclaration') { - return statement.declarations - .map((declaration) => { - const isSketchOrFirstPipeExpressionIsSketch = - declaration.init.type === 'SketchExpression' || - (declaration.init.type === 'PipeExpression' && - declaration.init.body[0].type === 'SketchExpression') - - const assignmentString = isSketchOrFirstPipeExpressionIsSketch - ? ' ' - : ' = ' - return `${statement.kind} ${ - declaration.id.name - }${assignmentString}${recastValue(declaration.init, '', tokens)}` - }) - .join('') - } else if (statement.type === 'ReturnStatement') { - return `return ${recastArgument(statement.argument, tokens)}` + return ast.body + .map((statement) => { + if (statement.type === 'ExpressionStatement') { + if (statement.expression.type === 'BinaryExpression') { + return recastBinaryExpression(statement.expression) + } else if (statement.expression.type === 'ArrayExpression') { + return recastArrayExpression(statement.expression) + } else if (statement.expression.type === 'ObjectExpression') { + return recastObjectExpression(statement.expression) + } else if (statement.expression.type === 'CallExpression') { + return recastCallExpression(statement.expression) } - return statement.type - }) - .map( - (statementString, index) => - indentation + - statementString + - getNonCodeString(ast.body, index, tokens) + } else if (statement.type === 'VariableDeclaration') { + return statement.declarations + .map((declaration) => { + const isSketchOrFirstPipeExpressionIsSketch = + declaration.init.type === 'SketchExpression' || + (declaration.init.type === 'PipeExpression' && + declaration.init.body[0].type === 'SketchExpression') + + const assignmentString = isSketchOrFirstPipeExpressionIsSketch + ? ' ' + : ' = ' + return `${statement.kind} ${ + declaration.id.name + }${assignmentString}${recastValue(declaration.init)}` + }) + .join('') + } else if (statement.type === 'ReturnStatement') { + return `return ${recastArgument(statement.argument)}` + } + return statement.type + }) + .map((recastStr, index, arr) => { + const isLegitCustomWhitespaceOrComment = (str: string) => + str !== ' ' && str !== '\n' && str !== ' ' + + // determine the value of startString + const lastWhiteSpaceOrComment = + index > 0 ? ast?.nonCodeMeta?.[index - 1]?.value : ' ' + // indentation of this line will be covered by the previous if we're using a custom whitespace or comment + let startString = isLegitCustomWhitespaceOrComment( + lastWhiteSpaceOrComment ) - .join('\n') - ) + ? '' + : indentation + if (index === 0) { + startString = ast?.nonCodeMeta?.start?.value || indentation + } + if (startString.endsWith('\n')) { + startString += indentation + } + + // determine the value of endString + const maybeLineBreak: string = + index === arr.length - 1 && !isWithBlock ? '' : '\n' + let customWhiteSpaceOrComment = ast?.nonCodeMeta?.[index]?.value + if (!isLegitCustomWhitespaceOrComment(customWhiteSpaceOrComment)) + customWhiteSpaceOrComment = '' + let endString = customWhiteSpaceOrComment || maybeLineBreak + + return startString + recastStr + endString + }) + .join('') } function recastBinaryExpression(expression: BinaryExpression): string { @@ -151,16 +160,13 @@ function recastLiteral(literal: Literal): string { return String(literal?.value) } -function recastCallExpression( - expression: CallExpression, - tokens: Token[] = [] -): string { +function recastCallExpression(expression: CallExpression): string { return `${expression.callee.name}(${expression.arguments - .map((arg) => recastArgument(arg, tokens)) + .map(recastArgument) .join(', ')})` } -function recastArgument(argument: Value, tokens: Token[] = []): string { +function recastArgument(argument: Value): string { if (argument.type === 'Literal') { return recastLiteral(argument) } else if (argument.type === 'Identifier') { @@ -172,33 +178,28 @@ function recastArgument(argument: Value, tokens: Token[] = []): string { } else if (argument.type === 'ObjectExpression') { return recastObjectExpression(argument) } else if (argument.type === 'CallExpression') { - return recastCallExpression(argument, tokens) + return recastCallExpression(argument) } else if (argument.type === 'FunctionExpression') { - return recastFunction(argument, tokens) + return recastFunction(argument) } else if (argument.type === 'PipeSubstitution') { return '%' } throw new Error(`Cannot recast argument ${argument}`) } -function recastFunction( - expression: FunctionExpression, - tokens: Token[] = [], - indentation = '' -): string { - return `(${expression.params.map((param) => param.name).join(', ')}) => { -${recast(expression.body, tokens, '', indentation + ' ')} -}` +function recastFunction(expression: FunctionExpression): string { + return `(${expression.params + .map((param) => param.name) + .join(', ')}) => {${recast(expression.body, '', '', true)}}` } function recastSketchExpression( expression: SketchExpression, - indentation: string, - tokens: Token[] = [] + indentation: string ): string { - return `{ -${recast(expression.body, tokens, '', indentation + ' ').trimEnd()} -}` + return `{${ + recast(expression.body, '', indentation + ' ', true) || '\n \n' + }}` } function recastMemberExpression( @@ -218,11 +219,7 @@ function recastMemberExpression( return expression.object.name + keyString } -function recastValue( - node: Value, - indentation = '', - tokens: Token[] = [] -): string { +function recastValue(node: Value, indentation = ''): string { if (node.type === 'BinaryExpression') { return recastBinaryExpression(node) } else if (node.type === 'ArrayExpression') { @@ -234,17 +231,38 @@ function recastValue( } else if (node.type === 'Literal') { return recastLiteral(node) } else if (node.type === 'FunctionExpression') { - return recastFunction(node, tokens) + return recastFunction(node) } else if (node.type === 'CallExpression') { - return recastCallExpression(node, tokens) + return recastCallExpression(node) } else if (node.type === 'Identifier') { return node.name } else if (node.type === 'SketchExpression') { - return recastSketchExpression(node, indentation, tokens) + return recastSketchExpression(node, indentation) } else if (node.type === 'PipeExpression') { - return node.body - .map((statement): string => recastValue(statement, indentation, tokens)) - .join('\n |> ') + return recastPipeExpression(node) } return '' } + +function recastPipeExpression(expression: PipeExpression): string { + return expression.body + .map((statement, index, arr): string => { + let str = '' + let indentation = ' ' + let maybeLineBreak = '\n' + str = recastValue(statement) + if ( + expression.nonCodeMeta?.[index]?.value && + expression.nonCodeMeta?.[index].value !== ' ' + ) { + str += expression.nonCodeMeta[index]?.value + indentation = '' + maybeLineBreak = '' + } + if (index !== arr.length - 1) { + str += maybeLineBreak + indentation + '|> ' + } + return str + }) + .join('') +} diff --git a/src/useStore.ts b/src/useStore.ts index 1c90ca41d..bec02fc05 100644 --- a/src/useStore.ts +++ b/src/useStore.ts @@ -7,8 +7,7 @@ import { } from './lang/abstractSyntaxTree' import { ProgramMemory, Position, PathToNode, Rotation } from './lang/executor' import { recast } from './lang/recast' -import { lexer, Token } from './lang/tokeniser' -import { processTokens } from './lang/recast' +import { lexer } from './lang/tokeniser' export type Range = [number, number] @@ -64,7 +63,7 @@ interface StoreState { addLog: (log: string) => void resetLogs: () => void ast: Program | null - setAst: (ast: Program | null, tokens?: Token[]) => void + setAst: (ast: Program | null) => void updateAst: (ast: Program, focusPath?: PathToNode) => void code: string setCode: (code: string) => void @@ -76,7 +75,6 @@ interface StoreState { setError: (error?: string) => void programMemory: ProgramMemory setProgramMemory: (programMemory: ProgramMemory) => void - tokens: Token[] } export const useStore = create()((set, get) => ({ @@ -121,16 +119,11 @@ export const useStore = create()((set, get) => ({ set({ logs: [] }) }, ast: null, - setAst: (ast, tokens) => { - if (tokens) { - set({ tokens: processTokens(tokens), ast }) - } else { - set({ ast, tokens: [] }) - } + setAst: (ast) => { + set({ ast }) }, updateAst: (ast, focusPath) => { - const tokens = get().tokens - const newCode = recast(ast, tokens) + const newCode = recast(ast) const astWithUpdatedSource = abstractSyntaxTree(lexer(newCode)) set({ ast: astWithUpdatedSource, code: newCode }) @@ -149,9 +142,8 @@ export const useStore = create()((set, get) => ({ }, formatCode: () => { const code = get().code - const tokens = lexer(code) - const ast = abstractSyntaxTree(tokens) - const newCode = recast(ast, processTokens(tokens)) + const ast = abstractSyntaxTree(lexer(code)) + const newCode = recast(ast) set({ code: newCode, ast }) }, errorState: { @@ -163,5 +155,4 @@ export const useStore = create()((set, get) => ({ }, programMemory: { root: {}, _sketch: [] }, setProgramMemory: (programMemory) => set({ programMemory }), - tokens: [], }))