ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add non-code metadata to the ast (#15)

Browse Source 
* Revert "wire up comments to ui (#11)"

This reverts commit bdf778530f.

* Revert "Add the ability to recast comments and some whitespace (#10)"

This reverts commit dd5022b38e.

* ast metadata

* clean up
This commit is contained in:
Kurt Hutten
2023-02-01 07:30:55 +11:00
committed by GitHub
parent bdf778530f
commit 29e06ec852
11 changed files with 433 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
src/lang/abstractSyntaxTree.ts
View File

@ -21,6 +21,7 @@ type syntaxType =
| 'PipeExpression'
| 'PipeSubstitution'
| 'Literal'
| 'NoneCodeNode'
// | 'NumberLiteral'
// | 'StringLiteral'
// | 'IfStatement'
@ -84,6 +85,7 @@ export interface Program {
start: number
end: number
body: BodyItem[]
nonCodeMeta: NoneCodeMeta
}
interface GeneralStatement {
type: syntaxType
@ -91,6 +93,44 @@ interface GeneralStatement {
end: number
}
interface NoneCodeNode extends GeneralStatement {
type: 'NoneCodeNode'
value: string
}
interface NoneCodeMeta {
// Stores the whitespace/comments that go after the statement who's index we're using here
[statementIndex: number]: NoneCodeNode
// Which is why we also need `start` for and whitespace at the start of the file/block
start?: NoneCodeNode
}
function makeNoneCodeNode(
tokens: Token[],
index: number
): { node?: NoneCodeNode; lastIndex: number } {
const currentToken = tokens[index]
const endIndex = findEndOfNonCodeNode(tokens, index)
const nonCodeTokens = tokens.slice(index, endIndex)
let value = nonCodeTokens.map((t) => t.value).join('')
const node: NoneCodeNode = {
type: 'NoneCodeNode',
start: currentToken.start,
end: tokens[endIndex - 1].end,
value,
}
return { node, lastIndex: endIndex - 1 }
}
export function findEndOfNonCodeNode(tokens: Token[], index: number): number {
const currentToken = tokens[index]
if (isNotCodeToken(currentToken)) {
return findEndOfNonCodeNode(tokens, index + 1)
}
return index
}
export interface ExpressionStatement extends GeneralStatement {
type: 'ExpressionStatement'
expression: Value
@ -828,6 +868,7 @@ function makeSketchExpression(
export interface PipeExpression extends GeneralStatement {
type: 'PipeExpression'
body: Value[]
nonCodeMeta: NoneCodeMeta
}
function makePipeExpression(
@ -835,7 +876,11 @@ function makePipeExpression(
index: number
): { expression: PipeExpression; lastIndex: number } {
const currentToken = tokens[index]
const { body, lastIndex: bodyLastIndex } = makePipeBody(tokens, index)
const {
body,
lastIndex: bodyLastIndex,
nonCodeMeta,
} = makePipeBody(tokens, index)
const endToken = tokens[bodyLastIndex]
return {
expression: {
@ -843,6 +888,7 @@ function makePipeExpression(
start: currentToken.start,
end: endToken.end,
body,
nonCodeMeta,
},
lastIndex: bodyLastIndex,
}
@ -851,8 +897,10 @@ function makePipeExpression(
function makePipeBody(
tokens: Token[],
index: number,
previousValues: Value[] = []
): { body: Value[]; lastIndex: number } {
previousValues: Value[] = [],
previousNonCodeMeta: NoneCodeMeta = {}
): { body: Value[]; lastIndex: number; nonCodeMeta: NoneCodeMeta } {
const nonCodeMeta = { ...previousNonCodeMeta }
const currentToken = tokens[index]
const expressionStart = nextMeaningfulToken(tokens, index)
let value: Value
@ -874,10 +922,18 @@ function makePipeBody(
return {
body: [...previousValues, value],
lastIndex,
nonCodeMeta,
}
}
// const nextToken = nextMeaningfulToken(tokens, nextPipeToken.index + 1)
return makePipeBody(tokens, nextPipeToken.index, [...previousValues, value])
if (nextPipeToken.bonusNonCodeNode) {
nonCodeMeta[previousValues.length] = nextPipeToken.bonusNonCodeNode
}
return makePipeBody(
tokens,
nextPipeToken.index,
[...previousValues, value],
nonCodeMeta
)
}
export interface FunctionExpression extends GeneralStatement {
@ -938,6 +994,7 @@ function makeParams(
export interface BlockStatement extends GeneralStatement {
type: 'BlockStatement'
body: BodyItem[]
nonCodeMeta: NoneCodeMeta
}
function makeBlockStatement(
@ -945,10 +1002,10 @@ function makeBlockStatement(
index: number
): { block: BlockStatement; lastIndex: number } {
const openingCurly = tokens[index]
const nextToken = nextMeaningfulToken(tokens, index)
const { body, lastIndex } =
const nextToken = { token: tokens[index + 1], index: index + 1 }
const { body, lastIndex, nonCodeMeta } =
nextToken.token.value === '}'
? { body: [], lastIndex: nextToken.index }
? { body: [], lastIndex: nextToken.index, nonCodeMeta: {} }
: makeBody({ tokens, tokenIndex: nextToken.index })
return {
block: {
@ -956,6 +1013,7 @@ function makeBlockStatement(
start: openingCurly.start,
end: tokens[lastIndex]?.end || 0,
body,
nonCodeMeta,
},
lastIndex,
}
@ -986,18 +1044,24 @@ function makeReturnStatement(
export type All = Program | ExpressionStatement[] | BinaryExpression | Literal
function nextMeaningfulToken(
export function nextMeaningfulToken(
tokens: Token[],
index: number,
offset: number = 1
): { token: Token; index: number } {
): { token: Token; index: number; bonusNonCodeNode?: NoneCodeNode } {
const newIndex = index + offset
const token = tokens[newIndex]
if (!token) {
return { token, index: tokens.length }
}
if (isNotCodeToken(token)) {
return nextMeaningfulToken(tokens, index, offset + 1)
const nonCodeNode = makeNoneCodeNode(tokens, newIndex)
const newnewIndex = nonCodeNode.lastIndex + 1
return {
token: tokens[newnewIndex],
index: newnewIndex,
bonusNonCodeNode: nonCodeNode?.node?.value ? nonCodeNode.node : undefined,
}
}
return { token, index: newIndex }
}
@ -1018,10 +1082,7 @@ function previousMeaningfulToken(
return { token, index: newIndex }
}
export type BodyItem =
| ExpressionStatement
| VariableDeclaration
| ReturnStatement
type BodyItem = ExpressionStatement | VariableDeclaration | ReturnStatement
function makeBody(
{
@ -1031,23 +1092,37 @@ function makeBody(
tokens: Token[]
tokenIndex?: number
},
previousBody: BodyItem[] = []
): { body: BodyItem[]; lastIndex: number } {
previousBody: BodyItem[] = [],
previousNonCodeMeta: NoneCodeMeta = {}
): { body: BodyItem[]; lastIndex: number; nonCodeMeta: NoneCodeMeta } {
const nonCodeMeta = { ...previousNonCodeMeta }
if (tokenIndex >= tokens.length) {
return { body: previousBody, lastIndex: tokenIndex }
return { body: previousBody, lastIndex: tokenIndex, nonCodeMeta }
}
const token = tokens[tokenIndex]
if (token.type === 'brace' && token.value === '}') {
return { body: previousBody, lastIndex: tokenIndex }
}
if (typeof token === 'undefined') {
console.log('probably should throw')
return { body: previousBody, lastIndex: tokenIndex, nonCodeMeta }
}
if (isNotCodeToken(token)) {
return makeBody({ tokens, tokenIndex: tokenIndex + 1 }, previousBody)
const nextToken = nextMeaningfulToken(tokens, tokenIndex, 0)
if (nextToken.bonusNonCodeNode) {
if (previousBody.length === 0) {
nonCodeMeta.start = nextToken.bonusNonCodeNode
} else {
nonCodeMeta[previousBody.length] = nextToken.bonusNonCodeNode
}
}
return makeBody(
{ tokens, tokenIndex: nextToken.index },
previousBody,
nonCodeMeta
)
}
const nextToken = nextMeaningfulToken(tokens, tokenIndex)
nextToken.bonusNonCodeNode &&
(nonCodeMeta[previousBody.length] = nextToken.bonusNonCodeNode)
if (
token.type === 'word' &&
(token.value === 'const' ||
@ -1060,18 +1135,26 @@ function makeBody(
tokenIndex
)
const nextThing = nextMeaningfulToken(tokens, lastIndex)
return makeBody({ tokens, tokenIndex: nextThing.index }, [
...previousBody,
declaration,
])
nextThing.bonusNonCodeNode &&
(nonCodeMeta[previousBody.length] = nextThing.bonusNonCodeNode)
return makeBody(
{ tokens, tokenIndex: nextThing.index },
[...previousBody, declaration],
nonCodeMeta
)
}
if (token.type === 'word' && token.value === 'return') {
const { statement, lastIndex } = makeReturnStatement(tokens, tokenIndex)
const nextThing = nextMeaningfulToken(tokens, lastIndex)
return makeBody({ tokens, tokenIndex: nextThing.index }, [
...previousBody,
statement,
])
nextThing.bonusNonCodeNode &&
(nonCodeMeta[previousBody.length] = nextThing.bonusNonCodeNode)
return makeBody(
{ tokens, tokenIndex: nextThing.index },
[...previousBody, statement],
nonCodeMeta
)
}
if (
token.type === 'word' &&
@ -1083,31 +1166,44 @@ function makeBody(
tokenIndex
)
const nextThing = nextMeaningfulToken(tokens, lastIndex)
return makeBody({ tokens, tokenIndex: nextThing.index }, [
...previousBody,
expression,
])
if (nextThing.bonusNonCodeNode) {
nonCodeMeta[previousBody.length] = nextThing.bonusNonCodeNode
}
return makeBody(
{ tokens, tokenIndex: nextThing.index },
[...previousBody, expression],
nonCodeMeta
)
}
const nextThing = nextMeaningfulToken(tokens, tokenIndex)
if (
(token.type === 'number' || token.type === 'word') &&
nextMeaningfulToken(tokens, tokenIndex).token.type === 'operator'
nextThing.token.type === 'operator'
) {
if (nextThing.bonusNonCodeNode) {
nonCodeMeta[previousBody.length] = nextThing.bonusNonCodeNode
}
const { expression, lastIndex } = makeExpressionStatement(
tokens,
tokenIndex
)
// return startTree(tokens, tokenIndex, [...previousBody, makeExpressionStatement(tokens, tokenIndex)]);
return { body: [...previousBody, expression], lastIndex }
return {
body: [...previousBody, expression],
nonCodeMeta: nonCodeMeta,
lastIndex,
}
}
throw new Error('Unexpected token')
}
export const abstractSyntaxTree = (tokens: Token[]): Program => {
const { body } = makeBody({ tokens })
const { body, nonCodeMeta } = makeBody({ tokens })
const program: Program = {
type: 'Program',
start: 0,
end: body[body.length - 1].end,
body: body,
nonCodeMeta,
}
return program
}
@ -1138,7 +1234,6 @@ export function findNextDeclarationKeyword(
) {
return nextToken
}
// return findNextDeclarationKeyword(tokens, nextToken.index)
// probably should do something else here
// throw new Error('Unexpected token')
}
@ -1190,7 +1285,7 @@ export function hasPipeOperator(
tokens: Token[],
index: number,
_limitIndex = -1
): { token: Token; index: number } | false {
): ReturnType<typeof nextMeaningfulToken> | false {
// this probably still needs some work
// should be called on expression statuments (i.e "lineTo" for lineTo(10, 10)) or "{" for sketch declarations
let limitIndex = _limitIndex
@ -1538,8 +1633,8 @@ export function getNodePathFromSourceRange(
export function isNotCodeToken(token: Token): boolean {
return (
token.type === 'whitespace' ||
token.type === 'linecomment' ||
token.type === 'blockcomment'
token?.type === 'whitespace' ||
token?.type === 'linecomment' ||
token?.type === 'blockcomment'
)
}