2024-02-23 11:24:22 -05:00
|
|
|
import { parse, recast, initPromise, Identifier } from './wasm'
|
2023-02-12 10:56:45 +11:00
|
|
|
import {
|
|
|
|
createLiteral,
|
|
|
|
createIdentifier,
|
|
|
|
createCallExpression,
|
|
|
|
createObjectExpression,
|
|
|
|
createArrayExpression,
|
|
|
|
createPipeSubstitution,
|
|
|
|
createVariableDeclaration,
|
|
|
|
createPipeExpression,
|
|
|
|
findUniqueName,
|
|
|
|
addSketchTo,
|
2023-03-02 21:19:11 +11:00
|
|
|
giveSketchFnCallTag,
|
2023-04-01 16:47:00 +11:00
|
|
|
moveValueIntoNewVariable,
|
2023-02-12 10:56:45 +11:00
|
|
|
} from './modifyAst'
|
2023-07-10 15:15:07 +10:00
|
|
|
import { enginelessExecutor } from '../lib/testHelpers'
|
2023-03-02 21:19:11 +11:00
|
|
|
|
|
|
|
beforeAll(() => initPromise)
|
2023-02-12 10:56:45 +11:00
|
|
|
|
|
|
|
describe('Testing createLiteral', () => {
|
|
|
|
it('should create a literal', () => {
|
|
|
|
const result = createLiteral(5)
|
|
|
|
expect(result.type).toBe('Literal')
|
|
|
|
expect(result.value).toBe(5)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
describe('Testing createIdentifier', () => {
|
|
|
|
it('should create an identifier', () => {
|
|
|
|
const result = createIdentifier('myVar')
|
|
|
|
expect(result.type).toBe('Identifier')
|
|
|
|
expect(result.name).toBe('myVar')
|
|
|
|
})
|
|
|
|
})
|
|
|
|
describe('Testing createCallExpression', () => {
|
|
|
|
it('should create a call expression', () => {
|
|
|
|
const result = createCallExpression('myFunc', [createLiteral(5)])
|
|
|
|
expect(result.type).toBe('CallExpression')
|
|
|
|
expect(result.callee.type).toBe('Identifier')
|
|
|
|
expect(result.callee.name).toBe('myFunc')
|
|
|
|
expect(result.arguments[0].type).toBe('Literal')
|
|
|
|
expect((result.arguments[0] as any).value).toBe(5)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
describe('Testing createObjectExpression', () => {
|
|
|
|
it('should create an object expression', () => {
|
|
|
|
const result = createObjectExpression({
|
|
|
|
myProp: createLiteral(5),
|
|
|
|
})
|
|
|
|
expect(result.type).toBe('ObjectExpression')
|
|
|
|
expect(result.properties[0].type).toBe('ObjectProperty')
|
|
|
|
expect(result.properties[0].key.name).toBe('myProp')
|
|
|
|
expect(result.properties[0].value.type).toBe('Literal')
|
|
|
|
expect((result.properties[0].value as any).value).toBe(5)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
describe('Testing createArrayExpression', () => {
|
|
|
|
it('should create an array expression', () => {
|
|
|
|
const result = createArrayExpression([createLiteral(5)])
|
|
|
|
expect(result.type).toBe('ArrayExpression')
|
|
|
|
expect(result.elements[0].type).toBe('Literal')
|
|
|
|
expect((result.elements[0] as any).value).toBe(5)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
describe('Testing createPipeSubstitution', () => {
|
|
|
|
it('should create a pipe substitution', () => {
|
|
|
|
const result = createPipeSubstitution()
|
|
|
|
expect(result.type).toBe('PipeSubstitution')
|
|
|
|
})
|
|
|
|
})
|
|
|
|
describe('Testing createVariableDeclaration', () => {
|
|
|
|
it('should create a variable declaration', () => {
|
|
|
|
const result = createVariableDeclaration('myVar', createLiteral(5))
|
|
|
|
expect(result.type).toBe('VariableDeclaration')
|
|
|
|
expect(result.declarations[0].type).toBe('VariableDeclarator')
|
|
|
|
expect(result.declarations[0].id.type).toBe('Identifier')
|
|
|
|
expect(result.declarations[0].id.name).toBe('myVar')
|
|
|
|
expect(result.declarations[0].init.type).toBe('Literal')
|
|
|
|
expect((result.declarations[0].init as any).value).toBe(5)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
describe('Testing createPipeExpression', () => {
|
|
|
|
it('should create a pipe expression', () => {
|
|
|
|
const result = createPipeExpression([createLiteral(5)])
|
|
|
|
expect(result.type).toBe('PipeExpression')
|
|
|
|
expect(result.body[0].type).toBe('Literal')
|
|
|
|
expect((result.body[0] as any).value).toBe(5)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
|
|
|
|
describe('Testing findUniqueName', () => {
|
|
|
|
it('should find a unique name', () => {
|
|
|
|
const result = findUniqueName(
|
2024-02-23 11:24:22 -05:00
|
|
|
JSON.stringify([
|
|
|
|
{ type: 'Identifier', name: 'yo01', start: 0, end: 0 },
|
|
|
|
{ type: 'Identifier', name: 'yo02', start: 0, end: 0 },
|
|
|
|
{ type: 'Identifier', name: 'yo03', start: 0, end: 0 },
|
|
|
|
{ type: 'Identifier', name: 'yo04', start: 0, end: 0 },
|
|
|
|
{ type: 'Identifier', name: 'yo05', start: 0, end: 0 },
|
|
|
|
{ type: 'Identifier', name: 'yo06', start: 0, end: 0 },
|
|
|
|
{ type: 'Identifier', name: 'yo07', start: 0, end: 0 },
|
|
|
|
{ type: 'Identifier', name: 'yo08', start: 0, end: 0 },
|
|
|
|
{ type: 'Identifier', name: 'yo09', start: 0, end: 0 },
|
|
|
|
] satisfies Identifier[]),
|
2023-02-12 10:56:45 +11:00
|
|
|
'yo',
|
|
|
|
2
|
|
|
|
)
|
|
|
|
expect(result).toBe('yo10')
|
|
|
|
})
|
|
|
|
})
|
|
|
|
describe('Testing addSketchTo', () => {
|
|
|
|
it('should add a sketch to a program', () => {
|
|
|
|
const result = addSketchTo(
|
|
|
|
{
|
|
|
|
body: [],
|
|
|
|
start: 0,
|
|
|
|
end: 0,
|
New parser built in Winnow (#731)
* New parser built with Winnow
This new parser uses [winnow](docs.rs/winnow) to replace the handwritten recursive parser.
## Differences
I think the Winnow parser is more readable than handwritten one, due to reusing standard combinators. If you have a parsre like `p` or `q` you can combine them with standard functions like `repeat(0..4, p)`, `opt(p)`, `alt((p, q))` and `separated1(p, ", ")`. This IMO makes it more readable once you know what those standard functions do.
It's also more accurate now -- e.g. the parser no longer swallows whitespace between comments, or inserts it where there was none before. It no longer changes // comments to /* comments depending on the surrounding whitespace.
Primary form of testing was running the same KCL program through both the old and new parsers and asserting that both parsers produce the same AST. See the test `parser::parser_impl::tests::check_parsers_work_the_same`. But occasionally the new and old parsers disagree. This is either:
- Innocuous (e.g. disagreeing on whether a comment starts at the preceding whitespace or at the //)
- Helpful (e.g. new parser recognizes comments more accurately, preserving the difference between // and /* comments)
- Acceptably bad (e.g. new parser sometimes outputs worse error messages, TODO in #784)
so those KCL programs have their own unit tests in `parser_impl.rs` demonstrating the behaviour.
If you'd like to review this PR, it's arguably more important to review changes to the existing unit tests rather than the new parser itself. Because changes to the unit tests show where my parser changes behaviour -- usually for the better, occasionally for the worse (e.g. a worse error message than before). I think overall the improvements are worth it that I'd like to merge it without spending another week fixing it up -- we can fix the error messages in a follow-up PR.
## Performance
| Benchmark | Old parser (this branch) | New parser (this branch) | Speedup |
| ------------- | ------------- | ------------- | ------------- |
| Pipes on pipes | 922 ms | 42 ms | 21x |
| Kitt SVG | 148 ms | 7 ms | 21x |
There's definitely still room to improve performance:
- https://github.com/KittyCAD/modeling-app/issues/839
- https://github.com/KittyCAD/modeling-app/issues/840
## Winnow
Y'all know I love [Nom](docs.rs/nom) and I've blogged about it a lot. But I'm very happy using Winnow, a fork. It's got some really nice usability improvements. While writing this PR I found some bugs or unclear docs in Winnow:
- https://github.com/winnow-rs/winnow/issues/339
- https://github.com/winnow-rs/winnow/issues/341
- https://github.com/winnow-rs/winnow/issues/342
- https://github.com/winnow-rs/winnow/issues/344
The maintainer was quick to close them and release new versions within a few hours, so I feel very confident building the parser on this library. It's a clear improvement over Nom and it's used in toml-edit (and therefore within Cargo) and Gitoxide, so it's becoming a staple of the Rust ecosystem, which adds confidence.
Closes #716
Closes #815
Closes #599
2023-10-12 09:42:37 -05:00
|
|
|
nonCodeMeta: { nonCodeNodes: {}, start: [] },
|
2023-02-12 10:56:45 +11:00
|
|
|
},
|
|
|
|
'yz'
|
|
|
|
)
|
|
|
|
const str = recast(result.modifiedAst)
|
2023-10-05 14:27:48 -07:00
|
|
|
expect(str).toBe(`const part001 = startSketchOn('YZ')
|
|
|
|
|> startProfileAt('default', %)
|
2023-03-17 15:53:20 +11:00
|
|
|
|> line('default', %)
|
2023-09-11 17:14:41 -07:00
|
|
|
`)
|
2023-02-12 10:56:45 +11:00
|
|
|
})
|
|
|
|
})
|
2023-03-02 21:19:11 +11:00
|
|
|
|
|
|
|
function giveSketchFnCallTagTestHelper(
|
|
|
|
code: string,
|
|
|
|
searchStr: string
|
2023-03-07 15:45:59 +11:00
|
|
|
): { tag: string; newCode: string; isTagExisting: boolean } {
|
2023-03-02 21:19:11 +11:00
|
|
|
// giveSketchFnCallTag inputs and outputs an ast, which is very verbose for testing
|
|
|
|
// this wrapper changes the input and output to code
|
|
|
|
// making it more of an integration test, but easier to read the test intention is the goal
|
2023-09-29 11:11:01 -07:00
|
|
|
const ast = parse(code)
|
2023-03-02 21:19:11 +11:00
|
|
|
const start = code.indexOf(searchStr)
|
|
|
|
const range: [number, number] = [start, start + searchStr.length]
|
2023-03-07 15:45:59 +11:00
|
|
|
const { modifiedAst, tag, isTagExisting } = giveSketchFnCallTag(ast, range)
|
2023-03-02 21:19:11 +11:00
|
|
|
const newCode = recast(modifiedAst)
|
2023-03-07 15:45:59 +11:00
|
|
|
return { tag, newCode, isTagExisting }
|
2023-03-02 21:19:11 +11:00
|
|
|
}
|
|
|
|
|
|
|
|
describe('Testing giveSketchFnCallTag', () => {
|
2023-10-05 14:27:48 -07:00
|
|
|
const code = `const part001 = startSketchOn('XY')
|
|
|
|
|> startProfileAt([0, 0], %)
|
2023-03-02 21:19:11 +11:00
|
|
|
|> line([-2.57, -0.13], %)
|
|
|
|
|> line([0, 0.83], %)
|
2024-03-01 17:16:18 -08:00
|
|
|
|> line([0.82, 0.34], %)`
|
2023-03-02 21:19:11 +11:00
|
|
|
it('Should add tag to a sketch function call', () => {
|
2023-03-07 15:45:59 +11:00
|
|
|
const { newCode, tag, isTagExisting } = giveSketchFnCallTagTestHelper(
|
2023-03-02 21:19:11 +11:00
|
|
|
code,
|
|
|
|
'line([0, 0.83], %)'
|
|
|
|
)
|
|
|
|
expect(newCode).toContain("line({ to: [0, 0.83], tag: 'seg01' }, %)")
|
|
|
|
expect(tag).toBe('seg01')
|
2023-03-07 15:45:59 +11:00
|
|
|
expect(isTagExisting).toBe(false)
|
2023-03-02 21:19:11 +11:00
|
|
|
})
|
|
|
|
it('Should create a unique tag if seg01 already exists', () => {
|
|
|
|
let _code = code.replace(
|
|
|
|
'line([-2.57, -0.13], %)',
|
|
|
|
"line({ to: [-2.57, -0.13], tag: 'seg01' }, %)"
|
|
|
|
)
|
2023-03-07 15:45:59 +11:00
|
|
|
const { newCode, tag, isTagExisting } = giveSketchFnCallTagTestHelper(
|
2023-03-02 21:19:11 +11:00
|
|
|
_code,
|
|
|
|
'line([0, 0.83], %)'
|
|
|
|
)
|
|
|
|
expect(newCode).toContain("line({ to: [0, 0.83], tag: 'seg02' }, %)")
|
|
|
|
expect(tag).toBe('seg02')
|
2023-03-07 15:45:59 +11:00
|
|
|
expect(isTagExisting).toBe(false)
|
2023-03-02 21:19:11 +11:00
|
|
|
})
|
|
|
|
it('Should return existing tag if it already exists', () => {
|
|
|
|
const lineButWithTag = "line({ to: [-2.57, -0.13], tag: 'butts' }, %)"
|
|
|
|
let _code = code.replace('line([-2.57, -0.13], %)', lineButWithTag)
|
2023-03-07 15:45:59 +11:00
|
|
|
const { newCode, tag, isTagExisting } = giveSketchFnCallTagTestHelper(
|
2023-03-02 21:19:11 +11:00
|
|
|
_code,
|
|
|
|
lineButWithTag
|
|
|
|
)
|
|
|
|
expect(newCode).toContain(lineButWithTag) // no change
|
|
|
|
expect(tag).toBe('butts')
|
2023-03-07 15:45:59 +11:00
|
|
|
expect(isTagExisting).toBe(true)
|
2023-03-02 21:19:11 +11:00
|
|
|
})
|
|
|
|
})
|
2023-04-01 16:47:00 +11:00
|
|
|
|
|
|
|
describe('Testing moveValueIntoNewVariable', () => {
|
2023-09-13 11:42:09 -07:00
|
|
|
const fn = (fnName: string) => `fn ${fnName} = (x) => {
|
2023-04-01 16:47:00 +11:00
|
|
|
return x
|
|
|
|
}
|
|
|
|
`
|
2023-09-11 15:15:37 -07:00
|
|
|
const code = `${fn('def')}${fn('jkl')}${fn('hmm')}
|
|
|
|
fn ghi = (x) => {
|
|
|
|
return 2
|
|
|
|
}
|
2023-04-01 16:47:00 +11:00
|
|
|
const abc = 3
|
|
|
|
const identifierGuy = 5
|
2023-08-24 15:34:51 -07:00
|
|
|
const yo = 5 + 6
|
2023-10-05 14:27:48 -07:00
|
|
|
const part001 = startSketchOn('XY')
|
|
|
|
|> startProfileAt([-1.2, 4.83], %)
|
2023-04-01 16:47:00 +11:00
|
|
|
|> line([2.8, 0], %)
|
|
|
|
|> angledLine([100 + 100, 3.09], %)
|
|
|
|
|> angledLine([abc, 3.09], %)
|
2023-08-24 15:34:51 -07:00
|
|
|
|> angledLine([def(yo), 3.09], %)
|
2023-04-01 16:47:00 +11:00
|
|
|
|> angledLine([ghi(%), 3.09], %)
|
2023-08-24 15:34:51 -07:00
|
|
|
|> angledLine([jkl(yo) + 2, 3.09], %)
|
2024-03-01 17:16:18 -08:00
|
|
|
const yo2 = hmm([identifierGuy + 5])`
|
2023-07-20 19:25:04 -04:00
|
|
|
it('should move a binary expression into a new variable', async () => {
|
2023-09-29 11:11:01 -07:00
|
|
|
const ast = parse(code)
|
2023-07-10 15:15:07 +10:00
|
|
|
const programMemory = await enginelessExecutor(ast)
|
2023-04-01 16:47:00 +11:00
|
|
|
const startIndex = code.indexOf('100 + 100') + 1
|
|
|
|
const { modifiedAst } = moveValueIntoNewVariable(
|
|
|
|
ast,
|
|
|
|
programMemory,
|
|
|
|
[startIndex, startIndex],
|
|
|
|
'newVar'
|
|
|
|
)
|
|
|
|
const newCode = recast(modifiedAst)
|
|
|
|
expect(newCode).toContain(`const newVar = 100 + 100`)
|
|
|
|
expect(newCode).toContain(`angledLine([newVar, 3.09], %)`)
|
|
|
|
})
|
2023-06-22 16:43:33 +10:00
|
|
|
it('should move a value into a new variable', async () => {
|
2023-09-29 11:11:01 -07:00
|
|
|
const ast = parse(code)
|
2023-07-10 15:15:07 +10:00
|
|
|
const programMemory = await enginelessExecutor(ast)
|
2023-04-01 16:47:00 +11:00
|
|
|
const startIndex = code.indexOf('2.8') + 1
|
|
|
|
const { modifiedAst } = moveValueIntoNewVariable(
|
|
|
|
ast,
|
|
|
|
programMemory,
|
|
|
|
[startIndex, startIndex],
|
|
|
|
'newVar'
|
|
|
|
)
|
|
|
|
const newCode = recast(modifiedAst)
|
|
|
|
expect(newCode).toContain(`const newVar = 2.8`)
|
|
|
|
expect(newCode).toContain(`line([newVar, 0], %)`)
|
|
|
|
})
|
2023-07-20 19:25:04 -04:00
|
|
|
it('should move a callExpression into a new variable', async () => {
|
2023-09-29 11:11:01 -07:00
|
|
|
const ast = parse(code)
|
2023-07-10 15:15:07 +10:00
|
|
|
const programMemory = await enginelessExecutor(ast)
|
2023-04-01 16:47:00 +11:00
|
|
|
const startIndex = code.indexOf('def(')
|
|
|
|
const { modifiedAst } = moveValueIntoNewVariable(
|
|
|
|
ast,
|
|
|
|
programMemory,
|
|
|
|
[startIndex, startIndex],
|
|
|
|
'newVar'
|
|
|
|
)
|
|
|
|
const newCode = recast(modifiedAst)
|
2023-08-24 15:34:51 -07:00
|
|
|
expect(newCode).toContain(`const newVar = def(yo)`)
|
2023-04-01 16:47:00 +11:00
|
|
|
expect(newCode).toContain(`angledLine([newVar, 3.09], %)`)
|
|
|
|
})
|
2023-07-20 19:25:04 -04:00
|
|
|
it('should move a binary expression with call expression into a new variable', async () => {
|
2023-09-29 11:11:01 -07:00
|
|
|
const ast = parse(code)
|
2023-07-10 15:15:07 +10:00
|
|
|
const programMemory = await enginelessExecutor(ast)
|
2023-04-01 16:47:00 +11:00
|
|
|
const startIndex = code.indexOf('jkl(') + 1
|
|
|
|
const { modifiedAst } = moveValueIntoNewVariable(
|
|
|
|
ast,
|
|
|
|
programMemory,
|
|
|
|
[startIndex, startIndex],
|
|
|
|
'newVar'
|
|
|
|
)
|
|
|
|
const newCode = recast(modifiedAst)
|
2023-08-24 15:34:51 -07:00
|
|
|
expect(newCode).toContain(`const newVar = jkl(yo) + 2`)
|
2023-04-01 16:47:00 +11:00
|
|
|
expect(newCode).toContain(`angledLine([newVar, 3.09], %)`)
|
|
|
|
})
|
2023-07-20 19:25:04 -04:00
|
|
|
it('should move a identifier into a new variable', async () => {
|
2023-09-29 11:11:01 -07:00
|
|
|
const ast = parse(code)
|
2023-07-10 15:15:07 +10:00
|
|
|
const programMemory = await enginelessExecutor(ast)
|
2023-04-01 16:47:00 +11:00
|
|
|
const startIndex = code.indexOf('identifierGuy +') + 1
|
|
|
|
const { modifiedAst } = moveValueIntoNewVariable(
|
|
|
|
ast,
|
|
|
|
programMemory,
|
|
|
|
[startIndex, startIndex],
|
|
|
|
'newVar'
|
|
|
|
)
|
|
|
|
const newCode = recast(modifiedAst)
|
|
|
|
expect(newCode).toContain(`const newVar = identifierGuy + 5`)
|
|
|
|
expect(newCode).toContain(`const yo2 = hmm([newVar])`)
|
|
|
|
})
|
|
|
|
})
|