ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tokeniser should handle negative and decimal place numbers

Browse Source
This commit is contained in:
Kurt Hutten IrevDev
2022-11-26 20:38:07 +11:00
parent 149633a5cb
commit 83b8333694
2 changed files with 50 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
src/lang/tokeniser.test.ts
View File

@ -18,7 +18,10 @@ describe('testing helpers', () => {
expect(isNumber('5?')).toBe(true)
expect(isNumber('5 + 6')).toBe(true)
expect(isNumber('5 + a')).toBe(true)
expect(isNumber('-5')).toBe(true)
expect(isNumber('5.5')).toBe(true)
expect(isNumber('-5.5')).toBe(true)
expect(isNumber('a')).toBe(false)
expect(isNumber('?')).toBe(false)
expect(isNumber('?5')).toBe(false)
@ -258,6 +261,46 @@ describe('testing lexer', () => {
"brace '}' from 35 to 36",
])
})
it('test negative and decimal numbers', () => {
expect(stringSummaryLexer('-1')).toEqual([
"number '-1' from 0 to 2",
])
expect(stringSummaryLexer('-1.5')).toEqual([
"number '-1.5' from 0 to 4",
])
expect(stringSummaryLexer('1.5')).toEqual([
"number '1.5' from 0 to 3",
])
expect(stringSummaryLexer('1.5 + 2.5')).toEqual([
"number '1.5' from 0 to 3",
"whitespace ' ' from 3 to 4",
"operator '+' from 4 to 5",
"whitespace ' ' from 5 to 6",
"number '2.5' from 6 to 9",
])
expect(stringSummaryLexer('1.5 - 2.5')).toEqual([
"number '1.5' from 0 to 3",
"whitespace ' ' from 3 to 4",
"operator '-' from 4 to 5",
"whitespace ' ' from 5 to 6",
"number '2.5' from 6 to 9",
])
expect(stringSummaryLexer('1.5 + -2.5')).toEqual([
"number '1.5' from 0 to 3",
"whitespace ' ' from 3 to 4",
"operator '+' from 4 to 5",
"whitespace ' ' from 5 to 6",
"number '-2.5' from 6 to 10",
])
expect(stringSummaryLexer('-1.5 + 2.5')).toEqual([
"number '-1.5' from 0 to 4",
"whitespace ' ' from 4 to 5",
"operator '+' from 5 to 6",
"whitespace ' ' from 6 to 7",
"number '2.5' from 7 to 10",
])
})
})
// helpers

10
src/lang/tokeniser.ts
View File

@ -1,4 +1,6 @@
const NUMBER = /^[0-9]+/
// regular expression for number that includes a decimal point or starts with a minus sign
const NUMBER = /^-?\d+(\.\d+)?/
const WHITESPACE = /\s+/
const WORD = /^[a-zA-Z_][a-zA-Z0-9_]*/
// regex that captures everything between two non escaped quotes and the quotes aren't captured in the match
@ -58,9 +60,6 @@ const makeToken = (
const returnTokenAtIndex = (str: string, startIndex: number): Token | null => {
const strFromIndex = str.slice(startIndex)
if (isOperator(strFromIndex)) {
return makeToken('operator', matchFirst(strFromIndex, OPERATOR), startIndex)
}
if (isString(strFromIndex)) {
return makeToken('string', matchFirst(strFromIndex, STRING), startIndex)
}
@ -82,6 +81,9 @@ const returnTokenAtIndex = (str: string, startIndex: number): Token | null => {
if (isNumber(strFromIndex)) {
return makeToken('number', matchFirst(strFromIndex, NUMBER), startIndex)
}
if (isOperator(strFromIndex)) {
return makeToken('operator', matchFirst(strFromIndex, OPERATOR), startIndex)
}
if (isWord(strFromIndex)) {
return makeToken('word', matchFirst(strFromIndex, WORD), startIndex)
}