diff --git a/src/lang/tokeniser.test.ts b/src/lang/tokeniser.test.ts index b2fb9a09c..402f6edc3 100644 --- a/src/lang/tokeniser.test.ts +++ b/src/lang/tokeniser.test.ts @@ -18,7 +18,10 @@ describe('testing helpers', () => { expect(isNumber('5?')).toBe(true) expect(isNumber('5 + 6')).toBe(true) expect(isNumber('5 + a')).toBe(true) - + expect(isNumber('-5')).toBe(true) + expect(isNumber('5.5')).toBe(true) + expect(isNumber('-5.5')).toBe(true) + expect(isNumber('a')).toBe(false) expect(isNumber('?')).toBe(false) expect(isNumber('?5')).toBe(false) @@ -258,6 +261,46 @@ describe('testing lexer', () => { "brace '}' from 35 to 36", ]) }) + it('test negative and decimal numbers', () => { + expect(stringSummaryLexer('-1')).toEqual([ + "number '-1' from 0 to 2", + ]) + expect(stringSummaryLexer('-1.5')).toEqual([ + "number '-1.5' from 0 to 4", + ]) + expect(stringSummaryLexer('1.5')).toEqual([ + "number '1.5' from 0 to 3", + ]) + expect(stringSummaryLexer('1.5 + 2.5')).toEqual([ + "number '1.5' from 0 to 3", + "whitespace ' ' from 3 to 4", + "operator '+' from 4 to 5", + "whitespace ' ' from 5 to 6", + "number '2.5' from 6 to 9", + ]) + expect(stringSummaryLexer('1.5 - 2.5')).toEqual([ + "number '1.5' from 0 to 3", + "whitespace ' ' from 3 to 4", + "operator '-' from 4 to 5", + "whitespace ' ' from 5 to 6", + "number '2.5' from 6 to 9", + ]) + expect(stringSummaryLexer('1.5 + -2.5')).toEqual([ + "number '1.5' from 0 to 3", + "whitespace ' ' from 3 to 4", + "operator '+' from 4 to 5", + "whitespace ' ' from 5 to 6", + "number '-2.5' from 6 to 10", + ]) + expect(stringSummaryLexer('-1.5 + 2.5')).toEqual([ + "number '-1.5' from 0 to 4", + "whitespace ' ' from 4 to 5", + "operator '+' from 5 to 6", + "whitespace ' ' from 6 to 7", + "number '2.5' from 7 to 10", + ]) + + }) }) // helpers diff --git a/src/lang/tokeniser.ts b/src/lang/tokeniser.ts index 1181a92e5..7cbcaac6c 100644 --- a/src/lang/tokeniser.ts +++ b/src/lang/tokeniser.ts @@ -1,4 +1,6 @@ -const NUMBER = /^[0-9]+/ +// regular expression for number that includes a decimal point or starts with a minus sign +const NUMBER = /^-?\d+(\.\d+)?/ + const WHITESPACE = /\s+/ const WORD = /^[a-zA-Z_][a-zA-Z0-9_]*/ // regex that captures everything between two non escaped quotes and the quotes aren't captured in the match @@ -58,9 +60,6 @@ const makeToken = ( const returnTokenAtIndex = (str: string, startIndex: number): Token | null => { const strFromIndex = str.slice(startIndex) - if (isOperator(strFromIndex)) { - return makeToken('operator', matchFirst(strFromIndex, OPERATOR), startIndex) - } if (isString(strFromIndex)) { return makeToken('string', matchFirst(strFromIndex, STRING), startIndex) } @@ -82,6 +81,9 @@ const returnTokenAtIndex = (str: string, startIndex: number): Token | null => { if (isNumber(strFromIndex)) { return makeToken('number', matchFirst(strFromIndex, NUMBER), startIndex) } + if (isOperator(strFromIndex)) { + return makeToken('operator', matchFirst(strFromIndex, OPERATOR), startIndex) + } if (isWord(strFromIndex)) { return makeToken('word', matchFirst(strFromIndex, WORD), startIndex) }