modeling-app/src/lang/tokeniser.test.ts

import {
  isBlockEnd,
  isBlockStart,
  isNumber,
  isOperator,
  isParanEnd,
  isParanStart,
  isString,
  isWhitespace,
  isWord,
  isComma,
  lexer,
} from './tokeniser'

describe('testing helpers', () => {
  it('test is number', () => {
    expect(isNumber('1')).toBe(true)
    expect(isNumber('5?')).toBe(true)
    expect(isNumber('5 + 6')).toBe(true)
    expect(isNumber('5 + a')).toBe(true)
    expect(isNumber('-5')).toBe(true)
    expect(isNumber('5.5')).toBe(true)
    expect(isNumber('-5.5')).toBe(true)

    expect(isNumber('a')).toBe(false)
    expect(isNumber('?')).toBe(false)
    expect(isNumber('?5')).toBe(false)
  })
  it('test is whitespace', () => {
    expect(isWhitespace(' ')).toBe(true)
    expect(isWhitespace('  ')).toBe(true)
    expect(isWhitespace(' a')).toBe(true)
    expect(isWhitespace('a ')).toBe(true)

    expect(isWhitespace('a')).toBe(false)
    expect(isWhitespace('?')).toBe(false)
  })
  it('test is word', () => {
    expect(isWord('a')).toBe(true)
    expect(isWord('a ')).toBe(true)
    expect(isWord('a5')).toBe(true)
    expect(isWord('a5a')).toBe(true)

    expect(isWord('5')).toBe(false)
    expect(isWord('5a')).toBe(false)
    expect(isWord('5a5')).toBe(false)
  })
  it('test is string', () => {
    expect(isString('""')).toBe(true)
    expect(isString('"a"')).toBe(true)
    expect(isString('"a" ')).toBe(true)
    expect(isString('"a"5')).toBe(true)
    expect(isString("'a'5")).toBe(true)
    expect(isString('"with escaped \\" backslash"')).toBe(true)

    expect(isString('"')).toBe(false)
    expect(isString('"a')).toBe(false)
    expect(isString('a"')).toBe(false)
    expect(isString(' "a"')).toBe(false)
    expect(isString('5"a"')).toBe(false)
  })
  it('test is operator', () => {
    expect(isOperator('+')).toBe(true)
    expect(isOperator('+ ')).toBe(true)
    expect(isOperator('-')).toBe(true)
    expect(isOperator('<=')).toBe(true)
    expect(isOperator('<= ')).toBe(true)
    expect(isOperator('>=')).toBe(true)
    expect(isOperator('>= ')).toBe(true)
    expect(isOperator('> ')).toBe(true)
    expect(isOperator('< ')).toBe(true)
    expect(isOperator('| ')).toBe(true)
    expect(isOperator('|> ')).toBe(true)
    expect(isOperator('^ ')).toBe(true)
    expect(isOperator('% ')).toBe(true)
    expect(isOperator('+* ')).toBe(true)

    expect(isOperator('5 + 5')).toBe(false)
    expect(isOperator('a')).toBe(false)
    expect(isOperator('a+')).toBe(false)
    expect(isOperator('a+5')).toBe(false)
    expect(isOperator('5a+5')).toBe(false)
    expect(isOperator(', newVar')).toBe(false)
    expect(isOperator(',')).toBe(false)
  })
  it('test is paran start', () => {
    expect(isParanStart('(')).toBe(true)
    expect(isParanStart('( ')).toBe(true)
    expect(isParanStart('(5')).toBe(true)
    expect(isParanStart('(5 ')).toBe(true)
    expect(isParanStart('(5 + 5')).toBe(true)
    expect(isParanStart('(5 + 5)')).toBe(true)
    expect(isParanStart('(5 + 5) ')).toBe(true)

    expect(isParanStart('5')).toBe(false)
    expect(isParanStart('5 + 5')).toBe(false)
    expect(isParanStart('5( + 5)')).toBe(false)
    expect(isParanStart(' ( + 5)')).toBe(false)
  })
  it('test is paran end', () => {
    expect(isParanEnd(')')).toBe(true)
    expect(isParanEnd(') ')).toBe(true)
    expect(isParanEnd(')5')).toBe(true)
    expect(isParanEnd(')5 ')).toBe(true)

    expect(isParanEnd('5')).toBe(false)
    expect(isParanEnd('5 + 5')).toBe(false)
    expect(isParanEnd('5) + 5')).toBe(false)
    expect(isParanEnd(' ) + 5')).toBe(false)
  })
  it('test is block start', () => {
    expect(isBlockStart('{')).toBe(true)
    expect(isBlockStart('{ ')).toBe(true)
    expect(isBlockStart('{5')).toBe(true)
    expect(isBlockStart('{a')).toBe(true)
    expect(isBlockStart('{5 ')).toBe(true)

    expect(isBlockStart('5')).toBe(false)
    expect(isBlockStart('5 + 5')).toBe(false)
    expect(isBlockStart('5{ + 5')).toBe(false)
    expect(isBlockStart('a{ + 5')).toBe(false)
    expect(isBlockStart(' { + 5')).toBe(false)
  })
  it('test is block end', () => {
    expect(isBlockEnd('}')).toBe(true)
    expect(isBlockEnd('} ')).toBe(true)
    expect(isBlockEnd('}5')).toBe(true)
    expect(isBlockEnd('}5 ')).toBe(true)

    expect(isBlockEnd('5')).toBe(false)
    expect(isBlockEnd('5 + 5')).toBe(false)
    expect(isBlockEnd('5} + 5')).toBe(false)
    expect(isBlockEnd(' } + 5')).toBe(false)
  })
  it('test is comma', () => {
    expect(isComma(',')).toBe(true)
    expect(isComma(', ')).toBe(true)
    expect(isComma(',5')).toBe(true)
    expect(isComma(',5 ')).toBe(true)

    expect(isComma('5')).toBe(false)
    expect(isComma('5 + 5')).toBe(false)
    expect(isComma('5, + 5')).toBe(false)
    expect(isComma(' , + 5')).toBe(false)
  })
})

describe('testing lexer', () => {
  it('test lexer', () => {
    expect(stringSummaryLexer('1  + 2')).toEqual([
      "number       '1'        from 0   to 1",
      "whitespace   '  '       from 1   to 3",
      "operator     '+'        from 3   to 4",
      "whitespace   ' '        from 4   to 5",
      "number       '2'        from 5   to 6",
    ])
    expect(stringSummaryLexer('54 + 22500 + 6')).toEqual([
      "number       '54'       from 0   to 2",
      "whitespace   ' '        from 2   to 3",
      "operator     '+'        from 3   to 4",
      "whitespace   ' '        from 4   to 5",
      "number       '22500'    from 5   to 10",
      "whitespace   ' '        from 10  to 11",
      "operator     '+'        from 11  to 12",
      "whitespace   ' '        from 12  to 13",
      "number       '6'        from 13  to 14",
    ])
    expect(stringSummaryLexer('a + bo + t5 - 6')).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "whitespace   ' '        from 3   to 4",
      "word         'bo'       from 4   to 6",
      "whitespace   ' '        from 6   to 7",
      "operator     '+'        from 7   to 8",
      "whitespace   ' '        from 8   to 9",
      "word         't5'       from 9   to 11",
      "whitespace   ' '        from 11  to 12",
      "operator     '-'        from 12  to 13",
      "whitespace   ' '        from 13  to 14",
      "number       '6'        from 14  to 15",
    ])
    expect(stringSummaryLexer('a + "a str" - 6')).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "whitespace   ' '        from 3   to 4",
      'string       \'"a str"\'  from 4   to 11',
      "whitespace   ' '        from 11  to 12",
      "operator     '-'        from 12  to 13",
      "whitespace   ' '        from 13  to 14",
      "number       '6'        from 14  to 15",
    ])
    expect(stringSummaryLexer("a + 'str'")).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "whitespace   ' '        from 3   to 4",
      "string       ''str''    from 4   to 9",
    ])
    expect(stringSummaryLexer("a +'str'")).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "string       ''str''    from 3   to 8",
    ])

    expect(stringSummaryLexer('a + (sick)')).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "whitespace   ' '        from 3   to 4",
      "brace        '('        from 4   to 5",
      "word         'sick'     from 5   to 9",
      "brace        ')'        from 9   to 10",
    ])

    expect(stringSummaryLexer('a + { sick}')).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "whitespace   ' '        from 3   to 4",
      "brace        '{'        from 4   to 5",
      "whitespace   ' '        from 5   to 6",
      "word         'sick'     from 6   to 10",
      "brace        '}'        from 10  to 11",
    ])

    expect(stringSummaryLexer("log('hi')")).toEqual([
      "word         'log'      from 0   to 3",
      "brace        '('        from 3   to 4",
      "string       ''hi''     from 4   to 8",
      "brace        ')'        from 8   to 9",
    ])
    expect(stringSummaryLexer("log('hi', 'hello')")).toEqual([
      "word         'log'      from 0   to 3",
      "brace        '('        from 3   to 4",
      "string       ''hi''     from 4   to 8",
      "comma        ','        from 8   to 9",
      "whitespace   ' '        from 9   to 10",
      "string       ''hello''  from 10  to 17",
      "brace        ')'        from 17  to 18",
    ])
    expect(stringSummaryLexer('fn funcName = (param1, param2) => {}')).toEqual([
      "word         'fn'       from 0   to 2",
      "whitespace   ' '        from 2   to 3",
      "word         'funcName' from 3   to 11",
      "whitespace   ' '        from 11  to 12",
      "operator     '='        from 12  to 13",
      "whitespace   ' '        from 13  to 14",
      "brace        '('        from 14  to 15",
      "word         'param1'   from 15  to 21",
      "comma        ','        from 21  to 22",
      "whitespace   ' '        from 22  to 23",
      "word         'param2'   from 23  to 29",
      "brace        ')'        from 29  to 30",
      "whitespace   ' '        from 30  to 31",
      "operator     '=>'       from 31  to 33",
      "whitespace   ' '        from 33  to 34",
      "brace        '{'        from 34  to 35",
      "brace        '}'        from 35  to 36",
    ])
  })
  it('test negative and decimal numbers', () => {
    expect(stringSummaryLexer('-1')).toEqual([
      "number       '-1'       from 0   to 2",
    ])
    expect(stringSummaryLexer('-1.5')).toEqual([
      "number       '-1.5'     from 0   to 4",
    ])
    expect(stringSummaryLexer('1.5')).toEqual([
      "number       '1.5'      from 0   to 3",
    ])
    expect(stringSummaryLexer('1.5 + 2.5')).toEqual([
      "number       '1.5'      from 0   to 3",
      "whitespace   ' '        from 3   to 4",
      "operator     '+'        from 4   to 5",
      "whitespace   ' '        from 5   to 6",
      "number       '2.5'      from 6   to 9",
    ])
    expect(stringSummaryLexer('1.5 - 2.5')).toEqual([
      "number       '1.5'      from 0   to 3",
      "whitespace   ' '        from 3   to 4",
      "operator     '-'        from 4   to 5",
      "whitespace   ' '        from 5   to 6",
      "number       '2.5'      from 6   to 9",
    ])
    expect(stringSummaryLexer('1.5 + -2.5')).toEqual([
      "number       '1.5'      from 0   to 3",
      "whitespace   ' '        from 3   to 4",
      "operator     '+'        from 4   to 5",
      "whitespace   ' '        from 5   to 6",
      "number       '-2.5'     from 6   to 10",
    ])
    expect(stringSummaryLexer('-1.5 + 2.5')).toEqual([
      "number       '-1.5'     from 0   to 4",
      "whitespace   ' '        from 4   to 5",
      "operator     '+'        from 5   to 6",
      "whitespace   ' '        from 6   to 7",
      "number       '2.5'      from 7   to 10",
    ])
  })
  it('testing piping operator', () => {
    const result = stringSummaryLexer(`sketch mySketch {
      lineTo(2, 3)
    } |> rx(45, %)`)
    expect(result).toEqual([
      "word         'sketch'   from 0   to 6",
      "whitespace   ' '        from 6   to 7",
      "word         'mySketch' from 7   to 15",
      "whitespace   ' '        from 15  to 16",
      "brace        '{'        from 16  to 17",
      "whitespace   '\n      '  from 17  to 24",
      "word         'lineTo'   from 24  to 30",
      "brace        '('        from 30  to 31",
      "number       '2'        from 31  to 32",
      "comma        ','        from 32  to 33",
      "whitespace   ' '        from 33  to 34",
      "number       '3'        from 34  to 35",
      "brace        ')'        from 35  to 36",
      "whitespace   '\n    '    from 36  to 41",
      "brace        '}'        from 41  to 42",
      "whitespace   ' '        from 42  to 43",
      "operator     '|>'       from 43  to 45",
      "whitespace   ' '        from 45  to 46",
      "word         'rx'       from 46  to 48",
      "brace        '('        from 48  to 49",
      "number       '45'       from 49  to 51",
      "comma        ','        from 51  to 52",
      "whitespace   ' '        from 52  to 53",
      "operator     '%'        from 53  to 54",
      "brace        ')'        from 54  to 55",
    ])
  })
  it('testing array declaration', () => {
    const result = stringSummaryLexer(`const yo = [1, 2]`)
    expect(result).toEqual([
      "word         'const'    from 0   to 5",
      "whitespace   ' '        from 5   to 6",
      "word         'yo'       from 6   to 8",
      "whitespace   ' '        from 8   to 9",
      "operator     '='        from 9   to 10",
      "whitespace   ' '        from 10  to 11",
      "brace        '['        from 11  to 12",
      "number       '1'        from 12  to 13",
      "comma        ','        from 13  to 14",
      "whitespace   ' '        from 14  to 15",
      "number       '2'        from 15  to 16",
      "brace        ']'        from 16  to 17",
    ])
  })
})

// helpers

const stringSummaryLexer = (input: string) =>
  lexer(input).map(
    ({ type, value, start, end }) =>
      `${type.padEnd(12, ' ')} ${`'${value}'`.padEnd(10, ' ')} from ${String(
        start
      ).padEnd(3, ' ')} to ${end}`
  )