modeling-app/src/lang/tokeniser.test.ts

import {
  isBlockEnd,
  isBlockStart,
  isNumber,
  isOperator,
  isParanEnd,
  isParanStart,
  isString,
  isWhitespace,
  isWord,
  isComma,
  lexer,
} from './tokeniser'

describe('testing helpers', () => {
  it('test is number', () => {
    expect(isNumber('1')).toBe(true)
    expect(isNumber('5?')).toBe(true)
    expect(isNumber('5 + 6')).toBe(true)
    expect(isNumber('5 + a')).toBe(true)
    expect(isNumber('-5')).toBe(true)
    expect(isNumber('5.5')).toBe(true)
    expect(isNumber('-5.5')).toBe(true)

    expect(isNumber('a')).toBe(false)
    expect(isNumber('?')).toBe(false)
    expect(isNumber('?5')).toBe(false)
  })
  it('test is whitespace', () => {
    expect(isWhitespace(' ')).toBe(true)
    expect(isWhitespace('  ')).toBe(true)
    expect(isWhitespace(' a')).toBe(true)
    expect(isWhitespace('a ')).toBe(true)

    expect(isWhitespace('a')).toBe(false)
    expect(isWhitespace('?')).toBe(false)
  })
  it('test is word', () => {
    expect(isWord('a')).toBe(true)
    expect(isWord('a ')).toBe(true)
    expect(isWord('a5')).toBe(true)
    expect(isWord('a5a')).toBe(true)

    expect(isWord('5')).toBe(false)
    expect(isWord('5a')).toBe(false)
    expect(isWord('5a5')).toBe(false)
  })
  it('test is string', () => {
    expect(isString('""')).toBe(true)
    expect(isString('"a"')).toBe(true)
    expect(isString('"a" ')).toBe(true)
    expect(isString('"a"5')).toBe(true)
    expect(isString("'a'5")).toBe(true)
    expect(isString('"with escaped \\" backslash"')).toBe(true)

    expect(isString('"')).toBe(false)
    expect(isString('"a')).toBe(false)
    expect(isString('a"')).toBe(false)
    expect(isString(' "a"')).toBe(false)
    expect(isString('5"a"')).toBe(false)
  })
  it('test is operator', () => {
    expect(isOperator('+')).toBe(true)
    expect(isOperator('+ ')).toBe(true)
    expect(isOperator('-')).toBe(true)
    expect(isOperator('<=')).toBe(true)
    expect(isOperator('<= ')).toBe(true)
    expect(isOperator('>=')).toBe(true)
    expect(isOperator('>= ')).toBe(true)
    expect(isOperator('> ')).toBe(true)
    expect(isOperator('< ')).toBe(true)
    expect(isOperator('| ')).toBe(true)
    expect(isOperator('|> ')).toBe(true)
    expect(isOperator('^ ')).toBe(true)
    expect(isOperator('% ')).toBe(true)
    expect(isOperator('+* ')).toBe(true)

    expect(isOperator('5 + 5')).toBe(false)
    expect(isOperator('a')).toBe(false)
    expect(isOperator('a+')).toBe(false)
    expect(isOperator('a+5')).toBe(false)
    expect(isOperator('5a+5')).toBe(false)
    expect(isOperator(', newVar')).toBe(false)
    expect(isOperator(',')).toBe(false)
  })
  it('test is paran start', () => {
    expect(isParanStart('(')).toBe(true)
    expect(isParanStart('( ')).toBe(true)
    expect(isParanStart('(5')).toBe(true)
    expect(isParanStart('(5 ')).toBe(true)
    expect(isParanStart('(5 + 5')).toBe(true)
    expect(isParanStart('(5 + 5)')).toBe(true)
    expect(isParanStart('(5 + 5) ')).toBe(true)

    expect(isParanStart('5')).toBe(false)
    expect(isParanStart('5 + 5')).toBe(false)
    expect(isParanStart('5( + 5)')).toBe(false)
    expect(isParanStart(' ( + 5)')).toBe(false)
  })
  it('test is paran end', () => {
    expect(isParanEnd(')')).toBe(true)
    expect(isParanEnd(') ')).toBe(true)
    expect(isParanEnd(')5')).toBe(true)
    expect(isParanEnd(')5 ')).toBe(true)

    expect(isParanEnd('5')).toBe(false)
    expect(isParanEnd('5 + 5')).toBe(false)
    expect(isParanEnd('5) + 5')).toBe(false)
    expect(isParanEnd(' ) + 5')).toBe(false)
  })
  it('test is block start', () => {
    expect(isBlockStart('{')).toBe(true)
    expect(isBlockStart('{ ')).toBe(true)
    expect(isBlockStart('{5')).toBe(true)
    expect(isBlockStart('{a')).toBe(true)
    expect(isBlockStart('{5 ')).toBe(true)

    expect(isBlockStart('5')).toBe(false)
    expect(isBlockStart('5 + 5')).toBe(false)
    expect(isBlockStart('5{ + 5')).toBe(false)
    expect(isBlockStart('a{ + 5')).toBe(false)
    expect(isBlockStart(' { + 5')).toBe(false)
  })
  it('test is block end', () => {
    expect(isBlockEnd('}')).toBe(true)
    expect(isBlockEnd('} ')).toBe(true)
    expect(isBlockEnd('}5')).toBe(true)
    expect(isBlockEnd('}5 ')).toBe(true)

    expect(isBlockEnd('5')).toBe(false)
    expect(isBlockEnd('5 + 5')).toBe(false)
    expect(isBlockEnd('5} + 5')).toBe(false)
    expect(isBlockEnd(' } + 5')).toBe(false)
  })
  it('test is comma', () => {
    expect(isComma(',')).toBe(true)
    expect(isComma(', ')).toBe(true)
    expect(isComma(',5')).toBe(true)
    expect(isComma(',5 ')).toBe(true)

    expect(isComma('5')).toBe(false)
    expect(isComma('5 + 5')).toBe(false)
    expect(isComma('5, + 5')).toBe(false)
    expect(isComma(' , + 5')).toBe(false)
  })
})

describe('testing lexer', () => {
  it('test lexer', () => {
    expect(stringSummaryLexer('1  + 2')).toEqual([
      "number       '1'        from 0   to 1",
      "whitespace   '  '       from 1   to 3",
      "operator     '+'        from 3   to 4",
      "whitespace   ' '        from 4   to 5",
      "number       '2'        from 5   to 6",
    ])
    expect(stringSummaryLexer('54 + 22500 + 6')).toEqual([
      "number       '54'       from 0   to 2",
      "whitespace   ' '        from 2   to 3",
      "operator     '+'        from 3   to 4",
      "whitespace   ' '        from 4   to 5",
      "number       '22500'    from 5   to 10",
      "whitespace   ' '        from 10  to 11",
      "operator     '+'        from 11  to 12",
      "whitespace   ' '        from 12  to 13",
      "number       '6'        from 13  to 14",
    ])
    expect(stringSummaryLexer('a + bo + t5 - 6')).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "whitespace   ' '        from 3   to 4",
      "word         'bo'       from 4   to 6",
      "whitespace   ' '        from 6   to 7",
      "operator     '+'        from 7   to 8",
      "whitespace   ' '        from 8   to 9",
      "word         't5'       from 9   to 11",
      "whitespace   ' '        from 11  to 12",
      "operator     '-'        from 12  to 13",
      "whitespace   ' '        from 13  to 14",
      "number       '6'        from 14  to 15",
    ])
    expect(stringSummaryLexer('a + "a str" - 6')).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "whitespace   ' '        from 3   to 4",
      'string       \'"a str"\'  from 4   to 11',
      "whitespace   ' '        from 11  to 12",
      "operator     '-'        from 12  to 13",
      "whitespace   ' '        from 13  to 14",
      "number       '6'        from 14  to 15",
    ])
    expect(stringSummaryLexer("a + 'str'")).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "whitespace   ' '        from 3   to 4",
      "string       ''str''    from 4   to 9",
    ])
    expect(stringSummaryLexer("a +'str'")).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "string       ''str''    from 3   to 8",
    ])

    expect(stringSummaryLexer('a + (sick)')).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "whitespace   ' '        from 3   to 4",
      "brace        '('        from 4   to 5",
      "word         'sick'     from 5   to 9",
      "brace        ')'        from 9   to 10",
    ])

    expect(stringSummaryLexer('a + { sick}')).toEqual([
      "word         'a'        from 0   to 1",
      "whitespace   ' '        from 1   to 2",
      "operator     '+'        from 2   to 3",
      "whitespace   ' '        from 3   to 4",
      "brace        '{'        from 4   to 5",
      "whitespace   ' '        from 5   to 6",
      "word         'sick'     from 6   to 10",
      "brace        '}'        from 10  to 11",
    ])

    expect(stringSummaryLexer("log('hi')")).toEqual([
      "word         'log'      from 0   to 3",
      "brace        '('        from 3   to 4",
      "string       ''hi''     from 4   to 8",
      "brace        ')'        from 8   to 9",
    ])
    expect(stringSummaryLexer("log('hi', 'hello')")).toEqual([
      "word         'log'      from 0   to 3",
      "brace        '('        from 3   to 4",
      "string       ''hi''     from 4   to 8",
      "comma        ','        from 8   to 9",
      "whitespace   ' '        from 9   to 10",
      "string       ''hello''  from 10  to 17",
      "brace        ')'        from 17  to 18",
    ])
    expect(stringSummaryLexer('fn funcName = (param1, param2) => {}')).toEqual([
      "word         'fn'       from 0   to 2",
      "whitespace   ' '        from 2   to 3",
      "word         'funcName' from 3   to 11",
      "whitespace   ' '        from 11  to 12",
      "operator     '='        from 12  to 13",
      "whitespace   ' '        from 13  to 14",
      "brace        '('        from 14  to 15",
      "word         'param1'   from 15  to 21",
      "comma        ','        from 21  to 22",
      "whitespace   ' '        from 22  to 23",
      "word         'param2'   from 23  to 29",
      "brace        ')'        from 29  to 30",
      "whitespace   ' '        from 30  to 31",
      "operator     '=>'       from 31  to 33",
      "whitespace   ' '        from 33  to 34",
      "brace        '{'        from 34  to 35",
      "brace        '}'        from 35  to 36",
    ])
  })
  it('test negative and decimal numbers', () => {
    expect(stringSummaryLexer('-1')).toEqual([
      "number       '-1'       from 0   to 2",
    ])
    expect(stringSummaryLexer('-1.5')).toEqual([
      "number       '-1.5'     from 0   to 4",
    ])
    expect(stringSummaryLexer('1.5')).toEqual([
      "number       '1.5'      from 0   to 3",
    ])
    expect(stringSummaryLexer('1.5 + 2.5')).toEqual([
      "number       '1.5'      from 0   to 3",
      "whitespace   ' '        from 3   to 4",
      "operator     '+'        from 4   to 5",
      "whitespace   ' '        from 5   to 6",
      "number       '2.5'      from 6   to 9",
    ])
    expect(stringSummaryLexer('1.5 - 2.5')).toEqual([
      "number       '1.5'      from 0   to 3",
      "whitespace   ' '        from 3   to 4",
      "operator     '-'        from 4   to 5",
      "whitespace   ' '        from 5   to 6",
      "number       '2.5'      from 6   to 9",
    ])
    expect(stringSummaryLexer('1.5 + -2.5')).toEqual([
      "number       '1.5'      from 0   to 3",
      "whitespace   ' '        from 3   to 4",
      "operator     '+'        from 4   to 5",
      "whitespace   ' '        from 5   to 6",
      "number       '-2.5'     from 6   to 10",
    ])
    expect(stringSummaryLexer('-1.5 + 2.5')).toEqual([
      "number       '-1.5'     from 0   to 4",
      "whitespace   ' '        from 4   to 5",
      "operator     '+'        from 5   to 6",
      "whitespace   ' '        from 6   to 7",
      "number       '2.5'      from 7   to 10",
    ])
  })
  it('testing piping operator', () => {
    const result = stringSummaryLexer(`sketch mySketch {
      lineTo(2, 3)
    } |> rx(45, %)`)
    expect(result).toEqual([
      "word         'sketch'   from 0   to 6",
      "whitespace   ' '        from 6   to 7",
      "word         'mySketch' from 7   to 15",
      "whitespace   ' '        from 15  to 16",
      "brace        '{'        from 16  to 17",
      "whitespace   '\n      '  from 17  to 24",
      "word         'lineTo'   from 24  to 30",
      "brace        '('        from 30  to 31",
      "number       '2'        from 31  to 32",
      "comma        ','        from 32  to 33",
      "whitespace   ' '        from 33  to 34",
      "number       '3'        from 34  to 35",
      "brace        ')'        from 35  to 36",
      "whitespace   '\n    '    from 36  to 41",
      "brace        '}'        from 41  to 42",
      "whitespace   ' '        from 42  to 43",
      "operator     '|>'       from 43  to 45",
      "whitespace   ' '        from 45  to 46",
      "word         'rx'       from 46  to 48",
      "brace        '('        from 48  to 49",
      "number       '45'       from 49  to 51",
      "comma        ','        from 51  to 52",
      "whitespace   ' '        from 52  to 53",
      "operator     '%'        from 53  to 54",
      "brace        ')'        from 54  to 55",
    ])
  })
  it('testing array declaration', () => {
    const result = stringSummaryLexer(`const yo = [1, 2]`)
    expect(result).toEqual([
      "word         'const'    from 0   to 5",
      "whitespace   ' '        from 5   to 6",
      "word         'yo'       from 6   to 8",
      "whitespace   ' '        from 8   to 9",
      "operator     '='        from 9   to 10",
      "whitespace   ' '        from 10  to 11",
      "brace        '['        from 11  to 12",
      "number       '1'        from 12  to 13",
      "comma        ','        from 13  to 14",
      "whitespace   ' '        from 14  to 15",
      "number       '2'        from 15  to 16",
      "brace        ']'        from 16  to 17",
    ])
  })
  it('testing object declaration', () => {
    const result = stringSummaryLexer(`const yo = {key: 'value'}`)
    expect(result).toEqual([
      "word         'const'    from 0   to 5",
      "whitespace   ' '        from 5   to 6",
      "word         'yo'       from 6   to 8",
      "whitespace   ' '        from 8   to 9",
      "operator     '='        from 9   to 10",
      "whitespace   ' '        from 10  to 11",
      "brace        '{'        from 11  to 12",
      "word         'key'      from 12  to 15",
      "colon        ':'        from 15  to 16",
      "whitespace   ' '        from 16  to 17",
      "string       ''value''  from 17  to 24",
      "brace        '}'        from 24  to 25",
    ])
  })
  it('testing object property access', () => {
    const result = stringSummaryLexer(`const yo = {key: 'value'}
const prop = yo.key
const prop2 = yo['key']
const key = 'key'
const prop3 = yo[key]`)
    expect(result).toEqual([
      "word         'const'    from 0   to 5",
      "whitespace   ' '        from 5   to 6",
      "word         'yo'       from 6   to 8",
      "whitespace   ' '        from 8   to 9",
      "operator     '='        from 9   to 10",
      "whitespace   ' '        from 10  to 11",
      "brace        '{'        from 11  to 12",
      "word         'key'      from 12  to 15",
      "colon        ':'        from 15  to 16",
      "whitespace   ' '        from 16  to 17",
      "string       ''value''  from 17  to 24",
      "brace        '}'        from 24  to 25",
      "whitespace   '\n'        from 25  to 26",
      "word         'const'    from 26  to 31",
      "whitespace   ' '        from 31  to 32",
      "word         'prop'     from 32  to 36",
      "whitespace   ' '        from 36  to 37",
      "operator     '='        from 37  to 38",
      "whitespace   ' '        from 38  to 39",
      "word         'yo'       from 39  to 41",
      "period       '.'        from 41  to 42",
      "word         'key'      from 42  to 45",
      "whitespace   '\n'        from 45  to 46",
      "word         'const'    from 46  to 51",
      "whitespace   ' '        from 51  to 52",
      "word         'prop2'    from 52  to 57",
      "whitespace   ' '        from 57  to 58",
      "operator     '='        from 58  to 59",
      "whitespace   ' '        from 59  to 60",
      "word         'yo'       from 60  to 62",
      "brace        '['        from 62  to 63",
      "string       ''key''    from 63  to 68",
      "brace        ']'        from 68  to 69",
      "whitespace   '\n'        from 69  to 70",
      "word         'const'    from 70  to 75",
      "whitespace   ' '        from 75  to 76",
      "word         'key'      from 76  to 79",
      "whitespace   ' '        from 79  to 80",
      "operator     '='        from 80  to 81",
      "whitespace   ' '        from 81  to 82",
      "string       ''key''    from 82  to 87",
      "whitespace   '\n'        from 87  to 88",
      "word         'const'    from 88  to 93",
      "whitespace   ' '        from 93  to 94",
      "word         'prop3'    from 94  to 99",
      "whitespace   ' '        from 99  to 100",
      "operator     '='        from 100 to 101",
      "whitespace   ' '        from 101 to 102",
      "word         'yo'       from 102 to 104",
      "brace        '['        from 104 to 105",
      "word         'key'      from 105 to 108",
      "brace        ']'        from 108 to 109",
    ])
  })
})

// helpers

const stringSummaryLexer = (input: string) =>
  lexer(input).map(
    ({ type, value, start, end }) =>
      `${type.padEnd(12, ' ')} ${`'${value}'`.padEnd(10, ' ')} from ${String(
        start
      ).padEnd(3, ' ')} to ${end}`
  )