modeling-app/src/lang/tokeniser.test.ts

import {
  isBlockEnd,
  isBlockStart,
  isNumber,
  isOperator,
  isParanEnd,
  isParanStart,
  isString,
  isWhitespace,
  isWord,
  lexer,
} from "./tokeniser";

describe("testing helpers", () => {
  it("test is number", () => {
    expect(isNumber("1")).toBe(true);
    expect(isNumber("5?")).toBe(true);
    expect(isNumber("5 + 6")).toBe(true);
    expect(isNumber("5 + a")).toBe(true);

    expect(isNumber("a")).toBe(false);
    expect(isNumber("?")).toBe(false);
    expect(isNumber("?5")).toBe(false);
  });
  it("test is whitespace", () => {
    expect(isWhitespace(" ")).toBe(true);
    expect(isWhitespace("  ")).toBe(true);
    expect(isWhitespace(" a")).toBe(true);
    expect(isWhitespace("a ")).toBe(true);

    expect(isWhitespace("a")).toBe(false);
    expect(isWhitespace("?")).toBe(false);
  });
  it("test is word", () => {
    expect(isWord("a")).toBe(true);
    expect(isWord("a ")).toBe(true);
    expect(isWord("a5")).toBe(true);
    expect(isWord("a5a")).toBe(true);

    expect(isWord("5")).toBe(false);
    expect(isWord("5a")).toBe(false);
    expect(isWord("5a5")).toBe(false);
  });
  it("test is string", () => {
    expect(isString('""')).toBe(true);
    expect(isString('"a"')).toBe(true);
    expect(isString('"a" ')).toBe(true);
    expect(isString('"a"5')).toBe(true);
    expect(isString("'a'5")).toBe(true);
    expect(isString('"with escaped \\" backslash"')).toBe(true);

    expect(isString('"')).toBe(false);
    expect(isString('"a')).toBe(false);
    expect(isString('a"')).toBe(false);
    expect(isString(' "a"')).toBe(false);
    expect(isString('5"a"')).toBe(false);
  });
  it("test is operator", () => {
    expect(isOperator("+")).toBe(true);
    expect(isOperator("+ ")).toBe(true);
    expect(isOperator("-")).toBe(true);
    expect(isOperator("<=")).toBe(true);
    expect(isOperator("<= ")).toBe(true);
    expect(isOperator(">=")).toBe(true);
    expect(isOperator(">= ")).toBe(true);
    expect(isOperator("> ")).toBe(true);
    expect(isOperator("< ")).toBe(true);
    expect(isOperator("| ")).toBe(true);
    expect(isOperator("|> ")).toBe(true);
    expect(isOperator("^ ")).toBe(true);
    expect(isOperator("% ")).toBe(true);
    expect(isOperator("+* ")).toBe(true);

    expect(isOperator("5 + 5")).toBe(false);
    expect(isOperator("a")).toBe(false);
    expect(isOperator("a+")).toBe(false);
    expect(isOperator("a+5")).toBe(false);
    expect(isOperator("5a+5")).toBe(false);
  });
  it("test is paran start", () => {
    expect(isParanStart("(")).toBe(true);
    expect(isParanStart("( ")).toBe(true);
    expect(isParanStart("(5")).toBe(true);
    expect(isParanStart("(5 ")).toBe(true);
    expect(isParanStart("(5 + 5")).toBe(true);
    expect(isParanStart("(5 + 5)")).toBe(true);
    expect(isParanStart("(5 + 5) ")).toBe(true);

    expect(isParanStart("5")).toBe(false);
    expect(isParanStart("5 + 5")).toBe(false);
    expect(isParanStart("5( + 5)")).toBe(false);
    expect(isParanStart(" ( + 5)")).toBe(false);
  });
  it("test is paran end", () => {
    expect(isParanEnd(")")).toBe(true);
    expect(isParanEnd(") ")).toBe(true);
    expect(isParanEnd(")5")).toBe(true);
    expect(isParanEnd(")5 ")).toBe(true);

    expect(isParanEnd("5")).toBe(false);
    expect(isParanEnd("5 + 5")).toBe(false);
    expect(isParanEnd("5) + 5")).toBe(false);
    expect(isParanEnd(" ) + 5")).toBe(false);
  });
  it("test is block start", () => {
    expect(isBlockStart("{")).toBe(true);
    expect(isBlockStart("{ ")).toBe(true);
    expect(isBlockStart("{5")).toBe(true);
    expect(isBlockStart("{a")).toBe(true);
    expect(isBlockStart("{5 ")).toBe(true);

    expect(isBlockStart("5")).toBe(false);
    expect(isBlockStart("5 + 5")).toBe(false);
    expect(isBlockStart("5{ + 5")).toBe(false);
    expect(isBlockStart("a{ + 5")).toBe(false);
    expect(isBlockStart(" { + 5")).toBe(false);
  });
  it("test is block end", () => {
    expect(isBlockEnd("}")).toBe(true);
    expect(isBlockEnd("} ")).toBe(true);
    expect(isBlockEnd("}5")).toBe(true);
    expect(isBlockEnd("}5 ")).toBe(true);

    expect(isBlockEnd("5")).toBe(false);
    expect(isBlockEnd("5 + 5")).toBe(false);
    expect(isBlockEnd("5} + 5")).toBe(false);
    expect(isBlockEnd(" } + 5")).toBe(false);
  });
});

describe("testing lexer", () => {
  it("test lexer", () => {
    expect(stringSummaryLexer("1  + 2")).toEqual([
      "number       '1'        from 0 to 1",
      "whitespace   '  '       from 1 to 3",
      "operator     '+'        from 3 to 4",
      "whitespace   ' '        from 4 to 5",
      "number       '2'        from 5 to 6",
    ]);
    expect(stringSummaryLexer("54 + 22500 + 6")).toEqual([
      "number       '54'       from 0 to 2",
      "whitespace   ' '        from 2 to 3",
      "operator     '+'        from 3 to 4",
      "whitespace   ' '        from 4 to 5",
      "number       '22500'    from 5 to 10",
      "whitespace   ' '        from 10 to 11",
      "operator     '+'        from 11 to 12",
      "whitespace   ' '        from 12 to 13",
      "number       '6'        from 13 to 14",
    ]);
    expect(stringSummaryLexer("a + bo + t5 - 6")).toEqual([
      "word         'a'        from 0 to 1",
      "whitespace   ' '        from 1 to 2",
      "operator     '+'        from 2 to 3",
      "whitespace   ' '        from 3 to 4",
      "word         'bo'       from 4 to 6",
      "whitespace   ' '        from 6 to 7",
      "operator     '+'        from 7 to 8",
      "whitespace   ' '        from 8 to 9",
      "word         't5'       from 9 to 11",
      "whitespace   ' '        from 11 to 12",
      "operator     '-'        from 12 to 13",
      "whitespace   ' '        from 13 to 14",
      "number       '6'        from 14 to 15",
    ]);
    expect(stringSummaryLexer('a + "a str" - 6')).toEqual([
      "word         'a'        from 0 to 1",
      "whitespace   ' '        from 1 to 2",
      "operator     '+'        from 2 to 3",
      "whitespace   ' '        from 3 to 4",
      "string       '\"a str\"'  from 4 to 11",
      "whitespace   ' '        from 11 to 12",
      "operator     '-'        from 12 to 13",
      "whitespace   ' '        from 13 to 14",
      "number       '6'        from 14 to 15",
    ]);
    expect(stringSummaryLexer("a + 'str'")).toEqual([
      "word         'a'        from 0 to 1",
      "whitespace   ' '        from 1 to 2",
      "operator     '+'        from 2 to 3",
      "whitespace   ' '        from 3 to 4",
      "string       ''str''    from 4 to 9",
    ]);
    expect(stringSummaryLexer("a +'str'")).toEqual([
      "word         'a'        from 0 to 1",
      "whitespace   ' '        from 1 to 2",
      "operator     '+'        from 2 to 3",
      "string       ''str''    from 3 to 8",
    ]);

    expect(stringSummaryLexer("a + (sick)")).toEqual([
      "word         'a'        from 0 to 1",
      "whitespace   ' '        from 1 to 2",
      "operator     '+'        from 2 to 3",
      "whitespace   ' '        from 3 to 4",
      "brace        '('        from 4 to 5",
      "word         'sick'     from 5 to 9",
      "brace        ')'        from 9 to 10",
    ]);

    expect(stringSummaryLexer("a + { sick}")).toEqual([
      "word         'a'        from 0 to 1",
      "whitespace   ' '        from 1 to 2",
      "operator     '+'        from 2 to 3",
      "whitespace   ' '        from 3 to 4",
      "brace        '{'        from 4 to 5",
      "whitespace   ' '        from 5 to 6",
      "word         'sick'     from 6 to 10",
      "brace        '}'        from 10 to 11",
    ]);
  });
});

// helpers

const stringSummaryLexer = (input: string) =>
  lexer(input).map(
    ({ type, value, start, end }) =>
      `${type.padEnd(12, " ")} ${`'${value}'`.padEnd(
        10,
        " "
      )} from ${start} to ${end}`
  );