re-arrange folder

src/lang/tokeniser.test.ts

@@ -0,0 +1,223 @@
+import {
+  isBlockEnd,
+  isBlockStart,
+  isNumber,
+  isOperator,
+  isParanEnd,
+  isParanStart,
+  isString,
+  isWhitespace,
+  isWord,
+  lexer,
+} from "./tokeniser";
+
+describe("testing helpers", () => {
+  it("test is number", () => {
+    expect(isNumber("1")).toBe(true);
+    expect(isNumber("5?")).toBe(true);
+    expect(isNumber("5 + 6")).toBe(true);
+    expect(isNumber("5 + a")).toBe(true);
+
+    expect(isNumber("a")).toBe(false);
+    expect(isNumber("?")).toBe(false);
+    expect(isNumber("?5")).toBe(false);
+  });
+  it("test is whitespace", () => {
+    expect(isWhitespace(" ")).toBe(true);
+    expect(isWhitespace("  ")).toBe(true);
+    expect(isWhitespace(" a")).toBe(true);
+    expect(isWhitespace("a ")).toBe(true);
+
+    expect(isWhitespace("a")).toBe(false);
+    expect(isWhitespace("?")).toBe(false);
+  });
+  it("test is word", () => {
+    expect(isWord("a")).toBe(true);
+    expect(isWord("a ")).toBe(true);
+    expect(isWord("a5")).toBe(true);
+    expect(isWord("a5a")).toBe(true);
+
+    expect(isWord("5")).toBe(false);
+    expect(isWord("5a")).toBe(false);
+    expect(isWord("5a5")).toBe(false);
+  });
+  it("test is string", () => {
+    expect(isString('""')).toBe(true);
+    expect(isString('"a"')).toBe(true);
+    expect(isString('"a" ')).toBe(true);
+    expect(isString('"a"5')).toBe(true);
+    expect(isString("'a'5")).toBe(true);
+    expect(isString('"with escaped \\" backslash"')).toBe(true);
+
+    expect(isString('"')).toBe(false);
+    expect(isString('"a')).toBe(false);
+    expect(isString('a"')).toBe(false);
+    expect(isString(' "a"')).toBe(false);
+    expect(isString('5"a"')).toBe(false);
+  });
+  it("test is operator", () => {
+    expect(isOperator("+")).toBe(true);
+    expect(isOperator("+ ")).toBe(true);
+    expect(isOperator("-")).toBe(true);
+    expect(isOperator("<=")).toBe(true);
+    expect(isOperator("<= ")).toBe(true);
+    expect(isOperator(">=")).toBe(true);
+    expect(isOperator(">= ")).toBe(true);
+    expect(isOperator("> ")).toBe(true);
+    expect(isOperator("< ")).toBe(true);
+    expect(isOperator("| ")).toBe(true);
+    expect(isOperator("|> ")).toBe(true);
+    expect(isOperator("^ ")).toBe(true);
+    expect(isOperator("% ")).toBe(true);
+    expect(isOperator("+* ")).toBe(true);
+
+    expect(isOperator("5 + 5")).toBe(false);
+    expect(isOperator("a")).toBe(false);
+    expect(isOperator("a+")).toBe(false);
+    expect(isOperator("a+5")).toBe(false);
+    expect(isOperator("5a+5")).toBe(false);
+  });
+  it("test is paran start", () => {
+    expect(isParanStart("(")).toBe(true);
+    expect(isParanStart("( ")).toBe(true);
+    expect(isParanStart("(5")).toBe(true);
+    expect(isParanStart("(5 ")).toBe(true);
+    expect(isParanStart("(5 + 5")).toBe(true);
+    expect(isParanStart("(5 + 5)")).toBe(true);
+    expect(isParanStart("(5 + 5) ")).toBe(true);
+
+    expect(isParanStart("5")).toBe(false);
+    expect(isParanStart("5 + 5")).toBe(false);
+    expect(isParanStart("5( + 5)")).toBe(false);
+    expect(isParanStart(" ( + 5)")).toBe(false);
+  });
+  it("test is paran end", () => {
+    expect(isParanEnd(")")).toBe(true);
+    expect(isParanEnd(") ")).toBe(true);
+    expect(isParanEnd(")5")).toBe(true);
+    expect(isParanEnd(")5 ")).toBe(true);
+
+    expect(isParanEnd("5")).toBe(false);
+    expect(isParanEnd("5 + 5")).toBe(false);
+    expect(isParanEnd("5) + 5")).toBe(false);
+    expect(isParanEnd(" ) + 5")).toBe(false);
+  });
+  it("test is block start", () => {
+    expect(isBlockStart("{")).toBe(true);
+    expect(isBlockStart("{ ")).toBe(true);
+    expect(isBlockStart("{5")).toBe(true);
+    expect(isBlockStart("{a")).toBe(true);
+    expect(isBlockStart("{5 ")).toBe(true);
+
+    expect(isBlockStart("5")).toBe(false);
+    expect(isBlockStart("5 + 5")).toBe(false);
+    expect(isBlockStart("5{ + 5")).toBe(false);
+    expect(isBlockStart("a{ + 5")).toBe(false);
+    expect(isBlockStart(" { + 5")).toBe(false);
+  });
+  it("test is block end", () => {
+    expect(isBlockEnd("}")).toBe(true);
+    expect(isBlockEnd("} ")).toBe(true);
+    expect(isBlockEnd("}5")).toBe(true);
+    expect(isBlockEnd("}5 ")).toBe(true);
+
+    expect(isBlockEnd("5")).toBe(false);
+    expect(isBlockEnd("5 + 5")).toBe(false);
+    expect(isBlockEnd("5} + 5")).toBe(false);
+    expect(isBlockEnd(" } + 5")).toBe(false);
+  });
+});
+
+describe("testing lexer", () => {
+  it("test lexer", () => {
+    expect(stringSummaryLexer("1  + 2")).toEqual([
+      "number       '1'        from 0 to 1",
+      "whitespace   '  '       from 1 to 3",
+      "operator     '+'        from 3 to 4",
+      "whitespace   ' '        from 4 to 5",
+      "number       '2'        from 5 to 6",
+    ]);
+    expect(stringSummaryLexer("54 + 22500 + 6")).toEqual([
+      "number       '54'       from 0 to 2",
+      "whitespace   ' '        from 2 to 3",
+      "operator     '+'        from 3 to 4",
+      "whitespace   ' '        from 4 to 5",
+      "number       '22500'    from 5 to 10",
+      "whitespace   ' '        from 10 to 11",
+      "operator     '+'        from 11 to 12",
+      "whitespace   ' '        from 12 to 13",
+      "number       '6'        from 13 to 14",
+    ]);
+    expect(stringSummaryLexer("a + bo + t5 - 6")).toEqual([
+      "word         'a'        from 0 to 1",
+      "whitespace   ' '        from 1 to 2",
+      "operator     '+'        from 2 to 3",
+      "whitespace   ' '        from 3 to 4",
+      "word         'bo'       from 4 to 6",
+      "whitespace   ' '        from 6 to 7",
+      "operator     '+'        from 7 to 8",
+      "whitespace   ' '        from 8 to 9",
+      "word         't5'       from 9 to 11",
+      "whitespace   ' '        from 11 to 12",
+      "operator     '-'        from 12 to 13",
+      "whitespace   ' '        from 13 to 14",
+      "number       '6'        from 14 to 15",
+    ]);
+    expect(stringSummaryLexer('a + "a str" - 6')).toEqual([
+      "word         'a'        from 0 to 1",
+      "whitespace   ' '        from 1 to 2",
+      "operator     '+'        from 2 to 3",
+      "whitespace   ' '        from 3 to 4",
+      "string       '\"a str\"'  from 4 to 11",
+      "whitespace   ' '        from 11 to 12",
+      "operator     '-'        from 12 to 13",
+      "whitespace   ' '        from 13 to 14",
+      "number       '6'        from 14 to 15",
+    ]);
+    expect(stringSummaryLexer("a + 'str'")).toEqual([
+      "word         'a'        from 0 to 1",
+      "whitespace   ' '        from 1 to 2",
+      "operator     '+'        from 2 to 3",
+      "whitespace   ' '        from 3 to 4",
+      "string       ''str''    from 4 to 9",
+    ]);
+    expect(stringSummaryLexer("a +'str'")).toEqual([
+      "word         'a'        from 0 to 1",
+      "whitespace   ' '        from 1 to 2",
+      "operator     '+'        from 2 to 3",
+      "string       ''str''    from 3 to 8",
+    ]);
+
+    expect(stringSummaryLexer("a + (sick)")).toEqual([
+      "word         'a'        from 0 to 1",
+      "whitespace   ' '        from 1 to 2",
+      "operator     '+'        from 2 to 3",
+      "whitespace   ' '        from 3 to 4",
+      "brace        '('        from 4 to 5",
+      "word         'sick'     from 5 to 9",
+      "brace        ')'        from 9 to 10",
+    ]);
+
+    expect(stringSummaryLexer("a + { sick}")).toEqual([
+      "word         'a'        from 0 to 1",
+      "whitespace   ' '        from 1 to 2",
+      "operator     '+'        from 2 to 3",
+      "whitespace   ' '        from 3 to 4",
+      "brace        '{'        from 4 to 5",
+      "whitespace   ' '        from 5 to 6",
+      "word         'sick'     from 6 to 10",
+      "brace        '}'        from 10 to 11",
+    ]);
+  });
+});
+
+// helpers
+
+const stringSummaryLexer = (input: string) =>
+  lexer(input).map(
+    ({ type, value, start, end }) =>
+      `${type.padEnd(12, " ")} ${`'${value}'`.padEnd(
+        10,
+        " "
+      )} from ${start} to ${end}`
+  );