add white spacetoken

src/tokeniser.test.ts

@@ -126,61 +126,98 @@ describe("testing helpers", () => {
     expect(isBlockEnd("5} + 5")).toBe(false);
     expect(isBlockEnd(" } + 5")).toBe(false);
   });
-
 });
 
 describe("testing lexer", () => {
   it("test lexer", () => {
-    expect(lexer("1 + 2")).toEqual([
+    expect(stringSummaryLexer("1  + 2")).toEqual([
-      { type: "number", value: "1" },
+      "number       '1'        from 0 to 1",
-      { type: "operator", value: "+" },
+      "whitespace   '  '       from 1 to 3",
-      { type: "number", value: "2" },
+      "operator     '+'        from 3 to 4",
+      "whitespace   ' '        from 4 to 5",
+      "number       '2'        from 5 to 6",
     ]);
-    expect(lexer("54 + 22500 + 6")).toEqual([
+    expect(stringSummaryLexer("54 + 22500 + 6")).toEqual([
-      { type: "number", value: "54" },
+      "number       '54'       from 0 to 2",
-      { type: "operator", value: "+" },
+      "whitespace   ' '        from 2 to 3",
-      { type: "number", value: "22500" },
+      "operator     '+'        from 3 to 4",
-      { type: "operator", value: "+" },
+      "whitespace   ' '        from 4 to 5",
-      { type: "number", value: "6" },
+      "number       '22500'    from 5 to 10",
+      "whitespace   ' '        from 10 to 11",
+      "operator     '+'        from 11 to 12",
+      "whitespace   ' '        from 12 to 13",
+      "number       '6'        from 13 to 14",
     ]);
-    expect(lexer("a + bo + t5 - 6")).toEqual([
+    expect(stringSummaryLexer("a + bo + t5 - 6")).toEqual([
-      { type: "word", value: "a" },
+      "word         'a'        from 0 to 1",
-      { type: "operator", value: "+" },
+      "whitespace   ' '        from 1 to 2",
-      { type: "word", value: "bo" },
+      "operator     '+'        from 2 to 3",
-      { type: "operator", value: "+" },
+      "whitespace   ' '        from 3 to 4",
-      { type: "word", value: "t5" },
+      "word         'bo'       from 4 to 6",
-      { type: "operator", value: "-" },
+      "whitespace   ' '        from 6 to 7",
-      { type: "number", value: "6" },
+      "operator     '+'        from 7 to 8",
+      "whitespace   ' '        from 8 to 9",
+      "word         't5'       from 9 to 11",
+      "whitespace   ' '        from 11 to 12",
+      "operator     '-'        from 12 to 13",
+      "whitespace   ' '        from 13 to 14",
+      "number       '6'        from 14 to 15",
     ]);
-    expect(lexer('a + "a str" - 6')).toEqual([
+    expect(stringSummaryLexer('a + "a str" - 6')).toEqual([
-      { type: "word", value: "a" },
+      "word         'a'        from 0 to 1",
-      { type: "operator", value: "+" },
+      "whitespace   ' '        from 1 to 2",
-      { type: "string", value: '"a str"' },
+      "operator     '+'        from 2 to 3",
-      { type: "operator", value: "-" },
+      "whitespace   ' '        from 3 to 4",
-      { type: "number", value: "6" },
+      "string       '\"a str\"'  from 4 to 11",
+      "whitespace   ' '        from 11 to 12",
+      "operator     '-'        from 12 to 13",
+      "whitespace   ' '        from 13 to 14",
+      "number       '6'        from 14 to 15",
     ]);
-    const sameWithOrWithoutWhiteSpaces = [
+    expect(stringSummaryLexer("a + 'str'")).toEqual([
-      { type: "word", value: "a" },
+      "word         'a'        from 0 to 1",
-      { type: "operator", value: "+" },
+      "whitespace   ' '        from 1 to 2",
-      { type: "string", value: "'str'" },
+      "operator     '+'        from 2 to 3",
-    ];
+      "whitespace   ' '        from 3 to 4",
-    expect(lexer("a + 'str'")).toEqual(sameWithOrWithoutWhiteSpaces);
+      "string       ''str''    from 4 to 9",
-    expect(lexer("a +'str'")).toEqual(sameWithOrWithoutWhiteSpaces);
+    ]);
-
+    expect(stringSummaryLexer("a +'str'")).toEqual([
-    expect(lexer("a + (sick)")).toEqual([
+      "word         'a'        from 0 to 1",
-      { type: "word", value: "a" },
+      "whitespace   ' '        from 1 to 2",
-      { type: "operator", value: "+" },
+      "operator     '+'        from 2 to 3",
-      { type: "brace", value: "(" },
+      "string       ''str''    from 3 to 8",
-      { type: "word", value: "sick" },
-      { type: "brace", value: ")" },
     ]);
 
-    expect(lexer("a + {sick}")).toEqual([
+    expect(stringSummaryLexer("a + (sick)")).toEqual([
-      { type: "word", value: "a" },
+      "word         'a'        from 0 to 1",
-      { type: "operator", value: "+" },
+      "whitespace   ' '        from 1 to 2",
-      { type: "brace", value: "{" },
+      "operator     '+'        from 2 to 3",
-      { type: "word", value: "sick" },
+      "whitespace   ' '        from 3 to 4",
-      { type: "brace", value: "}" },
+      "brace        '('        from 4 to 5",
+      "word         'sick'     from 5 to 9",
+      "brace        ')'        from 9 to 10",
+    ]);
+
+    expect(stringSummaryLexer("a + { sick}")).toEqual([
+      "word         'a'        from 0 to 1",
+      "whitespace   ' '        from 1 to 2",
+      "operator     '+'        from 2 to 3",
+      "whitespace   ' '        from 3 to 4",
+      "brace        '{'        from 4 to 5",
+      "whitespace   ' '        from 5 to 6",
+      "word         'sick'     from 6 to 10",
+      "brace        '}'        from 10 to 11",
     ]);
   });
 });
+
+// helpers
+
+const stringSummaryLexer = (input: string) =>
+  lexer(input).map(
+    ({ type, value, start, end }) =>
+      `${type.padEnd(12, " ")} ${`'${value}'`.padEnd(
+        10,
+        " "
+      )} from ${start} to ${end}`
+  );

src/tokeniser.ts

@@ -30,60 +30,48 @@ function matchFirst(str: string, regex: RegExp) {
   return theMatch[0];
 }
 
-interface Token {
+export interface Token {
-  type: "number" | "word" | "operator" | "string" | "brace";
+  type: "number" | "word" | "operator" | "string" | "brace" | "whitespace";
   value: string;
+  start: number;
+  end: number;
 }
 
+const makeToken = (type: Token["type"], value: string, start: number): Token => ({
+  type,
+  value,
+  start,
+  end: start + value.length,
+})
+
 const returnTokenAtIndex = (str: string, startIndex: number): Token | null => {
   const strFromIndex = str.slice(startIndex);
   if (isOperator(strFromIndex)) {
-    return {
+    return makeToken("operator", matchFirst(strFromIndex, OPERATOR), startIndex);
-      type: "operator",
-      value: matchFirst(strFromIndex, OPERATOR),
-    };
   }
   if (isString(strFromIndex)) {
-    return {
+    return makeToken("string", matchFirst(strFromIndex, STRING), startIndex);
-      type: "string",
-      value: matchFirst(strFromIndex, STRING),
-    };
   }
   if (isParanEnd(strFromIndex)) {
-    return {
+    return makeToken("brace", matchFirst(strFromIndex, PARAN_END), startIndex);
-      type: "brace",
-      value: matchFirst(strFromIndex, PARAN_END),
-    };
   }
   if (isParanStart(strFromIndex)) {
-    return {
+    return makeToken("brace", matchFirst(strFromIndex, PARAN_START), startIndex);
-      type: "brace",
-      value: matchFirst(strFromIndex, PARAN_START),
-    };
   }
   if (isBlockStart(strFromIndex)) {
-    return {
+    return makeToken("brace", matchFirst(strFromIndex, BLOCK_START), startIndex);
-      type: "brace",
-      value: matchFirst(strFromIndex, BLOCK_START),
-    };
   }
   if (isBlockEnd(strFromIndex)) {
-    return {
+    return makeToken("brace", matchFirst(strFromIndex, BLOCK_END), startIndex);
-      type: "brace",
-      value: matchFirst(strFromIndex, BLOCK_END),
-    };
   }
   if (isNumber(strFromIndex)) {
-    return {
+    return makeToken("number", matchFirst(strFromIndex, NUMBER), startIndex);
-      type: "number",
-      value: matchFirst(strFromIndex, NUMBER),
-    };
   }
   if (isWord(strFromIndex)) {
-    return {
+    return makeToken("word", matchFirst(strFromIndex, WORD), startIndex);
-      type: "word",
+  }
-      value: matchFirst(strFromIndex, WORD),
+  if (isWhitespace(strFromIndex)) {
-    };
+    return makeToken("whitespace", matchFirst(strFromIndex, WHITESPACE), startIndex);
   }
   return null;
 };