Merge pull request #7 from imteekay/semicolon-as-statement-ender

Semicolon as statement ender

README.md

@@ -34,9 +34,9 @@ npm run mtsc ./tests/singleVar.ts
 ## Exercises
 
 - [x] Add EmptyStatement (https://github.com/imteekay/mini-typescript/pull/2).
-- [ ] Make semicolon a statement ender, not statement separator.
+- [x] Make semicolon a statement ender, not statement separator (https://github.com/imteekay/mini-typescript/pull/7).
   - Hint: You'll need a predicate to peek at the next token and decide if it's the start of an element.
-  - Bonus: Switch from semicolon to newline as statement ender.
+  - [ ] Bonus: Switch from semicolon to newline as statement ender.
 - [x] Add string literals (https://github.com/imteekay/mini-typescript/pull/4).
 - [x] Refactor: rename `Literal` to `NumericLiteral` (https://github.com/imteekay/mini-typescript/pull/6).
 - [x] Add support for the lexer to report errors

baselines/reference/emptyStatement.tree.baseline

@@ -46,9 +46,6 @@
         "kind": "NumericLiteral",
         "value": 2
       }
-    },
-    {
-      "kind": "EmptyStatement"
     }
   ]
 }

baselines/reference/redeclare.tree.baseline

@@ -29,9 +29,6 @@
         "kind": "NumericLiteral",
         "value": 2
       }
-    },
-    {
-      "kind": "EmptyStatement"
     }
   ]
 }

baselines/reference/singleIdentifier.tree.baseline

@@ -7,9 +7,6 @@
         "kind": "Identifier",
         "text": "x"
       }
-    },
-    {
-      "kind": "EmptyStatement"
     }
   ]
 }

baselines/reference/singleTypedVar.tree.baseline

@@ -44,9 +44,6 @@
         "value": "test",
         "isSingleQuote": true
       }
-    },
-    {
-      "kind": "EmptyStatement"
     }
   ]
 }

baselines/reference/singleVar.tree.baseline

@@ -18,9 +18,6 @@
         "kind": "NumericLiteral",
         "value": 1
       }
-    },
-    {
-      "kind": "EmptyStatement"
     }
   ]
 }

baselines/reference/stringLiteral.tree.baseline

@@ -145,9 +145,6 @@
         "value": "escaped\rR",
         "isSingleQuote": true
       }
-    },
-    {
-      "kind": "EmptyStatement"
     }
   ]
 }

baselines/reference/terminator.errors.baseline

@@ -0,0 +1 @@
+[]

baselines/reference/terminator.js.baseline

@@ -0,0 +1 @@
+"var x = 1;\nvar y = 2;\nvar z = 3;\nx;\ny;\nz;\n"

baselines/reference/terminator.tree.baseline

@@ -0,0 +1,78 @@
+{
+  "locals": {
+    "x": [
+      {
+        "kind": "Var",
+        "pos": 3
+      }
+    ],
+    "y": [
+      {
+        "kind": "Var",
+        "pos": 14
+      }
+    ],
+    "z": [
+      {
+        "kind": "Var",
+        "pos": 25
+      }
+    ]
+  },
+  "statements": [
+    {
+      "kind": "Var",
+      "name": {
+        "kind": "Identifier",
+        "text": "x"
+      },
+      "init": {
+        "kind": "NumericLiteral",
+        "value": 1
+      }
+    },
+    {
+      "kind": "Var",
+      "name": {
+        "kind": "Identifier",
+        "text": "y"
+      },
+      "init": {
+        "kind": "NumericLiteral",
+        "value": 2
+      }
+    },
+    {
+      "kind": "Var",
+      "name": {
+        "kind": "Identifier",
+        "text": "z"
+      },
+      "init": {
+        "kind": "NumericLiteral",
+        "value": 3
+      }
+    },
+    {
+      "kind": "ExpressionStatement",
+      "expr": {
+        "kind": "Identifier",
+        "text": "x"
+      }
+    },
+    {
+      "kind": "ExpressionStatement",
+      "expr": {
+        "kind": "Identifier",
+        "text": "y"
+      }
+    },
+    {
+      "kind": "ExpressionStatement",
+      "expr": {
+        "kind": "Identifier",
+        "text": "z"
+      }
+    }
+  ]
+}

baselines/reference/twoStatements.tree.baseline

@@ -32,9 +32,6 @@
           "value": 2
         }
       }
-    },
-    {
-      "kind": "EmptyStatement"
     }
   ]
 }

baselines/reference/twoTypedStatements.tree.baseline

@@ -36,9 +36,6 @@
           "value": 2
         }
       }
-    },
-    {
-      "kind": "EmptyStatement"
     }
   ]
 }

baselines/reference/typeAlias.tree.baseline

@@ -192,9 +192,6 @@
         "kind": "Identifier",
         "text": "Net"
       }
-    },
-    {
-      "kind": "EmptyStatement"
     }
   ]
 }

src/emit.ts

@@ -18,7 +18,9 @@ const escapedCharsMap = new Map(
 );
 
 export function emit(statements: Statement[]) {
-  return statements.map(emitStatement).join(';\n');
+  return statements
+    .map((statement) => `${emitStatement(statement)};\n`)
+    .join('');
 }
 
 function emitStatement(statement: Statement): string {

src/lex.ts

@@ -23,21 +23,17 @@ export function lex(s: string): Lexer {
   };
 
   function scan() {
-    // scan forward all
-    // \t - tabs
-    // \b - empty strings at the beginning and end of a word
-    // \n - newline char
-    scanForward((c) => /[ \t\b\n]/.test(c));
+    scanForward(isEmptyStrings);
     const start = pos;
 
     if (pos === s.length) {
       token = Token.EOF;
     } else if (/[0-9]/.test(s.charAt(pos))) {
-      scanForward((c) => /[0-9]/.test(c));
+      scanForward(isNumber);
       text = s.slice(start, pos);
       token = Token.NumericLiteral;
     } else if (/[_a-zA-Z]/.test(s.charAt(pos))) {
-      scanForward((c) => /[_a-zA-Z0-9]/.test(c));
+      scanForward(isAlphanumerical);
       text = s.slice(start, pos);
       token =
         text in keywords
@@ -66,6 +62,22 @@ export function lex(s: string): Lexer {
     }
   }
 
+  function isEmptyStrings(c: string) {
+    // scan forward all
+    // \t - tabs
+    // \b - empty strings at the beginning and end of a word
+    // \n - newline char
+    return /[ \t\b\n]/.test(c);
+  }
+
+  function isNumber(c: string) {
+    return /[0-9]/.test(c);
+  }
+
+  function isAlphanumerical(c: string) {
+    return /[_a-zA-Z0-9]/.test(c);
+  }
+
   function scanForward(pred: (x: string) => boolean) {
     while (pos < s.length && pred(s.charAt(pos))) pos++;
   }

src/parse.ts

@@ -9,18 +9,19 @@ import {
 } from './types';
 import { error } from './error';
 
-const emptyTokens = [Token.EOF, Token.Semicolon];
-
 export function parse(lexer: Lexer): Module {
   lexer.scan();
   return parseModule();
 
   function parseModule(): Module {
-    const statements = parseSeparated(parseStatement, () =>
-      tryParseToken(Token.Semicolon),
-    );
-    parseExpected(Token.EOF);
-    return { statements, locals: new Map() };
+    return {
+      statements: parseStatements(
+        parseStatement,
+        () => tryParseToken(Token.Semicolon),
+        () => lexer.token() !== Token.EOF,
+      ),
+      locals: new Map(),
+    };
   }
 
   function parseExpression(): Expression {
@@ -65,6 +66,7 @@ export function parse(lexer: Lexer): Module {
 
   function parseStatement(): Statement {
     const pos = lexer.pos();
+
     if (tryParseToken(Token.Var)) {
       const name = parseIdentifier();
       const typename = tryParseToken(Token.Colon)
@@ -78,7 +80,7 @@ export function parse(lexer: Lexer): Module {
       parseExpected(Token.Equals);
       const typename = parseIdentifier();
       return { kind: Node.TypeAlias, name, typename, pos };
-    } else if (emptyTokens.includes(lexer.token())) {
+    } else if (lexer.token() === Token.Semicolon) {
       return { kind: Node.EmptyStatement };
     }
     return { kind: Node.ExpressionStatement, expr: parseExpression(), pos };
@@ -103,10 +105,15 @@ export function parse(lexer: Lexer): Module {
     }
   }
 
-  function parseSeparated<T>(element: () => T, separator: () => unknown) {
-    const list = [element()];
-    while (separator()) {
+  function parseStatements<T>(
+    element: () => T,
+    terminator: () => boolean,
+    peek: () => boolean,
+  ) {
+    const list = [];
+    while (peek()) {
       list.push(element());
+      terminator();
     }
     return list;
   }

src/types.ts

@@ -33,6 +33,7 @@ export enum Node {
   TypeAlias,
   StringLiteral,
   EmptyStatement,
+  EndOfFile,
 }
 
 export type Error = {

tests/terminator.ts

@@ -0,0 +1,4 @@
+var x = 1;
+var y = 2;
+var z = 3;
+x;y;z;