From 7d20eb783c1e608810926b40318708e820ce0bf8 Mon Sep 17 00:00:00 2001 From: liulinboyi <814921718@qq.com> Date: Sun, 10 Jan 2021 17:12:29 +0800 Subject: [PATCH] init --- .gitignore | 4 + .vscode/launch.json | 36 +++++++ README.md | 7 ++ package-lock.json | 78 +++++++++++++++ package.json | 9 ++ src/backend.js | 71 +++++++++++++ src/backend.ts | 82 +++++++++++++++ src/definition.js | 3 + src/definition.ts | 20 ++++ src/lexer.js | 222 ++++++++++++++++++++++++++++++++++++++++ src/lexer.ts | 239 ++++++++++++++++++++++++++++++++++++++++++++ src/parser.js | 124 +++++++++++++++++++++++ src/parser.ts | 161 +++++++++++++++++++++++++++++ tsconfig.json | 70 +++++++++++++ 14 files changed, 1126 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/launch.json create mode 100644 README.md create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 src/backend.js create mode 100644 src/backend.ts create mode 100644 src/definition.js create mode 100644 src/definition.ts create mode 100644 src/lexer.js create mode 100644 src/lexer.ts create mode 100644 src/parser.js create mode 100644 src/parser.ts create mode 100644 tsconfig.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4880a9f --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/node_modules/ +/src/*.js.map +/dist/*.js.map +/.cache/ diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..92be0a4 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,36 @@ +{ + // 使用 IntelliSense 了解相关属性。 + // 悬停以查看现有属性的描述。 + // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Program", + "program": "${workspaceFolder}/src/backend.js", + "request": "launch", + "skipFiles": [ + "/**" + ], + "type": "pwa-node" + }, + { + "name": "Current TS File", + "type": "node", + "request": "launch", + "args": [ + "${workspaceRoot}/src/parser.ts" // 入口文件 + ], + "runtimeArgs": [ + "--nolazy", + "-r", + "ts-node/register" + ], + "sourceMaps": true, + "cwd": "${workspaceRoot}", + "protocol": "inspector", + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen" + } + ] +} + diff --git a/README.md b/README.md new file mode 100644 index 0000000..07462c9 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# pineapple-ts + +TypeScript implementation of pineapple language (https://github.com/karminski/pineapple) as a personal exercise. + +[karminski/pineapple](https://github.com/karminski/pineapple) + +PINEAPPLE diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..67f1e33 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,78 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "arg": { + "version": "4.1.3", + "resolved": "https://registry.npm.taobao.org/arg/download/arg-4.1.3.tgz?cache=0&sync_timestamp=1605575011090&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Farg%2Fdownload%2Farg-4.1.3.tgz", + "integrity": "sha1-Jp/HrVuOQstjyJbVZmAXJhwUQIk=", + "dev": true + }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npm.taobao.org/buffer-from/download/buffer-from-1.1.1.tgz", + "integrity": "sha1-MnE7wCj3XAL9txDXx7zsHyxgcO8=", + "dev": true + }, + "create-require": { + "version": "1.1.1", + "resolved": "https://registry.npm.taobao.org/create-require/download/create-require-1.1.1.tgz", + "integrity": "sha1-wdfo8eX2z8n/ZfnNNS03NIdWwzM=", + "dev": true + }, + "diff": { + "version": "4.0.2", + "resolved": "https://registry.npm.taobao.org/diff/download/diff-4.0.2.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdiff%2Fdownload%2Fdiff-4.0.2.tgz", + "integrity": "sha1-YPOuy4nV+uUgwRqhnvwruYKq3n0=", + "dev": true + }, + "make-error": { + "version": "1.3.6", + "resolved": "https://registry.npm.taobao.org/make-error/download/make-error-1.3.6.tgz", + "integrity": "sha1-LrLjfqm2fEiR9oShOUeZr0hM96I=", + "dev": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npm.taobao.org/source-map/download/source-map-0.6.1.tgz", + "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=", + "dev": true + }, + "source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npm.taobao.org/source-map-support/download/source-map-support-0.5.19.tgz?cache=0&sync_timestamp=1587719517036&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fsource-map-support%2Fdownload%2Fsource-map-support-0.5.19.tgz", + "integrity": "sha1-qYti+G3K9PZzmWSMCFKRq56P7WE=", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "ts-node": { + "version": "9.1.1", + "resolved": "https://registry.npm.taobao.org/ts-node/download/ts-node-9.1.1.tgz?cache=0&sync_timestamp=1607350694094&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fts-node%2Fdownload%2Fts-node-9.1.1.tgz", + "integrity": "sha1-UamkUKPpWUAb2l8ASnLVS5NtN20=", + "dev": true, + "requires": { + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "source-map-support": "^0.5.17", + "yn": "3.1.1" + } + }, + "typescript": { + "version": "4.1.3", + "resolved": "https://registry.npm.taobao.org/typescript/download/typescript-4.1.3.tgz?cache=0&sync_timestamp=1610089571804&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Ftypescript%2Fdownload%2Ftypescript-4.1.3.tgz", + "integrity": "sha1-UZ1YK9lMugz4k0x9joRn5HP1O7c=", + "dev": true + }, + "yn": { + "version": "3.1.1", + "resolved": "https://registry.npm.taobao.org/yn/download/yn-3.1.1.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fyn%2Fdownload%2Fyn-3.1.1.tgz", + "integrity": "sha1-HodAGgnXZ8HV6rJqbkwYUYLS61A=", + "dev": true + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..afdcbac --- /dev/null +++ b/package.json @@ -0,0 +1,9 @@ +{ + "devDependencies": { + "ts-node": "^9.1.1", + "typescript": "^4.1.3" + }, + "scripts": { + "test": "ts-node ./src/backend.ts" + } +} diff --git a/src/backend.js b/src/backend.js new file mode 100644 index 0000000..3e9ea5d --- /dev/null +++ b/src/backend.js @@ -0,0 +1,71 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const parser_1 = require("./parser"); +let GlobalVariables = { + Variables: {} +}; +function NewGlobalVariables() { + var g = GlobalVariables; + g.Variables = {}; + return g; +} +function Execute(code) { + var ast = {}; + let g = NewGlobalVariables(); + // parse + ast = parser_1.parse(code); + // resolve + resolveAST(g, ast); +} +function resolveAST(g, ast) { + if (ast.Statements.length == 0) { + throw new Error("resolveAST(): no code to execute, please check your input."); + } + for (let statement of ast.Statements) { + resolveStatement(g, statement); + } + return null; +} +function resolveStatement(g, statement) { + if (statement instanceof parser_1.Assignment) { + let assignment = statement; + return resolveAssignment(g, assignment); + } + else if (statement instanceof parser_1.Print) { + let print = statement; + return resolvePrint(g, print); + } + else { + throw new Error("resolveStatement(): undefined statement type."); + } +} +function resolveAssignment(g, assignment) { + let varName = ""; + varName = assignment.Variable.Name; + if (varName == "") { + throw new Error("resolveAssignment(): variable name can NOT be empty."); + } + g.Variables[varName] = assignment.String; + return null; +} +function resolvePrint(g, print) { + let varName = ""; + varName = print.Variable.Name; + if (varName == "") { + throw new Error("resolvePrint(): variable name can NOT be empty."); + } + let str = ""; + let ok = false; + str = g.Variables[varName]; + ok = str ? true : false; + if (!ok) { + throw new Error(`resolvePrint(): variable '$${varName}'not found.`); + } + console.log(str); + return null; +} +// test +Execute(`$a = "你好，我是pineapple11" +print($a) +`); +//# sourceMappingURL=backend.js.map \ No newline at end of file diff --git a/src/backend.ts b/src/backend.ts new file mode 100644 index 0000000..f86b9c5 --- /dev/null +++ b/src/backend.ts @@ -0,0 +1,82 @@ +import { Assignment, parse, Print } from "./parser" +import { GlobalVariables } from './definition' + +let GlobalVariables: GlobalVariables = { + Variables: {} +} + + +function NewGlobalVariables() { + var g = GlobalVariables + g.Variables = {} + return g +} + + +function Execute(code: string) { + var ast = {} + + let g = NewGlobalVariables() + + // parse + ast = parse(code) + + // resolve + resolveAST(g, ast) +} + +function resolveAST(g: any, ast: any) { + if (ast.Statements.length == 0) { + throw new Error("resolveAST(): no code to execute, please check your input.") + } + for (let statement of ast.Statements) { + resolveStatement(g, statement) + } + return null +} + +function resolveStatement(g: any, statement: any) { + if (statement instanceof Assignment) { + let assignment = statement + return resolveAssignment(g, assignment) + } else if (statement instanceof Print) { + let print = statement + return resolvePrint(g, print) + } else { + throw new Error("resolveStatement(): undefined statement type.") + } +} + +function resolveAssignment(g: any, assignment: any) { + let varName = "" + varName = assignment.Variable.Name; + if (varName == "") { + throw new Error("resolveAssignment(): variable name can NOT be empty.") + } + g.Variables[varName] = assignment.String + return null +} + + +function resolvePrint(g: any, print: any) { + let varName = "" + varName = print.Variable.Name; + if (varName == "") { + throw new Error("resolvePrint(): variable name can NOT be empty.") + } + let str = "" + let ok = false + str = g.Variables[varName] + ok = str ? true : false + if (!ok) { + throw new Error(`resolvePrint(): variable '$${varName}'not found.`) + } + console.log(str) + return null +} + + +// test +Execute(`$a = "你好，我是pineapple11" +print($a) +`) diff --git a/src/definition.js b/src/definition.js new file mode 100644 index 0000000..ad2a0c8 --- /dev/null +++ b/src/definition.js @@ -0,0 +1,3 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +//# sourceMappingURL=definition.js.map \ No newline at end of file diff --git a/src/definition.ts b/src/definition.ts new file mode 100644 index 0000000..acccd88 --- /dev/null +++ b/src/definition.ts @@ -0,0 +1,20 @@ +export interface GlobalVariables { + Variables: { + [index: string]: string + } +} + + +export interface Keywords { + print: number, + [index: string]: any +} + +export interface TokenNameMap { + [index: number]: any +} + +export interface Variable { + LineNum?: number, + Name?: string, +} diff --git a/src/lexer.js b/src/lexer.js new file mode 100644 index 0000000..c7a5983 --- /dev/null +++ b/src/lexer.js @@ -0,0 +1,222 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.NewLexer = exports.Lexer = exports.keywords = exports.tokenNameMap = exports.Tokens = void 0; +// token const +var Tokens; +(function (Tokens) { + Tokens[Tokens["TOKEN_EOF"] = 0] = "TOKEN_EOF"; + Tokens[Tokens["TOKEN_VAR_PREFIX"] = 1] = "TOKEN_VAR_PREFIX"; + Tokens[Tokens["TOKEN_LEFT_PAREN"] = 2] = "TOKEN_LEFT_PAREN"; + Tokens[Tokens["TOKEN_RIGHT_PAREN"] = 3] = "TOKEN_RIGHT_PAREN"; + Tokens[Tokens["TOKEN_EQUAL"] = 4] = "TOKEN_EQUAL"; + Tokens[Tokens["TOKEN_QUOTE"] = 5] = "TOKEN_QUOTE"; + Tokens[Tokens["TOKEN_DUOQUOTE"] = 6] = "TOKEN_DUOQUOTE"; + Tokens[Tokens["TOKEN_NAME"] = 7] = "TOKEN_NAME"; + Tokens[Tokens["TOKEN_PRINT"] = 8] = "TOKEN_PRINT"; + Tokens[Tokens["TOKEN_IGNORED"] = 9] = "TOKEN_IGNORED"; +})(Tokens = exports.Tokens || (exports.Tokens = {})); +exports.tokenNameMap = { + [Tokens.TOKEN_EOF]: "EOF", + [Tokens.TOKEN_VAR_PREFIX]: "$", + [Tokens.TOKEN_LEFT_PAREN]: "(", + [Tokens.TOKEN_RIGHT_PAREN]: ")", + [Tokens.TOKEN_EQUAL]: "=", + [Tokens.TOKEN_QUOTE]: "\"", + [Tokens.TOKEN_DUOQUOTE]: "\"\"", + [Tokens.TOKEN_NAME]: "Name", + [Tokens.TOKEN_PRINT]: "print", + [Tokens.TOKEN_IGNORED]: "Ignored", +}; +exports.keywords = { + "print": Tokens.TOKEN_PRINT, +}; +// regex match patterns +const regexName = /^[_\d\w]+/; +class Lexer { + constructor(sourceCode, lineNum, nextToken, nextTokenType, nextTokenLineNum) { + this.sourceCode = sourceCode; + this.lineNum = lineNum; + this.nextToken = nextToken; + this.nextTokenType = nextTokenType; + this.nextTokenLineNum = nextTokenLineNum; + } + GetNextToken() { + // next token already loaded + if (this.nextTokenLineNum > 0) { + let lineNum = this.nextTokenLineNum; + let tokenType = this.nextTokenType; + let token = this.nextToken; + this.lineNum = this.nextTokenLineNum; + this.nextTokenLineNum = 0; + return { + lineNum, + tokenType, + token + }; + } + return this.MatchToken(); + } + isLetter(c) { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'; + } + skipSourceCode(n) { + this.sourceCode = this.sourceCode.slice(n); + } + nextSourceCodeIs(s) { + return this.sourceCode.startsWith(s); + } + isIgnored() { + let isIgnored = false; + // target pattern + let isNewLine = function (c) { + return c == '\r' || c == '\n'; + }; + let isWhiteSpace = function (c) { + if (['\t', '\n', '\v', '\f', '\r', ' '].includes(c)) { + return true; + } + return false; + }; + // matching + while (this.sourceCode.length > 0) { + if (this.nextSourceCodeIs("\r\n") || this.nextSourceCodeIs("\n\r")) { + this.skipSourceCode(2); + this.lineNum += 1; + isIgnored = true; + } + else if (isNewLine(this.sourceCode[0])) { + this.skipSourceCode(1); + this.lineNum += 1; + isIgnored = true; + } + else if (isWhiteSpace(this.sourceCode[0])) { + this.skipSourceCode(1); + isIgnored = true; + } + else { + break; + } + } + return isIgnored; + } + MatchToken() { + // check ignored + if (this.isIgnored()) { + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_IGNORED, token: "Ignored" }; + } + // finish + if (this.sourceCode.length == 0) { + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_EOF, token: exports.tokenNameMap[Tokens.TOKEN_EOF] }; + } + // check token + switch (this.sourceCode[0]) { + case '$': + this.skipSourceCode(1); + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_VAR_PREFIX, token: "$" }; + case '(': + this.skipSourceCode(1); + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_LEFT_PAREN, token: "(" }; + case ')': + this.skipSourceCode(1); + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_RIGHT_PAREN, token: ")" }; + case '=': + this.skipSourceCode(1); + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_EQUAL, token: "=" }; + case '"': + if (this.nextSourceCodeIs("\"\"")) { + this.skipSourceCode(2); + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_DUOQUOTE, token: "\"\"" }; + } + this.skipSourceCode(1); + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_QUOTE, token: "\"" }; + } + // check multiple character token + if (this.sourceCode[0] == '_' || this.isLetter(this.sourceCode[0])) { + let token = this.scanName(); + let tokenType = exports.keywords[token]; + let isMatch = tokenType !== undefined ? true : false; + if (isMatch) { + return { lineNum: this.lineNum, tokenType, token }; + } + else { + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_NAME, token }; + } + } + // unexpected symbol + throw new Error(`MatchToken(): unexpected symbol near '${this.sourceCode[0]}'.`); + } + scanName() { + return this.scan(regexName); + } + scan(regexp) { + let token; + let reg = this.sourceCode.match(regexp); + if (reg) { + token = reg[0]; + } + if (token) { + this.skipSourceCode(token.length); + return token; + } + console.log("unreachable!"); + return ""; + } + NextTokenIs(tokenType) { + const { lineNum: nowLineNum, tokenType: nowTokenType, token: nowToken } = this.GetNextToken(); + // syntax error + if (tokenType != nowTokenType) { + throw new Error(`NextTokenIs(): syntax error near '${exports.tokenNameMap[nowTokenType]}', expected token: {${exports.tokenNameMap[tokenType]}} but got {${exports.tokenNameMap[nowTokenType]}}.`); + } + return { nowLineNum, nowToken }; + } + GetLineNum() { + return this.lineNum; + } + LookAhead() { + // lexer.nextToken already setted + if (this.nextTokenLineNum > 0) { + return this.nextTokenType; + } + // set it + let nowLineNum = this.lineNum; + let { lineNum, tokenType, token } = this.GetNextToken(); + this.lineNum = nowLineNum; + this.nextTokenLineNum = lineNum; + this.nextTokenType = tokenType; + this.nextToken = token; + return tokenType; + } + LookAheadAndSkip(expectedType) { + // get next token + let nowLineNum = this.lineNum; + let { lineNum, tokenType, token } = this.GetNextToken(); + // not is expected type, reverse cursor + if (tokenType != expectedType) { + this.lineNum = nowLineNum; + this.nextTokenLineNum = lineNum; + this.nextTokenType = tokenType; + this.nextToken = token; + } + } + // return content before token + scanBeforeToken(token) { + let s = this.sourceCode.split(token); + if (s.length < 2) { + console.log("unreachable!"); + return ""; + } + this.skipSourceCode(s[0].length); + return s[0]; + } +} +exports.Lexer = Lexer; +function NewLexer(sourceCode) { + return new Lexer(sourceCode, 1, "", 0, 0); // start at line 1 in default. +} +exports.NewLexer = NewLexer; +// test +// let lexer = NewLexer(`$a = "你好，我是pineapple" +// print($a) +// `) +// console.log(lexer) +//# sourceMappingURL=lexer.js.map \ No newline at end of file diff --git a/src/lexer.ts b/src/lexer.ts new file mode 100644 index 0000000..4559c28 --- /dev/null +++ b/src/lexer.ts @@ -0,0 +1,239 @@ +import { Keywords, TokenNameMap } from "./definition"; + +// lexer struct +export interface Lexer { + sourceCode: string + lineNum: number + nextToken: string + nextTokenType: number + nextTokenLineNum: number +} + + +// token const +export enum Tokens { + TOKEN_EOF, // end-of-file + TOKEN_VAR_PREFIX, // $ + TOKEN_LEFT_PAREN, // ( + TOKEN_RIGHT_PAREN, // ) + TOKEN_EQUAL, // = + TOKEN_QUOTE, // " + TOKEN_DUOQUOTE, // "" + TOKEN_NAME, // Name ::= [_A-Za-z][_0-9A-Za-z]* + TOKEN_PRINT, // print + TOKEN_IGNORED, // Ignored +} + +export const tokenNameMap: TokenNameMap = { + [Tokens.TOKEN_EOF]: "EOF", + [Tokens.TOKEN_VAR_PREFIX]: "$", + [Tokens.TOKEN_LEFT_PAREN]: "(", + [Tokens.TOKEN_RIGHT_PAREN]: ")", + [Tokens.TOKEN_EQUAL]: "=", + [Tokens.TOKEN_QUOTE]: "\"", + [Tokens.TOKEN_DUOQUOTE]: "\"\"", + [Tokens.TOKEN_NAME]: "Name", + [Tokens.TOKEN_PRINT]: "print", + [Tokens.TOKEN_IGNORED]: "Ignored", +} + +export const keywords: Keywords = { + "print": Tokens.TOKEN_PRINT, +} + +// regex match patterns +const regexName = /^[_\d\w]+/ + + + +export class Lexer { + constructor(sourceCode: string, lineNum: number, nextToken: string, nextTokenType: number, nextTokenLineNum: number) { + this.sourceCode = sourceCode; + this.lineNum = lineNum; + this.nextToken = nextToken; + this.nextTokenType = nextTokenType; + this.nextTokenLineNum = nextTokenLineNum; + } + GetNextToken(): { lineNum: number, tokenType: number, token: string } { + // next token already loaded + if (this.nextTokenLineNum > 0) { + let lineNum = this.nextTokenLineNum + let tokenType = this.nextTokenType + let token = this.nextToken + this.lineNum = this.nextTokenLineNum + this.nextTokenLineNum = 0 + return { + lineNum, + tokenType, + token + } + } + return this.MatchToken() + } + isLetter(c: string): boolean { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' + } + skipSourceCode(n: number) { + this.sourceCode = this.sourceCode.slice(n) + } + nextSourceCodeIs(s: string): boolean { + return this.sourceCode.startsWith(s) + } + isIgnored(): boolean { + let isIgnored = false + // target pattern + let isNewLine = function (c: string): boolean { + return c == '\r' || c == '\n' + } + let isWhiteSpace = function (c: string): boolean { + if (['\t', '\n', '\v', '\f', '\r', ' '].includes(c)) { + return true + } + return false + } + // matching + while (this.sourceCode.length > 0) { + if (this.nextSourceCodeIs("\r\n") || this.nextSourceCodeIs("\n\r")) { + this.skipSourceCode(2) + this.lineNum += 1 + isIgnored = true + } else if (isNewLine(this.sourceCode[0])) { + this.skipSourceCode(1) + this.lineNum += 1 + isIgnored = true + } else if (isWhiteSpace(this.sourceCode[0])) { + this.skipSourceCode(1) + isIgnored = true + } else { + break + } + } + return isIgnored + } + MatchToken(): { lineNum: number, tokenType: number, token: string } { + // check ignored + if (this.isIgnored()) { + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_IGNORED, token: "Ignored" } + } + // finish + if (this.sourceCode.length == 0) { + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_EOF, token: tokenNameMap[Tokens.TOKEN_EOF] } + } + // check token + switch (this.sourceCode[0]) { + case '$': + this.skipSourceCode(1) + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_VAR_PREFIX, token: "$" } + case '(': + this.skipSourceCode(1) + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_LEFT_PAREN, token: "(" } + case ')': + this.skipSourceCode(1) + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_RIGHT_PAREN, token: ")" } + case '=': + this.skipSourceCode(1) + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_EQUAL, token: "=" } + case '"': + if (this.nextSourceCodeIs("\"\"")) { + this.skipSourceCode(2) + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_DUOQUOTE, token: "\"\"" } + } + this.skipSourceCode(1) + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_QUOTE, token: "\"" } + } + // check multiple character token + if (this.sourceCode[0] == '_' || this.isLetter(this.sourceCode[0])) { + let token = this.scanName() + let tokenType = keywords[token]; + let isMatch = tokenType !== undefined ? true : false + if (isMatch) { + return { lineNum: this.lineNum, tokenType, token } + } else { + return { lineNum: this.lineNum, tokenType: Tokens.TOKEN_NAME, token } + } + } + // unexpected symbol + throw new Error(`MatchToken(): unexpected symbol near '${this.sourceCode[0]}'.`); + } + scanName(): string { + return this.scan(regexName) + } + scan(regexp: RegExp): string { + let token; + let reg = this.sourceCode.match(regexp) + if (reg) { + token = reg[0]; + } + if (token) { + this.skipSourceCode(token.length) + return token + } + console.log("unreachable!") + return "" + } + + NextTokenIs(tokenType: number) { + const { + lineNum: nowLineNum, + tokenType: nowTokenType, + token: nowToken } = this.GetNextToken() + // syntax error + if (tokenType != nowTokenType) { + throw new Error(`NextTokenIs(): syntax error near '${tokenNameMap[nowTokenType]}', expected token: {${tokenNameMap[tokenType]}} but got {${tokenNameMap[nowTokenType]}}.`) + } + return { nowLineNum, nowToken } + } + GetLineNum(): number { + return this.lineNum + } + LookAhead(): number { + // lexer.nextToken already setted + if (this.nextTokenLineNum > 0) { + return this.nextTokenType + } + // set it + let nowLineNum = this.lineNum + let { lineNum, tokenType, token } = this.GetNextToken() + this.lineNum = nowLineNum + this.nextTokenLineNum = lineNum + this.nextTokenType = tokenType + this.nextToken = token + return tokenType + } + + LookAheadAndSkip(expectedType: number) { + // get next token + let nowLineNum = this.lineNum + let { lineNum, tokenType, token } = this.GetNextToken() + // not is expected type, reverse cursor + if (tokenType != expectedType) { + this.lineNum = nowLineNum + this.nextTokenLineNum = lineNum + this.nextTokenType = tokenType + this.nextToken = token + } + } + // return content before token + scanBeforeToken(token: string): string { + let s = this.sourceCode.split(token) + if (s.length < 2) { + console.log("unreachable!") + return "" + } + this.skipSourceCode(s[0].length) + return s[0] + } +} + + +export function NewLexer(sourceCode: string): Lexer { + return new Lexer(sourceCode, 1, "", 0, 0) // start at line 1 in default. +} + +// test + +// let lexer = NewLexer(`$a = "你好，我是pineapple" +// print($a) +// `) +// console.log(lexer) + diff --git a/src/parser.js b/src/parser.js new file mode 100644 index 0000000..887b742 --- /dev/null +++ b/src/parser.js @@ -0,0 +1,124 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.parse = exports.Assignment = exports.Print = void 0; +const lexer_1 = require("./lexer"); +// SourceCode ::= Statement+ +function parseSourceCode(lexer) { + let sourceCode = {}; + sourceCode.LineNum = lexer.GetLineNum(); + sourceCode.Statements = parseStatements(lexer); + return sourceCode; +} +// Statement ::= Print | Assignment +function parseStatements(lexer) { + let statements = []; + while (!isSourceCodeEnd(lexer.LookAhead())) { + let statement = {}; + statement = parseStatement(lexer); + statements.push(statement); + } + return statements; +} +function isSourceCodeEnd(token) { + if (token == lexer_1.Tokens.TOKEN_EOF) { + return true; + } + return false; +} +function parseStatement(lexer) { + lexer.LookAheadAndSkip(lexer_1.Tokens.TOKEN_IGNORED); // skip if source code start with ignored token + let look = lexer.LookAhead(); + switch (look) { + case lexer_1.Tokens.TOKEN_PRINT: + return parsePrint(lexer); + case lexer_1.Tokens.TOKEN_VAR_PREFIX: + return parseAssignment(lexer); + default: + throw new Error("parseStatement(): unknown Statement."); + } +} +class Print { + constructor(LineNum, Variable) { + this.LineNum = LineNum; + this.Variable = Variable; + } +} +exports.Print = Print; +// Print ::= "print" "(" Ignored Variable Ignored ")" Ignored +function parsePrint(lexer) { + let print = new Print(); + print.LineNum = lexer.GetLineNum(); + lexer.NextTokenIs(lexer_1.Tokens.TOKEN_PRINT); + lexer.NextTokenIs(lexer_1.Tokens.TOKEN_LEFT_PAREN); + lexer.LookAheadAndSkip(lexer_1.Tokens.TOKEN_IGNORED); + print.Variable = parseVariable(lexer); + lexer.LookAheadAndSkip(lexer_1.Tokens.TOKEN_IGNORED); + lexer.NextTokenIs(lexer_1.Tokens.TOKEN_RIGHT_PAREN); + lexer.LookAheadAndSkip(lexer_1.Tokens.TOKEN_IGNORED); + return print; +} +// Variable ::= "$" Name Ignored +function parseVariable(lexer) { + let variable = {}; + variable.LineNum = lexer.GetLineNum(); + lexer.NextTokenIs(lexer_1.Tokens.TOKEN_VAR_PREFIX); + variable.Name = parseName(lexer); + lexer.LookAheadAndSkip(lexer_1.Tokens.TOKEN_IGNORED); + return variable; +} +// Name ::= [_A-Za-z][_0-9A-Za-z]* +function parseName(lexer) { + let { nowLineNum: _, nowToken: name } = lexer.NextTokenIs(lexer_1.Tokens.TOKEN_NAME); + return name; +} +class Assignment { + constructor(LineNum, Variable, String) { + this.LineNum = LineNum; + this.Variable = Variable; + this.String = String; + } +} +exports.Assignment = Assignment; +// Assignment ::= Variable Ignored "=" Ignored String Ignored +function parseAssignment(lexer) { + let assignment = new Assignment(); + assignment.LineNum = lexer.GetLineNum(); + assignment.Variable = parseVariable(lexer); + lexer.LookAheadAndSkip(lexer_1.Tokens.TOKEN_IGNORED); + lexer.NextTokenIs(lexer_1.Tokens.TOKEN_EQUAL); + lexer.LookAheadAndSkip(lexer_1.Tokens.TOKEN_IGNORED); + assignment.String = parseString(lexer); + lexer.LookAheadAndSkip(lexer_1.Tokens.TOKEN_IGNORED); + return assignment; +} +// String ::= '"' '"' Ignored | '"' StringCharacter '"' Ignored +function parseString(lexer) { + let str = ""; + let look = lexer.LookAhead(); + switch (look) { + case lexer_1.Tokens.TOKEN_DUOQUOTE: + lexer.NextTokenIs(lexer_1.Tokens.TOKEN_DUOQUOTE); + lexer.LookAheadAndSkip(lexer_1.Tokens.TOKEN_IGNORED); + return str; + case lexer_1.Tokens.TOKEN_QUOTE: + lexer.NextTokenIs(lexer_1.Tokens.TOKEN_QUOTE); + str = lexer.scanBeforeToken(lexer_1.tokenNameMap[lexer_1.Tokens.TOKEN_QUOTE]); + lexer.NextTokenIs(lexer_1.Tokens.TOKEN_QUOTE); + lexer.LookAheadAndSkip(lexer_1.Tokens.TOKEN_IGNORED); + return str; + default: + return ""; + } +} +function parse(code) { + let lexer = lexer_1.NewLexer(code); + let sourceCode = parseSourceCode(lexer); + lexer.NextTokenIs(lexer_1.Tokens.TOKEN_EOF); + return sourceCode; +} +exports.parse = parse; +// test +// const paserResult = parse(`$a = "你好，我是pineapple" +// print($a)`) +// console.log(JSON.stringify(paserResult)) +//# sourceMappingURL=parser.js.map \ No newline at end of file diff --git a/src/parser.ts b/src/parser.ts new file mode 100644 index 0000000..7ad1ac7 --- /dev/null +++ b/src/parser.ts @@ -0,0 +1,161 @@ +import { Lexer, NewLexer, tokenNameMap, Tokens } from "./lexer" +import { Variable } from './definition' + + +export interface Assignment { + LineNum?: number, + Variable?: Variable, + String?: string +} + +export interface Print { + LineNum?: number, + Variable?: Variable +} + +// SourceCode ::= Statement+ +function parseSourceCode(lexer: Lexer) { + + let sourceCode: { LineNum?: number, Statements?: Array } = {} + sourceCode.LineNum = lexer.GetLineNum() + sourceCode.Statements = parseStatements(lexer) + return sourceCode +} + +// Statement ::= Print | Assignment +function parseStatements(lexer: Lexer) { + let statements: Array = [] + + while (!isSourceCodeEnd(lexer.LookAhead())) { + let statement = {} + statement = parseStatement(lexer) + statements.push(statement) + } + return statements +} + +function isSourceCodeEnd(token: number): boolean { + if (token == Tokens.TOKEN_EOF) { + return true + } + return false +} + +function parseStatement(lexer: Lexer) { + lexer.LookAheadAndSkip(Tokens.TOKEN_IGNORED) // skip if source code start with ignored token + let look = lexer.LookAhead() + switch (look) { + case Tokens.TOKEN_PRINT: + return parsePrint(lexer) + case Tokens.TOKEN_VAR_PREFIX: + return parseAssignment(lexer) + default: + throw new Error("parseStatement(): unknown Statement.") + } +} + +export class Print { + constructor(LineNum?: number, Variable?: Variable) { + this.LineNum = LineNum + this.Variable = Variable + } +} + +// Print ::= "print" "(" Ignored Variable Ignored ")" Ignored +function parsePrint(lexer: Lexer) { + let print = new Print() + + + print.LineNum = lexer.GetLineNum() + lexer.NextTokenIs(Tokens.TOKEN_PRINT) + lexer.NextTokenIs(Tokens.TOKEN_LEFT_PAREN) + lexer.LookAheadAndSkip(Tokens.TOKEN_IGNORED) + print.Variable = parseVariable(lexer) + + lexer.LookAheadAndSkip(Tokens.TOKEN_IGNORED) + lexer.NextTokenIs(Tokens.TOKEN_RIGHT_PAREN) + lexer.LookAheadAndSkip(Tokens.TOKEN_IGNORED) + return print +} + +// Variable ::= "$" Name Ignored +function parseVariable(lexer: Lexer) { + let variable: { + LineNum?: number + Name?: string + } = {} + + variable.LineNum = lexer.GetLineNum() + lexer.NextTokenIs(Tokens.TOKEN_VAR_PREFIX) + variable.Name = parseName(lexer) + + lexer.LookAheadAndSkip(Tokens.TOKEN_IGNORED) + return variable +} + +// Name ::= [_A-Za-z][_0-9A-Za-z]* +function parseName(lexer: Lexer) { + let { nowLineNum: _, nowToken: name } = lexer.NextTokenIs(Tokens.TOKEN_NAME) + return name +} + + +export class Assignment { + constructor(LineNum?: number, Variable?: Variable, String?: string) { + this.LineNum = LineNum + this.Variable = Variable + this.String = String + } +} + +// Assignment ::= Variable Ignored "=" Ignored String Ignored +function parseAssignment(lexer: Lexer) { + let assignment = new Assignment() + + assignment.LineNum = lexer.GetLineNum() + assignment.Variable = parseVariable(lexer) + + lexer.LookAheadAndSkip(Tokens.TOKEN_IGNORED) + lexer.NextTokenIs(Tokens.TOKEN_EQUAL) + lexer.LookAheadAndSkip(Tokens.TOKEN_IGNORED) + assignment.String = parseString(lexer) + + lexer.LookAheadAndSkip(Tokens.TOKEN_IGNORED) + return assignment +} + +// String ::= '"' '"' Ignored | '"' StringCharacter '"' Ignored +function parseString(lexer: Lexer) { + let str = "" + let look = lexer.LookAhead() + switch (look) { + case Tokens.TOKEN_DUOQUOTE: + lexer.NextTokenIs(Tokens.TOKEN_DUOQUOTE) + lexer.LookAheadAndSkip(Tokens.TOKEN_IGNORED) + return str + case Tokens.TOKEN_QUOTE: + lexer.NextTokenIs(Tokens.TOKEN_QUOTE) + str = lexer.scanBeforeToken(tokenNameMap[Tokens.TOKEN_QUOTE]) + lexer.NextTokenIs(Tokens.TOKEN_QUOTE) + lexer.LookAheadAndSkip(Tokens.TOKEN_IGNORED) + return str + default: + return "" + } +} + +export function parse(code: string) { + + let lexer = NewLexer(code) + let sourceCode = parseSourceCode(lexer); + + lexer.NextTokenIs(Tokens.TOKEN_EOF) + return sourceCode +} + +// test + +// const paserResult = parse(`$a = "你好，我是pineapple" +// print($a)`) + +// console.log(JSON.stringify(paserResult)) diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..b2da2df --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,70 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig.json to read more about this file */ + + /* Basic Options */ + // "incremental": true, /* Enable incremental compilation */ + "target": "ESNEXT", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ + "lib": ["ESNext", "DOM"], /* Specify library files to be included in the compilation. */ + // "allowJs": true, /* Allow javascript files to be compiled. */ + // "checkJs": true, /* Report errors in .js files. */ + // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ + // "declaration": true, /* Generates corresponding '.d.ts' file. */ + // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ + "sourceMap": true, /* Generates corresponding '.map' file. */ + // "outFile": "./", /* Concatenate and emit output to single file. */ + // "outDir": "./", /* Redirect output structure to the directory. */ + // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ + // "composite": true, /* Enable project compilation */ + // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ + // "removeComments": true, /* Do not emit comments to output. */ + // "noEmit": true, /* Do not emit outputs. */ + // "importHelpers": true, /* Import emit helpers from 'tslib'. */ + // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ + // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ + + /* Strict Type-Checking Options */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* Enable strict null checks. */ + // "strictFunctionTypes": true, /* Enable strict checking of function types. */ + // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ + // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ + // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ + // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ + + /* Additional Checks */ + // "noUnusedLocals": true, /* Report errors on unused locals. */ + // "noUnusedParameters": true, /* Report errors on unused parameters. */ + // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ + // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ + // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ + + /* Module Resolution Options */ + // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ + // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ + // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ + // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ + // "typeRoots": [], /* List of folders to include type definitions from. */ + // "types": [], /* Type declaration files to be included in compilation. */ + // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ + "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ + // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + + /* Source Map Options */ + // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ + // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ + + /* Experimental Options */ + // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ + // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + + /* Advanced Options */ + "skipLibCheck": true, /* Skip type checking of declaration files. */ + "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ + } +}