From 349fc823faec00adc10e2c68421d4553bac8a85a Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sun, 3 Nov 2024 14:27:48 -0600
Subject: [PATCH] Handle `*COMMAND`s in the tokeniser

Uses the new Monaco's ability to handle `\n` specially. Adds
tests for the new capabilities.

With thanks to microsoft/monaco-editor#2265
---
 src/bbcbasic.js       | 13 +++++++++++--
 src/owlet.js          |  5 +++++
 test/bbcbasic_test.js | 33 ++++++++++++++++++++++++++++++---
 3 files changed, 46 insertions(+), 5 deletions(-)

diff --git a/src/bbcbasic.js b/src/bbcbasic.js
index e5fca15..de30938 100644
--- a/src/bbcbasic.js
+++ b/src/bbcbasic.js
@@ -75,6 +75,7 @@ export function registerBbcBasicLanguage() {
     // Register a tokens provider for the language
     languages.setMonarchTokensProvider("BBCBASIC", {
         defaultToken: "invalid",
+        includeLF: true,
         brackets: [["(", ")", "delimiter.parenthesis"]],
         operators: [
             "+",
@@ -104,6 +105,14 @@ export function registerBbcBasicLanguage() {
         symbols: /[-+#=><!*/{}:?$;,~^']+/,
         tokenizer: {
             root: [
+                {include: "@whitespace"},
+                [/\d+/, "constant.linenum"], // line numbers
+                [/(?=[^*])/, "", "@statement"], // As soon as we know this statement doesn't start with * we can process it with statement
+                [/\*.*/, {token: "keyword.oscli"}],
+            ],
+            statement: [
+                ["\n", "", "@pop"],
+                [":", "symbol", "@pop"],
                 [/(\bREM|\xf4)$/, {token: "keyword"}], // A REM on its own line
                 [/(\bREM|\xf4)/, {token: "keyword", next: "@remStatement"}], // A REM consumes to EOL
                 [/(FN|PROC|\xa4|\xf2)/, {token: "keyword", next: "@fnProcName"}],
@@ -117,7 +126,6 @@ export function registerBbcBasicLanguage() {
                 ["\\[", {token: "delimiter.square", next: "@asm"}],
             ],
             common: [
-                [/^\s*\d+/, "enum"], // line numbers
                 {include: "@whitespace"},
                 // immediate
                 [
@@ -155,7 +163,7 @@ export function registerBbcBasicLanguage() {
             remStatement: [[/.*/, "comment", "@pop"]],
             asm: [
                 [
-                    /ADC|AND|ASL|B(CC|CS|EQ|MI|NE|PL|VC|VS)|BIT|BRK|CL[CDIV]|CMP|CP[XY]|DE[CXY]|EOR|IN[CXY]|JMP|JSR|LD[AXY]|LSR|NOP|ORA|PH[AP]|PL[AP]|RO[LR]|RTI|RTS|SBC|SE[CDI]|ST[AXY]|TA[XY]|TSX|TX[AS]|TYA/,
+                    /ADC|AND|ASL|B(CC|CS|EQ|MI|NE|PL|VC|VS)|BIT|BR[AK]|CL[CDIV]|CMP|CP[XY]|DE[CXY]|EOR|IN[CXY]|JMP|JSR|LD[AXY]|LSR|NOP|ORA|PH[AP]|PL[AP]|RO[LR]|RTI|RTS|SBC|SE[CDI]|ST[AXY]|TA[XY]|TSX|TX[AS]|TYA/,
                     "keyword",
                 ],
                 [/OPT|EQU[BDSW]/, "keyword.directive"],
@@ -164,6 +172,7 @@ export function registerBbcBasicLanguage() {
                 // labels
                 [/\.([a-zA-Z_][\w]*%?|@%)/, "type.identifier"],
                 [allTokensForAsmRegex, "keyword"],
+                [/^\d+/, "constant.linenum"], // line numbers
                 {include: "@common"},
                 ["]", {token: "delimiter.square", next: "@pop"}],
             ],
diff --git a/src/owlet.js b/src/owlet.js
index 3e05841..06dff4f 100644
--- a/src/owlet.js
+++ b/src/owlet.js
@@ -37,6 +37,11 @@ export class OwletEditor {
             rules: [
                 {token: "variable", foreground: "bb8844"},
                 {token: "number", foreground: "22bb88"},
+                {token: "number.hex", fontStyle: "bold"},
+                {token: "number.float", fontStyle: "italic"},
+                {token: "constant.linenum", foreground: "44eeaa"},
+                {token: "keyword.oscli", foreground: "bb66bb"},
+                {token: "keyword.directive", fontStyle: "bold"},
             ],
         });
 
diff --git a/test/bbcbasic_test.js b/test/bbcbasic_test.js
index 8bf1f71..43a8139 100644
--- a/test/bbcbasic_test.js
+++ b/test/bbcbasic_test.js
@@ -157,15 +157,15 @@ describe("Tokenisation", () => {
     it("should not recognize 6502 outside of []", () => {
         checkTokens(["LDA"], [{offset: 0, type: "variable"}]);
     });
-    it("should recognize 6502 inside []", () => {
+    it("should recognise 6502 inside []", () => {
         checkTokens(
             ["0[", "1LDA"],
             [
-                {offset: 0, type: "enum"},
+                {offset: 0, type: "constant.linenum"},
                 {offset: 1, type: "delimiter.square"},
             ],
             [
-                {offset: 0, type: "enum"},
+                {offset: 0, type: "constant.linenum"},
                 {offset: 1, type: "keyword"},
             ],
         );
@@ -355,11 +355,38 @@ describe("Tokenisation", () => {
                 {offset: 8, type: "keyword"},
                 {offset: 12, type: "variable"},
                 {offset: 16, type: "symbol"},
+                {offset: 17, type: "operator"},
                 {offset: 18, type: "keyword"},
                 {offset: 20, type: "variable"},
             ],
         );
     });
+    it("should handle OSCLI at start of line", () => {
+        checkTokens(["*INFO"], [{offset: 0, type: "keyword.oscli"}]);
+    });
+    it("should handle OSCLI after line number", () => {
+        checkTokens(
+            ["10 *INFO"],
+            [
+                {offset: 0, type: "constant.linenum"},
+                {offset: 2, type: "white"},
+                {offset: 3, type: "keyword.oscli"},
+            ],
+        );
+    });
+    it("should handle OSCLI at end of line", () => {
+        checkTokens(
+            ["P.2*3:*.0:blah"],
+            [
+                {offset: 0, type: "keyword"},
+                {offset: 2, type: "number"},
+                {offset: 3, type: "operator"},
+                {offset: 4, type: "number"},
+                {offset: 5, type: "symbol"},
+                {offset: 6, type: "keyword.oscli"},
+            ],
+        );
+    });
 });
 
 function checkWarnings(text, ...expected) {