support typed array in parser

src/EscapeSequenceParser.ts

@@ -6,6 +6,7 @@
 import { ParserState, ParserAction, IParsingState, IDcsHandler, IEscapeSequenceParser } from './Types';
 import { IDisposable } from 'xterm';
 import { Disposable } from './common/Lifecycle';
+import { utf32ToString } from './core/input/TextDecoder';
 
 interface IHandlerCollection<T> {
   [key: string]: T[];
@@ -134,6 +135,7 @@ export const VT500_TRANSITION_TABLE = (function (): TransitionTable {
   table.addMany(PRINTABLES, ParserState.SOS_PM_APC_STRING, ParserAction.IGNORE, ParserState.SOS_PM_APC_STRING);
   table.addMany(EXECUTABLES, ParserState.SOS_PM_APC_STRING, ParserAction.IGNORE, ParserState.SOS_PM_APC_STRING);
   table.add(0x9c, ParserState.SOS_PM_APC_STRING, ParserAction.IGNORE, ParserState.GROUND);
+  table.add(0x7f, ParserState.SOS_PM_APC_STRING, ParserAction.IGNORE, ParserState.SOS_PM_APC_STRING);
   // csi entries
   table.add(0x5b, ParserState.ESCAPE, ParserAction.CLEAR, ParserState.CSI_ENTRY);
   table.addMany(r(0x40, 0x7f), ParserState.CSI_ENTRY, ParserAction.CSI_DISPATCH, ParserState.GROUND);
@@ -202,7 +204,7 @@ export const VT500_TRANSITION_TABLE = (function (): TransitionTable {
  */
 class DcsDummy implements IDcsHandler {
   hook(collect: string, params: number[], flag: number): void { }
-  put(data: string, start: number, end: number): void { }
+  put(data: Uint32Array, start: number, end: number): void { }
   unhook(): void { }
 }
 
@@ -228,7 +230,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
   protected _collect: string;
 
   // handler lookup containers
-  protected _printHandler: (data: string, start: number, end: number) => void;
+  protected _printHandler: (data: Uint32Array, start: number, end: number) => void;
   protected _executeHandlers: any;
   protected _csiHandlers: IHandlerCollection<CsiHandler>;
   protected _escHandlers: any;
@@ -238,7 +240,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
   protected _errorHandler: (state: IParsingState) => IParsingState;
 
   // fallback handlers
-  protected _printHandlerFb: (data: string, start: number, end: number) => void;
+  protected _printHandlerFb: (data: Uint32Array, start: number, end: number) => void;
   protected _executeHandlerFb: (code: number) => void;
   protected _csiHandlerFb: (collect: string, params: number[], flag: number) => void;
   protected _escHandlerFb: (collect: string, flag: number) => void;
@@ -294,7 +296,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
     this._errorHandler = null;
   }
 
-  setPrintHandler(callback: (data: string, start: number, end: number) => void): void {
+  setPrintHandler(callback: (data: Uint32Array, start: number, end: number) => void): void {
     this._printHandler = callback;
   }
   clearPrintHandler(): void {
@@ -397,7 +399,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
     this._activeDcsHandler = null;
   }
 
-  parse(data: string): void {
+  parse(data: Uint32Array, length: number): void {
     let code = 0;
     let transition = 0;
     let error = false;
@@ -412,15 +414,14 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
     let callback: Function | null = null;
 
     // process input string
-    const l = data.length;
-    for (let i = 0; i < l; ++i) {
-      code = data.charCodeAt(i);
+    for (let i = 0; i < length; ++i) {
+      code = data[i];
 
       // shortcut for most chars (print action)
       if (currentState === ParserState.GROUND && code > 0x1f && code < 0x80) {
         print = (~print) ? print : i;
         do i++;
-        while (i < l && data.charCodeAt(i) > 0x1f && data.charCodeAt(i) < 0x80);
+        while (i < length && data[i] > 0x1f && data[i] < 0x80);
         i--;
         continue;
       }
@@ -563,10 +564,10 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
           break;
         case ParserAction.OSC_PUT:
           for (let j = i + 1; ; j++) {
-            if (j >= l
-                || (code = data.charCodeAt(j)) < 0x20
+            if (j >= length
+                || (code = data[j]) < 0x20
                 || (code > 0x7f && code <= 0x9f)) {
-              osc += data.substring(i, j);
+              osc += utf32ToString(data, i, j);
               i = j - 1;
               break;
             }
@@ -610,9 +611,9 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
 
     // push leftover pushable buffers to terminal
     if (currentState === ParserState.GROUND && ~print) {
-      this._printHandler(data, print, data.length);
+      this._printHandler(data, print, length);
     } else if (currentState === ParserState.DCS_PASSTHROUGH && ~dcs && dcsHandler) {
-      dcsHandler.put(data, dcs, data.length);
+      dcsHandler.put(data, dcs, length);
     }
 
     // save non pushable buffers

src/InputHandler.test.ts

@@ -337,7 +337,9 @@ describe('InputHandler', () => {
     it('should not cause an infinite loop (regression test)', () => {
       const term = new Terminal();
       const inputHandler = new InputHandler(term);
-      inputHandler.print(String.fromCharCode(0x200B), 0, 1);
+      const container = new Uint32Array(10);
+      container[0] = 0x200B;
+      inputHandler.print(container, 0, 1);
     });
   });
 

src/InputHandler.ts

@@ -14,6 +14,8 @@ import { EscapeSequenceParser } from './EscapeSequenceParser';
 import { ICharset } from './core/Types';
 import { IDisposable } from 'xterm';
 import { Disposable } from './common/Lifecycle';
+import { concat } from './common/TypedArrayUtils';
+import { StringToUtf32, stringFromCodePoint, utf32ToString } from './core/input/TextDecoder';
 
 /**
  * Map collect to glevel. Used in `selectCharset`.
@@ -32,21 +34,22 @@ const GLEVEL: {[key: string]: number} = {'(': 0, ')': 1, '*': 2, '+': 3, '-': 1,
  *   Response: DECRPSS (https://vt100.net/docs/vt510-rm/DECRPSS.html)
  */
 class DECRQSS implements IDcsHandler {
-  private _data: string;
+  private _data: Uint32Array = new Uint32Array(0);
 
   constructor(private _terminal: any) { }
 
   hook(collect: string, params: number[], flag: number): void {
-    // reset data
-    this._data = '';
+    this._data = new Uint32Array(0);
   }
 
-  put(data: string, start: number, end: number): void {
-    this._data += data.substring(start, end);
+  put(data: Uint32Array, start: number, end: number): void {
+    this._data = concat(this._data, data.subarray(start, end));
   }
 
   unhook(): void {
-    switch (this._data) {
+    const data = utf32ToString(this._data);
+    this._data = new Uint32Array(0);
+    switch (data) {
       // valid: DCS 1 $ r Pt ST (xterm)
       case '"q': // DECSCA
         return this._terminal.handler(`${C0.ESC}P1$r0"q${C0.ESC}\\`);
@@ -66,7 +69,7 @@ class DECRQSS implements IDcsHandler {
         return this._terminal.handler(`${C0.ESC}P1$r${style} q${C0.ESC}\\`);
       default:
         // invalid: DCS 0 $ r Pt ST (xterm)
-        this._terminal.error('Unknown DCS $q %s', this._data);
+        this._terminal.error('Unknown DCS $q %s', data);
         this._terminal.handler(`${C0.ESC}P0$r${C0.ESC}\\`);
     }
   }
@@ -78,11 +81,17 @@ class DECRQSS implements IDcsHandler {
  *   not supported
  */
 
- /**
-  * DCS + p Pt ST (xterm)
-  *   Set Terminfo Data
-  *   not supported
-  */
+/**
+ * DCS + q Pt ST (xterm)
+ *   Request Terminfo String
+ *   not implemented
+ */
+
+/**
+ * DCS + p Pt ST (xterm)
+ *   Set Terminfo Data
+ *   not supported
+ */
 
 
 
@@ -94,7 +103,8 @@ class DECRQSS implements IDcsHandler {
  * each function's header comment.
  */
 export class InputHandler extends Disposable implements IInputHandler {
-  private _surrogateFirst: string;
+  private _parseBuffer: Uint32Array = new Uint32Array(4096);
+  private _stringDecoder: StringToUtf32 = new StringToUtf32();
 
   constructor(
       protected _terminal: IInputHandlingTerminal,
@@ -104,8 +114,6 @@ export class InputHandler extends Disposable implements IInputHandler {
 
     this.register(this._parser);
 
-    this._surrogateFirst = '';
-
     /**
      * custom fallback handlers
      */
@@ -290,23 +298,23 @@ export class InputHandler extends Disposable implements IInputHandler {
       this._terminal.log('data: ' + data);
     }
 
-    // apply leftover surrogate high from last write
-    if (this._surrogateFirst) {
-      data = this._surrogateFirst + data;
-      this._surrogateFirst = '';
+    if (this._parseBuffer.length < data.length) {
+      this._parseBuffer = new Uint32Array(data.length);
     }
-
-    this._parser.parse(data);
+    for (let i = 0; i < data.length; ++i) {
+      this._parseBuffer[i] = data.charCodeAt(i);
+    }
+    this._parser.parse(this._parseBuffer, this._stringDecoder.decode(data, this._parseBuffer));
 
     buffer = this._terminal.buffer;
     if (buffer.x !== cursorStartX || buffer.y !== cursorStartY) {
       this._terminal.emit('cursormove');
     }
   }
 
-  public print(data: string, start: number, end: number): void {
-    let char: string;
+  public print(data: Uint32Array, start: number, end: number): void {
     let code: number;
+    let char: string;
     let chWidth: number;
     const buffer: IBuffer = this._terminal.buffer;
     const charset: ICharset = this._terminal.charset;
@@ -318,41 +326,23 @@ export class InputHandler extends Disposable implements IInputHandler {
     let bufferRow = buffer.lines.get(buffer.y + buffer.ybase);
 
     this._terminal.updateRange(buffer.y);
-    for (let stringPosition = start; stringPosition < end; ++stringPosition) {
-      char = data.charAt(stringPosition);
-      code = data.charCodeAt(stringPosition);
-
-      // surrogate pair handling
-      if (0xD800 <= code && code <= 0xDBFF) {
-        if (++stringPosition >= end) {
-          // end of input:
-          // handle pairs as true UTF-16 and wait for the second part
-          // since we expect the input comming from a stream there is
-          // a small chance that the surrogate pair got split
-          // therefore we dont process the first char here, instead
-          // it gets added as first char to the next processed chunk
-          this._surrogateFirst = char;
-          continue;
-        }
-        const second = data.charCodeAt(stringPosition);
-        // if the second part is in surrogate pair range create the high codepoint
-        // otherwise fall back to UCS-2 behavior (handle codepoints independently)
-        if (0xDC00 <= second && second <= 0xDFFF) {
-          code = (code - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
-          char += data.charAt(stringPosition);
-        } else {
-          stringPosition--;
-        }
-      }
+    for (let pos = start; pos < end; ++pos) {
+      code = data[pos];
+      char = stringFromCodePoint(code);
 
       // calculate print space
       // expensive call, therefore we save width in line buffer
       chWidth = wcwidth(code);
 
       // get charset replacement character
-      if (charset) {
-        char = charset[char] || char;
-        code = char.charCodeAt(0);
+      // charset are only defined for ASCII, therefore we only
+      // search for an replacement char if code < 127
+      if (code < 127 && charset) {
+        const ch = charset[char];
+        if (ch) {
+          code = ch.charCodeAt(0);
+          char = ch;
+        }
       }
 
       if (screenReaderMode) {

src/Types.ts

@@ -111,7 +111,7 @@ export interface ICompositionHelper {
  */
 export interface IInputHandler {
   parse(data: string): void;
-  print(data: string, start: number, end: number): void;
+  print(data: Uint32Array, start: number, end: number): void;
 
   /** C0 BEL */ bell(): void;
   /** C0 LF */ lineFeed(): void;
@@ -452,18 +452,26 @@ export interface IParsingState {
 * DCS handler signature for EscapeSequenceParser.
 * EscapeSequenceParser handles DCS commands via separate
 * subparsers that get hook/unhooked and can handle
-* arbitrary amount of print data.
+* arbitrary amount of data.
+*
 * On entering a DSC sequence `hook` is called by
 * `EscapeSequenceParser`. Use it to initialize or reset
 * states needed to handle the current DCS sequence.
+* Note: A DCS parser is only instantiated once, therefore
+* you cannot rely on the ctor to reinitialize state.
+*
 * EscapeSequenceParser will call `put` several times if the
-* parsed string got splitted, therefore you might have to collect
-* `data` until `unhook` is called. `unhook` marks the end
-* of the current DCS sequence.
+* parsed data got split, therefore you might have to collect
+* `data` until `unhook` is called.
+* Note: `data` is borrowed, if you cannot process the data
+* in chunks you have to copy it, doing otherwise will lead to
+* data losses or corruption.
+*
+* `unhook` marks the end of the current DCS sequence.
 */
 export interface IDcsHandler {
   hook(collect: string, params: number[], flag: number): void;
-  put(data: string, start: number, end: number): void;
+  put(data: Uint32Array, start: number, end: number): void;
   unhook(): void;
 }
 
@@ -480,9 +488,9 @@ export interface IEscapeSequenceParser extends IDisposable {
    * Parse string `data`.
    * @param data The data to parse.
    */
-  parse(data: string): void;
+  parse(data: Uint32Array, length: number): void;
 
-  setPrintHandler(callback: (data: string, start: number, end: number) => void): void;
+  setPrintHandler(callback: (data: Uint32Array, start: number, end: number) => void): void;
   clearPrintHandler(): void;
 
   setExecuteHandler(flag: string, callback: () => void): void;

src/common/TypedArrayUtils.test.ts

@@ -3,21 +3,20 @@
  * @license MIT
  */
 import { assert } from 'chai';
-import { fillFallback } from './TypedArrayUtils';
+import { fillFallback, concat } from './TypedArrayUtils';
 
 type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray
   | Int8Array | Int16Array | Int32Array
   | Float32Array | Float64Array;
 
-describe('polyfill conformance tests', function(): void {
-
-  function deepEquals(a: TypedArray, b: TypedArray): void {
-    assert.equal(a.length, b.length);
-    for (let i = 0; i < a.length; ++i) {
-      assert.equal(a[i], b[i]);
-    }
+function deepEquals(a: TypedArray, b: TypedArray): void {
+  assert.equal(a.length, b.length);
+  for (let i = 0; i < a.length; ++i) {
+    assert.equal(a[i], b[i]);
   }
+}
 
+describe('polyfill conformance tests', function(): void {
   describe('TypedArray.fill', function(): void {
     it('should work with all typed array types', function(): void {
       const u81 = new Uint8Array(5);
@@ -87,3 +86,12 @@ describe('polyfill conformance tests', function(): void {
     });
   });
 });
+
+describe('typed array convenience functions', () => {
+  it('concat', () => {
+    const a = new Uint8Array([1, 2, 3, 4, 5]);
+    const b = new Uint8Array([6, 7, 8, 9, 0]);
+    const merged = concat(a, b);
+    deepEquals(merged, new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]));
+  });
+});