Utf8 input

demo/client.ts

@@ -139,6 +139,7 @@ function createTerminal(): void {
         pid = processId;
         socketURL += processId;
         socket = new WebSocket(socketURL);
+        socket.binaryType = 'arraybuffer';
         socket.onopen = runRealTerminal;
         socket.onclose = runFakeTerminal;
         socket.onerror = runFakeTerminal;

demo/server.js

@@ -36,7 +36,8 @@ function startServer() {
           cols: cols || 80,
           rows: rows || 24,
           cwd: process.env.PWD,
-          env: process.env
+          env: process.env,
+          encoding: null
         });
 
     console.log('Created terminal with PID: ' + term.pid);
@@ -66,20 +67,20 @@ function startServer() {
     ws.send(logs[term.pid]);
 
     function buffer(socket, timeout) {
-      let s = '';
+      let buffer = [];
       let sender = null;
       return (data) => {
-        s += data;
+        buffer.push(data);
         if (!sender) {
           sender = setTimeout(() => {
-            socket.send(s);
-            s = '';
+            socket.send(Buffer.concat(buffer));
+            buffer = [];
             sender = null;
           }, timeout);
         }
       };
     }
-    const send = buffer(ws, 5);
+  const send = buffer(ws, 5);
 
     term.on('data', function(data) {
       try {

package.json

@@ -13,6 +13,7 @@
     "@types/mocha": "^2.2.33",
     "@types/node": "6.0.108",
     "@types/puppeteer": "^1.12.4",
+    "@types/utf8": "^2.1.6",
     "@types/webpack": "^4.4.11",
     "browserify": "^13.3.0",
     "chai": "3.5.0",
@@ -39,6 +40,7 @@
     "ts-loader": "^4.5.0",
     "tslint": "^5.9.1",
     "tslint-consistent-codestyle": "^1.13.0",
+    "utf8": "^3.0.0",
     "typescript": "3.4",
     "vinyl-buffer": "^1.0.0",
     "vinyl-source-stream": "^1.1.0",

src/InputHandler.ts

@@ -12,7 +12,7 @@ import { EscapeSequenceParser } from './EscapeSequenceParser';
 import { IDisposable } from 'xterm';
 import { Disposable } from './common/Lifecycle';
 import { concat } from './common/TypedArrayUtils';
-import { StringToUtf32, stringFromCodePoint, utf32ToString } from './core/input/TextDecoder';
+import { StringToUtf32, stringFromCodePoint, utf32ToString, Utf8ToUtf32 } from './core/input/TextDecoder';
 import { CellData, Attributes, FgFlags, BgFlags, AttributeData, NULL_CELL_WIDTH, NULL_CELL_CODE, DEFAULT_ATTR_DATA } from './core/buffer/BufferLine';
 import { EventEmitter2, IEvent } from './common/EventEmitter2';
 
@@ -104,6 +104,7 @@ class DECRQSS implements IDcsHandler {
 export class InputHandler extends Disposable implements IInputHandler {
   private _parseBuffer: Uint32Array = new Uint32Array(4096);
   private _stringDecoder: StringToUtf32 = new StringToUtf32();
+  private _utf8Decoder: Utf8ToUtf32 = new Utf8ToUtf32();
   private _workCell: CellData = new CellData();
 
   private _onCursorMove = new EventEmitter2<void>();
@@ -318,6 +319,32 @@ export class InputHandler extends Disposable implements IInputHandler {
     }
   }
 
+  public parseUtf8(data: Uint8Array): void {
+    // Ensure the terminal is not disposed
+    if (!this._terminal) {
+      return;
+    }
+
+    let buffer = this._terminal.buffer;
+    const cursorStartX = buffer.x;
+    const cursorStartY = buffer.y;
+
+    // TODO: Consolidate debug/logging #1560
+    if ((<any>this._terminal).debug) {
+      this._terminal.log('data: ' + data);
+    }
+
+    if (this._parseBuffer.length < data.length) {
+      this._parseBuffer = new Uint32Array(data.length);
+    }
+    this._parser.parse(this._parseBuffer, this._utf8Decoder.decode(data, this._parseBuffer));
+
+    buffer = this._terminal.buffer;
+    if (buffer.x !== cursorStartX || buffer.y !== cursorStartY) {
+      this._terminal.emit('cursormove');
+    }
+  }
+
   public print(data: Uint32Array, start: number, end: number): void {
     let code: number;
     let chWidth: number;

src/Terminal.ts

@@ -183,6 +183,7 @@ export class Terminal extends EventEmitter implements ITerminal, IDisposable, II
 
   // user input states
   public writeBuffer: string[];
+  public writeBufferUtf8: Uint8Array[];
   private _writeInProgress: boolean;
 
   /**
@@ -340,6 +341,7 @@ export class Terminal extends EventEmitter implements ITerminal, IDisposable, II
 
     // user input states
     this.writeBuffer = [];
+    this.writeBufferUtf8 = [];
     this._writeInProgress = false;
 
     this._xoffSentToCatchUp = false;
@@ -1365,6 +1367,88 @@ export class Terminal extends EventEmitter implements ITerminal, IDisposable, II
     }
   }
 
+  /**
+   * Writes raw utf8 bytes to the terminal.
+   * @param data UintArray with UTF8 bytes to write to the terminal.
+   */
+  public writeUtf8(data: Uint8Array): void {
+    // Ensure the terminal isn't disposed
+    if (this._isDisposed) {
+      return;
+    }
+
+    // Ignore falsy data values (including the empty string)
+    if (!data) {
+      return;
+    }
+
+    this.writeBufferUtf8.push(data);
+
+    // Send XOFF to pause the pty process if the write buffer becomes too large so
+    // xterm.js can catch up before more data is sent. This is necessary in order
+    // to keep signals such as ^C responsive.
+    if (this.options.useFlowControl && !this._xoffSentToCatchUp && this.writeBufferUtf8.length >= WRITE_BUFFER_PAUSE_THRESHOLD) {
+      // XOFF - stop pty pipe
+      // XON will be triggered by emulator before processing data chunk
+      this.handler(C0.DC3);
+      this._xoffSentToCatchUp = true;
+    }
+
+    if (!this._writeInProgress && this.writeBufferUtf8.length > 0) {
+      // Kick off a write which will write all data in sequence recursively
+      this._writeInProgress = true;
+      // Kick off an async innerWrite so more writes can come in while processing data
+      setTimeout(() => {
+        this._innerWriteUtf8();
+      });
+    }
+  }
+
+  protected _innerWriteUtf8(bufferOffset: number = 0): void {
+    // Ensure the terminal isn't disposed
+    if (this._isDisposed) {
+      this.writeBufferUtf8 = [];
+    }
+
+    const startTime = Date.now();
+    while (this.writeBufferUtf8.length > bufferOffset) {
+      const data = this.writeBufferUtf8[bufferOffset];
+      bufferOffset++;
+
+      // If XOFF was sent in order to catch up with the pty process, resume it if
+      // we reached the end of the writeBuffer to allow more data to come in.
+      if (this._xoffSentToCatchUp && this.writeBufferUtf8.length === bufferOffset) {
+        this.handler(C0.DC1);
+        this._xoffSentToCatchUp = false;
+      }
+
+      this._refreshStart = this.buffer.y;
+      this._refreshEnd = this.buffer.y;
+
+      // HACK: Set the parser state based on it's state at the time of return.
+      // This works around the bug #662 which saw the parser state reset in the
+      // middle of parsing escape sequence in two chunks. For some reason the
+      // state of the parser resets to 0 after exiting parser.parse. This change
+      // just sets the state back based on the correct return statement.
+
+      this._inputHandler.parseUtf8(data);
+
+      this.updateRange(this.buffer.y);
+      this.refresh(this._refreshStart, this._refreshEnd);
+
+      if (Date.now() - startTime >= WRITE_TIMEOUT_MS) {
+        break;
+      }
+    }
+    if (this.writeBufferUtf8.length > bufferOffset) {
+      // Allow renderer to catch up before processing the next batch
+      setTimeout(() => this._innerWriteUtf8(bufferOffset), 0);
+    } else {
+      this._writeInProgress = false;
+      this.writeBufferUtf8 = [];
+    }
+  }
+
   /**
    * Writes text to the terminal.
    * @param data The text to write to the terminal.

src/TestUtils.test.ts

@@ -110,6 +110,9 @@ export class MockTerminal implements ITerminal {
   write(data: string): void {
     throw new Error('Method not implemented.');
   }
+  writeUtf8(data: Uint8Array): void {
+    throw new Error('Method not implemented.');
+  }
   bracketedPasteMode: boolean;
   mouseHelper: IMouseHelper;
   renderer: IRenderer;

src/Types.ts

@@ -103,6 +103,7 @@ export interface ICompositionHelper {
  */
 export interface IInputHandler {
   parse(data: string): void;
+  parseUtf8(data: Uint8Array): void;
   print(data: Uint32Array, start: number, end: number): void;
 
   /** C0 BEL */ bell(): void;
@@ -262,6 +263,7 @@ export interface IPublicTerminal extends IDisposable, IEventEmitter {
     scrollToLine(line: number): void;
     clear(): void;
     write(data: string): void;
+    writeUtf8(data: Uint8Array): void;
     getOption(key: string): any;
     setOption(key: string, value: any): void;
     refresh(start: number, end: number): void;

src/addons/attach/attach.ts

@@ -42,6 +42,11 @@ export function attach(term: Terminal, socket: WebSocket, bidirectional: boolean
   addonTerminal.__getMessage = function(ev: MessageEvent): void {
     let str: string;
 
+    if (ev.data instanceof ArrayBuffer) {
+      addonTerminal.writeUtf8(new Uint8Array(ev.data));
+      return;
+    }
+
     if (typeof ev.data === 'object') {
       if (!myTextDecoder) {
         myTextDecoder = new TextDecoder();