Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Utf32 buffer #1878

Merged
merged 41 commits into from
Apr 2, 2019
Merged

Utf32 buffer #1878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
ba1582c
apply utf32 buffer layout
jerch Jan 3, 2019
a2a8c3d
extend buffer line with more direct access methods
jerch Jan 3, 2019
f63223f
Merge branch 'master' into utf32_buffer
jerch Jan 3, 2019
b0eecea
partially apply fast data access in InputHandler.print
jerch Jan 3, 2019
75fbda1
further opt for InputHandler.print
jerch Jan 4, 2019
2f66efa
first try to optimize read access
jerch Jan 4, 2019
f3619ab
fix linter error
jerch Jan 4, 2019
802543f
replace set(CharData) with setCell
jerch Jan 4, 2019
d7e5977
remove leftover
jerch Jan 4, 2019
13ffed3
remove get calls from renderer
jerch Jan 4, 2019
6552a02
remove get from linkifier
jerch Jan 4, 2019
b93b774
direct cell attrs getter
jerch Jan 4, 2019
62dfa84
remove get CharData calls from codebase (beside tests)
jerch Jan 5, 2019
907a6b9
Merge branch 'master' into utf32_buffer
jerch Jan 12, 2019
ad67f47
Merge branch 'typedarray_parser' into utf32_buffer
jerch Jan 12, 2019
88a037b
change insertCells to new interface
jerch Jan 12, 2019
523ff6e
add null and whitespace placeholder cells to buffer
jerch Jan 12, 2019
4bf7f3e
change deleteCells to new interface
jerch Jan 12, 2019
fe6919b
change replaceCells to new interface
jerch Jan 12, 2019
56dec28
change resize, fill to new interface, remove _cell on buffer line
jerch Jan 12, 2019
60a591e
use getNullChar in InputHandler.print
jerch Jan 12, 2019
3e45697
change buffer line ctor to new interface
jerch Jan 12, 2019
ece7192
remove set with CharData from codebase and deprecate method
jerch Jan 12, 2019
711ae65
remove get CharData from codebase and deprecate method
jerch Jan 12, 2019
35a6aa8
some docs
jerch Jan 12, 2019
40b231e
tests for CellData
jerch Jan 12, 2019
7c6dad5
test cases
jerch Jan 12, 2019
200f7bf
Merge branch 'master' into utf32_buffer
jerch Jan 17, 2019
4f531f8
Merge branch 'master' into utf32_buffer
jerch Jan 25, 2019
acea73c
Merge branch 'master' into utf32_buffer
jerch Jan 27, 2019
b20730e
add more docs
jerch Jan 27, 2019
e77e36c
fix typo in HAS_CONTENT
jerch Jan 27, 2019
c60f9b1
code formatting
jerch Jan 27, 2019
39fd6c4
name polishing, docs
jerch Jan 28, 2019
a1b61e0
Merge branch 'master' into utf32_buffer
jerch Jan 31, 2019
594797e
Merge branch 'master' into utf32_buffer
jerch Jan 31, 2019
079729f
change property getter into methods
jerch Feb 1, 2019
179e5d5
Merge branch 'master' into utf32_buffer
jerch Feb 1, 2019
ba9ed43
Merge branch 'master' into utf32_buffer
Tyriar Feb 5, 2019
69f8700
Merge branch 'master' into pr/jerch/1878
Tyriar Apr 1, 2019
5c8c680
Fix feedback
Tyriar Apr 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 30 additions & 30 deletions src/Buffer.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@

import { assert, expect } from 'chai';
import { ITerminal } from './Types';
import { Buffer, DEFAULT_ATTR, CHAR_DATA_CHAR_INDEX } from './Buffer';
import { Buffer, DEFAULT_ATTR } from './Buffer';
import { CircularList } from './common/CircularList';
import { MockTerminal, TestTerminal } from './ui/TestUtils.test';
import { BufferLine } from './BufferLine';
import { BufferLine, CellData } from './BufferLine';

const INIT_COLS = 80;
const INIT_ROWS = 24;
Expand Down Expand Up @@ -37,13 +37,13 @@ describe('Buffer', () => {

describe('fillViewportRows', () => {
it('should fill the buffer with blank lines based on the size of the viewport', () => {
const blankLineChar = buffer.getBlankLine(DEFAULT_ATTR).get(0);
const blankLineChar = buffer.getBlankLine(DEFAULT_ATTR).loadCell(0, new CellData()).getAsCharData;
buffer.fillViewportRows();
assert.equal(buffer.lines.length, INIT_ROWS);
for (let y = 0; y < INIT_ROWS; y++) {
assert.equal(buffer.lines.get(y).length, INIT_COLS);
for (let x = 0; x < INIT_COLS; x++) {
assert.deepEqual(buffer.lines.get(y).get(x), blankLineChar);
assert.deepEqual(buffer.lines.get(y).loadCell(x, new CellData()).getAsCharData, blankLineChar);
}
}
});
Expand Down Expand Up @@ -155,15 +155,15 @@ describe('Buffer', () => {
assert.equal(buffer.lines.maxLength, INIT_ROWS);
buffer.y = INIT_ROWS - 1;
buffer.fillViewportRows();
let chData = buffer.lines.get(5).get(0);
let chData = buffer.lines.get(5).loadCell(0, new CellData()).getAsCharData();
chData[1] = 'a';
buffer.lines.get(5).set(0, chData);
chData = buffer.lines.get(INIT_ROWS - 1).get(0);
buffer.lines.get(5).setCell(0, CellData.fromCharData(chData));
chData = buffer.lines.get(INIT_ROWS - 1).loadCell(0, new CellData()).getAsCharData();
chData[1] = 'b';
buffer.lines.get(INIT_ROWS - 1).set(0, chData);
buffer.lines.get(INIT_ROWS - 1).setCell(0, CellData.fromCharData(chData));
buffer.resize(INIT_COLS, INIT_ROWS - 5);
assert.equal(buffer.lines.get(0).get(0)[1], 'a');
assert.equal(buffer.lines.get(INIT_ROWS - 1 - 5).get(0)[1], 'b');
assert.equal(buffer.lines.get(0).loadCell(0, new CellData()).getAsCharData()[1], 'a');
assert.equal(buffer.lines.get(INIT_ROWS - 1 - 5).loadCell(0, new CellData()).getAsCharData()[1], 'b');
});
});
});
Expand Down Expand Up @@ -1045,10 +1045,10 @@ describe('Buffer', () => {
describe ('translateBufferLineToString', () => {
it('should handle selecting a section of ascii text', () => {
const line = new BufferLine(4);
line.set(0, [ null, 'a', 1, 'a'.charCodeAt(0)]);
line.set(1, [ null, 'b', 1, 'b'.charCodeAt(0)]);
line.set(2, [ null, 'c', 1, 'c'.charCodeAt(0)]);
line.set(3, [ null, 'd', 1, 'd'.charCodeAt(0)]);
line.setCell(0, CellData.fromCharData([ null, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(1, CellData.fromCharData([ null, 'b', 1, 'b'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([ null, 'c', 1, 'c'.charCodeAt(0)]));
line.setCell(3, CellData.fromCharData([ null, 'd', 1, 'd'.charCodeAt(0)]));
buffer.lines.set(0, line);

const str = buffer.translateBufferLineToString(0, true, 0, 2);
Expand All @@ -1057,9 +1057,9 @@ describe('Buffer', () => {

it('should handle a cut-off double width character by including it', () => {
const line = new BufferLine(3);
line.set(0, [ null, '語', 2, 35486 ]);
line.set(1, [ null, '', 0, null]);
line.set(2, [ null, 'a', 1, 'a'.charCodeAt(0)]);
line.setCell(0, CellData.fromCharData([ null, '語', 2, 35486 ]));
line.setCell(1, CellData.fromCharData([ null, '', 0, null]));
line.setCell(2, CellData.fromCharData([ null, 'a', 1, 'a'.charCodeAt(0)]));
buffer.lines.set(0, line);

const str1 = buffer.translateBufferLineToString(0, true, 0, 1);
Expand All @@ -1068,9 +1068,9 @@ describe('Buffer', () => {

it('should handle a zero width character in the middle of the string by not including it', () => {
const line = new BufferLine(3);
line.set(0, [ null, '語', 2, '語'.charCodeAt(0) ]);
line.set(1, [ null, '', 0, null]);
line.set(2, [ null, 'a', 1, 'a'.charCodeAt(0)]);
line.setCell(0, CellData.fromCharData([ null, '語', 2, '語'.charCodeAt(0) ]));
line.setCell(1, CellData.fromCharData([ null, '', 0, null]));
line.setCell(2, CellData.fromCharData([ null, 'a', 1, 'a'.charCodeAt(0)]));
buffer.lines.set(0, line);

const str0 = buffer.translateBufferLineToString(0, true, 0, 1);
Expand All @@ -1085,8 +1085,8 @@ describe('Buffer', () => {

it('should handle single width emojis', () => {
const line = new BufferLine(2);
line.set(0, [ null, '😁', 1, '😁'.charCodeAt(0) ]);
line.set(1, [ null, 'a', 1, 'a'.charCodeAt(0)]);
line.setCell(0, CellData.fromCharData([ null, '😁', 1, '😁'.charCodeAt(0) ]));
line.setCell(1, CellData.fromCharData([ null, 'a', 1, 'a'.charCodeAt(0)]));
buffer.lines.set(0, line);

const str1 = buffer.translateBufferLineToString(0, true, 0, 1);
Expand All @@ -1098,8 +1098,8 @@ describe('Buffer', () => {

it('should handle double width emojis', () => {
const line = new BufferLine(2);
line.set(0, [ null, '😁', 2, '😁'.charCodeAt(0) ]);
line.set(1, [ null, '', 0, null]);
line.setCell(0, CellData.fromCharData([ null, '😁', 2, '😁'.charCodeAt(0) ]));
line.setCell(1, CellData.fromCharData([ null, '', 0, null]));
buffer.lines.set(0, line);

const str1 = buffer.translateBufferLineToString(0, true, 0, 1);
Expand All @@ -1109,9 +1109,9 @@ describe('Buffer', () => {
assert.equal(str2, '😁');

const line2 = new BufferLine(3);
line2.set(0, [ null, '😁', 2, '😁'.charCodeAt(0) ]);
line2.set(1, [ null, '', 0, null]);
line2.set(2, [ null, 'a', 1, 'a'.charCodeAt(0)]);
line2.setCell(0, CellData.fromCharData([ null, '😁', 2, '😁'.charCodeAt(0) ]));
line2.setCell(1, CellData.fromCharData([ null, '', 0, null]));
line2.setCell(2, CellData.fromCharData([ null, 'a', 1, 'a'.charCodeAt(0)]));
buffer.lines.set(0, line2);

const str3 = buffer.translateBufferLineToString(0, true, 0, 3);
Expand Down Expand Up @@ -1264,7 +1264,7 @@ describe('Buffer', () => {
assert.equal(input, s);
const stringIndex = s.match(/😃/).index;
const bufferIndex = terminal.buffer.stringIndexToBufferIndex(0, stringIndex);
assert(terminal.buffer.lines.get(bufferIndex[0]).get(bufferIndex[1])[CHAR_DATA_CHAR_INDEX], '😃');
assert(terminal.buffer.lines.get(bufferIndex[0]).loadCell(bufferIndex[1], new CellData()).getChars(), '😃');
});

it('multiline fullwidth chars with offset 1 (currently tests for broken behavior)', () => {
Expand All @@ -1291,7 +1291,7 @@ describe('Buffer', () => {
assert.equal(input, s);
for (let i = 0; i < input.length; ++i) {
const bufferIndex = terminal.buffer.stringIndexToBufferIndex(0, i, true);
assert.equal(input[i], terminal.buffer.lines.get(bufferIndex[0]).get(bufferIndex[1])[CHAR_DATA_CHAR_INDEX]);
assert.equal(input[i], terminal.buffer.lines.get(bufferIndex[0]).loadCell(bufferIndex[1], new CellData()).getChars());
}
});

Expand All @@ -1309,7 +1309,7 @@ describe('Buffer', () => {
: (i % 3 === 1)
? input.substr(i, 2)
: input.substr(i - 1, 2),
terminal.buffer.lines.get(bufferIndex[0]).get(bufferIndex[1])[CHAR_DATA_CHAR_INDEX]);
terminal.buffer.lines.get(bufferIndex[0]).loadCell(bufferIndex[1], new CellData()).getChars());
}
});

Expand Down
53 changes: 40 additions & 13 deletions src/Buffer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@
* @license MIT
*/

import { CircularList, IInsertEvent, IDeleteEvent } from './common/CircularList';
import { ITerminal, IBuffer, IBufferLine, BufferIndex, IBufferStringIterator, IBufferStringIteratorResult, ICellData } from './Types';
import { EventEmitter } from './common/EventEmitter';
import { IMarker } from 'xterm';
import { BufferLine } from './BufferLine';
import { BufferLine, CellData } from './BufferLine';
import { reflowLargerApplyNewLayout, reflowLargerCreateNewLayout, reflowLargerGetLinesToRemove, reflowSmallerGetNewLineLengths } from './BufferReflow';
import { CircularList, IDeleteEvent, IInsertEvent } from './common/CircularList';
import { EventEmitter } from './common/EventEmitter';
import { DEFAULT_COLOR } from './renderer/atlas/Types';
import { BufferIndex, CharData, IBuffer, IBufferLine, IBufferStringIterator, IBufferStringIteratorResult, ITerminal } from './Types';


export const DEFAULT_ATTR = (0 << 18) | (DEFAULT_COLOR << 9) | (256 << 0);
export const CHAR_DATA_ATTR_INDEX = 0;
Expand All @@ -18,16 +19,24 @@ export const CHAR_DATA_WIDTH_INDEX = 2;
export const CHAR_DATA_CODE_INDEX = 3;
export const MAX_BUFFER_SIZE = 4294967295; // 2^32 - 1

/**
* Null cell - a real empty cell (containing nothing).
* Note that code should always be 0 for a null cell as
* several test condition of the buffer line rely on this.
*/
export const NULL_CELL_CHAR = '';
export const NULL_CELL_WIDTH = 1;
export const NULL_CELL_CODE = 0;

/**
* Whitespace cell.
* This is meant as a replacement for empty cells when needed
* during rendering lines to preserve correct aligment.
*/
export const WHITESPACE_CELL_CHAR = ' ';
export const WHITESPACE_CELL_WIDTH = 1;
export const WHITESPACE_CELL_CODE = 32;

export const FILL_CHAR_DATA: CharData = [DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE];

/**
* This class represents a terminal buffer (an internal state of the terminal), where the
* following information is stored (in high-level):
Expand All @@ -48,6 +57,8 @@ export class Buffer implements IBuffer {
public savedX: number;
public savedCurAttr: number;
public markers: Marker[] = [];
private _nullCell: ICellData = CellData.fromCharData([0, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]);
private _whitespaceCell: ICellData = CellData.fromCharData([0, WHITESPACE_CELL_CHAR, WHITESPACE_CELL_WIDTH, WHITESPACE_CELL_CODE]);
private _cols: number;
private _rows: number;

Expand All @@ -66,9 +77,20 @@ export class Buffer implements IBuffer {
this.clear();
}

public getNullCell(fg: number = 0, bg: number = 0): ICellData {
this._nullCell.fg = fg;
this._nullCell.bg = bg;
return this._nullCell;
}

public getWhitespaceCell(fg: number = 0, bg: number = 0): ICellData {
this._whitespaceCell.fg = fg;
this._whitespaceCell.bg = bg;
return this._whitespaceCell;
}

public getBlankLine(attr: number, isWrapped?: boolean): IBufferLine {
const fillCharData: CharData = [attr, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE];
return new BufferLine(this._cols, fillCharData, isWrapped);
return new BufferLine(this._terminal.cols, this.getNullCell(attr), isWrapped);
}

public get hasScrollback(): boolean {
Expand Down Expand Up @@ -131,6 +153,9 @@ export class Buffer implements IBuffer {
* @param newRows The new number of rows.
*/
public resize(newCols: number, newRows: number): void {
// store reference to null cell with default attrs
const nullCell = this.getNullCell(DEFAULT_ATTR);

// Increase max length if needed before adjustments to allow space to fill
// as required.
const newMaxLength = this._getCorrectBufferLength(newRows);
Expand All @@ -144,7 +169,7 @@ export class Buffer implements IBuffer {
// Deal with columns increasing (reducing needs to happen after reflow)
if (this._cols < newCols) {
for (let i = 0; i < this.lines.length; i++) {
this.lines.get(i).resize(newCols, FILL_CHAR_DATA);
this.lines.get(i).resize(newCols, nullCell);
}
}

Expand All @@ -165,7 +190,7 @@ export class Buffer implements IBuffer {
} else {
// Add a blank line if there is no buffer left at the top to scroll to, or if there
// are blank lines after the cursor
this.lines.push(new BufferLine(newCols, FILL_CHAR_DATA));
this.lines.push(new BufferLine(newCols, nullCell));
}
}
}
Expand Down Expand Up @@ -217,7 +242,7 @@ export class Buffer implements IBuffer {
// Trim the end of the line off if cols shrunk
if (this._cols > newCols) {
for (let i = 0; i < this.lines.length; i++) {
this.lines.get(i).resize(newCols, FILL_CHAR_DATA);
this.lines.get(i).resize(newCols, nullCell);
}
}
}
Expand Down Expand Up @@ -253,6 +278,7 @@ export class Buffer implements IBuffer {
}

private _reflowLargerAdjustViewport(newCols: number, newRows: number, countRemoved: number): void {
const nullCell = this.getNullCell(DEFAULT_ATTR);
// Adjust viewport based on number of items removed
let viewportAdjustments = countRemoved;
while (viewportAdjustments-- > 0) {
Expand All @@ -262,7 +288,7 @@ export class Buffer implements IBuffer {
}
if (this.lines.length < newRows) {
// Add an extra row at the bottom of the viewport
this.lines.push(new BufferLine(newCols, FILL_CHAR_DATA));
this.lines.push(new BufferLine(newCols, nullCell));
}
} else {
if (this.ydisp === this.ybase) {
Expand All @@ -274,6 +300,7 @@ export class Buffer implements IBuffer {
}

private _reflowSmaller(newCols: number, newRows: number): void {
const nullCell = this.getNullCell(DEFAULT_ATTR);
// Gather all BufferLines that need to be inserted into the Buffer here so that they can be
// batched up and only committed once
const toInsert = [];
Expand Down Expand Up @@ -356,7 +383,7 @@ export class Buffer implements IBuffer {
// Null out the end of the line ends if a wide character wrapped to the following line
for (let i = 0; i < wrappedLines.length; i++) {
if (destLineLengths[i] < newCols) {
wrappedLines[i].set(destLineLengths[i], FILL_CHAR_DATA);
wrappedLines[i].setCell(destLineLengths[i], nullCell);
}
}

Expand Down
Loading