Merge VS Code 1.21 source code (#1067)

* Initial VS Code 1.21 file copy with patches

* A few more merges

* Post npm install

* Fix batch of build breaks

* Fix more build breaks

* Fix more build errors

* Fix more build breaks

* Runtime fixes 1

* Get connection dialog working with some todos

* Fix a few packaging issues

* Copy several node_modules to package build to fix loader issues

* Fix breaks from master

* A few more fixes

* Make tests pass

* First pass of license header updates

* Second pass of license header updates

* Fix restore dialog issues

* Remove add additional themes menu items

* fix select box issues where the list doesn't show up

* formatting

* Fix editor dispose issue

* Copy over node modules to correct location on all platforms

src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts

@@ -0,0 +1,494 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { Range } from 'vs/editor/common/core/range';
+import { Position } from 'vs/editor/common/core/position';
+import * as strings from 'vs/base/common/strings';
+import { IValidatedEditOperation } from 'vs/editor/common/model/linesTextBuffer/linesTextBuffer';
+import { PieceTreeBase, StringBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
+import { IIdentifiedSingleEditOperation, EndOfLinePreference, ITextBuffer, ApplyEditsResult, IInternalModelContentChange } from 'vs/editor/common/model';
+import { ITextSnapshot } from 'vs/platform/files/common/files';
+
+export class PieceTreeTextBuffer implements ITextBuffer {
+	private _pieceTree: PieceTreeBase;
+	private _BOM: string;
+	private _mightContainRTL: boolean;
+	private _mightContainNonBasicASCII: boolean;
+
+	constructor(chunks: StringBuffer[], BOM: string, eol: '\r\n' | '\n', containsRTL: boolean, isBasicASCII: boolean, eolNormalized: boolean) {
+		this._BOM = BOM;
+		this._mightContainNonBasicASCII = !isBasicASCII;
+		this._mightContainRTL = containsRTL;
+		this._pieceTree = new PieceTreeBase(chunks, eol, eolNormalized);
+	}
+
+	// #region TextBuffer
+	public equals(other: ITextBuffer): boolean {
+		if (!(other instanceof PieceTreeTextBuffer)) {
+			return false;
+		}
+		if (this._BOM !== other._BOM) {
+			return false;
+		}
+		if (this.getEOL() !== other.getEOL()) {
+			return false;
+		}
+		return this._pieceTree.equal(other._pieceTree);
+	}
+	public mightContainRTL(): boolean {
+		return this._mightContainRTL;
+	}
+	public mightContainNonBasicASCII(): boolean {
+		return this._mightContainNonBasicASCII;
+	}
+	public getBOM(): string {
+		return this._BOM;
+	}
+	public getEOL(): string {
+		return this._pieceTree.getEOL();
+	}
+
+	public createSnapshot(preserveBOM: boolean): ITextSnapshot {
+		return this._pieceTree.createSnapshot(preserveBOM ? this._BOM : '');
+	}
+
+	public getOffsetAt(lineNumber: number, column: number): number {
+		return this._pieceTree.getOffsetAt(lineNumber, column);
+	}
+
+	public getPositionAt(offset: number): Position {
+		return this._pieceTree.getPositionAt(offset);
+	}
+
+	public getRangeAt(start: number, length: number): Range {
+		let end = start + length;
+		const startPosition = this.getPositionAt(start);
+		const endPosition = this.getPositionAt(end);
+		return new Range(startPosition.lineNumber, startPosition.column, endPosition.lineNumber, endPosition.column);
+	}
+
+	public getValueInRange(range: Range, eol: EndOfLinePreference = EndOfLinePreference.TextDefined): string {
+		if (range.isEmpty()) {
+			return '';
+		}
+
+		const lineEnding = this._getEndOfLine(eol);
+		const text = this._pieceTree.getValueInRange(range);
+		return text.replace(/\r\n|\r|\n/g, lineEnding);
+	}
+
+	public getValueLengthInRange(range: Range, eol: EndOfLinePreference = EndOfLinePreference.TextDefined): number {
+		if (range.isEmpty()) {
+			return 0;
+		}
+
+		if (range.startLineNumber === range.endLineNumber) {
+			return (range.endColumn - range.startColumn);
+		}
+
+		let startOffset = this.getOffsetAt(range.startLineNumber, range.startColumn);
+		let endOffset = this.getOffsetAt(range.endLineNumber, range.endColumn);
+		return endOffset - startOffset;
+	}
+
+	public getLength(): number {
+		return this._pieceTree.getLength();
+	}
+
+	public getLineCount(): number {
+		return this._pieceTree.getLineCount();
+	}
+
+	public getLinesContent(): string[] {
+		return this._pieceTree.getLinesContent();
+	}
+
+	public getLineContent(lineNumber: number): string {
+		return this._pieceTree.getLineContent(lineNumber);
+	}
+
+	public getLineCharCode(lineNumber: number, index: number): number {
+		return this._pieceTree.getLineCharCode(lineNumber, index);
+	}
+
+	public getLineLength(lineNumber: number): number {
+		return this._pieceTree.getLineLength(lineNumber);
+	}
+
+	public getLineMinColumn(lineNumber: number): number {
+		return 1;
+	}
+
+	public getLineMaxColumn(lineNumber: number): number {
+		return this.getLineLength(lineNumber) + 1;
+	}
+
+	public getLineFirstNonWhitespaceColumn(lineNumber: number): number {
+		const result = strings.firstNonWhitespaceIndex(this.getLineContent(lineNumber));
+		if (result === -1) {
+			return 0;
+		}
+		return result + 1;
+	}
+
+	public getLineLastNonWhitespaceColumn(lineNumber: number): number {
+		const result = strings.lastNonWhitespaceIndex(this.getLineContent(lineNumber));
+		if (result === -1) {
+			return 0;
+		}
+		return result + 2;
+	}
+
+	private _getEndOfLine(eol: EndOfLinePreference): string {
+		switch (eol) {
+			case EndOfLinePreference.LF:
+				return '\n';
+			case EndOfLinePreference.CRLF:
+				return '\r\n';
+			case EndOfLinePreference.TextDefined:
+				return this.getEOL();
+		}
+		throw new Error('Unknown EOL preference');
+	}
+
+	public setEOL(newEOL: '\r\n' | '\n'): void {
+		this._pieceTree.setEOL(newEOL);
+	}
+
+	public applyEdits(rawOperations: IIdentifiedSingleEditOperation[], recordTrimAutoWhitespace: boolean): ApplyEditsResult {
+		let mightContainRTL = this._mightContainRTL;
+		let mightContainNonBasicASCII = this._mightContainNonBasicASCII;
+		let canReduceOperations = true;
+
+		let operations: IValidatedEditOperation[] = [];
+		for (let i = 0; i < rawOperations.length; i++) {
+			let op = rawOperations[i];
+			if (canReduceOperations && op._isTracked) {
+				canReduceOperations = false;
+			}
+			let validatedRange = op.range;
+			if (!mightContainRTL && op.text) {
+				// check if the new inserted text contains RTL
+				mightContainRTL = strings.containsRTL(op.text);
+			}
+			if (!mightContainNonBasicASCII && op.text) {
+				mightContainNonBasicASCII = !strings.isBasicASCII(op.text);
+			}
+			operations[i] = {
+				sortIndex: i,
+				identifier: op.identifier,
+				range: validatedRange,
+				rangeOffset: this.getOffsetAt(validatedRange.startLineNumber, validatedRange.startColumn),
+				rangeLength: this.getValueLengthInRange(validatedRange),
+				lines: op.text ? op.text.split(/\r\n|\r|\n/) : null,
+				forceMoveMarkers: op.forceMoveMarkers,
+				isAutoWhitespaceEdit: op.isAutoWhitespaceEdit || false
+			};
+		}
+
+		// Sort operations ascending
+		operations.sort(PieceTreeTextBuffer._sortOpsAscending);
+
+		for (let i = 0, count = operations.length - 1; i < count; i++) {
+			let rangeEnd = operations[i].range.getEndPosition();
+			let nextRangeStart = operations[i + 1].range.getStartPosition();
+
+			if (nextRangeStart.isBefore(rangeEnd)) {
+				// overlapping ranges
+				throw new Error('Overlapping ranges are not allowed!');
+			}
+		}
+
+		if (canReduceOperations) {
+			operations = this._reduceOperations(operations);
+		}
+
+		// Delta encode operations
+		let reverseRanges = PieceTreeTextBuffer._getInverseEditRanges(operations);
+		let newTrimAutoWhitespaceCandidates: { lineNumber: number, oldContent: string }[] = [];
+
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+			let reverseRange = reverseRanges[i];
+
+			if (recordTrimAutoWhitespace && op.isAutoWhitespaceEdit && op.range.isEmpty()) {
+				// Record already the future line numbers that might be auto whitespace removal candidates on next edit
+				for (let lineNumber = reverseRange.startLineNumber; lineNumber <= reverseRange.endLineNumber; lineNumber++) {
+					let currentLineContent = '';
+					if (lineNumber === reverseRange.startLineNumber) {
+						currentLineContent = this.getLineContent(op.range.startLineNumber);
+						if (strings.firstNonWhitespaceIndex(currentLineContent) !== -1) {
+							continue;
+						}
+					}
+					newTrimAutoWhitespaceCandidates.push({ lineNumber: lineNumber, oldContent: currentLineContent });
+				}
+			}
+		}
+
+		let reverseOperations: IIdentifiedSingleEditOperation[] = [];
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+			let reverseRange = reverseRanges[i];
+
+			reverseOperations[i] = {
+				identifier: op.identifier,
+				range: reverseRange,
+				text: this.getValueInRange(op.range),
+				forceMoveMarkers: op.forceMoveMarkers
+			};
+		}
+
+		this._mightContainRTL = mightContainRTL;
+		this._mightContainNonBasicASCII = mightContainNonBasicASCII;
+
+		const contentChanges = this._doApplyEdits(operations);
+
+		let trimAutoWhitespaceLineNumbers: number[] = null;
+		if (recordTrimAutoWhitespace && newTrimAutoWhitespaceCandidates.length > 0) {
+			// sort line numbers auto whitespace removal candidates for next edit descending
+			newTrimAutoWhitespaceCandidates.sort((a, b) => b.lineNumber - a.lineNumber);
+
+			trimAutoWhitespaceLineNumbers = [];
+			for (let i = 0, len = newTrimAutoWhitespaceCandidates.length; i < len; i++) {
+				let lineNumber = newTrimAutoWhitespaceCandidates[i].lineNumber;
+				if (i > 0 && newTrimAutoWhitespaceCandidates[i - 1].lineNumber === lineNumber) {
+					// Do not have the same line number twice
+					continue;
+				}
+
+				let prevContent = newTrimAutoWhitespaceCandidates[i].oldContent;
+				let lineContent = this.getLineContent(lineNumber);
+
+				if (lineContent.length === 0 || lineContent === prevContent || strings.firstNonWhitespaceIndex(lineContent) !== -1) {
+					continue;
+				}
+
+				trimAutoWhitespaceLineNumbers.push(lineNumber);
+			}
+		}
+
+		return new ApplyEditsResult(
+			reverseOperations,
+			contentChanges,
+			trimAutoWhitespaceLineNumbers
+		);
+	}
+
+	/**
+ * Transform operations such that they represent the same logic edit,
+ * but that they also do not cause OOM crashes.
+ */
+	private _reduceOperations(operations: IValidatedEditOperation[]): IValidatedEditOperation[] {
+		if (operations.length < 1000) {
+			// We know from empirical testing that a thousand edits work fine regardless of their shape.
+			return operations;
+		}
+
+		// At one point, due to how events are emitted and how each operation is handled,
+		// some operations can trigger a high ammount of temporary string allocations,
+		// that will immediately get edited again.
+		// e.g. a formatter inserting ridiculous ammounts of \n on a model with a single line
+		// Therefore, the strategy is to collapse all the operations into a huge single edit operation
+		return [this._toSingleEditOperation(operations)];
+	}
+
+	_toSingleEditOperation(operations: IValidatedEditOperation[]): IValidatedEditOperation {
+		let forceMoveMarkers = false,
+			firstEditRange = operations[0].range,
+			lastEditRange = operations[operations.length - 1].range,
+			entireEditRange = new Range(firstEditRange.startLineNumber, firstEditRange.startColumn, lastEditRange.endLineNumber, lastEditRange.endColumn),
+			lastEndLineNumber = firstEditRange.startLineNumber,
+			lastEndColumn = firstEditRange.startColumn,
+			result: string[] = [];
+
+		for (let i = 0, len = operations.length; i < len; i++) {
+			let operation = operations[i],
+				range = operation.range;
+
+			forceMoveMarkers = forceMoveMarkers || operation.forceMoveMarkers;
+
+			// (1) -- Push old text
+			for (let lineNumber = lastEndLineNumber; lineNumber < range.startLineNumber; lineNumber++) {
+				if (lineNumber === lastEndLineNumber) {
+					result.push(this.getLineContent(lineNumber).substring(lastEndColumn - 1));
+				} else {
+					result.push('\n');
+					result.push(this.getLineContent(lineNumber));
+				}
+			}
+
+			if (range.startLineNumber === lastEndLineNumber) {
+				result.push(this.getLineContent(range.startLineNumber).substring(lastEndColumn - 1, range.startColumn - 1));
+			} else {
+				result.push('\n');
+				result.push(this.getLineContent(range.startLineNumber).substring(0, range.startColumn - 1));
+			}
+
+			// (2) -- Push new text
+			if (operation.lines) {
+				for (let j = 0, lenJ = operation.lines.length; j < lenJ; j++) {
+					if (j !== 0) {
+						result.push('\n');
+					}
+					result.push(operation.lines[j]);
+				}
+			}
+
+			lastEndLineNumber = operation.range.endLineNumber;
+			lastEndColumn = operation.range.endColumn;
+		}
+
+		return {
+			sortIndex: 0,
+			identifier: operations[0].identifier,
+			range: entireEditRange,
+			rangeOffset: this.getOffsetAt(entireEditRange.startLineNumber, entireEditRange.startColumn),
+			rangeLength: this.getValueLengthInRange(entireEditRange, EndOfLinePreference.TextDefined),
+			lines: result.join('').split('\n'),
+			forceMoveMarkers: forceMoveMarkers,
+			isAutoWhitespaceEdit: false
+		};
+	}
+
+	private _doApplyEdits(operations: IValidatedEditOperation[]): IInternalModelContentChange[] {
+		operations.sort(PieceTreeTextBuffer._sortOpsDescending);
+
+		let contentChanges: IInternalModelContentChange[] = [];
+
+		// operations are from bottom to top
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+
+			const startLineNumber = op.range.startLineNumber;
+			const startColumn = op.range.startColumn;
+			const endLineNumber = op.range.endLineNumber;
+			const endColumn = op.range.endColumn;
+
+			if (startLineNumber === endLineNumber && startColumn === endColumn && (!op.lines || op.lines.length === 0)) {
+				// no-op
+				continue;
+			}
+
+			const deletingLinesCnt = endLineNumber - startLineNumber;
+			const insertingLinesCnt = (op.lines ? op.lines.length - 1 : 0);
+			const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt);
+
+			const text = (op.lines ? op.lines.join(this.getEOL()) : '');
+
+			if (text) {
+				// replacement
+				this._pieceTree.delete(op.rangeOffset, op.rangeLength);
+				this._pieceTree.insert(op.rangeOffset, text, true);
+
+			} else {
+				// deletion
+				this._pieceTree.delete(op.rangeOffset, op.rangeLength);
+			}
+
+			if (editingLinesCnt < insertingLinesCnt) {
+				let newLinesContent: string[] = [];
+				for (let j = editingLinesCnt + 1; j <= insertingLinesCnt; j++) {
+					newLinesContent.push(op.lines[j]);
+				}
+
+				newLinesContent[newLinesContent.length - 1] = this.getLineContent(startLineNumber + insertingLinesCnt - 1);
+			}
+
+			const contentChangeRange = new Range(startLineNumber, startColumn, endLineNumber, endColumn);
+			contentChanges.push({
+				range: contentChangeRange,
+				rangeLength: op.rangeLength,
+				text: text,
+				rangeOffset: op.rangeOffset,
+				forceMoveMarkers: op.forceMoveMarkers
+			});
+		}
+		return contentChanges;
+	}
+
+	// #endregion
+
+	// #region helper
+	// testing purpose.
+	public getPieceTree(): PieceTreeBase {
+		return this._pieceTree;
+	}
+	/**
+	 * Assumes `operations` are validated and sorted ascending
+	 */
+	public static _getInverseEditRanges(operations: IValidatedEditOperation[]): Range[] {
+		let result: Range[] = [];
+
+		let prevOpEndLineNumber: number;
+		let prevOpEndColumn: number;
+		let prevOp: IValidatedEditOperation = null;
+		for (let i = 0, len = operations.length; i < len; i++) {
+			let op = operations[i];
+
+			let startLineNumber: number;
+			let startColumn: number;
+
+			if (prevOp) {
+				if (prevOp.range.endLineNumber === op.range.startLineNumber) {
+					startLineNumber = prevOpEndLineNumber;
+					startColumn = prevOpEndColumn + (op.range.startColumn - prevOp.range.endColumn);
+				} else {
+					startLineNumber = prevOpEndLineNumber + (op.range.startLineNumber - prevOp.range.endLineNumber);
+					startColumn = op.range.startColumn;
+				}
+			} else {
+				startLineNumber = op.range.startLineNumber;
+				startColumn = op.range.startColumn;
+			}
+
+			let resultRange: Range;
+
+			if (op.lines && op.lines.length > 0) {
+				// the operation inserts something
+				let lineCount = op.lines.length;
+				let firstLine = op.lines[0];
+				let lastLine = op.lines[lineCount - 1];
+
+				if (lineCount === 1) {
+					// single line insert
+					resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn + firstLine.length);
+				} else {
+					// multi line insert
+					resultRange = new Range(startLineNumber, startColumn, startLineNumber + lineCount - 1, lastLine.length + 1);
+				}
+			} else {
+				// There is nothing to insert
+				resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn);
+			}
+
+			prevOpEndLineNumber = resultRange.endLineNumber;
+			prevOpEndColumn = resultRange.endColumn;
+
+			result.push(resultRange);
+			prevOp = op;
+		}
+
+		return result;
+	}
+
+	private static _sortOpsAscending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
+		let r = Range.compareRangesUsingEnds(a.range, b.range);
+		if (r === 0) {
+			return a.sortIndex - b.sortIndex;
+		}
+		return r;
+	}
+
+	private static _sortOpsDescending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
+		let r = Range.compareRangesUsingEnds(a.range, b.range);
+		if (r === 0) {
+			return b.sortIndex - a.sortIndex;
+		}
+		return -r;
+	}
+	// #endregion
+}

src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder.ts

@@ -0,0 +1,180 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as strings from 'vs/base/common/strings';
+import { ITextBufferBuilder, DefaultEndOfLine, ITextBufferFactory, ITextBuffer } from 'vs/editor/common/model';
+import { PieceTreeTextBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer';
+import { StringBuffer, createLineStarts, createLineStartsFast } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
+import { CharCode } from 'vs/base/common/charCode';
+
+export class PieceTreeTextBufferFactory implements ITextBufferFactory {
+
+	constructor(
+		private readonly _chunks: StringBuffer[],
+		private readonly _bom: string,
+		private readonly _cr: number,
+		private readonly _lf: number,
+		private readonly _crlf: number,
+		private readonly _containsRTL: boolean,
+		private readonly _isBasicASCII: boolean,
+		private readonly _normalizeEOL: boolean
+	) { }
+
+	private _getEOL(defaultEOL: DefaultEndOfLine): '\r\n' | '\n' {
+		const totalEOLCount = this._cr + this._lf + this._crlf;
+		const totalCRCount = this._cr + this._crlf;
+		if (totalEOLCount === 0) {
+			// This is an empty file or a file with precisely one line
+			return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n');
+		}
+		if (totalCRCount > totalEOLCount / 2) {
+			// More than half of the file contains \r\n ending lines
+			return '\r\n';
+		}
+		// At least one line more ends in \n
+		return '\n';
+	}
+
+	public create(defaultEOL: DefaultEndOfLine): ITextBuffer {
+		const eol = this._getEOL(defaultEOL);
+		let chunks = this._chunks;
+
+		if (this._normalizeEOL &&
+			((eol === '\r\n' && (this._cr > 0 || this._lf > 0))
+				|| (eol === '\n' && (this._cr > 0 || this._crlf > 0)))
+		) {
+			// Normalize pieces
+			for (let i = 0, len = chunks.length; i < len; i++) {
+				let str = chunks[i].buffer.replace(/\r\n|\r|\n/g, eol);
+				let newLineStart = createLineStartsFast(str);
+				chunks[i] = new StringBuffer(str, newLineStart);
+			}
+		}
+
+		return new PieceTreeTextBuffer(chunks, this._bom, eol, this._containsRTL, this._isBasicASCII, this._normalizeEOL);
+	}
+
+	public getFirstLineText(lengthLimit: number): string {
+		return this._chunks[0].buffer.substr(0, 100).split(/\r\n|\r|\n/)[0];
+	}
+}
+
+export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
+	private chunks: StringBuffer[];
+	private BOM: string;
+
+	private _hasPreviousChar: boolean;
+	private _previousChar: number;
+	private _tmpLineStarts: number[];
+
+	private cr: number;
+	private lf: number;
+	private crlf: number;
+	private containsRTL: boolean;
+	private isBasicASCII: boolean;
+
+	constructor() {
+		this.chunks = [];
+		this.BOM = '';
+
+		this._hasPreviousChar = false;
+		this._previousChar = 0;
+		this._tmpLineStarts = [];
+
+		this.cr = 0;
+		this.lf = 0;
+		this.crlf = 0;
+		this.containsRTL = false;
+		this.isBasicASCII = true;
+	}
+
+	public acceptChunk(chunk: string): void {
+		if (chunk.length === 0) {
+			return;
+		}
+
+		if (this.chunks.length === 0) {
+			if (strings.startsWithUTF8BOM(chunk)) {
+				this.BOM = strings.UTF8_BOM_CHARACTER;
+				chunk = chunk.substr(1);
+			}
+		}
+
+		const lastChar = chunk.charCodeAt(chunk.length - 1);
+		if (lastChar === CharCode.CarriageReturn || (lastChar >= 0xd800 && lastChar <= 0xdbff)) {
+			// last character is \r or a high surrogate => keep it back
+			this._acceptChunk1(chunk.substr(0, chunk.length - 1), false);
+			this._hasPreviousChar = true;
+			this._previousChar = lastChar;
+		} else {
+			this._acceptChunk1(chunk, false);
+			this._hasPreviousChar = false;
+			this._previousChar = lastChar;
+		}
+	}
+
+	private _acceptChunk1(chunk: string, allowEmptyStrings: boolean): void {
+		if (!allowEmptyStrings && chunk.length === 0) {
+			// Nothing to do
+			return;
+		}
+
+		if (this._hasPreviousChar) {
+			this._acceptChunk2(String.fromCharCode(this._previousChar) + chunk);
+		} else {
+			this._acceptChunk2(chunk);
+		}
+	}
+
+	private _acceptChunk2(chunk: string): void {
+		const lineStarts = createLineStarts(this._tmpLineStarts, chunk);
+
+		this.chunks.push(new StringBuffer(chunk, lineStarts.lineStarts));
+		this.cr += lineStarts.cr;
+		this.lf += lineStarts.lf;
+		this.crlf += lineStarts.crlf;
+
+		if (this.isBasicASCII) {
+			this.isBasicASCII = lineStarts.isBasicASCII;
+		}
+		if (!this.isBasicASCII && !this.containsRTL) {
+			// No need to check if is basic ASCII
+			this.containsRTL = strings.containsRTL(chunk);
+		}
+	}
+
+	public finish(normalizeEOL: boolean = true): PieceTreeTextBufferFactory {
+		this._finish();
+		return new PieceTreeTextBufferFactory(
+			this.chunks,
+			this.BOM,
+			this.cr,
+			this.lf,
+			this.crlf,
+			this.containsRTL,
+			this.isBasicASCII,
+			normalizeEOL
+		);
+	}
+
+	private _finish(): void {
+		if (this.chunks.length === 0) {
+			this._acceptChunk1('', true);
+		}
+
+		if (this._hasPreviousChar) {
+			this._hasPreviousChar = false;
+			// recreate last chunk
+			let lastChunk = this.chunks[this.chunks.length - 1];
+			lastChunk.buffer += String.fromCharCode(this._previousChar);
+			let newLineStarts = createLineStartsFast(lastChunk.buffer);
+			lastChunk.lineStarts = newLineStarts;
+			if (this._previousChar === CharCode.CarriageReturn) {
+				this.cr++;
+			}
+		}
+	}
+}

src/vs/editor/common/model/pieceTreeTextBuffer/rbTreeBase.ts

@@ -0,0 +1,427 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { Piece, PieceTreeBase } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
+
+export class TreeNode {
+	parent: TreeNode;
+	left: TreeNode;
+	right: TreeNode;
+	color: NodeColor;
+
+	// Piece
+	piece: Piece;
+	size_left: number; // size of the left subtree (not inorder)
+	lf_left: number; // line feeds cnt in the left subtree (not in order)
+
+	constructor(piece: Piece, color: NodeColor) {
+		this.piece = piece;
+		this.color = color;
+		this.size_left = 0;
+		this.lf_left = 0;
+		this.parent = null;
+		this.left = null;
+		this.right = null;
+	}
+
+	public next(): TreeNode {
+		if (this.right !== SENTINEL) {
+			return leftest(this.right);
+		}
+
+		let node: TreeNode = this;
+
+		while (node.parent !== SENTINEL) {
+			if (node.parent.left === node) {
+				break;
+			}
+
+			node = node.parent;
+		}
+
+		if (node.parent === SENTINEL) {
+			return SENTINEL;
+		} else {
+			return node.parent;
+		}
+	}
+
+	public prev(): TreeNode {
+		if (this.left !== SENTINEL) {
+			return righttest(this.left);
+		}
+
+		let node: TreeNode = this;
+
+		while (node.parent !== SENTINEL) {
+			if (node.parent.right === node) {
+				break;
+			}
+
+			node = node.parent;
+		}
+
+		if (node.parent === SENTINEL) {
+			return SENTINEL;
+		} else {
+			return node.parent;
+		}
+	}
+
+	public detach(): void {
+		this.parent = null;
+		this.left = null;
+		this.right = null;
+	}
+}
+
+export const SENTINEL: TreeNode = new TreeNode(null, NodeColor.Black);
+SENTINEL.parent = SENTINEL;
+SENTINEL.left = SENTINEL;
+SENTINEL.right = SENTINEL;
+SENTINEL.color = NodeColor.Black;
+
+export const enum NodeColor {
+	Black = 0,
+	Red = 1,
+}
+
+export function leftest(node: TreeNode): TreeNode {
+	while (node.left !== SENTINEL) {
+		node = node.left;
+	}
+	return node;
+}
+
+export function righttest(node: TreeNode): TreeNode {
+	while (node.right !== SENTINEL) {
+		node = node.right;
+	}
+	return node;
+}
+
+export function calculateSize(node: TreeNode): number {
+	if (node === SENTINEL) {
+		return 0;
+	}
+
+	return node.size_left + node.piece.length + calculateSize(node.right);
+}
+
+export function calculateLF(node: TreeNode): number {
+	if (node === SENTINEL) {
+		return 0;
+	}
+
+	return node.lf_left + node.piece.lineFeedCnt + calculateLF(node.right);
+}
+
+export function resetSentinel(): void {
+	SENTINEL.parent = SENTINEL;
+}
+
+export function leftRotate(tree: PieceTreeBase, x: TreeNode) {
+	let y = x.right;
+
+	// fix size_left
+	y.size_left += x.size_left + (x.piece ? x.piece.length : 0);
+	y.lf_left += x.lf_left + (x.piece ? x.piece.lineFeedCnt : 0);
+	x.right = y.left;
+
+	if (y.left !== SENTINEL) {
+		y.left.parent = x;
+	}
+	y.parent = x.parent;
+	if (x.parent === SENTINEL) {
+		tree.root = y;
+	} else if (x.parent.left === x) {
+		x.parent.left = y;
+	} else {
+		x.parent.right = y;
+	}
+	y.left = x;
+	x.parent = y;
+}
+
+export function rightRotate(tree: PieceTreeBase, y: TreeNode) {
+	let x = y.left;
+	y.left = x.right;
+	if (x.right !== SENTINEL) {
+		x.right.parent = y;
+	}
+	x.parent = y.parent;
+
+	// fix size_left
+	y.size_left -= x.size_left + (x.piece ? x.piece.length : 0);
+	y.lf_left -= x.lf_left + (x.piece ? x.piece.lineFeedCnt : 0);
+
+	if (y.parent === SENTINEL) {
+		tree.root = x;
+	} else if (y === y.parent.right) {
+		y.parent.right = x;
+	} else {
+		y.parent.left = x;
+	}
+
+	x.right = y;
+	y.parent = x;
+}
+
+export function rbDelete(tree: PieceTreeBase, z: TreeNode) {
+	let x: TreeNode;
+	let y: TreeNode;
+
+	if (z.left === SENTINEL) {
+		y = z;
+		x = y.right;
+	} else if (z.right === SENTINEL) {
+		y = z;
+		x = y.left;
+	} else {
+		y = leftest(z.right);
+		x = y.right;
+	}
+
+	if (y === tree.root) {
+		tree.root = x;
+
+		// if x is null, we are removing the only node
+		x.color = NodeColor.Black;
+		z.detach();
+		resetSentinel();
+		tree.root.parent = SENTINEL;
+
+		return;
+	}
+
+	let yWasRed = (y.color === NodeColor.Red);
+
+	if (y === y.parent.left) {
+		y.parent.left = x;
+	} else {
+		y.parent.right = x;
+	}
+
+	if (y === z) {
+		x.parent = y.parent;
+		recomputeTreeMetadata(tree, x);
+	} else {
+		if (y.parent === z) {
+			x.parent = y;
+		} else {
+			x.parent = y.parent;
+		}
+
+		// as we make changes to x's hierarchy, update size_left of subtree first
+		recomputeTreeMetadata(tree, x);
+
+		y.left = z.left;
+		y.right = z.right;
+		y.parent = z.parent;
+		y.color = z.color;
+
+		if (z === tree.root) {
+			tree.root = y;
+		} else {
+			if (z === z.parent.left) {
+				z.parent.left = y;
+			} else {
+				z.parent.right = y;
+			}
+		}
+
+		if (y.left !== SENTINEL) {
+			y.left.parent = y;
+		}
+		if (y.right !== SENTINEL) {
+			y.right.parent = y;
+		}
+		// update metadata
+		// we replace z with y, so in this sub tree, the length change is z.item.length
+		y.size_left = z.size_left;
+		y.lf_left = z.lf_left;
+		recomputeTreeMetadata(tree, y);
+	}
+
+	z.detach();
+
+	if (x.parent.left === x) {
+		let newSizeLeft = calculateSize(x);
+		let newLFLeft = calculateLF(x);
+		if (newSizeLeft !== x.parent.size_left || newLFLeft !== x.parent.lf_left) {
+			let delta = newSizeLeft - x.parent.size_left;
+			let lf_delta = newLFLeft - x.parent.lf_left;
+			x.parent.size_left = newSizeLeft;
+			x.parent.lf_left = newLFLeft;
+			updateTreeMetadata(tree, x.parent, delta, lf_delta);
+		}
+	}
+
+	recomputeTreeMetadata(tree, x.parent);
+
+	if (yWasRed) {
+		resetSentinel();
+		return;
+	}
+
+	// RB-DELETE-FIXUP
+	let w: TreeNode;
+	while (x !== tree.root && x.color === NodeColor.Black) {
+		if (x === x.parent.left) {
+			w = x.parent.right;
+
+			if (w.color === NodeColor.Red) {
+				w.color = NodeColor.Black;
+				x.parent.color = NodeColor.Red;
+				leftRotate(tree, x.parent);
+				w = x.parent.right;
+			}
+
+			if (w.left.color === NodeColor.Black && w.right.color === NodeColor.Black) {
+				w.color = NodeColor.Red;
+				x = x.parent;
+			} else {
+				if (w.right.color === NodeColor.Black) {
+					w.left.color = NodeColor.Black;
+					w.color = NodeColor.Red;
+					rightRotate(tree, w);
+					w = x.parent.right;
+				}
+
+				w.color = x.parent.color;
+				x.parent.color = NodeColor.Black;
+				w.right.color = NodeColor.Black;
+				leftRotate(tree, x.parent);
+				x = tree.root;
+			}
+		} else {
+			w = x.parent.left;
+
+			if (w.color === NodeColor.Red) {
+				w.color = NodeColor.Black;
+				x.parent.color = NodeColor.Red;
+				rightRotate(tree, x.parent);
+				w = x.parent.left;
+			}
+
+			if (w.left.color === NodeColor.Black && w.right.color === NodeColor.Black) {
+				w.color = NodeColor.Red;
+				x = x.parent;
+
+			} else {
+				if (w.left.color === NodeColor.Black) {
+					w.right.color = NodeColor.Black;
+					w.color = NodeColor.Red;
+					leftRotate(tree, w);
+					w = x.parent.left;
+				}
+
+				w.color = x.parent.color;
+				x.parent.color = NodeColor.Black;
+				w.left.color = NodeColor.Black;
+				rightRotate(tree, x.parent);
+				x = tree.root;
+			}
+		}
+	}
+	x.color = NodeColor.Black;
+	resetSentinel();
+}
+
+export function fixInsert(tree: PieceTreeBase, x: TreeNode) {
+	recomputeTreeMetadata(tree, x);
+
+	while (x !== tree.root && x.parent.color === NodeColor.Red) {
+		if (x.parent === x.parent.parent.left) {
+			const y = x.parent.parent.right;
+
+			if (y.color === NodeColor.Red) {
+				x.parent.color = NodeColor.Black;
+				y.color = NodeColor.Black;
+				x.parent.parent.color = NodeColor.Red;
+				x = x.parent.parent;
+			} else {
+				if (x === x.parent.right) {
+					x = x.parent;
+					leftRotate(tree, x);
+				}
+
+				x.parent.color = NodeColor.Black;
+				x.parent.parent.color = NodeColor.Red;
+				rightRotate(tree, x.parent.parent);
+			}
+		} else {
+			const y = x.parent.parent.left;
+
+			if (y.color === NodeColor.Red) {
+				x.parent.color = NodeColor.Black;
+				y.color = NodeColor.Black;
+				x.parent.parent.color = NodeColor.Red;
+				x = x.parent.parent;
+			} else {
+				if (x === x.parent.left) {
+					x = x.parent;
+					rightRotate(tree, x);
+				}
+				x.parent.color = NodeColor.Black;
+				x.parent.parent.color = NodeColor.Red;
+				leftRotate(tree, x.parent.parent);
+			}
+		}
+	}
+
+	tree.root.color = NodeColor.Black;
+}
+
+export function updateTreeMetadata(tree: PieceTreeBase, x: TreeNode, delta: number, lineFeedCntDelta: number): void {
+	// node length change or line feed count change
+	while (x !== tree.root && x !== SENTINEL) {
+		if (x.parent.left === x) {
+			x.parent.size_left += delta;
+			x.parent.lf_left += lineFeedCntDelta;
+		}
+
+		x = x.parent;
+	}
+}
+
+export function recomputeTreeMetadata(tree: PieceTreeBase, x: TreeNode) {
+	let delta = 0;
+	let lf_delta = 0;
+	if (x === tree.root) {
+		return;
+	}
+
+	if (delta === 0) {
+		// go upwards till the node whose left subtree is changed.
+		while (x !== tree.root && x === x.parent.right) {
+			x = x.parent;
+		}
+
+		if (x === tree.root) {
+			// well, it means we add a node to the end (inorder)
+			return;
+		}
+
+		// x is the node whose right subtree is changed.
+		x = x.parent;
+
+		delta = calculateSize(x.left) - x.size_left;
+		lf_delta = calculateLF(x.left) - x.lf_left;
+		x.size_left += delta;
+		x.lf_left += lf_delta;
+	}
+
+	// go upwards till root. O(logN)
+	while (x !== tree.root && (delta !== 0 || lf_delta !== 0)) {
+		if (x.parent.left === x) {
+			x.parent.size_left += delta;
+			x.parent.lf_left += lf_delta;
+		}
+
+		x = x.parent;
+	}
+}