Merge VS Code 1.21 source code (#1067)

* Initial VS Code 1.21 file copy with patches

* A few more merges

* Post npm install

* Fix batch of build breaks

* Fix more build breaks

* Fix more build errors

* Fix more build breaks

* Runtime fixes 1

* Get connection dialog working with some todos

* Fix a few packaging issues

* Copy several node_modules to package build to fix loader issues

* Fix breaks from master

* A few more fixes

* Make tests pass

* First pass of license header updates

* Second pass of license header updates

* Fix restore dialog issues

* Remove add additional themes menu items

* fix select box issues where the list doesn't show up

* formatting

* Fix editor dispose issue

* Copy over node modules to correct location on all platforms

src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts

@@ -0,0 +1,323 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { CharCode } from 'vs/base/common/charCode';
+
+export class LeafOffsetLenEdit {
+	constructor(
+		public readonly start: number,
+		public readonly length: number,
+		public readonly text: string
+	) { }
+}
+
+export class BufferPiece {
+	private readonly _str: string;
+	public get text(): string { return this._str; }
+
+	private readonly _lineStarts: Uint32Array;
+
+	constructor(str: string, lineStarts: Uint32Array = null) {
+		this._str = str;
+		if (lineStarts === null) {
+			this._lineStarts = createLineStartsFast(str);
+		} else {
+			this._lineStarts = lineStarts;
+		}
+	}
+
+	public length(): number {
+		return this._str.length;
+	}
+
+	public newLineCount(): number {
+		return this._lineStarts.length;
+	}
+
+	public lineStartFor(relativeLineIndex: number): number {
+		return this._lineStarts[relativeLineIndex];
+	}
+
+	public charCodeAt(index: number): number {
+		return this._str.charCodeAt(index);
+	}
+
+	public substr(from: number, length: number): string {
+		return this._str.substr(from, length);
+	}
+
+	public findLineStartBeforeOffset(offset: number): number {
+		if (this._lineStarts.length === 0 || offset < this._lineStarts[0]) {
+			return -1;
+		}
+
+		let low = 0, high = this._lineStarts.length - 1;
+
+		while (low < high) {
+			let mid = low + Math.ceil((high - low) / 2);
+			let lineStart = this._lineStarts[mid];
+
+			if (offset === lineStart) {
+				return mid;
+			} else if (offset < lineStart) {
+				high = mid - 1;
+			} else {
+				low = mid;
+			}
+		}
+
+		return low;
+	}
+
+	public findLineFirstNonWhitespaceIndex(searchStartOffset: number): number {
+		for (let i = searchStartOffset, len = this._str.length; i < len; i++) {
+			const chCode = this._str.charCodeAt(i);
+			if (chCode === CharCode.CarriageReturn || chCode === CharCode.LineFeed) {
+				// Reached EOL
+				return -2;
+			}
+			if (chCode !== CharCode.Space && chCode !== CharCode.Tab) {
+				return i;
+			}
+		}
+		return -1;
+	}
+
+	public findLineLastNonWhitespaceIndex(searchStartOffset: number): number {
+		for (let i = searchStartOffset - 1; i >= 0; i--) {
+			const chCode = this._str.charCodeAt(i);
+			if (chCode === CharCode.CarriageReturn || chCode === CharCode.LineFeed) {
+				// Reached EOL
+				return -2;
+			}
+			if (chCode !== CharCode.Space && chCode !== CharCode.Tab) {
+				return i;
+			}
+		}
+		return -1;
+	}
+
+	public static normalizeEOL(target: BufferPiece, eol: '\r\n' | '\n'): BufferPiece {
+		return new BufferPiece(target._str.replace(/\r\n|\r|\n/g, eol));
+	}
+
+	public static deleteLastChar(target: BufferPiece): BufferPiece {
+		const targetCharsLength = target.length();
+		const targetLineStartsLength = target.newLineCount();
+		const targetLineStarts = target._lineStarts;
+
+		let newLineStartsLength;
+		if (targetLineStartsLength > 0 && targetLineStarts[targetLineStartsLength - 1] === targetCharsLength) {
+			newLineStartsLength = targetLineStartsLength - 1;
+		} else {
+			newLineStartsLength = targetLineStartsLength;
+		}
+
+		let newLineStarts = new Uint32Array(newLineStartsLength);
+		newLineStarts.set(targetLineStarts);
+
+		return new BufferPiece(
+			target._str.substr(0, targetCharsLength - 1),
+			newLineStarts
+		);
+	}
+
+	public static insertFirstChar(target: BufferPiece, character: number): BufferPiece {
+		const targetLineStartsLength = target.newLineCount();
+		const targetLineStarts = target._lineStarts;
+		const insertLineStart = ((character === CharCode.CarriageReturn && (targetLineStartsLength === 0 || targetLineStarts[0] !== 1 || target.charCodeAt(0) !== CharCode.LineFeed)) || (character === CharCode.LineFeed));
+
+		const newLineStartsLength = (insertLineStart ? targetLineStartsLength + 1 : targetLineStartsLength);
+		let newLineStarts = new Uint32Array(newLineStartsLength);
+
+		if (insertLineStart) {
+			newLineStarts[0] = 1;
+			for (let i = 0; i < targetLineStartsLength; i++) {
+				newLineStarts[i + 1] = targetLineStarts[i] + 1;
+			}
+		} else {
+			for (let i = 0; i < targetLineStartsLength; i++) {
+				newLineStarts[i] = targetLineStarts[i] + 1;
+			}
+		}
+
+		return new BufferPiece(
+			String.fromCharCode(character) + target._str,
+			newLineStarts
+		);
+	}
+
+	public static join(first: BufferPiece, second: BufferPiece): BufferPiece {
+		const firstCharsLength = first._str.length;
+
+		const firstLineStartsLength = first._lineStarts.length;
+		const secondLineStartsLength = second._lineStarts.length;
+
+		const firstLineStarts = first._lineStarts;
+		const secondLineStarts = second._lineStarts;
+
+		const newLineStartsLength = firstLineStartsLength + secondLineStartsLength;
+		let newLineStarts = new Uint32Array(newLineStartsLength);
+		newLineStarts.set(firstLineStarts, 0);
+		for (let i = 0; i < secondLineStartsLength; i++) {
+			newLineStarts[i + firstLineStartsLength] = secondLineStarts[i] + firstCharsLength;
+		}
+
+		return new BufferPiece(first._str + second._str, newLineStarts);
+	}
+
+	public static replaceOffsetLen(target: BufferPiece, edits: LeafOffsetLenEdit[], idealLeafLength: number, maxLeafLength: number, result: BufferPiece[]): void {
+		const editsSize = edits.length;
+		const originalCharsLength = target.length();
+		if (editsSize === 1 && edits[0].text.length === 0 && edits[0].start === 0 && edits[0].length === originalCharsLength) {
+			// special case => deleting everything
+			return;
+		}
+
+		let pieces: string[] = new Array<string>(2 * editsSize + 1);
+		let originalFromIndex = 0;
+		let piecesTextLength = 0;
+		for (let i = 0; i < editsSize; i++) {
+			const edit = edits[i];
+
+			const originalText = target._str.substr(originalFromIndex, edit.start - originalFromIndex);
+			pieces[2 * i] = originalText;
+			piecesTextLength += originalText.length;
+
+			originalFromIndex = edit.start + edit.length;
+			pieces[2 * i + 1] = edit.text;
+			piecesTextLength += edit.text.length;
+		}
+
+		// maintain the chars that survive to the right of the last edit
+		let text = target._str.substr(originalFromIndex, originalCharsLength - originalFromIndex);
+		pieces[2 * editsSize] = text;
+		piecesTextLength += text.length;
+
+		let targetDataLength = piecesTextLength > maxLeafLength ? idealLeafLength : piecesTextLength;
+		let targetDataOffset = 0;
+
+		let data: string = '';
+
+		for (let pieceIndex = 0, pieceCount = pieces.length; pieceIndex < pieceCount; pieceIndex++) {
+			const pieceText = pieces[pieceIndex];
+			const pieceLength = pieceText.length;
+			if (pieceLength === 0) {
+				continue;
+			}
+
+			let pieceOffset = 0;
+			while (pieceOffset < pieceLength) {
+				if (targetDataOffset >= targetDataLength) {
+					result.push(new BufferPiece(data));
+					targetDataLength = piecesTextLength > maxLeafLength ? idealLeafLength : piecesTextLength;
+					targetDataOffset = 0;
+					data = '';
+				}
+
+				let writingCnt = min(pieceLength - pieceOffset, targetDataLength - targetDataOffset);
+				data += pieceText.substr(pieceOffset, writingCnt);
+				pieceOffset += writingCnt;
+				targetDataOffset += writingCnt;
+				piecesTextLength -= writingCnt;
+
+				// check that the buffer piece does not end in a \r or high surrogate
+				if (targetDataOffset === targetDataLength && piecesTextLength > 0) {
+					const lastChar = data.charCodeAt(targetDataLength - 1);
+					if (lastChar === CharCode.CarriageReturn || (0xD800 <= lastChar && lastChar <= 0xDBFF)) {
+						// move lastChar over to next buffer piece
+						targetDataLength -= 1;
+						pieceOffset -= 1;
+						targetDataOffset -= 1;
+						piecesTextLength += 1;
+						data = data.substr(0, data.length - 1);
+					}
+				}
+			}
+		}
+
+		result.push(new BufferPiece(data));
+	}
+}
+
+function min(a: number, b: number): number {
+	return (a < b ? a : b);
+}
+
+export function createUint32Array(arr: number[]): Uint32Array {
+	let r = new Uint32Array(arr.length);
+	r.set(arr, 0);
+	return r;
+}
+
+export class LineStarts {
+	constructor(
+		public readonly lineStarts: Uint32Array,
+		public readonly cr: number,
+		public readonly lf: number,
+		public readonly crlf: number,
+		public readonly isBasicASCII: boolean
+	) { }
+}
+
+export function createLineStartsFast(str: string): Uint32Array {
+	let r: number[] = [], rLength = 0;
+	for (let i = 0, len = str.length; i < len; i++) {
+		const chr = str.charCodeAt(i);
+
+		if (chr === CharCode.CarriageReturn) {
+			if (i + 1 < len && str.charCodeAt(i + 1) === CharCode.LineFeed) {
+				// \r\n... case
+				r[rLength++] = i + 2;
+				i++; // skip \n
+			} else {
+				// \r... case
+				r[rLength++] = i + 1;
+			}
+		} else if (chr === CharCode.LineFeed) {
+			r[rLength++] = i + 1;
+		}
+	}
+	return createUint32Array(r);
+}
+
+export function createLineStarts(r: number[], str: string): LineStarts {
+	r.length = 0;
+
+	let rLength = 0;
+	let cr = 0, lf = 0, crlf = 0;
+	let isBasicASCII = true;
+	for (let i = 0, len = str.length; i < len; i++) {
+		const chr = str.charCodeAt(i);
+
+		if (chr === CharCode.CarriageReturn) {
+			if (i + 1 < len && str.charCodeAt(i + 1) === CharCode.LineFeed) {
+				// \r\n... case
+				crlf++;
+				r[rLength++] = i + 2;
+				i++; // skip \n
+			} else {
+				cr++;
+				// \r... case
+				r[rLength++] = i + 1;
+			}
+		} else if (chr === CharCode.LineFeed) {
+			lf++;
+			r[rLength++] = i + 1;
+		} else {
+			if (isBasicASCII) {
+				if (chr !== CharCode.Tab && (chr < 32 || chr > 126)) {
+					isBasicASCII = false;
+				}
+			}
+		}
+	}
+
+	const result = new LineStarts(createUint32Array(r), cr, lf, crlf, isBasicASCII);
+	r.length = 0;
+
+	return result;
+}

src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts

@@ -0,0 +1,186 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as strings from 'vs/base/common/strings';
+import { ITextBufferBuilder, ITextBufferFactory, ITextBuffer, DefaultEndOfLine } from 'vs/editor/common/model';
+import { BufferPiece, createLineStarts } from 'vs/editor/common/model/chunksTextBuffer/bufferPiece';
+import { ChunksTextBuffer } from 'vs/editor/common/model/chunksTextBuffer/chunksTextBuffer';
+import { CharCode } from 'vs/base/common/charCode';
+
+export class TextBufferFactory implements ITextBufferFactory {
+
+	constructor(
+		private readonly _pieces: BufferPiece[],
+		private readonly _averageChunkSize: number,
+		private readonly _BOM: string,
+		private readonly _cr: number,
+		private readonly _lf: number,
+		private readonly _crlf: number,
+		private readonly _containsRTL: boolean,
+		private readonly _isBasicASCII: boolean,
+	) {
+	}
+
+	/**
+	 * if text source is empty or with precisely one line, returns null. No end of line is detected.
+	 * if text source contains more lines ending with '\r\n', returns '\r\n'.
+	 * Otherwise returns '\n'. More lines end with '\n'.
+	 */
+	private _getEOL(defaultEOL: DefaultEndOfLine): '\r\n' | '\n' {
+		const totalEOLCount = this._cr + this._lf + this._crlf;
+		const totalCRCount = this._cr + this._crlf;
+		if (totalEOLCount === 0) {
+			// This is an empty file or a file with precisely one line
+			return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n');
+		}
+		if (totalCRCount > totalEOLCount / 2) {
+			// More than half of the file contains \r\n ending lines
+			return '\r\n';
+		}
+		// At least one line more ends in \n
+		return '\n';
+	}
+
+	public create(defaultEOL: DefaultEndOfLine): ITextBuffer {
+		const eol = this._getEOL(defaultEOL);
+		let pieces = this._pieces;
+
+		if (
+			(eol === '\r\n' && (this._cr > 0 || this._lf > 0))
+			|| (eol === '\n' && (this._cr > 0 || this._crlf > 0))
+		) {
+			// Normalize pieces
+			for (let i = 0, len = pieces.length; i < len; i++) {
+				pieces[i] = BufferPiece.normalizeEOL(pieces[i], eol);
+			}
+		}
+		return new ChunksTextBuffer(pieces, this._averageChunkSize, this._BOM, eol, this._containsRTL, this._isBasicASCII);
+	}
+
+	public getFirstLineText(lengthLimit: number): string {
+		const firstPiece = this._pieces[0];
+		if (firstPiece.newLineCount() === 0) {
+			return firstPiece.substr(0, lengthLimit);
+		}
+
+		const firstEOLOffset = firstPiece.lineStartFor(0);
+		return firstPiece.substr(0, Math.min(lengthLimit, firstEOLOffset));
+	}
+}
+
+export class ChunksTextBufferBuilder implements ITextBufferBuilder {
+
+	private _rawPieces: BufferPiece[];
+	private _hasPreviousChar: boolean;
+	private _previousChar: number;
+	private _averageChunkSize: number;
+	private _tmpLineStarts: number[];
+
+	private BOM: string;
+	private cr: number;
+	private lf: number;
+	private crlf: number;
+	private containsRTL: boolean;
+	private isBasicASCII: boolean;
+
+	constructor() {
+		this._rawPieces = [];
+		this._hasPreviousChar = false;
+		this._previousChar = 0;
+		this._averageChunkSize = 0;
+		this._tmpLineStarts = [];
+
+		this.BOM = '';
+		this.cr = 0;
+		this.lf = 0;
+		this.crlf = 0;
+		this.containsRTL = false;
+		this.isBasicASCII = true;
+	}
+
+	public acceptChunk(chunk: string): void {
+		if (chunk.length === 0) {
+			return;
+		}
+
+		if (this._rawPieces.length === 0) {
+			if (strings.startsWithUTF8BOM(chunk)) {
+				this.BOM = strings.UTF8_BOM_CHARACTER;
+				chunk = chunk.substr(1);
+			}
+		}
+
+		this._averageChunkSize = (this._averageChunkSize * this._rawPieces.length + chunk.length) / (this._rawPieces.length + 1);
+
+		const lastChar = chunk.charCodeAt(chunk.length - 1);
+		if (lastChar === CharCode.CarriageReturn || (lastChar >= 0xd800 && lastChar <= 0xdbff)) {
+			// last character is \r or a high surrogate => keep it back
+			this._acceptChunk1(chunk.substr(0, chunk.length - 1), false);
+			this._hasPreviousChar = true;
+			this._previousChar = lastChar;
+		} else {
+			this._acceptChunk1(chunk, false);
+			this._hasPreviousChar = false;
+			this._previousChar = lastChar;
+		}
+	}
+
+	private _acceptChunk1(chunk: string, allowEmptyStrings: boolean): void {
+		if (!allowEmptyStrings && chunk.length === 0) {
+			// Nothing to do
+			return;
+		}
+
+		if (this._hasPreviousChar) {
+			this._acceptChunk2(chunk + String.fromCharCode(this._previousChar));
+		} else {
+			this._acceptChunk2(chunk);
+		}
+	}
+
+	private _acceptChunk2(chunk: string): void {
+		const lineStarts = createLineStarts(this._tmpLineStarts, chunk);
+
+		this._rawPieces.push(new BufferPiece(chunk, lineStarts.lineStarts));
+		this.cr += lineStarts.cr;
+		this.lf += lineStarts.lf;
+		this.crlf += lineStarts.crlf;
+
+		if (this.isBasicASCII) {
+			this.isBasicASCII = lineStarts.isBasicASCII;
+		}
+		if (!this.isBasicASCII && !this.containsRTL) {
+			// No need to check if is basic ASCII
+			this.containsRTL = strings.containsRTL(chunk);
+		}
+	}
+
+	public finish(): TextBufferFactory {
+		this._finish();
+		return new TextBufferFactory(this._rawPieces, this._averageChunkSize, this.BOM, this.cr, this.lf, this.crlf, this.containsRTL, this.isBasicASCII);
+	}
+
+	private _finish(): void {
+		if (this._rawPieces.length === 0) {
+			// no chunks => forcefully go through accept chunk
+			this._acceptChunk1('', true);
+			return;
+		}
+
+		if (this._hasPreviousChar) {
+			this._hasPreviousChar = false;
+
+			// recreate last chunk
+			const lastPiece = this._rawPieces[this._rawPieces.length - 1];
+			const tmp = new BufferPiece(String.fromCharCode(this._previousChar));
+			const newLastPiece = BufferPiece.join(lastPiece, tmp);
+			this._rawPieces[this._rawPieces.length - 1] = newLastPiece;
+			if (this._previousChar === CharCode.CarriageReturn) {
+				this.cr++;
+			}
+		}
+	}
+}

src/vs/editor/common/model/editStack.ts

@@ -5,8 +5,9 @@
 'use strict';
 
 import { onUnexpectedError } from 'vs/base/common/errors';
-import { ICursorStateComputer, IEditableTextModel, IIdentifiedSingleEditOperation } from 'vs/editor/common/editorCommon';
+import { ICursorStateComputer, IIdentifiedSingleEditOperation } from 'vs/editor/common/model';
 import { Selection } from 'vs/editor/common/core/selection';
+import { TextModel } from 'vs/editor/common/model/textModel';
 
 interface IEditOperation {
 	operations: IIdentifiedSingleEditOperation[];
@@ -29,12 +30,12 @@ export interface IUndoRedoResult {
 
 export class EditStack {
 
-	private model: IEditableTextModel;
+	private model: TextModel;
 	private currentOpenStackElement: IStackElement;
 	private past: IStackElement[];
 	private future: IStackElement[];
 
-	constructor(model: IEditableTextModel) {
+	constructor(model: TextModel) {
 		this.model = model;
 		this.currentOpenStackElement = null;
 		this.past = [];
@@ -68,7 +69,7 @@ export class EditStack {
 			};
 		}
 
-		var inverseEditOperation: IEditOperation = {
+		const inverseEditOperation: IEditOperation = {
 			operations: this.model.applyEdits(editOperations)
 		};
 
@@ -89,11 +90,11 @@ export class EditStack {
 		this.pushStackElement();
 
 		if (this.past.length > 0) {
-			var pastStackElement = this.past.pop();
+			const pastStackElement = this.past.pop();
 
 			try {
 				// Apply all operations in reverse order
-				for (var i = pastStackElement.editOperations.length - 1; i >= 0; i--) {
+				for (let i = pastStackElement.editOperations.length - 1; i >= 0; i--) {
 					pastStackElement.editOperations[i] = {
 						operations: this.model.applyEdits(pastStackElement.editOperations[i].operations)
 					};
@@ -121,11 +122,11 @@ export class EditStack {
 				throw new Error('How is this possible?');
 			}
 
-			var futureStackElement = this.future.pop();
+			const futureStackElement = this.future.pop();
 
 			try {
 				// Apply all operations
-				for (var i = 0; i < futureStackElement.editOperations.length; i++) {
+				for (let i = 0; i < futureStackElement.editOperations.length; i++) {
 					futureStackElement.editOperations[i] = {
 						operations: this.model.applyEdits(futureStackElement.editOperations[i].operations)
 					};

src/vs/editor/common/model/editableTextModel.ts

@@ -1,745 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import { Range, IRange } from 'vs/editor/common/core/range';
-import * as editorCommon from 'vs/editor/common/editorCommon';
-import { EditStack } from 'vs/editor/common/model/editStack';
-import { ILineEdit, IModelLine } from 'vs/editor/common/model/modelLine';
-import { TextModelWithDecorations, ModelDecorationOptions } from 'vs/editor/common/model/textModelWithDecorations';
-import * as strings from 'vs/base/common/strings';
-import * as arrays from 'vs/base/common/arrays';
-import { Selection } from 'vs/editor/common/core/selection';
-import { LanguageIdentifier } from 'vs/editor/common/modes';
-import { ITextSource, IRawTextSource } from 'vs/editor/common/model/textSource';
-import { ModelRawContentChangedEvent, ModelRawChange, IModelContentChange, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents';
-
-export interface IValidatedEditOperation {
-	sortIndex: number;
-	identifier: editorCommon.ISingleEditOperationIdentifier;
-	range: Range;
-	rangeOffset: number;
-	rangeLength: number;
-	lines: string[];
-	forceMoveMarkers: boolean;
-	isAutoWhitespaceEdit: boolean;
-}
-
-interface IIdentifiedLineEdit extends ILineEdit {
-	lineNumber: number;
-}
-
-export class EditableTextModel extends TextModelWithDecorations implements editorCommon.IEditableTextModel {
-
-	private _commandManager: EditStack;
-
-	// for extra details about change events:
-	private _isUndoing: boolean;
-	private _isRedoing: boolean;
-
-	// editable range
-	private _hasEditableRange: boolean;
-	private _editableRangeId: string;
-
-	private _trimAutoWhitespaceLines: number[];
-
-	constructor(rawTextSource: IRawTextSource, creationOptions: editorCommon.ITextModelCreationOptions, languageIdentifier: LanguageIdentifier) {
-		super(rawTextSource, creationOptions, languageIdentifier);
-
-		this._commandManager = new EditStack(this);
-
-		this._isUndoing = false;
-		this._isRedoing = false;
-
-		this._hasEditableRange = false;
-		this._editableRangeId = null;
-		this._trimAutoWhitespaceLines = null;
-	}
-
-	public dispose(): void {
-		this._commandManager = null;
-		super.dispose();
-	}
-
-	protected _resetValue(newValue: ITextSource): void {
-		super._resetValue(newValue);
-
-		// Destroy my edit history and settings
-		this._commandManager = new EditStack(this);
-		this._hasEditableRange = false;
-		this._editableRangeId = null;
-		this._trimAutoWhitespaceLines = null;
-	}
-
-	public pushStackElement(): void {
-		this._commandManager.pushStackElement();
-	}
-
-	public pushEditOperations(beforeCursorState: Selection[], editOperations: editorCommon.IIdentifiedSingleEditOperation[], cursorStateComputer: editorCommon.ICursorStateComputer): Selection[] {
-		try {
-			this._eventEmitter.beginDeferredEmit();
-			this._onDidChangeDecorations.beginDeferredEmit();
-			return this._pushEditOperations(beforeCursorState, editOperations, cursorStateComputer);
-		} finally {
-			this._onDidChangeDecorations.endDeferredEmit();
-			this._eventEmitter.endDeferredEmit();
-		}
-	}
-
-	private _pushEditOperations(beforeCursorState: Selection[], editOperations: editorCommon.IIdentifiedSingleEditOperation[], cursorStateComputer: editorCommon.ICursorStateComputer): Selection[] {
-		if (this._options.trimAutoWhitespace && this._trimAutoWhitespaceLines) {
-			// Go through each saved line number and insert a trim whitespace edit
-			// if it is safe to do so (no conflicts with other edits).
-
-			let incomingEdits = editOperations.map((op) => {
-				return {
-					range: this.validateRange(op.range),
-					text: op.text
-				};
-			});
-
-			// Sometimes, auto-formatters change ranges automatically which can cause undesired auto whitespace trimming near the cursor
-			// We'll use the following heuristic: if the edits occur near the cursor, then it's ok to trim auto whitespace
-			let editsAreNearCursors = true;
-			for (let i = 0, len = beforeCursorState.length; i < len; i++) {
-				let sel = beforeCursorState[i];
-				let foundEditNearSel = false;
-				for (let j = 0, lenJ = incomingEdits.length; j < lenJ; j++) {
-					let editRange = incomingEdits[j].range;
-					let selIsAbove = editRange.startLineNumber > sel.endLineNumber;
-					let selIsBelow = sel.startLineNumber > editRange.endLineNumber;
-					if (!selIsAbove && !selIsBelow) {
-						foundEditNearSel = true;
-						break;
-					}
-				}
-				if (!foundEditNearSel) {
-					editsAreNearCursors = false;
-					break;
-				}
-			}
-
-			if (editsAreNearCursors) {
-				for (let i = 0, len = this._trimAutoWhitespaceLines.length; i < len; i++) {
-					let trimLineNumber = this._trimAutoWhitespaceLines[i];
-					let maxLineColumn = this.getLineMaxColumn(trimLineNumber);
-
-					let allowTrimLine = true;
-					for (let j = 0, lenJ = incomingEdits.length; j < lenJ; j++) {
-						let editRange = incomingEdits[j].range;
-						let editText = incomingEdits[j].text;
-
-						if (trimLineNumber < editRange.startLineNumber || trimLineNumber > editRange.endLineNumber) {
-							// `trimLine` is completely outside this edit
-							continue;
-						}
-
-						// At this point:
-						//   editRange.startLineNumber <= trimLine <= editRange.endLineNumber
-
-						if (
-							trimLineNumber === editRange.startLineNumber && editRange.startColumn === maxLineColumn
-							&& editRange.isEmpty() && editText && editText.length > 0 && editText.charAt(0) === '\n'
-						) {
-							// This edit inserts a new line (and maybe other text) after `trimLine`
-							continue;
-						}
-
-						// Looks like we can't trim this line as it would interfere with an incoming edit
-						allowTrimLine = false;
-						break;
-					}
-
-					if (allowTrimLine) {
-						editOperations.push({
-							identifier: null,
-							range: new Range(trimLineNumber, 1, trimLineNumber, maxLineColumn),
-							text: null,
-							forceMoveMarkers: false,
-							isAutoWhitespaceEdit: false
-						});
-					}
-
-				}
-			}
-
-			this._trimAutoWhitespaceLines = null;
-		}
-		return this._commandManager.pushEditOperation(beforeCursorState, editOperations, cursorStateComputer);
-	}
-
-	/**
-	 * Transform operations such that they represent the same logic edit,
-	 * but that they also do not cause OOM crashes.
-	 */
-	private _reduceOperations(operations: IValidatedEditOperation[]): IValidatedEditOperation[] {
-		if (operations.length < 1000) {
-			// We know from empirical testing that a thousand edits work fine regardless of their shape.
-			return operations;
-		}
-
-		// At one point, due to how events are emitted and how each operation is handled,
-		// some operations can trigger a high ammount of temporary string allocations,
-		// that will immediately get edited again.
-		// e.g. a formatter inserting ridiculous ammounts of \n on a model with a single line
-		// Therefore, the strategy is to collapse all the operations into a huge single edit operation
-		return [this._toSingleEditOperation(operations)];
-	}
-
-	_toSingleEditOperation(operations: IValidatedEditOperation[]): IValidatedEditOperation {
-		let forceMoveMarkers = false,
-			firstEditRange = operations[0].range,
-			lastEditRange = operations[operations.length - 1].range,
-			entireEditRange = new Range(firstEditRange.startLineNumber, firstEditRange.startColumn, lastEditRange.endLineNumber, lastEditRange.endColumn),
-			lastEndLineNumber = firstEditRange.startLineNumber,
-			lastEndColumn = firstEditRange.startColumn,
-			result: string[] = [];
-
-		for (let i = 0, len = operations.length; i < len; i++) {
-			let operation = operations[i],
-				range = operation.range;
-
-			forceMoveMarkers = forceMoveMarkers || operation.forceMoveMarkers;
-
-			// (1) -- Push old text
-			for (let lineNumber = lastEndLineNumber; lineNumber < range.startLineNumber; lineNumber++) {
-				if (lineNumber === lastEndLineNumber) {
-					result.push(this._lines[lineNumber - 1].text.substring(lastEndColumn - 1));
-				} else {
-					result.push('\n');
-					result.push(this._lines[lineNumber - 1].text);
-				}
-			}
-
-			if (range.startLineNumber === lastEndLineNumber) {
-				result.push(this._lines[range.startLineNumber - 1].text.substring(lastEndColumn - 1, range.startColumn - 1));
-			} else {
-				result.push('\n');
-				result.push(this._lines[range.startLineNumber - 1].text.substring(0, range.startColumn - 1));
-			}
-
-			// (2) -- Push new text
-			if (operation.lines) {
-				for (let j = 0, lenJ = operation.lines.length; j < lenJ; j++) {
-					if (j !== 0) {
-						result.push('\n');
-					}
-					result.push(operation.lines[j]);
-				}
-			}
-
-			lastEndLineNumber = operation.range.endLineNumber;
-			lastEndColumn = operation.range.endColumn;
-		}
-
-		return {
-			sortIndex: 0,
-			identifier: operations[0].identifier,
-			range: entireEditRange,
-			rangeOffset: this.getOffsetAt(entireEditRange.getStartPosition()),
-			rangeLength: this.getValueLengthInRange(entireEditRange),
-			lines: result.join('').split('\n'),
-			forceMoveMarkers: forceMoveMarkers,
-			isAutoWhitespaceEdit: false
-		};
-	}
-
-	private static _sortOpsAscending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
-		let r = Range.compareRangesUsingEnds(a.range, b.range);
-		if (r === 0) {
-			return a.sortIndex - b.sortIndex;
-		}
-		return r;
-	}
-
-	private static _sortOpsDescending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
-		let r = Range.compareRangesUsingEnds(a.range, b.range);
-		if (r === 0) {
-			return b.sortIndex - a.sortIndex;
-		}
-		return -r;
-	}
-
-	public applyEdits(rawOperations: editorCommon.IIdentifiedSingleEditOperation[]): editorCommon.IIdentifiedSingleEditOperation[] {
-		try {
-			this._eventEmitter.beginDeferredEmit();
-			this._onDidChangeDecorations.beginDeferredEmit();
-			return this._applyEdits(rawOperations);
-		} finally {
-			this._onDidChangeDecorations.endDeferredEmit();
-			this._eventEmitter.endDeferredEmit();
-		}
-	}
-
-	private _applyEdits(rawOperations: editorCommon.IIdentifiedSingleEditOperation[]): editorCommon.IIdentifiedSingleEditOperation[] {
-		if (rawOperations.length === 0) {
-			return [];
-		}
-
-		let mightContainRTL = this._mightContainRTL;
-		let mightContainNonBasicASCII = this._mightContainNonBasicASCII;
-		let canReduceOperations = true;
-
-		let operations: IValidatedEditOperation[] = [];
-		for (let i = 0; i < rawOperations.length; i++) {
-			let op = rawOperations[i];
-			if (canReduceOperations && op._isTracked) {
-				canReduceOperations = false;
-			}
-			let validatedRange = this.validateRange(op.range);
-			if (!mightContainRTL && op.text) {
-				// check if the new inserted text contains RTL
-				mightContainRTL = strings.containsRTL(op.text);
-			}
-			if (!mightContainNonBasicASCII && op.text) {
-				mightContainNonBasicASCII = !strings.isBasicASCII(op.text);
-			}
-			operations[i] = {
-				sortIndex: i,
-				identifier: op.identifier,
-				range: validatedRange,
-				rangeOffset: this.getOffsetAt(validatedRange.getStartPosition()),
-				rangeLength: this.getValueLengthInRange(validatedRange),
-				lines: op.text ? op.text.split(/\r\n|\r|\n/) : null,
-				forceMoveMarkers: op.forceMoveMarkers,
-				isAutoWhitespaceEdit: op.isAutoWhitespaceEdit || false
-			};
-		}
-
-		// Sort operations ascending
-		operations.sort(EditableTextModel._sortOpsAscending);
-
-		for (let i = 0, count = operations.length - 1; i < count; i++) {
-			let rangeEnd = operations[i].range.getEndPosition();
-			let nextRangeStart = operations[i + 1].range.getStartPosition();
-
-			if (nextRangeStart.isBefore(rangeEnd)) {
-				// overlapping ranges
-				throw new Error('Overlapping ranges are not allowed!');
-			}
-		}
-
-		if (canReduceOperations) {
-			operations = this._reduceOperations(operations);
-		}
-
-		let editableRange = this.getEditableRange();
-		let editableRangeStart = editableRange.getStartPosition();
-		let editableRangeEnd = editableRange.getEndPosition();
-		for (let i = 0; i < operations.length; i++) {
-			let operationRange = operations[i].range;
-			if (!editableRangeStart.isBeforeOrEqual(operationRange.getStartPosition()) || !operationRange.getEndPosition().isBeforeOrEqual(editableRangeEnd)) {
-				throw new Error('Editing outside of editable range not allowed!');
-			}
-		}
-
-		// Delta encode operations
-		let reverseRanges = EditableTextModel._getInverseEditRanges(operations);
-		let reverseOperations: editorCommon.IIdentifiedSingleEditOperation[] = [];
-
-		let newTrimAutoWhitespaceCandidates: { lineNumber: number, oldContent: string }[] = [];
-
-		for (let i = 0; i < operations.length; i++) {
-			let op = operations[i];
-			let reverseRange = reverseRanges[i];
-
-			reverseOperations[i] = {
-				identifier: op.identifier,
-				range: reverseRange,
-				text: this.getValueInRange(op.range),
-				forceMoveMarkers: op.forceMoveMarkers
-			};
-
-			if (this._options.trimAutoWhitespace && op.isAutoWhitespaceEdit && op.range.isEmpty()) {
-				// Record already the future line numbers that might be auto whitespace removal candidates on next edit
-				for (let lineNumber = reverseRange.startLineNumber; lineNumber <= reverseRange.endLineNumber; lineNumber++) {
-					let currentLineContent = '';
-					if (lineNumber === reverseRange.startLineNumber) {
-						currentLineContent = this.getLineContent(op.range.startLineNumber);
-						if (strings.firstNonWhitespaceIndex(currentLineContent) !== -1) {
-							continue;
-						}
-					}
-					newTrimAutoWhitespaceCandidates.push({ lineNumber: lineNumber, oldContent: currentLineContent });
-				}
-			}
-		}
-
-		this._mightContainRTL = mightContainRTL;
-		this._mightContainNonBasicASCII = mightContainNonBasicASCII;
-		this._doApplyEdits(operations);
-
-		this._trimAutoWhitespaceLines = null;
-		if (this._options.trimAutoWhitespace && newTrimAutoWhitespaceCandidates.length > 0) {
-			// sort line numbers auto whitespace removal candidates for next edit descending
-			newTrimAutoWhitespaceCandidates.sort((a, b) => b.lineNumber - a.lineNumber);
-
-			this._trimAutoWhitespaceLines = [];
-			for (let i = 0, len = newTrimAutoWhitespaceCandidates.length; i < len; i++) {
-				let lineNumber = newTrimAutoWhitespaceCandidates[i].lineNumber;
-				if (i > 0 && newTrimAutoWhitespaceCandidates[i - 1].lineNumber === lineNumber) {
-					// Do not have the same line number twice
-					continue;
-				}
-
-				let prevContent = newTrimAutoWhitespaceCandidates[i].oldContent;
-				let lineContent = this.getLineContent(lineNumber);
-
-				if (lineContent.length === 0 || lineContent === prevContent || strings.firstNonWhitespaceIndex(lineContent) !== -1) {
-					continue;
-				}
-
-				this._trimAutoWhitespaceLines.push(lineNumber);
-			}
-		}
-
-		return reverseOperations;
-	}
-
-	/**
-	 * Assumes `operations` are validated and sorted ascending
-	 */
-	public static _getInverseEditRanges(operations: IValidatedEditOperation[]): Range[] {
-		let result: Range[] = [];
-
-		let prevOpEndLineNumber: number;
-		let prevOpEndColumn: number;
-		let prevOp: IValidatedEditOperation = null;
-		for (let i = 0, len = operations.length; i < len; i++) {
-			let op = operations[i];
-
-			let startLineNumber: number;
-			let startColumn: number;
-
-			if (prevOp) {
-				if (prevOp.range.endLineNumber === op.range.startLineNumber) {
-					startLineNumber = prevOpEndLineNumber;
-					startColumn = prevOpEndColumn + (op.range.startColumn - prevOp.range.endColumn);
-				} else {
-					startLineNumber = prevOpEndLineNumber + (op.range.startLineNumber - prevOp.range.endLineNumber);
-					startColumn = op.range.startColumn;
-				}
-			} else {
-				startLineNumber = op.range.startLineNumber;
-				startColumn = op.range.startColumn;
-			}
-
-			let resultRange: Range;
-
-			if (op.lines && op.lines.length > 0) {
-				// the operation inserts something
-				let lineCount = op.lines.length;
-				let firstLine = op.lines[0];
-				let lastLine = op.lines[lineCount - 1];
-
-				if (lineCount === 1) {
-					// single line insert
-					resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn + firstLine.length);
-				} else {
-					// multi line insert
-					resultRange = new Range(startLineNumber, startColumn, startLineNumber + lineCount - 1, lastLine.length + 1);
-				}
-			} else {
-				// There is nothing to insert
-				resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn);
-			}
-
-			prevOpEndLineNumber = resultRange.endLineNumber;
-			prevOpEndColumn = resultRange.endColumn;
-
-			result.push(resultRange);
-			prevOp = op;
-		}
-
-		return result;
-	}
-
-	private _doApplyEdits(operations: IValidatedEditOperation[]): void {
-
-		// Sort operations descending
-		operations.sort(EditableTextModel._sortOpsDescending);
-
-		let rawContentChanges: ModelRawChange[] = [];
-		let contentChanges: IModelContentChange[] = [];
-		let lineEditsQueue: IIdentifiedLineEdit[] = [];
-
-		const queueLineEdit = (lineEdit: IIdentifiedLineEdit) => {
-			if (lineEdit.startColumn === lineEdit.endColumn && lineEdit.text.length === 0) {
-				// empty edit => ignore it
-				return;
-			}
-			lineEditsQueue.push(lineEdit);
-		};
-
-		const flushLineEdits = () => {
-			if (lineEditsQueue.length === 0) {
-				return;
-			}
-
-			lineEditsQueue.reverse();
-
-			// `lineEditsQueue` now contains edits from smaller (line number,column) to larger (line number,column)
-			let currentLineNumber = lineEditsQueue[0].lineNumber;
-			let currentLineNumberStart = 0;
-
-			for (let i = 1, len = lineEditsQueue.length; i < len; i++) {
-				const lineNumber = lineEditsQueue[i].lineNumber;
-
-				if (lineNumber === currentLineNumber) {
-					continue;
-				}
-
-				this._invalidateLine(currentLineNumber - 1);
-				this._lines[currentLineNumber - 1].applyEdits(lineEditsQueue.slice(currentLineNumberStart, i));
-				this._lineStarts.changeValue(currentLineNumber - 1, this._lines[currentLineNumber - 1].text.length + this._EOL.length);
-				rawContentChanges.push(
-					new ModelRawLineChanged(currentLineNumber, this._lines[currentLineNumber - 1].text)
-				);
-
-				currentLineNumber = lineNumber;
-				currentLineNumberStart = i;
-			}
-
-			this._invalidateLine(currentLineNumber - 1);
-			this._lines[currentLineNumber - 1].applyEdits(lineEditsQueue.slice(currentLineNumberStart, lineEditsQueue.length));
-			this._lineStarts.changeValue(currentLineNumber - 1, this._lines[currentLineNumber - 1].text.length + this._EOL.length);
-			rawContentChanges.push(
-				new ModelRawLineChanged(currentLineNumber, this._lines[currentLineNumber - 1].text)
-			);
-
-			lineEditsQueue = [];
-		};
-
-		for (let i = 0, len = operations.length; i < len; i++) {
-			const op = operations[i];
-
-			// console.log();
-			// console.log('-------------------');
-			// console.log('OPERATION #' + (i));
-			// console.log('op: ', op);
-			// console.log('<<<\n' + this._lines.map(l => l.text).join('\n') + '\n>>>');
-
-			const startLineNumber = op.range.startLineNumber;
-			const startColumn = op.range.startColumn;
-			const endLineNumber = op.range.endLineNumber;
-			const endColumn = op.range.endColumn;
-
-			if (startLineNumber === endLineNumber && startColumn === endColumn && (!op.lines || op.lines.length === 0)) {
-				// no-op
-				continue;
-			}
-
-			const deletingLinesCnt = endLineNumber - startLineNumber;
-			const insertingLinesCnt = (op.lines ? op.lines.length - 1 : 0);
-			const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt);
-
-			// Iterating descending to overlap with previous op
-			// in case there are common lines being edited in both
-			for (let j = editingLinesCnt; j >= 0; j--) {
-				const editLineNumber = startLineNumber + j;
-
-				queueLineEdit({
-					lineNumber: editLineNumber,
-					startColumn: (editLineNumber === startLineNumber ? startColumn : 1),
-					endColumn: (editLineNumber === endLineNumber ? endColumn : this.getLineMaxColumn(editLineNumber)),
-					text: (op.lines ? op.lines[j] : '')
-				});
-			}
-
-			if (editingLinesCnt < deletingLinesCnt) {
-				// Must delete some lines
-
-				// Flush any pending line edits
-				flushLineEdits();
-
-				const spliceStartLineNumber = startLineNumber + editingLinesCnt;
-
-				const endLineRemains = this._lines[endLineNumber - 1].split(endColumn);
-				this._invalidateLine(spliceStartLineNumber - 1);
-
-				const spliceCnt = endLineNumber - spliceStartLineNumber;
-
-				this._lines.splice(spliceStartLineNumber, spliceCnt);
-				this._lineStarts.removeValues(spliceStartLineNumber, spliceCnt);
-
-				// Reconstruct first line
-				this._lines[spliceStartLineNumber - 1].append(endLineRemains);
-				this._lineStarts.changeValue(spliceStartLineNumber - 1, this._lines[spliceStartLineNumber - 1].text.length + this._EOL.length);
-
-				rawContentChanges.push(
-					new ModelRawLineChanged(spliceStartLineNumber, this._lines[spliceStartLineNumber - 1].text)
-				);
-
-				rawContentChanges.push(
-					new ModelRawLinesDeleted(spliceStartLineNumber + 1, spliceStartLineNumber + spliceCnt)
-				);
-			}
-
-			if (editingLinesCnt < insertingLinesCnt) {
-				// Must insert some lines
-
-				// Flush any pending line edits
-				flushLineEdits();
-
-				const spliceLineNumber = startLineNumber + editingLinesCnt;
-				let spliceColumn = (spliceLineNumber === startLineNumber ? startColumn : 1);
-				if (op.lines) {
-					spliceColumn += op.lines[editingLinesCnt].length;
-				}
-
-				// Split last line
-				let leftoverLine = this._lines[spliceLineNumber - 1].split(spliceColumn);
-				this._lineStarts.changeValue(spliceLineNumber - 1, this._lines[spliceLineNumber - 1].text.length + this._EOL.length);
-				rawContentChanges.push(
-					new ModelRawLineChanged(spliceLineNumber, this._lines[spliceLineNumber - 1].text)
-				);
-				this._invalidateLine(spliceLineNumber - 1);
-
-				// Lines in the middle
-				let newLines: IModelLine[] = [];
-				let newLinesContent: string[] = [];
-				let newLinesLengths = new Uint32Array(insertingLinesCnt - editingLinesCnt);
-				for (let j = editingLinesCnt + 1; j <= insertingLinesCnt; j++) {
-					newLines.push(this._createModelLine(op.lines[j]));
-					newLinesContent.push(op.lines[j]);
-					newLinesLengths[j - editingLinesCnt - 1] = op.lines[j].length + this._EOL.length;
-				}
-				this._lines = arrays.arrayInsert(this._lines, startLineNumber + editingLinesCnt, newLines);
-				newLinesContent[newLinesContent.length - 1] += leftoverLine.text;
-				this._lineStarts.insertValues(startLineNumber + editingLinesCnt, newLinesLengths);
-
-				// Last line
-				this._lines[startLineNumber + insertingLinesCnt - 1].append(leftoverLine);
-				this._lineStarts.changeValue(startLineNumber + insertingLinesCnt - 1, this._lines[startLineNumber + insertingLinesCnt - 1].text.length + this._EOL.length);
-				rawContentChanges.push(
-					new ModelRawLinesInserted(spliceLineNumber + 1, startLineNumber + insertingLinesCnt, newLinesContent.join('\n'))
-				);
-			}
-
-			const text = (op.lines ? op.lines.join(this.getEOL()) : '');
-			contentChanges.push({
-				range: new Range(startLineNumber, startColumn, endLineNumber, endColumn),
-				rangeLength: op.rangeLength,
-				text: text
-			});
-
-			this._adjustDecorationsForEdit(op.rangeOffset, op.rangeLength, text.length, op.forceMoveMarkers);
-
-			// console.log('AFTER:');
-			// console.log('<<<\n' + this._lines.map(l => l.text).join('\n') + '\n>>>');
-		}
-
-		flushLineEdits();
-
-		if (rawContentChanges.length !== 0 || contentChanges.length !== 0) {
-			this._increaseVersionId();
-
-			this._emitContentChangedEvent(
-				new ModelRawContentChangedEvent(
-					rawContentChanges,
-					this.getVersionId(),
-					this._isUndoing,
-					this._isRedoing
-				),
-				{
-					changes: contentChanges,
-					eol: this._EOL,
-					versionId: this.getVersionId(),
-					isUndoing: this._isUndoing,
-					isRedoing: this._isRedoing,
-					isFlush: false
-				}
-			);
-		}
-	}
-
-	private _undo(): Selection[] {
-		this._isUndoing = true;
-		let r = this._commandManager.undo();
-		this._isUndoing = false;
-
-		if (!r) {
-			return null;
-		}
-
-		this._overwriteAlternativeVersionId(r.recordedVersionId);
-
-		return r.selections;
-	}
-
-	public undo(): Selection[] {
-		try {
-			this._eventEmitter.beginDeferredEmit();
-			this._onDidChangeDecorations.beginDeferredEmit();
-			return this._undo();
-		} finally {
-			this._onDidChangeDecorations.endDeferredEmit();
-			this._eventEmitter.endDeferredEmit();
-		}
-	}
-
-	private _redo(): Selection[] {
-		this._isRedoing = true;
-		let r = this._commandManager.redo();
-		this._isRedoing = false;
-
-		if (!r) {
-			return null;
-		}
-
-		this._overwriteAlternativeVersionId(r.recordedVersionId);
-
-		return r.selections;
-	}
-
-	public redo(): Selection[] {
-		try {
-			this._eventEmitter.beginDeferredEmit();
-			this._onDidChangeDecorations.beginDeferredEmit();
-			return this._redo();
-		} finally {
-			this._onDidChangeDecorations.endDeferredEmit();
-			this._eventEmitter.endDeferredEmit();
-		}
-	}
-
-	public setEditableRange(range: IRange): void {
-		this._commandManager.clear();
-
-		if (!this._hasEditableRange && !range) {
-			// Nothing to do
-			return;
-		}
-
-		this.changeDecorations((changeAccessor) => {
-			if (this._hasEditableRange) {
-				changeAccessor.removeDecoration(this._editableRangeId);
-				this._editableRangeId = null;
-				this._hasEditableRange = false;
-			}
-
-			if (range) {
-				this._hasEditableRange = true;
-				this._editableRangeId = changeAccessor.addDecoration(range, EditableTextModel._DECORATION_OPTION);
-			}
-		});
-	}
-
-	private static readonly _DECORATION_OPTION = ModelDecorationOptions.register({
-		stickiness: editorCommon.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges
-	});
-
-	public hasEditableRange(): boolean {
-		return this._hasEditableRange;
-	}
-
-	public getEditableRange(): Range {
-		if (this._hasEditableRange) {
-			return this.getDecorationRange(this._editableRangeId);
-		} else {
-			return this.getFullModelRange();
-		}
-	}
-}

src/vs/editor/common/model/indentationGuesser.ts

@@ -5,6 +5,7 @@
 'use strict';
 
 import { CharCode } from 'vs/base/common/charCode';
+import { ITextBuffer } from 'vs/editor/common/model';
 
 /**
  * Compute the diff in spaces between two line's indentation.
@@ -80,9 +81,9 @@ export interface IGuessedIndentation {
 	insertSpaces: boolean;
 }
 
-export function guessIndentation(lines: string[], defaultTabSize: number, defaultInsertSpaces: boolean): IGuessedIndentation {
+export function guessIndentation(source: ITextBuffer, defaultTabSize: number, defaultInsertSpaces: boolean): IGuessedIndentation {
 	// Look at most at the first 10k lines
-	const linesLen = Math.min(lines.length, 10000);
+	const linesCount = Math.min(source.getLineCount(), 10000);
 
 	let linesIndentedWithTabsCount = 0;				// number of lines that contain at least one tab in indentation
 	let linesIndentedWithSpacesCount = 0;			// number of lines that contain only spaces in indentation
@@ -95,15 +96,24 @@ export function guessIndentation(lines: string[], defaultTabSize: number, defaul
 
 	let spacesDiffCount = [0, 0, 0, 0, 0, 0, 0, 0, 0];		// `tabSize` scores
 
-	for (let i = 0; i < linesLen; i++) {
-		let currentLineText = lines[i];
+	for (let lineNumber = 1; lineNumber <= linesCount; lineNumber++) {
+		let currentLineLength = source.getLineLength(lineNumber);
+		let currentLineText = source.getLineContent(lineNumber);
+		let charCodeAt: (offset: number) => number;
+		if (currentLineLength > 65536) {
+			// if the text buffer is chunk based, so long lines are cons-string, v8 will flattern the string when we check charCode.
+			// checking charCode on chunks directly is cheaper.
+			charCodeAt = (offset: number) => source.getLineCharCode(lineNumber, offset);
+		} else {
+			charCodeAt = (offset: number) => currentLineText.charCodeAt(offset);
+		}
 
 		let currentLineHasContent = false;			// does `currentLineText` contain non-whitespace chars
 		let currentLineIndentation = 0;				// index at which `currentLineText` contains the first non-whitespace char
 		let currentLineSpacesCount = 0;				// count of spaces found in `currentLineText` indentation
 		let currentLineTabsCount = 0;				// count of tabs found in `currentLineText` indentation
-		for (let j = 0, lenJ = currentLineText.length; j < lenJ; j++) {
-			let charCode = currentLineText.charCodeAt(j);
+		for (let j = 0, lenJ = currentLineLength; j < lenJ; j++) {
+			let charCode = charCodeAt(j);
 
 			if (charCode === CharCode.Tab) {
 				currentLineTabsCount++;
@@ -149,7 +159,7 @@ export function guessIndentation(lines: string[], defaultTabSize: number, defaul
 	}
 
 	let tabSize = defaultTabSize;
-	let tabSizeScore = (insertSpaces ? 0 : 0.1 * linesLen);
+	let tabSizeScore = (insertSpaces ? 0 : 0.1 * linesCount);
 
 	// console.log("score threshold: " + tabSizeScore);
 

src/vs/editor/common/model/intervalTree.ts

@@ -4,9 +4,9 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import { ModelDecorationOptions } from 'vs/editor/common/model/textModelWithDecorations';
+import { ModelDecorationOptions } from 'vs/editor/common/model/textModel';
 import { Range } from 'vs/editor/common/core/range';
-import { IModelDecoration } from 'vs/editor/common/editorCommon';
+import { IModelDecoration } from 'vs/editor/common/model';
 
 //
 // The red-black tree is based on the "Introduction to Algorithms" by Cormen, Leiserson and Rivest.
@@ -25,7 +25,7 @@ export const ClassName = {
  * Describes the behavior of decorations when typing/editing near their edges.
  * Note: Please do not edit the values, as they very carefully match `DecorationRangeBehavior`
  */
-export const enum TrackedRangeStickiness {
+const enum TrackedRangeStickiness {
 	AlwaysGrowsWhenTypingAtEdges = 0,
 	NeverGrowsWhenTypingAtEdges = 1,
 	GrowsOnlyWhenTypingBefore = 2,

src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts

@@ -0,0 +1,660 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { Range } from 'vs/editor/common/core/range';
+import { Position } from 'vs/editor/common/core/position';
+import * as strings from 'vs/base/common/strings';
+import * as arrays from 'vs/base/common/arrays';
+import { PrefixSumComputer } from 'vs/editor/common/viewModel/prefixSumComputer';
+import { ISingleEditOperationIdentifier, IIdentifiedSingleEditOperation, EndOfLinePreference, ITextBuffer, ApplyEditsResult, IInternalModelContentChange } from 'vs/editor/common/model';
+import { ITextSnapshot } from 'vs/platform/files/common/files';
+
+export interface IValidatedEditOperation {
+	sortIndex: number;
+	identifier: ISingleEditOperationIdentifier;
+	range: Range;
+	rangeOffset: number;
+	rangeLength: number;
+	lines: string[];
+	forceMoveMarkers: boolean;
+	isAutoWhitespaceEdit: boolean;
+}
+
+/**
+ * A processed string with its EOL resolved ready to be turned into an editor model.
+ */
+export interface ITextSource {
+	/**
+	 * The text split into lines.
+	 */
+	readonly lines: string[];
+	/**
+	 * The BOM (leading character sequence of the file).
+	 */
+	readonly BOM: string;
+	/**
+	 * The end of line sequence.
+	 */
+	readonly EOL: string;
+	/**
+	 * The text contains Unicode characters classified as "R" or "AL".
+	 */
+	readonly containsRTL: boolean;
+	/**
+	 * The text contains only characters inside the ASCII range 32-126 or \t \r \n
+	 */
+	readonly isBasicASCII: boolean;
+}
+
+class LinesTextBufferSnapshot implements ITextSnapshot {
+
+	private readonly _lines: string[];
+	private readonly _linesLength: number;
+	private readonly _eol: string;
+	private readonly _bom: string;
+	private _lineIndex: number;
+
+	constructor(lines: string[], eol: string, bom: string) {
+		this._lines = lines;
+		this._linesLength = this._lines.length;
+		this._eol = eol;
+		this._bom = bom;
+		this._lineIndex = 0;
+	}
+
+	public read(): string {
+		if (this._lineIndex >= this._linesLength) {
+			return null;
+		}
+
+		let result: string = null;
+
+		if (this._lineIndex === 0) {
+			result = this._bom + this._lines[this._lineIndex];
+		} else {
+			result = this._lines[this._lineIndex];
+		}
+
+		this._lineIndex++;
+
+		if (this._lineIndex < this._linesLength) {
+			result += this._eol;
+		}
+
+		return result;
+	}
+}
+
+export class LinesTextBuffer implements ITextBuffer {
+
+	private _lines: string[];
+	private _BOM: string;
+	private _EOL: string;
+	private _mightContainRTL: boolean;
+	private _mightContainNonBasicASCII: boolean;
+	private _lineStarts: PrefixSumComputer;
+
+	constructor(textSource: ITextSource) {
+		this._lines = textSource.lines.slice(0);
+		this._BOM = textSource.BOM;
+		this._EOL = textSource.EOL;
+		this._mightContainRTL = textSource.containsRTL;
+		this._mightContainNonBasicASCII = !textSource.isBasicASCII;
+		this._constructLineStarts();
+	}
+
+	private _constructLineStarts(): void {
+		const eolLength = this._EOL.length;
+		const linesLength = this._lines.length;
+		const lineStartValues = new Uint32Array(linesLength);
+		for (let i = 0; i < linesLength; i++) {
+			lineStartValues[i] = this._lines[i].length + eolLength;
+		}
+		this._lineStarts = new PrefixSumComputer(lineStartValues);
+	}
+
+	public equals(other: ITextBuffer): boolean {
+		if (!(other instanceof LinesTextBuffer)) {
+			return false;
+		}
+		if (this._BOM !== other._BOM) {
+			return false;
+		}
+		if (this._EOL !== other._EOL) {
+			return false;
+		}
+		if (this._lines.length !== other._lines.length) {
+			return false;
+		}
+		for (let i = 0, len = this._lines.length; i < len; i++) {
+			if (this._lines[i] !== other._lines[i]) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	public mightContainRTL(): boolean {
+		return this._mightContainRTL;
+	}
+
+	public mightContainNonBasicASCII(): boolean {
+		return this._mightContainNonBasicASCII;
+	}
+
+	public getBOM(): string {
+		return this._BOM;
+	}
+
+	public getEOL(): string {
+		return this._EOL;
+	}
+
+	public getOffsetAt(lineNumber: number, column: number): number {
+		return this._lineStarts.getAccumulatedValue(lineNumber - 2) + column - 1;
+	}
+
+	public getPositionAt(offset: number): Position {
+		offset = Math.floor(offset);
+		offset = Math.max(0, offset);
+
+		let out = this._lineStarts.getIndexOf(offset);
+
+		let lineLength = this._lines[out.index].length;
+
+		// Ensure we return a valid position
+		return new Position(out.index + 1, Math.min(out.remainder + 1, lineLength + 1));
+	}
+
+	public getRangeAt(offset: number, length: number): Range {
+		const startResult = this._lineStarts.getIndexOf(offset);
+		const startLineLength = this._lines[startResult.index].length;
+		const startColumn = Math.min(startResult.remainder + 1, startLineLength + 1);
+
+		const endResult = this._lineStarts.getIndexOf(offset + length);
+		const endLineLength = this._lines[endResult.index].length;
+		const endColumn = Math.min(endResult.remainder + 1, endLineLength + 1);
+
+		return new Range(startResult.index + 1, startColumn, endResult.index + 1, endColumn);
+	}
+
+	private _getEndOfLine(eol: EndOfLinePreference): string {
+		switch (eol) {
+			case EndOfLinePreference.LF:
+				return '\n';
+			case EndOfLinePreference.CRLF:
+				return '\r\n';
+			case EndOfLinePreference.TextDefined:
+				return this.getEOL();
+		}
+		throw new Error('Unknown EOL preference');
+	}
+
+	public getValueInRange(range: Range, eol: EndOfLinePreference): string {
+		if (range.isEmpty()) {
+			return '';
+		}
+
+		if (range.startLineNumber === range.endLineNumber) {
+			return this._lines[range.startLineNumber - 1].substring(range.startColumn - 1, range.endColumn - 1);
+		}
+
+		const lineEnding = this._getEndOfLine(eol);
+		const startLineIndex = range.startLineNumber - 1;
+		const endLineIndex = range.endLineNumber - 1;
+		let resultLines: string[] = [];
+
+		resultLines.push(this._lines[startLineIndex].substring(range.startColumn - 1));
+		for (let i = startLineIndex + 1; i < endLineIndex; i++) {
+			resultLines.push(this._lines[i]);
+		}
+		resultLines.push(this._lines[endLineIndex].substring(0, range.endColumn - 1));
+
+		return resultLines.join(lineEnding);
+	}
+
+	public createSnapshot(preserveBOM: boolean): ITextSnapshot {
+		return new LinesTextBufferSnapshot(this._lines.slice(0), this._EOL, preserveBOM ? this._BOM : '');
+	}
+
+	public getValueLengthInRange(range: Range, eol: EndOfLinePreference): number {
+		if (range.isEmpty()) {
+			return 0;
+		}
+
+		if (range.startLineNumber === range.endLineNumber) {
+			return (range.endColumn - range.startColumn);
+		}
+
+		let startOffset = this.getOffsetAt(range.startLineNumber, range.startColumn);
+		let endOffset = this.getOffsetAt(range.endLineNumber, range.endColumn);
+		return endOffset - startOffset;
+	}
+
+	public getLineCount(): number {
+		return this._lines.length;
+	}
+
+	public getLinesContent(): string[] {
+		return this._lines.slice(0);
+	}
+
+	public getLength(): number {
+		return this._lineStarts.getTotalValue();
+	}
+
+	public getLineContent(lineNumber: number): string {
+		return this._lines[lineNumber - 1];
+	}
+
+	public getLineCharCode(lineNumber: number, index: number): number {
+		return this._lines[lineNumber - 1].charCodeAt(index);
+	}
+
+	public getLineLength(lineNumber: number): number {
+		return this._lines[lineNumber - 1].length;
+	}
+
+	public getLineFirstNonWhitespaceColumn(lineNumber: number): number {
+		const result = strings.firstNonWhitespaceIndex(this._lines[lineNumber - 1]);
+		if (result === -1) {
+			return 0;
+		}
+		return result + 1;
+	}
+
+	public getLineLastNonWhitespaceColumn(lineNumber: number): number {
+		const result = strings.lastNonWhitespaceIndex(this._lines[lineNumber - 1]);
+		if (result === -1) {
+			return 0;
+		}
+		return result + 2;
+	}
+
+	//#region Editing
+
+	public setEOL(newEOL: '\r\n' | '\n'): void {
+		this._EOL = newEOL;
+		this._constructLineStarts();
+	}
+
+	private static _sortOpsAscending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
+		let r = Range.compareRangesUsingEnds(a.range, b.range);
+		if (r === 0) {
+			return a.sortIndex - b.sortIndex;
+		}
+		return r;
+	}
+
+	private static _sortOpsDescending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
+		let r = Range.compareRangesUsingEnds(a.range, b.range);
+		if (r === 0) {
+			return b.sortIndex - a.sortIndex;
+		}
+		return -r;
+	}
+
+	public applyEdits(rawOperations: IIdentifiedSingleEditOperation[], recordTrimAutoWhitespace: boolean): ApplyEditsResult {
+		if (rawOperations.length === 0) {
+			return new ApplyEditsResult([], [], []);
+		}
+
+		let mightContainRTL = this._mightContainRTL;
+		let mightContainNonBasicASCII = this._mightContainNonBasicASCII;
+		let canReduceOperations = true;
+
+		let operations: IValidatedEditOperation[] = [];
+		for (let i = 0; i < rawOperations.length; i++) {
+			let op = rawOperations[i];
+			if (canReduceOperations && op._isTracked) {
+				canReduceOperations = false;
+			}
+			let validatedRange = op.range;
+			if (!mightContainRTL && op.text) {
+				// check if the new inserted text contains RTL
+				mightContainRTL = strings.containsRTL(op.text);
+			}
+			if (!mightContainNonBasicASCII && op.text) {
+				mightContainNonBasicASCII = !strings.isBasicASCII(op.text);
+			}
+			operations[i] = {
+				sortIndex: i,
+				identifier: op.identifier || null,
+				range: validatedRange,
+				rangeOffset: this.getOffsetAt(validatedRange.startLineNumber, validatedRange.startColumn),
+				rangeLength: this.getValueLengthInRange(validatedRange, EndOfLinePreference.TextDefined),
+				lines: op.text ? op.text.split(/\r\n|\r|\n/) : null,
+				forceMoveMarkers: op.forceMoveMarkers || false,
+				isAutoWhitespaceEdit: op.isAutoWhitespaceEdit || false
+			};
+		}
+
+		// Sort operations ascending
+		operations.sort(LinesTextBuffer._sortOpsAscending);
+
+		for (let i = 0, count = operations.length - 1; i < count; i++) {
+			let rangeEnd = operations[i].range.getEndPosition();
+			let nextRangeStart = operations[i + 1].range.getStartPosition();
+
+			if (nextRangeStart.isBefore(rangeEnd)) {
+				// overlapping ranges
+				throw new Error('Overlapping ranges are not allowed!');
+			}
+		}
+
+		if (canReduceOperations) {
+			operations = this._reduceOperations(operations);
+		}
+
+		// Delta encode operations
+		let reverseRanges = LinesTextBuffer._getInverseEditRanges(operations);
+		let newTrimAutoWhitespaceCandidates: { lineNumber: number, oldContent: string }[] = [];
+
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+			let reverseRange = reverseRanges[i];
+
+			if (recordTrimAutoWhitespace && op.isAutoWhitespaceEdit && op.range.isEmpty()) {
+				// Record already the future line numbers that might be auto whitespace removal candidates on next edit
+				for (let lineNumber = reverseRange.startLineNumber; lineNumber <= reverseRange.endLineNumber; lineNumber++) {
+					let currentLineContent = '';
+					if (lineNumber === reverseRange.startLineNumber) {
+						currentLineContent = this.getLineContent(op.range.startLineNumber);
+						if (strings.firstNonWhitespaceIndex(currentLineContent) !== -1) {
+							continue;
+						}
+					}
+					newTrimAutoWhitespaceCandidates.push({ lineNumber: lineNumber, oldContent: currentLineContent });
+				}
+			}
+		}
+
+		let reverseOperations: IIdentifiedSingleEditOperation[] = [];
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+			let reverseRange = reverseRanges[i];
+
+			reverseOperations[i] = {
+				identifier: op.identifier,
+				range: reverseRange,
+				text: this.getValueInRange(op.range, EndOfLinePreference.TextDefined),
+				forceMoveMarkers: op.forceMoveMarkers
+			};
+		}
+
+		this._mightContainRTL = mightContainRTL;
+		this._mightContainNonBasicASCII = mightContainNonBasicASCII;
+
+		const contentChanges = this._doApplyEdits(operations);
+
+		let trimAutoWhitespaceLineNumbers: number[] = null;
+		if (recordTrimAutoWhitespace && newTrimAutoWhitespaceCandidates.length > 0) {
+			// sort line numbers auto whitespace removal candidates for next edit descending
+			newTrimAutoWhitespaceCandidates.sort((a, b) => b.lineNumber - a.lineNumber);
+
+			trimAutoWhitespaceLineNumbers = [];
+			for (let i = 0, len = newTrimAutoWhitespaceCandidates.length; i < len; i++) {
+				let lineNumber = newTrimAutoWhitespaceCandidates[i].lineNumber;
+				if (i > 0 && newTrimAutoWhitespaceCandidates[i - 1].lineNumber === lineNumber) {
+					// Do not have the same line number twice
+					continue;
+				}
+
+				let prevContent = newTrimAutoWhitespaceCandidates[i].oldContent;
+				let lineContent = this.getLineContent(lineNumber);
+
+				if (lineContent.length === 0 || lineContent === prevContent || strings.firstNonWhitespaceIndex(lineContent) !== -1) {
+					continue;
+				}
+
+				trimAutoWhitespaceLineNumbers.push(lineNumber);
+			}
+		}
+
+		return new ApplyEditsResult(
+			reverseOperations,
+			contentChanges,
+			trimAutoWhitespaceLineNumbers
+		);
+	}
+
+	/**
+	 * Transform operations such that they represent the same logic edit,
+	 * but that they also do not cause OOM crashes.
+	 */
+	private _reduceOperations(operations: IValidatedEditOperation[]): IValidatedEditOperation[] {
+		if (operations.length < 1000) {
+			// We know from empirical testing that a thousand edits work fine regardless of their shape.
+			return operations;
+		}
+
+		// At one point, due to how events are emitted and how each operation is handled,
+		// some operations can trigger a high ammount of temporary string allocations,
+		// that will immediately get edited again.
+		// e.g. a formatter inserting ridiculous ammounts of \n on a model with a single line
+		// Therefore, the strategy is to collapse all the operations into a huge single edit operation
+		return [this._toSingleEditOperation(operations)];
+	}
+
+	_toSingleEditOperation(operations: IValidatedEditOperation[]): IValidatedEditOperation {
+		let forceMoveMarkers = false,
+			firstEditRange = operations[0].range,
+			lastEditRange = operations[operations.length - 1].range,
+			entireEditRange = new Range(firstEditRange.startLineNumber, firstEditRange.startColumn, lastEditRange.endLineNumber, lastEditRange.endColumn),
+			lastEndLineNumber = firstEditRange.startLineNumber,
+			lastEndColumn = firstEditRange.startColumn,
+			result: string[] = [];
+
+		for (let i = 0, len = operations.length; i < len; i++) {
+			let operation = operations[i],
+				range = operation.range;
+
+			forceMoveMarkers = forceMoveMarkers || operation.forceMoveMarkers;
+
+			// (1) -- Push old text
+			for (let lineNumber = lastEndLineNumber; lineNumber < range.startLineNumber; lineNumber++) {
+				if (lineNumber === lastEndLineNumber) {
+					result.push(this._lines[lineNumber - 1].substring(lastEndColumn - 1));
+				} else {
+					result.push('\n');
+					result.push(this._lines[lineNumber - 1]);
+				}
+			}
+
+			if (range.startLineNumber === lastEndLineNumber) {
+				result.push(this._lines[range.startLineNumber - 1].substring(lastEndColumn - 1, range.startColumn - 1));
+			} else {
+				result.push('\n');
+				result.push(this._lines[range.startLineNumber - 1].substring(0, range.startColumn - 1));
+			}
+
+			// (2) -- Push new text
+			if (operation.lines) {
+				for (let j = 0, lenJ = operation.lines.length; j < lenJ; j++) {
+					if (j !== 0) {
+						result.push('\n');
+					}
+					result.push(operation.lines[j]);
+				}
+			}
+
+			lastEndLineNumber = operation.range.endLineNumber;
+			lastEndColumn = operation.range.endColumn;
+		}
+
+		return {
+			sortIndex: 0,
+			identifier: operations[0].identifier,
+			range: entireEditRange,
+			rangeOffset: this.getOffsetAt(entireEditRange.startLineNumber, entireEditRange.startColumn),
+			rangeLength: this.getValueLengthInRange(entireEditRange, EndOfLinePreference.TextDefined),
+			lines: result.join('').split('\n'),
+			forceMoveMarkers: forceMoveMarkers,
+			isAutoWhitespaceEdit: false
+		};
+	}
+
+	private _setLineContent(lineNumber: number, content: string): void {
+		this._lines[lineNumber - 1] = content;
+		this._lineStarts.changeValue(lineNumber - 1, content.length + this._EOL.length);
+	}
+
+	private _doApplyEdits(operations: IValidatedEditOperation[]): IInternalModelContentChange[] {
+
+		// Sort operations descending
+		operations.sort(LinesTextBuffer._sortOpsDescending);
+
+		let contentChanges: IInternalModelContentChange[] = [];
+
+		for (let i = 0, len = operations.length; i < len; i++) {
+			const op = operations[i];
+
+			const startLineNumber = op.range.startLineNumber;
+			const startColumn = op.range.startColumn;
+			const endLineNumber = op.range.endLineNumber;
+			const endColumn = op.range.endColumn;
+
+			if (startLineNumber === endLineNumber && startColumn === endColumn && (!op.lines || op.lines.length === 0)) {
+				// no-op
+				continue;
+			}
+
+			const deletingLinesCnt = endLineNumber - startLineNumber;
+			const insertingLinesCnt = (op.lines ? op.lines.length - 1 : 0);
+			const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt);
+
+			for (let j = editingLinesCnt; j >= 0; j--) {
+				const editLineNumber = startLineNumber + j;
+				let editText = (op.lines ? op.lines[j] : '');
+
+				if (editLineNumber === startLineNumber || editLineNumber === endLineNumber) {
+					const editStartColumn = (editLineNumber === startLineNumber ? startColumn : 1);
+					const editEndColumn = (editLineNumber === endLineNumber ? endColumn : this.getLineLength(editLineNumber) + 1);
+
+					editText = (
+						this._lines[editLineNumber - 1].substring(0, editStartColumn - 1)
+						+ editText
+						+ this._lines[editLineNumber - 1].substring(editEndColumn - 1)
+					);
+				}
+
+				this._setLineContent(editLineNumber, editText);
+			}
+
+			if (editingLinesCnt < deletingLinesCnt) {
+				// Must delete some lines
+
+				const spliceStartLineNumber = startLineNumber + editingLinesCnt;
+				const endLineRemains = this._lines[endLineNumber - 1].substring(endColumn - 1);
+
+				// Reconstruct first line
+				this._setLineContent(spliceStartLineNumber, this._lines[spliceStartLineNumber - 1] + endLineRemains);
+
+				this._lines.splice(spliceStartLineNumber, endLineNumber - spliceStartLineNumber);
+				this._lineStarts.removeValues(spliceStartLineNumber, endLineNumber - spliceStartLineNumber);
+			}
+
+			if (editingLinesCnt < insertingLinesCnt) {
+				// Must insert some lines
+
+				const spliceLineNumber = startLineNumber + editingLinesCnt;
+				let spliceColumn = (spliceLineNumber === startLineNumber ? startColumn : 1);
+				if (op.lines) {
+					spliceColumn += op.lines[editingLinesCnt].length;
+				}
+
+				// Split last line
+				const leftoverLine = this._lines[spliceLineNumber - 1].substring(spliceColumn - 1);
+
+				this._setLineContent(spliceLineNumber, this._lines[spliceLineNumber - 1].substring(0, spliceColumn - 1));
+
+				// Lines in the middle
+				let newLines: string[] = new Array<string>(insertingLinesCnt - editingLinesCnt);
+				let newLinesLengths = new Uint32Array(insertingLinesCnt - editingLinesCnt);
+				for (let j = editingLinesCnt + 1; j <= insertingLinesCnt; j++) {
+					newLines[j - editingLinesCnt - 1] = op.lines[j];
+					newLinesLengths[j - editingLinesCnt - 1] = op.lines[j].length + this._EOL.length;
+				}
+				newLines[newLines.length - 1] += leftoverLine;
+				newLinesLengths[newLines.length - 1] += leftoverLine.length;
+				this._lines = arrays.arrayInsert(this._lines, startLineNumber + editingLinesCnt, newLines);
+				this._lineStarts.insertValues(startLineNumber + editingLinesCnt, newLinesLengths);
+			}
+
+			const contentChangeRange = new Range(startLineNumber, startColumn, endLineNumber, endColumn);
+			const text = (op.lines ? op.lines.join(this.getEOL()) : '');
+			contentChanges.push({
+				range: contentChangeRange,
+				rangeLength: op.rangeLength,
+				text: text,
+				rangeOffset: op.rangeOffset,
+				forceMoveMarkers: op.forceMoveMarkers
+			});
+		}
+
+		return contentChanges;
+	}
+
+	/**
+	 * Assumes `operations` are validated and sorted ascending
+	 */
+	public static _getInverseEditRanges(operations: IValidatedEditOperation[]): Range[] {
+		let result: Range[] = [];
+
+		let prevOpEndLineNumber: number;
+		let prevOpEndColumn: number;
+		let prevOp: IValidatedEditOperation = null;
+		for (let i = 0, len = operations.length; i < len; i++) {
+			let op = operations[i];
+
+			let startLineNumber: number;
+			let startColumn: number;
+
+			if (prevOp) {
+				if (prevOp.range.endLineNumber === op.range.startLineNumber) {
+					startLineNumber = prevOpEndLineNumber;
+					startColumn = prevOpEndColumn + (op.range.startColumn - prevOp.range.endColumn);
+				} else {
+					startLineNumber = prevOpEndLineNumber + (op.range.startLineNumber - prevOp.range.endLineNumber);
+					startColumn = op.range.startColumn;
+				}
+			} else {
+				startLineNumber = op.range.startLineNumber;
+				startColumn = op.range.startColumn;
+			}
+
+			let resultRange: Range;
+
+			if (op.lines && op.lines.length > 0) {
+				// the operation inserts something
+				let lineCount = op.lines.length;
+				let firstLine = op.lines[0];
+				let lastLine = op.lines[lineCount - 1];
+
+				if (lineCount === 1) {
+					// single line insert
+					resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn + firstLine.length);
+				} else {
+					// multi line insert
+					resultRange = new Range(startLineNumber, startColumn, startLineNumber + lineCount - 1, lastLine.length + 1);
+				}
+			} else {
+				// There is nothing to insert
+				resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn);
+			}
+
+			prevOpEndLineNumber = resultRange.endLineNumber;
+			prevOpEndColumn = resultRange.endColumn;
+
+			result.push(resultRange);
+			prevOp = op;
+		}
+
+		return result;
+	}
+
+	//#endregion
+}

src/vs/editor/common/model/linesTextBuffer/linesTextBufferBuilder.ts

@@ -0,0 +1,139 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as strings from 'vs/base/common/strings';
+import { CharCode } from 'vs/base/common/charCode';
+import { ITextBufferBuilder, ITextBufferFactory, ITextBuffer, DefaultEndOfLine } from 'vs/editor/common/model';
+import { IRawTextSource, TextSource } from 'vs/editor/common/model/linesTextBuffer/textSource';
+import { LinesTextBuffer } from 'vs/editor/common/model/linesTextBuffer/linesTextBuffer';
+
+export class TextBufferFactory implements ITextBufferFactory {
+
+	constructor(public readonly rawTextSource: IRawTextSource) {
+	}
+
+	public create(defaultEOL: DefaultEndOfLine): ITextBuffer {
+		const textSource = TextSource.fromRawTextSource(this.rawTextSource, defaultEOL);
+		return new LinesTextBuffer(textSource);
+	}
+
+	public getFirstLineText(lengthLimit: number): string {
+		return this.rawTextSource.lines[0].substr(0, lengthLimit);
+	}
+}
+
+class ModelLineBasedBuilder {
+
+	private BOM: string;
+	private lines: string[];
+	private currLineIndex: number;
+
+	constructor() {
+		this.BOM = '';
+		this.lines = [];
+		this.currLineIndex = 0;
+	}
+
+	public acceptLines(lines: string[]): void {
+		if (this.currLineIndex === 0) {
+			// Remove the BOM (if present)
+			if (strings.startsWithUTF8BOM(lines[0])) {
+				this.BOM = strings.UTF8_BOM_CHARACTER;
+				lines[0] = lines[0].substr(1);
+			}
+		}
+
+		for (let i = 0, len = lines.length; i < len; i++) {
+			this.lines[this.currLineIndex++] = lines[i];
+		}
+	}
+
+	public finish(carriageReturnCnt: number, containsRTL: boolean, isBasicASCII: boolean): TextBufferFactory {
+		return new TextBufferFactory({
+			BOM: this.BOM,
+			lines: this.lines,
+			containsRTL: containsRTL,
+			totalCRCount: carriageReturnCnt,
+			isBasicASCII,
+		});
+	}
+}
+
+export class LinesTextBufferBuilder implements ITextBufferBuilder {
+
+	private leftoverPrevChunk: string;
+	private leftoverEndsInCR: boolean;
+	private totalCRCount: number;
+	private lineBasedBuilder: ModelLineBasedBuilder;
+	private containsRTL: boolean;
+	private isBasicASCII: boolean;
+
+	constructor() {
+		this.leftoverPrevChunk = '';
+		this.leftoverEndsInCR = false;
+		this.totalCRCount = 0;
+		this.lineBasedBuilder = new ModelLineBasedBuilder();
+		this.containsRTL = false;
+		this.isBasicASCII = true;
+	}
+
+	private _updateCRCount(chunk: string): void {
+		// Count how many \r are present in chunk to determine the majority EOL sequence
+		let chunkCarriageReturnCnt = 0;
+		let lastCarriageReturnIndex = -1;
+		while ((lastCarriageReturnIndex = chunk.indexOf('\r', lastCarriageReturnIndex + 1)) !== -1) {
+			chunkCarriageReturnCnt++;
+		}
+		this.totalCRCount += chunkCarriageReturnCnt;
+	}
+
+	public acceptChunk(chunk: string): void {
+		if (chunk.length === 0) {
+			return;
+		}
+
+		this._updateCRCount(chunk);
+
+		if (!this.containsRTL) {
+			this.containsRTL = strings.containsRTL(chunk);
+		}
+		if (this.isBasicASCII) {
+			this.isBasicASCII = strings.isBasicASCII(chunk);
+		}
+
+		// Avoid dealing with a chunk that ends in \r (push the \r to the next chunk)
+		if (this.leftoverEndsInCR) {
+			chunk = '\r' + chunk;
+		}
+		if (chunk.charCodeAt(chunk.length - 1) === CharCode.CarriageReturn) {
+			this.leftoverEndsInCR = true;
+			chunk = chunk.substr(0, chunk.length - 1);
+		} else {
+			this.leftoverEndsInCR = false;
+		}
+
+		let lines = chunk.split(/\r\n|\r|\n/);
+
+		if (lines.length === 1) {
+			// no \r or \n encountered
+			this.leftoverPrevChunk += lines[0];
+			return;
+		}
+
+		lines[0] = this.leftoverPrevChunk + lines[0];
+		this.lineBasedBuilder.acceptLines(lines.slice(0, lines.length - 1));
+		this.leftoverPrevChunk = lines[lines.length - 1];
+	}
+
+	public finish(): TextBufferFactory {
+		let finalLines = [this.leftoverPrevChunk];
+		if (this.leftoverEndsInCR) {
+			finalLines.push('');
+		}
+		this.lineBasedBuilder.acceptLines(finalLines);
+		return this.lineBasedBuilder.finish(this.totalCRCount, this.containsRTL, this.isBasicASCII);
+	}
+}

src/vs/editor/common/model/linesTextBuffer/textSource.ts

@@ -0,0 +1,66 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { DefaultEndOfLine } from 'vs/editor/common/model';
+import { ITextSource } from 'vs/editor/common/model/linesTextBuffer/linesTextBuffer';
+
+/**
+ * A processed string ready to be turned into an editor model.
+ */
+export interface IRawTextSource {
+	/**
+	 * The text split into lines.
+	 */
+	readonly lines: string[];
+	/**
+	 * The BOM (leading character sequence of the file).
+	 */
+	readonly BOM: string;
+	/**
+	 * The number of lines ending with '\r\n'
+	 */
+	readonly totalCRCount: number;
+	/**
+	 * The text contains Unicode characters classified as "R" or "AL".
+	 */
+	readonly containsRTL: boolean;
+	/**
+	 * The text contains only characters inside the ASCII range 32-126 or \t \r \n
+	 */
+	readonly isBasicASCII: boolean;
+}
+
+export class TextSource {
+
+	/**
+	 * if text source is empty or with precisely one line, returns null. No end of line is detected.
+	 * if text source contains more lines ending with '\r\n', returns '\r\n'.
+	 * Otherwise returns '\n'. More lines end with '\n'.
+	 */
+	private static _getEOL(rawTextSource: IRawTextSource, defaultEOL: DefaultEndOfLine): '\r\n' | '\n' {
+		const lineFeedCnt = rawTextSource.lines.length - 1;
+		if (lineFeedCnt === 0) {
+			// This is an empty file or a file with precisely one line
+			return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n');
+		}
+		if (rawTextSource.totalCRCount > lineFeedCnt / 2) {
+			// More than half of the file contains \r\n ending lines
+			return '\r\n';
+		}
+		// At least one line more ends in \n
+		return '\n';
+	}
+
+	public static fromRawTextSource(rawTextSource: IRawTextSource, defaultEOL: DefaultEndOfLine): ITextSource {
+		return {
+			lines: rawTextSource.lines,
+			BOM: rawTextSource.BOM,
+			EOL: TextSource._getEOL(rawTextSource, defaultEOL),
+			containsRTL: rawTextSource.containsRTL,
+			isBasicASCII: rawTextSource.isBasicASCII,
+		};
+	}
+}

src/vs/editor/common/model/mirrorTextModel.ts

@@ -8,7 +8,7 @@ import URI from 'vs/base/common/uri';
 import { IRange } from 'vs/editor/common/core/range';
 import { PrefixSumComputer } from 'vs/editor/common/viewModel/prefixSumComputer';
 import { IModelContentChange } from 'vs/editor/common/model/textModelEvents';
-import { IPosition } from 'vs/editor/common/core/position';
+import { Position } from 'vs/editor/common/core/position';
 
 export interface IModelChangedEvent {
 	/**
@@ -25,7 +25,7 @@ export interface IModelChangedEvent {
 	readonly versionId: number;
 }
 
-export class MirrorModel {
+export class MirrorTextModel {
 
 	protected _uri: URI;
 	protected _lines: string[];
@@ -63,10 +63,7 @@ export class MirrorModel {
 		for (let i = 0, len = changes.length; i < len; i++) {
 			const change = changes[i];
 			this._acceptDeleteRange(change.range);
-			this._acceptInsertText({
-				lineNumber: change.range.startLineNumber,
-				column: change.range.startColumn
-			}, change.text);
+			this._acceptInsertText(new Position(change.range.startLineNumber, change.range.startColumn), change.text);
 		}
 
 		this._versionId = e.versionId;
@@ -124,7 +121,7 @@ export class MirrorModel {
 		}
 	}
 
-	private _acceptInsertText(position: IPosition, insertText: string): void {
+	private _acceptInsertText(position: Position, insertText: string): void {
 		if (insertText.length === 0) {
 			// Nothing to insert
 			return;

src/vs/editor/common/model/model.ts

@@ -1,78 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import URI from 'vs/base/common/uri';
-import Event, { Emitter } from 'vs/base/common/event';
-import { IModel, ITextModelCreationOptions } from 'vs/editor/common/editorCommon';
-import { EditableTextModel } from 'vs/editor/common/model/editableTextModel';
-import { TextModel } from 'vs/editor/common/model/textModel';
-import { LanguageIdentifier } from 'vs/editor/common/modes';
-import { IRawTextSource, RawTextSource } from 'vs/editor/common/model/textSource';
-
-// The hierarchy is:
-// Model -> EditableTextModel -> TextModelWithDecorations -> TextModelWithTokens -> TextModel
-
-var MODEL_ID = 0;
-
-export class Model extends EditableTextModel implements IModel {
-
-	private readonly _onWillDispose: Emitter<void> = this._register(new Emitter<void>());
-	public readonly onWillDispose: Event<void> = this._onWillDispose.event;
-
-	public static createFromString(text: string, options: ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageIdentifier: LanguageIdentifier = null, uri: URI = null): Model {
-		return new Model(RawTextSource.fromString(text), options, languageIdentifier, uri);
-	}
-
-	public readonly id: string;
-
-	private readonly _associatedResource: URI;
-	private _attachedEditorCount: number;
-
-	constructor(rawTextSource: IRawTextSource, creationOptions: ITextModelCreationOptions, languageIdentifier: LanguageIdentifier, associatedResource: URI = null) {
-		super(rawTextSource, creationOptions, languageIdentifier);
-
-		// Generate a new unique model id
-		MODEL_ID++;
-		this.id = '$model' + MODEL_ID;
-
-		if (typeof associatedResource === 'undefined' || associatedResource === null) {
-			this._associatedResource = URI.parse('inmemory://model/' + MODEL_ID);
-		} else {
-			this._associatedResource = associatedResource;
-		}
-
-		this._attachedEditorCount = 0;
-	}
-
-	public dispose(): void {
-		this._isDisposing = true;
-		this._onWillDispose.fire();
-		super.dispose();
-		this._isDisposing = false;
-	}
-
-	public onBeforeAttached(): void {
-		this._attachedEditorCount++;
-		// Warm up tokens for the editor
-		this._warmUpTokens();
-	}
-
-	public onBeforeDetached(): void {
-		this._attachedEditorCount--;
-	}
-
-	protected _shouldAutoTokenize(): boolean {
-		return this.isAttachedToEditor();
-	}
-
-	public isAttachedToEditor(): boolean {
-		return this._attachedEditorCount > 0;
-	}
-
-	public get uri(): URI {
-		return this._associatedResource;
-	}
-}

src/vs/editor/common/model/modelLine.ts

@@ -1,482 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import { IState, FontStyle, StandardTokenType, MetadataConsts, ColorId, LanguageId } from 'vs/editor/common/modes';
-import { CharCode } from 'vs/base/common/charCode';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
-import { Constants } from 'vs/editor/common/core/uint';
-
-export interface ILineEdit {
-	startColumn: number;
-	endColumn: number;
-	text: string;
-}
-
-export interface ITokensAdjuster {
-	adjust(toColumn: number, delta: number, minimumAllowedColumn: number): void;
-	finish(delta: number, lineTextLength: number): void;
-}
-
-var NO_OP_TOKENS_ADJUSTER: ITokensAdjuster = {
-	adjust: () => { },
-	finish: () => { }
-};
-
-/**
- * Returns:
- *  - -1 => the line consists of whitespace
- *  - otherwise => the indent level is returned value
- */
-export function computeIndentLevel(line: string, tabSize: number): number {
-	let indent = 0;
-	let i = 0;
-	let len = line.length;
-
-	while (i < len) {
-		let chCode = line.charCodeAt(i);
-		if (chCode === CharCode.Space) {
-			indent++;
-		} else if (chCode === CharCode.Tab) {
-			indent = indent - indent % tabSize + tabSize;
-		} else {
-			break;
-		}
-		i++;
-	}
-
-	if (i === len) {
-		return -1; // line only consists of whitespace
-	}
-
-	return indent;
-}
-
-export interface IModelLine {
-	readonly text: string;
-
-	// --- tokenization
-	resetTokenizationState(): void;
-	isInvalid(): boolean;
-	setIsInvalid(isInvalid: boolean): void;
-	getState(): IState;
-	setState(state: IState): void;
-	getTokens(topLevelLanguageId: LanguageId): LineTokens;
-	setTokens(topLevelLanguageId: LanguageId, tokens: Uint32Array): void;
-
-	// --- editing
-	applyEdits(edits: ILineEdit[]): number;
-	append(other: IModelLine): void;
-	split(splitColumn: number): IModelLine;
-}
-
-export abstract class AbstractModelLine {
-
-	constructor() {
-	}
-
-	///
-
-	public abstract get text(): string;
-	protected abstract _setText(text: string): void;
-	protected abstract _createTokensAdjuster(): ITokensAdjuster;
-	protected abstract _createModelLine(text: string): IModelLine;
-
-	///
-
-	public applyEdits(edits: ILineEdit[]): number {
-		let deltaColumn = 0;
-		let resultText = this.text;
-
-		let tokensAdjuster = this._createTokensAdjuster();
-
-		for (let i = 0, len = edits.length; i < len; i++) {
-			let edit = edits[i];
-
-			// console.log();
-			// console.log('=============================');
-			// console.log('EDIT #' + i + ' [ ' + edit.startColumn + ' -> ' + edit.endColumn + ' ] : <<<' + edit.text + '>>>');
-			// console.log('deltaColumn: ' + deltaColumn);
-
-			let startColumn = deltaColumn + edit.startColumn;
-			let endColumn = deltaColumn + edit.endColumn;
-			let deletingCnt = endColumn - startColumn;
-			let insertingCnt = edit.text.length;
-
-			// Adjust tokens before this edit
-			// console.log('Adjust tokens before this edit');
-			tokensAdjuster.adjust(edit.startColumn - 1, deltaColumn, 1);
-
-			// Adjust tokens for the common part of this edit
-			let commonLength = Math.min(deletingCnt, insertingCnt);
-			if (commonLength > 0) {
-				// console.log('Adjust tokens for the common part of this edit');
-				tokensAdjuster.adjust(edit.startColumn - 1 + commonLength, deltaColumn, startColumn);
-			}
-
-			// Perform the edit & update `deltaColumn`
-			resultText = resultText.substring(0, startColumn - 1) + edit.text + resultText.substring(endColumn - 1);
-			deltaColumn += insertingCnt - deletingCnt;
-
-			// Adjust tokens inside this edit
-			// console.log('Adjust tokens inside this edit');
-			tokensAdjuster.adjust(edit.endColumn, deltaColumn, startColumn);
-		}
-
-		// Wrap up tokens; adjust remaining if needed
-		tokensAdjuster.finish(deltaColumn, resultText.length);
-
-		// Save the resulting text
-		this._setText(resultText);
-
-		return deltaColumn;
-	}
-
-	public split(splitColumn: number): IModelLine {
-		const myText = this.text.substring(0, splitColumn - 1);
-		const otherText = this.text.substring(splitColumn - 1);
-
-		this._setText(myText);
-		return this._createModelLine(otherText);
-	}
-
-	public append(other: IModelLine): void {
-		this._setText(this.text + other.text);
-	}
-}
-
-export class ModelLine extends AbstractModelLine implements IModelLine {
-
-	private _text: string;
-	public get text(): string { return this._text; }
-
-	private _isInvalid: boolean;
-
-	public isInvalid(): boolean {
-		return this._isInvalid;
-	}
-
-	public setIsInvalid(isInvalid: boolean): void {
-		this._isInvalid = isInvalid;
-	}
-
-	private _state: IState;
-	private _lineTokens: ArrayBuffer;
-
-	constructor(text: string) {
-		super();
-		this._isInvalid = false;
-		this._setText(text);
-		this._state = null;
-		this._lineTokens = null;
-	}
-
-	protected _createModelLine(text: string): IModelLine {
-		return new ModelLine(text);
-	}
-
-	public split(splitColumn: number): IModelLine {
-		let result = super.split(splitColumn);
-
-		// Mark overflowing tokens for deletion & delete marked tokens
-		this._deleteMarkedTokens(this._markOverflowingTokensForDeletion(0, this.text.length));
-
-		return result;
-	}
-
-	public append(other: IModelLine): void {
-		let thisTextLength = this.text.length;
-
-		super.append(other);
-
-		if (other instanceof ModelLine) {
-			let otherRawTokens = other._lineTokens;
-			if (otherRawTokens) {
-				// Other has real tokens
-
-				let otherTokens = new Uint32Array(otherRawTokens);
-
-				// Adjust other tokens
-				if (thisTextLength > 0) {
-					for (let i = 0, len = (otherTokens.length >>> 1); i < len; i++) {
-						otherTokens[(i << 1)] = otherTokens[(i << 1)] + thisTextLength;
-					}
-				}
-
-				// Append other tokens
-				let myRawTokens = this._lineTokens;
-				if (myRawTokens) {
-					// I have real tokens
-					let myTokens = new Uint32Array(myRawTokens);
-					let result = new Uint32Array(myTokens.length + otherTokens.length);
-					result.set(myTokens, 0);
-					result.set(otherTokens, myTokens.length);
-					this._lineTokens = result.buffer;
-				} else {
-					// I don't have real tokens
-					this._lineTokens = otherTokens.buffer;
-				}
-			}
-		}
-	}
-
-	// --- BEGIN STATE
-
-	public resetTokenizationState(): void {
-		this._state = null;
-		this._lineTokens = null;
-	}
-
-	public setState(state: IState): void {
-		this._state = state;
-	}
-
-	public getState(): IState {
-		return this._state || null;
-	}
-
-	// --- END STATE
-
-	// --- BEGIN TOKENS
-
-	public setTokens(topLevelLanguageId: LanguageId, tokens: Uint32Array): void {
-		if (!tokens || tokens.length === 0) {
-			this._lineTokens = null;
-			return;
-		}
-		if (tokens.length === 2) {
-			// there is one token
-			if (tokens[0] === 0 && tokens[1] === getDefaultMetadata(topLevelLanguageId)) {
-				this._lineTokens = null;
-				return;
-			}
-		}
-		this._lineTokens = tokens.buffer;
-	}
-
-	public getTokens(topLevelLanguageId: LanguageId): LineTokens {
-		let rawLineTokens = this._lineTokens;
-		if (rawLineTokens) {
-			return new LineTokens(new Uint32Array(rawLineTokens), this._text);
-		}
-
-		let lineTokens = new Uint32Array(2);
-		lineTokens[0] = 0;
-		lineTokens[1] = getDefaultMetadata(topLevelLanguageId);
-		return new LineTokens(lineTokens, this._text);
-	}
-
-	// --- END TOKENS
-
-	protected _createTokensAdjuster(): ITokensAdjuster {
-		if (!this._lineTokens) {
-			// This line does not have real tokens, so there is nothing to adjust
-			return NO_OP_TOKENS_ADJUSTER;
-		}
-
-		let lineTokens = new Uint32Array(this._lineTokens);
-		let tokensLength = (lineTokens.length >>> 1);
-		let tokenIndex = 0;
-		let tokenStartOffset = 0;
-		let removeTokensCount = 0;
-
-		let adjust = (toColumn: number, delta: number, minimumAllowedColumn: number) => {
-			// console.log(`------------------------------------------------------------------`);
-			// console.log(`before call: tokenIndex: ${tokenIndex}: ${lineTokens}`);
-			// console.log(`adjustTokens: ${toColumn} with delta: ${delta} and [${minimumAllowedColumn}]`);
-			// console.log(`tokenStartOffset: ${tokenStartOffset}`);
-			let minimumAllowedIndex = minimumAllowedColumn - 1;
-
-			while (tokenStartOffset < toColumn && tokenIndex < tokensLength) {
-
-				if (tokenStartOffset > 0 && delta !== 0) {
-					// adjust token's `startIndex` by `delta`
-					let newTokenStartOffset = Math.max(minimumAllowedIndex, tokenStartOffset + delta);
-					lineTokens[(tokenIndex << 1)] = newTokenStartOffset;
-
-					// console.log(` * adjusted token start offset for token at ${tokenIndex}: ${newTokenStartOffset}`);
-
-					if (delta < 0) {
-						let tmpTokenIndex = tokenIndex;
-						while (tmpTokenIndex > 0) {
-							let prevTokenStartOffset = lineTokens[((tmpTokenIndex - 1) << 1)];
-							if (prevTokenStartOffset >= newTokenStartOffset) {
-								if (prevTokenStartOffset !== Constants.MAX_UINT_32) {
-									// console.log(` * marking for deletion token at ${tmpTokenIndex - 1}`);
-									lineTokens[((tmpTokenIndex - 1) << 1)] = Constants.MAX_UINT_32;
-									removeTokensCount++;
-								}
-								tmpTokenIndex--;
-							} else {
-								break;
-							}
-						}
-					}
-				}
-
-				tokenIndex++;
-				if (tokenIndex < tokensLength) {
-					tokenStartOffset = lineTokens[(tokenIndex << 1)];
-				}
-			}
-			// console.log(`after call: tokenIndex: ${tokenIndex}: ${lineTokens}`);
-		};
-
-		let finish = (delta: number, lineTextLength: number) => {
-			adjust(Constants.MAX_SAFE_SMALL_INTEGER, delta, 1);
-
-			// Mark overflowing tokens for deletion & delete marked tokens
-			this._deleteMarkedTokens(this._markOverflowingTokensForDeletion(removeTokensCount, lineTextLength));
-		};
-
-		return {
-			adjust: adjust,
-			finish: finish
-		};
-	}
-
-	private _markOverflowingTokensForDeletion(removeTokensCount: number, lineTextLength: number): number {
-		if (!this._lineTokens) {
-			return removeTokensCount;
-		}
-
-		let lineTokens = new Uint32Array(this._lineTokens);
-		let tokensLength = (lineTokens.length >>> 1);
-
-		if (removeTokensCount + 1 === tokensLength) {
-			// no more removing, cannot end up without any tokens for mode transition reasons
-			return removeTokensCount;
-		}
-
-		for (let tokenIndex = tokensLength - 1; tokenIndex > 0; tokenIndex--) {
-			let tokenStartOffset = lineTokens[(tokenIndex << 1)];
-			if (tokenStartOffset < lineTextLength) {
-				// valid token => stop iterating
-				return removeTokensCount;
-			}
-
-			// this token now overflows the text => mark it for removal
-			if (tokenStartOffset !== Constants.MAX_UINT_32) {
-				// console.log(` * marking for deletion token at ${tokenIndex}`);
-				lineTokens[(tokenIndex << 1)] = Constants.MAX_UINT_32;
-				removeTokensCount++;
-
-				if (removeTokensCount + 1 === tokensLength) {
-					// no more removing, cannot end up without any tokens for mode transition reasons
-					return removeTokensCount;
-				}
-			}
-		}
-
-		return removeTokensCount;
-	}
-
-	private _deleteMarkedTokens(removeTokensCount: number): void {
-		if (removeTokensCount === 0) {
-			return;
-		}
-
-		let lineTokens = new Uint32Array(this._lineTokens);
-		let tokensLength = (lineTokens.length >>> 1);
-		let newTokens = new Uint32Array(((tokensLength - removeTokensCount) << 1)), newTokenIdx = 0;
-		for (let i = 0; i < tokensLength; i++) {
-			let startOffset = lineTokens[(i << 1)];
-			if (startOffset === Constants.MAX_UINT_32) {
-				// marked for deletion
-				continue;
-			}
-			let metadata = lineTokens[(i << 1) + 1];
-			newTokens[newTokenIdx++] = startOffset;
-			newTokens[newTokenIdx++] = metadata;
-		}
-		this._lineTokens = newTokens.buffer;
-	}
-
-	protected _setText(text: string): void {
-		this._text = text;
-	}
-
-}
-
-/**
- * A model line that cannot store any tokenization state.
- * It has no fields except the text.
- */
-export class MinimalModelLine extends AbstractModelLine implements IModelLine {
-
-	private _text: string;
-	public get text(): string { return this._text; }
-
-	public isInvalid(): boolean {
-		return false;
-	}
-
-	public setIsInvalid(isInvalid: boolean): void {
-	}
-
-	constructor(text: string) {
-		super();
-		this._setText(text);
-	}
-
-	protected _createModelLine(text: string): IModelLine {
-		return new MinimalModelLine(text);
-	}
-
-	public split(splitColumn: number): IModelLine {
-		return super.split(splitColumn);
-	}
-
-	public append(other: IModelLine): void {
-		super.append(other);
-	}
-
-	// --- BEGIN STATE
-
-	public resetTokenizationState(): void {
-	}
-
-	public setState(state: IState): void {
-	}
-
-	public getState(): IState {
-		return null;
-	}
-
-	// --- END STATE
-
-	// --- BEGIN TOKENS
-
-	public setTokens(topLevelLanguageId: LanguageId, tokens: Uint32Array): void {
-	}
-
-	public getTokens(topLevelLanguageId: LanguageId): LineTokens {
-		let lineTokens = new Uint32Array(2);
-		lineTokens[0] = 0;
-		lineTokens[1] = getDefaultMetadata(topLevelLanguageId);
-		return new LineTokens(lineTokens, this._text);
-	}
-
-	// --- END TOKENS
-
-	protected _createTokensAdjuster(): ITokensAdjuster {
-		// This line does not have real tokens, so there is nothing to adjust
-		return NO_OP_TOKENS_ADJUSTER;
-	}
-
-	protected _setText(text: string): void {
-		this._text = text;
-	}
-}
-
-function getDefaultMetadata(topLevelLanguageId: LanguageId): number {
-	return (
-		(topLevelLanguageId << MetadataConsts.LANGUAGEID_OFFSET)
-		| (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET)
-		| (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET)
-		| (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET)
-		| (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET)
-	) >>> 0;
-}

src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts

@@ -0,0 +1,494 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { Range } from 'vs/editor/common/core/range';
+import { Position } from 'vs/editor/common/core/position';
+import * as strings from 'vs/base/common/strings';
+import { IValidatedEditOperation } from 'vs/editor/common/model/linesTextBuffer/linesTextBuffer';
+import { PieceTreeBase, StringBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
+import { IIdentifiedSingleEditOperation, EndOfLinePreference, ITextBuffer, ApplyEditsResult, IInternalModelContentChange } from 'vs/editor/common/model';
+import { ITextSnapshot } from 'vs/platform/files/common/files';
+
+export class PieceTreeTextBuffer implements ITextBuffer {
+	private _pieceTree: PieceTreeBase;
+	private _BOM: string;
+	private _mightContainRTL: boolean;
+	private _mightContainNonBasicASCII: boolean;
+
+	constructor(chunks: StringBuffer[], BOM: string, eol: '\r\n' | '\n', containsRTL: boolean, isBasicASCII: boolean, eolNormalized: boolean) {
+		this._BOM = BOM;
+		this._mightContainNonBasicASCII = !isBasicASCII;
+		this._mightContainRTL = containsRTL;
+		this._pieceTree = new PieceTreeBase(chunks, eol, eolNormalized);
+	}
+
+	// #region TextBuffer
+	public equals(other: ITextBuffer): boolean {
+		if (!(other instanceof PieceTreeTextBuffer)) {
+			return false;
+		}
+		if (this._BOM !== other._BOM) {
+			return false;
+		}
+		if (this.getEOL() !== other.getEOL()) {
+			return false;
+		}
+		return this._pieceTree.equal(other._pieceTree);
+	}
+	public mightContainRTL(): boolean {
+		return this._mightContainRTL;
+	}
+	public mightContainNonBasicASCII(): boolean {
+		return this._mightContainNonBasicASCII;
+	}
+	public getBOM(): string {
+		return this._BOM;
+	}
+	public getEOL(): string {
+		return this._pieceTree.getEOL();
+	}
+
+	public createSnapshot(preserveBOM: boolean): ITextSnapshot {
+		return this._pieceTree.createSnapshot(preserveBOM ? this._BOM : '');
+	}
+
+	public getOffsetAt(lineNumber: number, column: number): number {
+		return this._pieceTree.getOffsetAt(lineNumber, column);
+	}
+
+	public getPositionAt(offset: number): Position {
+		return this._pieceTree.getPositionAt(offset);
+	}
+
+	public getRangeAt(start: number, length: number): Range {
+		let end = start + length;
+		const startPosition = this.getPositionAt(start);
+		const endPosition = this.getPositionAt(end);
+		return new Range(startPosition.lineNumber, startPosition.column, endPosition.lineNumber, endPosition.column);
+	}
+
+	public getValueInRange(range: Range, eol: EndOfLinePreference = EndOfLinePreference.TextDefined): string {
+		if (range.isEmpty()) {
+			return '';
+		}
+
+		const lineEnding = this._getEndOfLine(eol);
+		const text = this._pieceTree.getValueInRange(range);
+		return text.replace(/\r\n|\r|\n/g, lineEnding);
+	}
+
+	public getValueLengthInRange(range: Range, eol: EndOfLinePreference = EndOfLinePreference.TextDefined): number {
+		if (range.isEmpty()) {
+			return 0;
+		}
+
+		if (range.startLineNumber === range.endLineNumber) {
+			return (range.endColumn - range.startColumn);
+		}
+
+		let startOffset = this.getOffsetAt(range.startLineNumber, range.startColumn);
+		let endOffset = this.getOffsetAt(range.endLineNumber, range.endColumn);
+		return endOffset - startOffset;
+	}
+
+	public getLength(): number {
+		return this._pieceTree.getLength();
+	}
+
+	public getLineCount(): number {
+		return this._pieceTree.getLineCount();
+	}
+
+	public getLinesContent(): string[] {
+		return this._pieceTree.getLinesContent();
+	}
+
+	public getLineContent(lineNumber: number): string {
+		return this._pieceTree.getLineContent(lineNumber);
+	}
+
+	public getLineCharCode(lineNumber: number, index: number): number {
+		return this._pieceTree.getLineCharCode(lineNumber, index);
+	}
+
+	public getLineLength(lineNumber: number): number {
+		return this._pieceTree.getLineLength(lineNumber);
+	}
+
+	public getLineMinColumn(lineNumber: number): number {
+		return 1;
+	}
+
+	public getLineMaxColumn(lineNumber: number): number {
+		return this.getLineLength(lineNumber) + 1;
+	}
+
+	public getLineFirstNonWhitespaceColumn(lineNumber: number): number {
+		const result = strings.firstNonWhitespaceIndex(this.getLineContent(lineNumber));
+		if (result === -1) {
+			return 0;
+		}
+		return result + 1;
+	}
+
+	public getLineLastNonWhitespaceColumn(lineNumber: number): number {
+		const result = strings.lastNonWhitespaceIndex(this.getLineContent(lineNumber));
+		if (result === -1) {
+			return 0;
+		}
+		return result + 2;
+	}
+
+	private _getEndOfLine(eol: EndOfLinePreference): string {
+		switch (eol) {
+			case EndOfLinePreference.LF:
+				return '\n';
+			case EndOfLinePreference.CRLF:
+				return '\r\n';
+			case EndOfLinePreference.TextDefined:
+				return this.getEOL();
+		}
+		throw new Error('Unknown EOL preference');
+	}
+
+	public setEOL(newEOL: '\r\n' | '\n'): void {
+		this._pieceTree.setEOL(newEOL);
+	}
+
+	public applyEdits(rawOperations: IIdentifiedSingleEditOperation[], recordTrimAutoWhitespace: boolean): ApplyEditsResult {
+		let mightContainRTL = this._mightContainRTL;
+		let mightContainNonBasicASCII = this._mightContainNonBasicASCII;
+		let canReduceOperations = true;
+
+		let operations: IValidatedEditOperation[] = [];
+		for (let i = 0; i < rawOperations.length; i++) {
+			let op = rawOperations[i];
+			if (canReduceOperations && op._isTracked) {
+				canReduceOperations = false;
+			}
+			let validatedRange = op.range;
+			if (!mightContainRTL && op.text) {
+				// check if the new inserted text contains RTL
+				mightContainRTL = strings.containsRTL(op.text);
+			}
+			if (!mightContainNonBasicASCII && op.text) {
+				mightContainNonBasicASCII = !strings.isBasicASCII(op.text);
+			}
+			operations[i] = {
+				sortIndex: i,
+				identifier: op.identifier,
+				range: validatedRange,
+				rangeOffset: this.getOffsetAt(validatedRange.startLineNumber, validatedRange.startColumn),
+				rangeLength: this.getValueLengthInRange(validatedRange),
+				lines: op.text ? op.text.split(/\r\n|\r|\n/) : null,
+				forceMoveMarkers: op.forceMoveMarkers,
+				isAutoWhitespaceEdit: op.isAutoWhitespaceEdit || false
+			};
+		}
+
+		// Sort operations ascending
+		operations.sort(PieceTreeTextBuffer._sortOpsAscending);
+
+		for (let i = 0, count = operations.length - 1; i < count; i++) {
+			let rangeEnd = operations[i].range.getEndPosition();
+			let nextRangeStart = operations[i + 1].range.getStartPosition();
+
+			if (nextRangeStart.isBefore(rangeEnd)) {
+				// overlapping ranges
+				throw new Error('Overlapping ranges are not allowed!');
+			}
+		}
+
+		if (canReduceOperations) {
+			operations = this._reduceOperations(operations);
+		}
+
+		// Delta encode operations
+		let reverseRanges = PieceTreeTextBuffer._getInverseEditRanges(operations);
+		let newTrimAutoWhitespaceCandidates: { lineNumber: number, oldContent: string }[] = [];
+
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+			let reverseRange = reverseRanges[i];
+
+			if (recordTrimAutoWhitespace && op.isAutoWhitespaceEdit && op.range.isEmpty()) {
+				// Record already the future line numbers that might be auto whitespace removal candidates on next edit
+				for (let lineNumber = reverseRange.startLineNumber; lineNumber <= reverseRange.endLineNumber; lineNumber++) {
+					let currentLineContent = '';
+					if (lineNumber === reverseRange.startLineNumber) {
+						currentLineContent = this.getLineContent(op.range.startLineNumber);
+						if (strings.firstNonWhitespaceIndex(currentLineContent) !== -1) {
+							continue;
+						}
+					}
+					newTrimAutoWhitespaceCandidates.push({ lineNumber: lineNumber, oldContent: currentLineContent });
+				}
+			}
+		}
+
+		let reverseOperations: IIdentifiedSingleEditOperation[] = [];
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+			let reverseRange = reverseRanges[i];
+
+			reverseOperations[i] = {
+				identifier: op.identifier,
+				range: reverseRange,
+				text: this.getValueInRange(op.range),
+				forceMoveMarkers: op.forceMoveMarkers
+			};
+		}
+
+		this._mightContainRTL = mightContainRTL;
+		this._mightContainNonBasicASCII = mightContainNonBasicASCII;
+
+		const contentChanges = this._doApplyEdits(operations);
+
+		let trimAutoWhitespaceLineNumbers: number[] = null;
+		if (recordTrimAutoWhitespace && newTrimAutoWhitespaceCandidates.length > 0) {
+			// sort line numbers auto whitespace removal candidates for next edit descending
+			newTrimAutoWhitespaceCandidates.sort((a, b) => b.lineNumber - a.lineNumber);
+
+			trimAutoWhitespaceLineNumbers = [];
+			for (let i = 0, len = newTrimAutoWhitespaceCandidates.length; i < len; i++) {
+				let lineNumber = newTrimAutoWhitespaceCandidates[i].lineNumber;
+				if (i > 0 && newTrimAutoWhitespaceCandidates[i - 1].lineNumber === lineNumber) {
+					// Do not have the same line number twice
+					continue;
+				}
+
+				let prevContent = newTrimAutoWhitespaceCandidates[i].oldContent;
+				let lineContent = this.getLineContent(lineNumber);
+
+				if (lineContent.length === 0 || lineContent === prevContent || strings.firstNonWhitespaceIndex(lineContent) !== -1) {
+					continue;
+				}
+
+				trimAutoWhitespaceLineNumbers.push(lineNumber);
+			}
+		}
+
+		return new ApplyEditsResult(
+			reverseOperations,
+			contentChanges,
+			trimAutoWhitespaceLineNumbers
+		);
+	}
+
+	/**
+ * Transform operations such that they represent the same logic edit,
+ * but that they also do not cause OOM crashes.
+ */
+	private _reduceOperations(operations: IValidatedEditOperation[]): IValidatedEditOperation[] {
+		if (operations.length < 1000) {
+			// We know from empirical testing that a thousand edits work fine regardless of their shape.
+			return operations;
+		}
+
+		// At one point, due to how events are emitted and how each operation is handled,
+		// some operations can trigger a high ammount of temporary string allocations,
+		// that will immediately get edited again.
+		// e.g. a formatter inserting ridiculous ammounts of \n on a model with a single line
+		// Therefore, the strategy is to collapse all the operations into a huge single edit operation
+		return [this._toSingleEditOperation(operations)];
+	}
+
+	_toSingleEditOperation(operations: IValidatedEditOperation[]): IValidatedEditOperation {
+		let forceMoveMarkers = false,
+			firstEditRange = operations[0].range,
+			lastEditRange = operations[operations.length - 1].range,
+			entireEditRange = new Range(firstEditRange.startLineNumber, firstEditRange.startColumn, lastEditRange.endLineNumber, lastEditRange.endColumn),
+			lastEndLineNumber = firstEditRange.startLineNumber,
+			lastEndColumn = firstEditRange.startColumn,
+			result: string[] = [];
+
+		for (let i = 0, len = operations.length; i < len; i++) {
+			let operation = operations[i],
+				range = operation.range;
+
+			forceMoveMarkers = forceMoveMarkers || operation.forceMoveMarkers;
+
+			// (1) -- Push old text
+			for (let lineNumber = lastEndLineNumber; lineNumber < range.startLineNumber; lineNumber++) {
+				if (lineNumber === lastEndLineNumber) {
+					result.push(this.getLineContent(lineNumber).substring(lastEndColumn - 1));
+				} else {
+					result.push('\n');
+					result.push(this.getLineContent(lineNumber));
+				}
+			}
+
+			if (range.startLineNumber === lastEndLineNumber) {
+				result.push(this.getLineContent(range.startLineNumber).substring(lastEndColumn - 1, range.startColumn - 1));
+			} else {
+				result.push('\n');
+				result.push(this.getLineContent(range.startLineNumber).substring(0, range.startColumn - 1));
+			}
+
+			// (2) -- Push new text
+			if (operation.lines) {
+				for (let j = 0, lenJ = operation.lines.length; j < lenJ; j++) {
+					if (j !== 0) {
+						result.push('\n');
+					}
+					result.push(operation.lines[j]);
+				}
+			}
+
+			lastEndLineNumber = operation.range.endLineNumber;
+			lastEndColumn = operation.range.endColumn;
+		}
+
+		return {
+			sortIndex: 0,
+			identifier: operations[0].identifier,
+			range: entireEditRange,
+			rangeOffset: this.getOffsetAt(entireEditRange.startLineNumber, entireEditRange.startColumn),
+			rangeLength: this.getValueLengthInRange(entireEditRange, EndOfLinePreference.TextDefined),
+			lines: result.join('').split('\n'),
+			forceMoveMarkers: forceMoveMarkers,
+			isAutoWhitespaceEdit: false
+		};
+	}
+
+	private _doApplyEdits(operations: IValidatedEditOperation[]): IInternalModelContentChange[] {
+		operations.sort(PieceTreeTextBuffer._sortOpsDescending);
+
+		let contentChanges: IInternalModelContentChange[] = [];
+
+		// operations are from bottom to top
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+
+			const startLineNumber = op.range.startLineNumber;
+			const startColumn = op.range.startColumn;
+			const endLineNumber = op.range.endLineNumber;
+			const endColumn = op.range.endColumn;
+
+			if (startLineNumber === endLineNumber && startColumn === endColumn && (!op.lines || op.lines.length === 0)) {
+				// no-op
+				continue;
+			}
+
+			const deletingLinesCnt = endLineNumber - startLineNumber;
+			const insertingLinesCnt = (op.lines ? op.lines.length - 1 : 0);
+			const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt);
+
+			const text = (op.lines ? op.lines.join(this.getEOL()) : '');
+
+			if (text) {
+				// replacement
+				this._pieceTree.delete(op.rangeOffset, op.rangeLength);
+				this._pieceTree.insert(op.rangeOffset, text, true);
+
+			} else {
+				// deletion
+				this._pieceTree.delete(op.rangeOffset, op.rangeLength);
+			}
+
+			if (editingLinesCnt < insertingLinesCnt) {
+				let newLinesContent: string[] = [];
+				for (let j = editingLinesCnt + 1; j <= insertingLinesCnt; j++) {
+					newLinesContent.push(op.lines[j]);
+				}
+
+				newLinesContent[newLinesContent.length - 1] = this.getLineContent(startLineNumber + insertingLinesCnt - 1);
+			}
+
+			const contentChangeRange = new Range(startLineNumber, startColumn, endLineNumber, endColumn);
+			contentChanges.push({
+				range: contentChangeRange,
+				rangeLength: op.rangeLength,
+				text: text,
+				rangeOffset: op.rangeOffset,
+				forceMoveMarkers: op.forceMoveMarkers
+			});
+		}
+		return contentChanges;
+	}
+
+	// #endregion
+
+	// #region helper
+	// testing purpose.
+	public getPieceTree(): PieceTreeBase {
+		return this._pieceTree;
+	}
+	/**
+	 * Assumes `operations` are validated and sorted ascending
+	 */
+	public static _getInverseEditRanges(operations: IValidatedEditOperation[]): Range[] {
+		let result: Range[] = [];
+
+		let prevOpEndLineNumber: number;
+		let prevOpEndColumn: number;
+		let prevOp: IValidatedEditOperation = null;
+		for (let i = 0, len = operations.length; i < len; i++) {
+			let op = operations[i];
+
+			let startLineNumber: number;
+			let startColumn: number;
+
+			if (prevOp) {
+				if (prevOp.range.endLineNumber === op.range.startLineNumber) {
+					startLineNumber = prevOpEndLineNumber;
+					startColumn = prevOpEndColumn + (op.range.startColumn - prevOp.range.endColumn);
+				} else {
+					startLineNumber = prevOpEndLineNumber + (op.range.startLineNumber - prevOp.range.endLineNumber);
+					startColumn = op.range.startColumn;
+				}
+			} else {
+				startLineNumber = op.range.startLineNumber;
+				startColumn = op.range.startColumn;
+			}
+
+			let resultRange: Range;
+
+			if (op.lines && op.lines.length > 0) {
+				// the operation inserts something
+				let lineCount = op.lines.length;
+				let firstLine = op.lines[0];
+				let lastLine = op.lines[lineCount - 1];
+
+				if (lineCount === 1) {
+					// single line insert
+					resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn + firstLine.length);
+				} else {
+					// multi line insert
+					resultRange = new Range(startLineNumber, startColumn, startLineNumber + lineCount - 1, lastLine.length + 1);
+				}
+			} else {
+				// There is nothing to insert
+				resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn);
+			}
+
+			prevOpEndLineNumber = resultRange.endLineNumber;
+			prevOpEndColumn = resultRange.endColumn;
+
+			result.push(resultRange);
+			prevOp = op;
+		}
+
+		return result;
+	}
+
+	private static _sortOpsAscending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
+		let r = Range.compareRangesUsingEnds(a.range, b.range);
+		if (r === 0) {
+			return a.sortIndex - b.sortIndex;
+		}
+		return r;
+	}
+
+	private static _sortOpsDescending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
+		let r = Range.compareRangesUsingEnds(a.range, b.range);
+		if (r === 0) {
+			return b.sortIndex - a.sortIndex;
+		}
+		return -r;
+	}
+	// #endregion
+}

src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder.ts

@@ -0,0 +1,180 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as strings from 'vs/base/common/strings';
+import { ITextBufferBuilder, DefaultEndOfLine, ITextBufferFactory, ITextBuffer } from 'vs/editor/common/model';
+import { PieceTreeTextBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer';
+import { StringBuffer, createLineStarts, createLineStartsFast } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
+import { CharCode } from 'vs/base/common/charCode';
+
+export class PieceTreeTextBufferFactory implements ITextBufferFactory {
+
+	constructor(
+		private readonly _chunks: StringBuffer[],
+		private readonly _bom: string,
+		private readonly _cr: number,
+		private readonly _lf: number,
+		private readonly _crlf: number,
+		private readonly _containsRTL: boolean,
+		private readonly _isBasicASCII: boolean,
+		private readonly _normalizeEOL: boolean
+	) { }
+
+	private _getEOL(defaultEOL: DefaultEndOfLine): '\r\n' | '\n' {
+		const totalEOLCount = this._cr + this._lf + this._crlf;
+		const totalCRCount = this._cr + this._crlf;
+		if (totalEOLCount === 0) {
+			// This is an empty file or a file with precisely one line
+			return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n');
+		}
+		if (totalCRCount > totalEOLCount / 2) {
+			// More than half of the file contains \r\n ending lines
+			return '\r\n';
+		}
+		// At least one line more ends in \n
+		return '\n';
+	}
+
+	public create(defaultEOL: DefaultEndOfLine): ITextBuffer {
+		const eol = this._getEOL(defaultEOL);
+		let chunks = this._chunks;
+
+		if (this._normalizeEOL &&
+			((eol === '\r\n' && (this._cr > 0 || this._lf > 0))
+				|| (eol === '\n' && (this._cr > 0 || this._crlf > 0)))
+		) {
+			// Normalize pieces
+			for (let i = 0, len = chunks.length; i < len; i++) {
+				let str = chunks[i].buffer.replace(/\r\n|\r|\n/g, eol);
+				let newLineStart = createLineStartsFast(str);
+				chunks[i] = new StringBuffer(str, newLineStart);
+			}
+		}
+
+		return new PieceTreeTextBuffer(chunks, this._bom, eol, this._containsRTL, this._isBasicASCII, this._normalizeEOL);
+	}
+
+	public getFirstLineText(lengthLimit: number): string {
+		return this._chunks[0].buffer.substr(0, 100).split(/\r\n|\r|\n/)[0];
+	}
+}
+
+export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
+	private chunks: StringBuffer[];
+	private BOM: string;
+
+	private _hasPreviousChar: boolean;
+	private _previousChar: number;
+	private _tmpLineStarts: number[];
+
+	private cr: number;
+	private lf: number;
+	private crlf: number;
+	private containsRTL: boolean;
+	private isBasicASCII: boolean;
+
+	constructor() {
+		this.chunks = [];
+		this.BOM = '';
+
+		this._hasPreviousChar = false;
+		this._previousChar = 0;
+		this._tmpLineStarts = [];
+
+		this.cr = 0;
+		this.lf = 0;
+		this.crlf = 0;
+		this.containsRTL = false;
+		this.isBasicASCII = true;
+	}
+
+	public acceptChunk(chunk: string): void {
+		if (chunk.length === 0) {
+			return;
+		}
+
+		if (this.chunks.length === 0) {
+			if (strings.startsWithUTF8BOM(chunk)) {
+				this.BOM = strings.UTF8_BOM_CHARACTER;
+				chunk = chunk.substr(1);
+			}
+		}
+
+		const lastChar = chunk.charCodeAt(chunk.length - 1);
+		if (lastChar === CharCode.CarriageReturn || (lastChar >= 0xd800 && lastChar <= 0xdbff)) {
+			// last character is \r or a high surrogate => keep it back
+			this._acceptChunk1(chunk.substr(0, chunk.length - 1), false);
+			this._hasPreviousChar = true;
+			this._previousChar = lastChar;
+		} else {
+			this._acceptChunk1(chunk, false);
+			this._hasPreviousChar = false;
+			this._previousChar = lastChar;
+		}
+	}
+
+	private _acceptChunk1(chunk: string, allowEmptyStrings: boolean): void {
+		if (!allowEmptyStrings && chunk.length === 0) {
+			// Nothing to do
+			return;
+		}
+
+		if (this._hasPreviousChar) {
+			this._acceptChunk2(String.fromCharCode(this._previousChar) + chunk);
+		} else {
+			this._acceptChunk2(chunk);
+		}
+	}
+
+	private _acceptChunk2(chunk: string): void {
+		const lineStarts = createLineStarts(this._tmpLineStarts, chunk);
+
+		this.chunks.push(new StringBuffer(chunk, lineStarts.lineStarts));
+		this.cr += lineStarts.cr;
+		this.lf += lineStarts.lf;
+		this.crlf += lineStarts.crlf;
+
+		if (this.isBasicASCII) {
+			this.isBasicASCII = lineStarts.isBasicASCII;
+		}
+		if (!this.isBasicASCII && !this.containsRTL) {
+			// No need to check if is basic ASCII
+			this.containsRTL = strings.containsRTL(chunk);
+		}
+	}
+
+	public finish(normalizeEOL: boolean = true): PieceTreeTextBufferFactory {
+		this._finish();
+		return new PieceTreeTextBufferFactory(
+			this.chunks,
+			this.BOM,
+			this.cr,
+			this.lf,
+			this.crlf,
+			this.containsRTL,
+			this.isBasicASCII,
+			normalizeEOL
+		);
+	}
+
+	private _finish(): void {
+		if (this.chunks.length === 0) {
+			this._acceptChunk1('', true);
+		}
+
+		if (this._hasPreviousChar) {
+			this._hasPreviousChar = false;
+			// recreate last chunk
+			let lastChunk = this.chunks[this.chunks.length - 1];
+			lastChunk.buffer += String.fromCharCode(this._previousChar);
+			let newLineStarts = createLineStartsFast(lastChunk.buffer);
+			lastChunk.lineStarts = newLineStarts;
+			if (this._previousChar === CharCode.CarriageReturn) {
+				this.cr++;
+			}
+		}
+	}
+}

src/vs/editor/common/model/pieceTreeTextBuffer/rbTreeBase.ts

@@ -0,0 +1,427 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { Piece, PieceTreeBase } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
+
+export class TreeNode {
+	parent: TreeNode;
+	left: TreeNode;
+	right: TreeNode;
+	color: NodeColor;
+
+	// Piece
+	piece: Piece;
+	size_left: number; // size of the left subtree (not inorder)
+	lf_left: number; // line feeds cnt in the left subtree (not in order)
+
+	constructor(piece: Piece, color: NodeColor) {
+		this.piece = piece;
+		this.color = color;
+		this.size_left = 0;
+		this.lf_left = 0;
+		this.parent = null;
+		this.left = null;
+		this.right = null;
+	}
+
+	public next(): TreeNode {
+		if (this.right !== SENTINEL) {
+			return leftest(this.right);
+		}
+
+		let node: TreeNode = this;
+
+		while (node.parent !== SENTINEL) {
+			if (node.parent.left === node) {
+				break;
+			}
+
+			node = node.parent;
+		}
+
+		if (node.parent === SENTINEL) {
+			return SENTINEL;
+		} else {
+			return node.parent;
+		}
+	}
+
+	public prev(): TreeNode {
+		if (this.left !== SENTINEL) {
+			return righttest(this.left);
+		}
+
+		let node: TreeNode = this;
+
+		while (node.parent !== SENTINEL) {
+			if (node.parent.right === node) {
+				break;
+			}
+
+			node = node.parent;
+		}
+
+		if (node.parent === SENTINEL) {
+			return SENTINEL;
+		} else {
+			return node.parent;
+		}
+	}
+
+	public detach(): void {
+		this.parent = null;
+		this.left = null;
+		this.right = null;
+	}
+}
+
+export const SENTINEL: TreeNode = new TreeNode(null, NodeColor.Black);
+SENTINEL.parent = SENTINEL;
+SENTINEL.left = SENTINEL;
+SENTINEL.right = SENTINEL;
+SENTINEL.color = NodeColor.Black;
+
+export const enum NodeColor {
+	Black = 0,
+	Red = 1,
+}
+
+export function leftest(node: TreeNode): TreeNode {
+	while (node.left !== SENTINEL) {
+		node = node.left;
+	}
+	return node;
+}
+
+export function righttest(node: TreeNode): TreeNode {
+	while (node.right !== SENTINEL) {
+		node = node.right;
+	}
+	return node;
+}
+
+export function calculateSize(node: TreeNode): number {
+	if (node === SENTINEL) {
+		return 0;
+	}
+
+	return node.size_left + node.piece.length + calculateSize(node.right);
+}
+
+export function calculateLF(node: TreeNode): number {
+	if (node === SENTINEL) {
+		return 0;
+	}
+
+	return node.lf_left + node.piece.lineFeedCnt + calculateLF(node.right);
+}
+
+export function resetSentinel(): void {
+	SENTINEL.parent = SENTINEL;
+}
+
+export function leftRotate(tree: PieceTreeBase, x: TreeNode) {
+	let y = x.right;
+
+	// fix size_left
+	y.size_left += x.size_left + (x.piece ? x.piece.length : 0);
+	y.lf_left += x.lf_left + (x.piece ? x.piece.lineFeedCnt : 0);
+	x.right = y.left;
+
+	if (y.left !== SENTINEL) {
+		y.left.parent = x;
+	}
+	y.parent = x.parent;
+	if (x.parent === SENTINEL) {
+		tree.root = y;
+	} else if (x.parent.left === x) {
+		x.parent.left = y;
+	} else {
+		x.parent.right = y;
+	}
+	y.left = x;
+	x.parent = y;
+}
+
+export function rightRotate(tree: PieceTreeBase, y: TreeNode) {
+	let x = y.left;
+	y.left = x.right;
+	if (x.right !== SENTINEL) {
+		x.right.parent = y;
+	}
+	x.parent = y.parent;
+
+	// fix size_left
+	y.size_left -= x.size_left + (x.piece ? x.piece.length : 0);
+	y.lf_left -= x.lf_left + (x.piece ? x.piece.lineFeedCnt : 0);
+
+	if (y.parent === SENTINEL) {
+		tree.root = x;
+	} else if (y === y.parent.right) {
+		y.parent.right = x;
+	} else {
+		y.parent.left = x;
+	}
+
+	x.right = y;
+	y.parent = x;
+}
+
+export function rbDelete(tree: PieceTreeBase, z: TreeNode) {
+	let x: TreeNode;
+	let y: TreeNode;
+
+	if (z.left === SENTINEL) {
+		y = z;
+		x = y.right;
+	} else if (z.right === SENTINEL) {
+		y = z;
+		x = y.left;
+	} else {
+		y = leftest(z.right);
+		x = y.right;
+	}
+
+	if (y === tree.root) {
+		tree.root = x;
+
+		// if x is null, we are removing the only node
+		x.color = NodeColor.Black;
+		z.detach();
+		resetSentinel();
+		tree.root.parent = SENTINEL;
+
+		return;
+	}
+
+	let yWasRed = (y.color === NodeColor.Red);
+
+	if (y === y.parent.left) {
+		y.parent.left = x;
+	} else {
+		y.parent.right = x;
+	}
+
+	if (y === z) {
+		x.parent = y.parent;
+		recomputeTreeMetadata(tree, x);
+	} else {
+		if (y.parent === z) {
+			x.parent = y;
+		} else {
+			x.parent = y.parent;
+		}
+
+		// as we make changes to x's hierarchy, update size_left of subtree first
+		recomputeTreeMetadata(tree, x);
+
+		y.left = z.left;
+		y.right = z.right;
+		y.parent = z.parent;
+		y.color = z.color;
+
+		if (z === tree.root) {
+			tree.root = y;
+		} else {
+			if (z === z.parent.left) {
+				z.parent.left = y;
+			} else {
+				z.parent.right = y;
+			}
+		}
+
+		if (y.left !== SENTINEL) {
+			y.left.parent = y;
+		}
+		if (y.right !== SENTINEL) {
+			y.right.parent = y;
+		}
+		// update metadata
+		// we replace z with y, so in this sub tree, the length change is z.item.length
+		y.size_left = z.size_left;
+		y.lf_left = z.lf_left;
+		recomputeTreeMetadata(tree, y);
+	}
+
+	z.detach();
+
+	if (x.parent.left === x) {
+		let newSizeLeft = calculateSize(x);
+		let newLFLeft = calculateLF(x);
+		if (newSizeLeft !== x.parent.size_left || newLFLeft !== x.parent.lf_left) {
+			let delta = newSizeLeft - x.parent.size_left;
+			let lf_delta = newLFLeft - x.parent.lf_left;
+			x.parent.size_left = newSizeLeft;
+			x.parent.lf_left = newLFLeft;
+			updateTreeMetadata(tree, x.parent, delta, lf_delta);
+		}
+	}
+
+	recomputeTreeMetadata(tree, x.parent);
+
+	if (yWasRed) {
+		resetSentinel();
+		return;
+	}
+
+	// RB-DELETE-FIXUP
+	let w: TreeNode;
+	while (x !== tree.root && x.color === NodeColor.Black) {
+		if (x === x.parent.left) {
+			w = x.parent.right;
+
+			if (w.color === NodeColor.Red) {
+				w.color = NodeColor.Black;
+				x.parent.color = NodeColor.Red;
+				leftRotate(tree, x.parent);
+				w = x.parent.right;
+			}
+
+			if (w.left.color === NodeColor.Black && w.right.color === NodeColor.Black) {
+				w.color = NodeColor.Red;
+				x = x.parent;
+			} else {
+				if (w.right.color === NodeColor.Black) {
+					w.left.color = NodeColor.Black;
+					w.color = NodeColor.Red;
+					rightRotate(tree, w);
+					w = x.parent.right;
+				}
+
+				w.color = x.parent.color;
+				x.parent.color = NodeColor.Black;
+				w.right.color = NodeColor.Black;
+				leftRotate(tree, x.parent);
+				x = tree.root;
+			}
+		} else {
+			w = x.parent.left;
+
+			if (w.color === NodeColor.Red) {
+				w.color = NodeColor.Black;
+				x.parent.color = NodeColor.Red;
+				rightRotate(tree, x.parent);
+				w = x.parent.left;
+			}
+
+			if (w.left.color === NodeColor.Black && w.right.color === NodeColor.Black) {
+				w.color = NodeColor.Red;
+				x = x.parent;
+
+			} else {
+				if (w.left.color === NodeColor.Black) {
+					w.right.color = NodeColor.Black;
+					w.color = NodeColor.Red;
+					leftRotate(tree, w);
+					w = x.parent.left;
+				}
+
+				w.color = x.parent.color;
+				x.parent.color = NodeColor.Black;
+				w.left.color = NodeColor.Black;
+				rightRotate(tree, x.parent);
+				x = tree.root;
+			}
+		}
+	}
+	x.color = NodeColor.Black;
+	resetSentinel();
+}
+
+export function fixInsert(tree: PieceTreeBase, x: TreeNode) {
+	recomputeTreeMetadata(tree, x);
+
+	while (x !== tree.root && x.parent.color === NodeColor.Red) {
+		if (x.parent === x.parent.parent.left) {
+			const y = x.parent.parent.right;
+
+			if (y.color === NodeColor.Red) {
+				x.parent.color = NodeColor.Black;
+				y.color = NodeColor.Black;
+				x.parent.parent.color = NodeColor.Red;
+				x = x.parent.parent;
+			} else {
+				if (x === x.parent.right) {
+					x = x.parent;
+					leftRotate(tree, x);
+				}
+
+				x.parent.color = NodeColor.Black;
+				x.parent.parent.color = NodeColor.Red;
+				rightRotate(tree, x.parent.parent);
+			}
+		} else {
+			const y = x.parent.parent.left;
+
+			if (y.color === NodeColor.Red) {
+				x.parent.color = NodeColor.Black;
+				y.color = NodeColor.Black;
+				x.parent.parent.color = NodeColor.Red;
+				x = x.parent.parent;
+			} else {
+				if (x === x.parent.left) {
+					x = x.parent;
+					rightRotate(tree, x);
+				}
+				x.parent.color = NodeColor.Black;
+				x.parent.parent.color = NodeColor.Red;
+				leftRotate(tree, x.parent.parent);
+			}
+		}
+	}
+
+	tree.root.color = NodeColor.Black;
+}
+
+export function updateTreeMetadata(tree: PieceTreeBase, x: TreeNode, delta: number, lineFeedCntDelta: number): void {
+	// node length change or line feed count change
+	while (x !== tree.root && x !== SENTINEL) {
+		if (x.parent.left === x) {
+			x.parent.size_left += delta;
+			x.parent.lf_left += lineFeedCntDelta;
+		}
+
+		x = x.parent;
+	}
+}
+
+export function recomputeTreeMetadata(tree: PieceTreeBase, x: TreeNode) {
+	let delta = 0;
+	let lf_delta = 0;
+	if (x === tree.root) {
+		return;
+	}
+
+	if (delta === 0) {
+		// go upwards till the node whose left subtree is changed.
+		while (x !== tree.root && x === x.parent.right) {
+			x = x.parent;
+		}
+
+		if (x === tree.root) {
+			// well, it means we add a node to the end (inorder)
+			return;
+		}
+
+		// x is the node whose right subtree is changed.
+		x = x.parent;
+
+		delta = calculateSize(x.left) - x.size_left;
+		lf_delta = calculateLF(x.left) - x.lf_left;
+		x.size_left += delta;
+		x.lf_left += lf_delta;
+	}
+
+	// go upwards till root. O(logN)
+	while (x !== tree.root && (delta !== 0 || lf_delta !== 0)) {
+		if (x.parent.left === x) {
+			x.parent.size_left += delta;
+			x.parent.lf_left += lf_delta;
+		}
+
+		x = x.parent;
+	}
+}

src/vs/editor/common/model/textModelEvents.ts

@@ -176,9 +176,9 @@ export class ModelRawLinesInserted {
 	/**
 	 * The text that was inserted
 	 */
-	public readonly detail: string;
+	public readonly detail: string[];
 
-	constructor(fromLineNumber: number, toLineNumber: number, detail: string) {
+	constructor(fromLineNumber: number, toLineNumber: number, detail: string[]) {
 		this.fromLineNumber = fromLineNumber;
 		this.toLineNumber = toLineNumber;
 		this.detail = detail;
@@ -234,6 +234,14 @@ export class ModelRawContentChangedEvent {
 		}
 		return false;
 	}
+
+	public static merge(a: ModelRawContentChangedEvent, b: ModelRawContentChangedEvent): ModelRawContentChangedEvent {
+		const changes = [].concat(a.changes).concat(b.changes);
+		const versionId = b.versionId;
+		const isUndoing = (a.isUndoing || b.isUndoing);
+		const isRedoing = (a.isRedoing || b.isRedoing);
+		return new ModelRawContentChangedEvent(changes, versionId, isUndoing, isRedoing);
+	}
 }
 
 /**
@@ -244,4 +252,27 @@ export class InternalModelContentChangeEvent {
 		public readonly rawContentChangedEvent: ModelRawContentChangedEvent,
 		public readonly contentChangedEvent: IModelContentChangedEvent,
 	) { }
+
+	public merge(other: InternalModelContentChangeEvent): InternalModelContentChangeEvent {
+		const rawContentChangedEvent = ModelRawContentChangedEvent.merge(this.rawContentChangedEvent, other.rawContentChangedEvent);
+		const contentChangedEvent = InternalModelContentChangeEvent._mergeChangeEvents(this.contentChangedEvent, other.contentChangedEvent);
+		return new InternalModelContentChangeEvent(rawContentChangedEvent, contentChangedEvent);
+	}
+
+	private static _mergeChangeEvents(a: IModelContentChangedEvent, b: IModelContentChangedEvent): IModelContentChangedEvent {
+		const changes = [].concat(a.changes).concat(b.changes);
+		const eol = b.eol;
+		const versionId = b.versionId;
+		const isUndoing = (a.isUndoing || b.isUndoing);
+		const isRedoing = (a.isRedoing || b.isRedoing);
+		const isFlush = (a.isFlush || b.isFlush);
+		return {
+			changes: changes,
+			eol: eol,
+			versionId: versionId,
+			isUndoing: isUndoing,
+			isRedoing: isRedoing,
+			isFlush: isFlush
+		};
+	}
 }

src/vs/editor/common/model/textModelSearch.ts

@@ -7,7 +7,7 @@
 import * as strings from 'vs/base/common/strings';
 import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
-import { FindMatch, EndOfLinePreference } from 'vs/editor/common/editorCommon';
+import { FindMatch, EndOfLinePreference } from 'vs/editor/common/model';
 import { CharCode } from 'vs/base/common/charCode';
 import { TextModel } from 'vs/editor/common/model/textModel';
 import { getMapForWordSeparators, WordCharacterClassifier, WordCharacterClass } from 'vs/editor/common/controller/wordCharacterClassifier';
@@ -136,6 +136,23 @@ export class TextModelSearch {
 		}
 
 		if (searchData.regex.multiline) {
+			if (searchData.regex.source === '\\n') {
+				// Fast path for searching for EOL
+				let result: FindMatch[] = [], resultLen = 0;
+				for (let lineNumber = 1, lineCount = model.getLineCount(); lineNumber < lineCount; lineNumber++) {
+					const range = new Range(lineNumber, model.getLineMaxColumn(lineNumber), lineNumber + 1, 1);
+					if (captureMatches) {
+						result[resultLen++] = new FindMatch(range, null);
+					} else {
+						result[resultLen++] = new FindMatch(range, ['\n']);
+					}
+
+					if (resultLen >= limitResultCount) {
+						break;
+					}
+				}
+				return result;
+			}
 			return this._doFindMatchesMultiline(model, searchRange, new Searcher(searchData.wordSeparators, searchData.regex), captureMatches, limitResultCount);
 		}
 		return this._doFindMatchesLineByLine(model, searchRange, searchData, captureMatches, limitResultCount);

src/vs/editor/common/model/textModelTokens.ts

@@ -0,0 +1,476 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { IState, FontStyle, StandardTokenType, MetadataConsts, ColorId, LanguageId, ITokenizationSupport, LanguageIdentifier } from 'vs/editor/common/modes';
+import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import * as arrays from 'vs/base/common/arrays';
+import { Position } from 'vs/editor/common/core/position';
+import { Range } from 'vs/editor/common/core/range';
+import { IModelTokensChangedEvent } from 'vs/editor/common/model/textModelEvents';
+import { onUnexpectedError } from 'vs/base/common/errors';
+import { TokenizationResult2 } from 'vs/editor/common/core/token';
+import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
+import { ITextBuffer } from 'vs/editor/common/model';
+
+function getDefaultMetadata(topLevelLanguageId: LanguageId): number {
+	return (
+		(topLevelLanguageId << MetadataConsts.LANGUAGEID_OFFSET)
+		| (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET)
+		| (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET)
+		| (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET)
+		| (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET)
+	) >>> 0;
+}
+
+const EMPTY_LINE_TOKENS = new Uint32Array(0);
+
+class ModelLineTokens {
+	_state: IState;
+	_lineTokens: ArrayBuffer;
+	_invalid: boolean;
+
+	constructor(state: IState) {
+		this._state = state;
+		this._lineTokens = null;
+		this._invalid = true;
+	}
+
+	public deleteBeginning(toChIndex: number): void {
+		if (this._lineTokens === null || this._lineTokens === EMPTY_LINE_TOKENS) {
+			return;
+		}
+		this.delete(0, toChIndex);
+	}
+
+	public deleteEnding(fromChIndex: number): void {
+		if (this._lineTokens === null || this._lineTokens === EMPTY_LINE_TOKENS) {
+			return;
+		}
+
+		const tokens = new Uint32Array(this._lineTokens);
+		const lineTextLength = tokens[tokens.length - 2];
+		this.delete(fromChIndex, lineTextLength);
+	}
+
+	public delete(fromChIndex: number, toChIndex: number): void {
+		if (this._lineTokens === null || this._lineTokens === EMPTY_LINE_TOKENS || fromChIndex === toChIndex) {
+			return;
+		}
+
+		const tokens = new Uint32Array(this._lineTokens);
+		const tokensCount = (tokens.length >>> 1);
+
+		// special case: deleting everything
+		if (fromChIndex === 0 && tokens[tokens.length - 2] === toChIndex) {
+			this._lineTokens = EMPTY_LINE_TOKENS;
+			return;
+		}
+
+		const fromTokenIndex = LineTokens.findIndexInTokensArray(tokens, fromChIndex);
+		const fromTokenStartOffset = (fromTokenIndex > 0 ? tokens[(fromTokenIndex - 1) << 1] : 0);
+		const fromTokenEndOffset = tokens[fromTokenIndex << 1];
+
+		if (toChIndex < fromTokenEndOffset) {
+			// the delete range is inside a single token
+			const delta = (toChIndex - fromChIndex);
+			for (let i = fromTokenIndex; i < tokensCount; i++) {
+				tokens[i << 1] -= delta;
+			}
+			return;
+		}
+
+		let dest: number;
+		let lastEnd: number;
+		if (fromTokenStartOffset !== fromChIndex) {
+			tokens[fromTokenIndex << 1] = fromChIndex;
+			dest = ((fromTokenIndex + 1) << 1);
+			lastEnd = fromChIndex;
+		} else {
+			dest = (fromTokenIndex << 1);
+			lastEnd = fromTokenStartOffset;
+		}
+
+		const delta = (toChIndex - fromChIndex);
+		for (let tokenIndex = fromTokenIndex + 1; tokenIndex < tokensCount; tokenIndex++) {
+			const tokenEndOffset = tokens[tokenIndex << 1] - delta;
+			if (tokenEndOffset > lastEnd) {
+				tokens[dest++] = tokenEndOffset;
+				tokens[dest++] = tokens[(tokenIndex << 1) + 1];
+				lastEnd = tokenEndOffset;
+			}
+		}
+
+		if (dest === tokens.length) {
+			// nothing to trim
+			return;
+		}
+
+		let tmp = new Uint32Array(dest);
+		tmp.set(tokens.subarray(0, dest), 0);
+		this._lineTokens = tmp.buffer;
+	}
+
+	public append(_otherTokens: ArrayBuffer): void {
+		if (_otherTokens === EMPTY_LINE_TOKENS) {
+			return;
+		}
+		if (this._lineTokens === EMPTY_LINE_TOKENS) {
+			this._lineTokens = _otherTokens;
+			return;
+		}
+		if (this._lineTokens === null) {
+			return;
+		}
+		if (_otherTokens === null) {
+			// cannot determine combined line length...
+			this._lineTokens = null;
+			return;
+		}
+		const myTokens = new Uint32Array(this._lineTokens);
+		const otherTokens = new Uint32Array(_otherTokens);
+		const otherTokensCount = (otherTokens.length >>> 1);
+
+		let result = new Uint32Array(myTokens.length + otherTokens.length);
+		result.set(myTokens, 0);
+		let dest = myTokens.length;
+		const delta = myTokens[myTokens.length - 2];
+		for (let i = 0; i < otherTokensCount; i++) {
+			result[dest++] = otherTokens[(i << 1)] + delta;
+			result[dest++] = otherTokens[(i << 1) + 1];
+		}
+		this._lineTokens = result.buffer;
+	}
+
+	public insert(chIndex: number, textLength: number): void {
+		if (!this._lineTokens) {
+			// nothing to do
+			return;
+		}
+
+		const tokens = new Uint32Array(this._lineTokens);
+		const tokensCount = (tokens.length >>> 1);
+
+		let fromTokenIndex = LineTokens.findIndexInTokensArray(tokens, chIndex);
+		if (fromTokenIndex > 0) {
+			const fromTokenStartOffset = (fromTokenIndex > 0 ? tokens[(fromTokenIndex - 1) << 1] : 0);
+			if (fromTokenStartOffset === chIndex) {
+				fromTokenIndex--;
+			}
+		}
+		for (let tokenIndex = fromTokenIndex; tokenIndex < tokensCount; tokenIndex++) {
+			tokens[tokenIndex << 1] += textLength;
+		}
+	}
+}
+
+export class ModelLinesTokens {
+
+	public readonly languageIdentifier: LanguageIdentifier;
+	public readonly tokenizationSupport: ITokenizationSupport;
+	private _tokens: ModelLineTokens[];
+	private _invalidLineStartIndex: number;
+	private _lastState: IState;
+
+	constructor(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport) {
+		this.languageIdentifier = languageIdentifier;
+		this.tokenizationSupport = tokenizationSupport;
+		this._tokens = [];
+		if (this.tokenizationSupport) {
+			let initialState: IState = null;
+			try {
+				initialState = this.tokenizationSupport.getInitialState();
+			} catch (e) {
+				onUnexpectedError(e);
+				this.tokenizationSupport = null;
+			}
+
+			if (initialState) {
+				this._tokens[0] = new ModelLineTokens(initialState);
+			}
+		}
+
+		this._invalidLineStartIndex = 0;
+		this._lastState = null;
+	}
+
+	public getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens {
+		let rawLineTokens: ArrayBuffer = null;
+		if (lineIndex < this._tokens.length) {
+			rawLineTokens = this._tokens[lineIndex]._lineTokens;
+		}
+
+		if (rawLineTokens !== null && rawLineTokens !== EMPTY_LINE_TOKENS) {
+			return new LineTokens(new Uint32Array(rawLineTokens), lineText);
+		}
+
+		let lineTokens = new Uint32Array(2);
+		lineTokens[0] = lineText.length;
+		lineTokens[1] = getDefaultMetadata(topLevelLanguageId);
+		return new LineTokens(lineTokens, lineText);
+	}
+
+	public isCheapToTokenize(lineNumber: number): boolean {
+		const firstInvalidLineNumber = this._invalidLineStartIndex + 1;
+		return (firstInvalidLineNumber >= lineNumber);
+	}
+
+	public hasLinesToTokenize(buffer: ITextBuffer): boolean {
+		return (this._invalidLineStartIndex < buffer.getLineCount());
+	}
+
+	public invalidateLine(lineIndex: number): void {
+		this._setIsInvalid(lineIndex, true);
+		if (lineIndex < this._invalidLineStartIndex) {
+			this._setIsInvalid(this._invalidLineStartIndex, true);
+			this._invalidLineStartIndex = lineIndex;
+		}
+	}
+
+	private _setIsInvalid(lineIndex: number, invalid: boolean): void {
+		if (lineIndex < this._tokens.length) {
+			this._tokens[lineIndex]._invalid = invalid;
+		}
+	}
+
+	_isInvalid(lineIndex: number): boolean {
+		if (lineIndex < this._tokens.length) {
+			return this._tokens[lineIndex]._invalid;
+		}
+		return true;
+	}
+
+	_getState(lineIndex: number): IState {
+		if (lineIndex < this._tokens.length) {
+			return this._tokens[lineIndex]._state;
+		}
+		return null;
+	}
+
+	_setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
+		let target: ModelLineTokens;
+		if (lineIndex < this._tokens.length) {
+			target = this._tokens[lineIndex];
+		} else {
+			target = new ModelLineTokens(null);
+			this._tokens[lineIndex] = target;
+		}
+
+		if (lineTextLength === 0) {
+			target._lineTokens = EMPTY_LINE_TOKENS;
+			return;
+		}
+
+		if (!tokens || tokens.length === 0) {
+			tokens = new Uint32Array(2);
+			tokens[0] = 0;
+			tokens[1] = getDefaultMetadata(topLevelLanguageId);
+		}
+
+		LineTokens.convertToEndOffset(tokens, lineTextLength);
+
+		target._lineTokens = tokens.buffer;
+	}
+
+	private _setState(lineIndex: number, state: IState): void {
+		if (lineIndex < this._tokens.length) {
+			this._tokens[lineIndex]._state = state;
+		} else {
+			const tmp = new ModelLineTokens(state);
+			this._tokens[lineIndex] = tmp;
+		}
+	}
+
+	//#region Editing
+
+	public applyEdits(range: Range, eolCount: number, firstLineLength: number): void {
+
+		const deletingLinesCnt = range.endLineNumber - range.startLineNumber;
+		const insertingLinesCnt = eolCount;
+		const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt);
+
+		for (let j = editingLinesCnt; j >= 0; j--) {
+			this.invalidateLine(range.startLineNumber + j - 1);
+		}
+
+		this._acceptDeleteRange(range);
+		this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength);
+	}
+
+	private _acceptDeleteRange(range: Range): void {
+
+		const firstLineIndex = range.startLineNumber - 1;
+		if (firstLineIndex >= this._tokens.length) {
+			return;
+		}
+
+		if (range.startLineNumber === range.endLineNumber) {
+			if (range.startColumn === range.endColumn) {
+				// Nothing to delete
+				return;
+			}
+
+			this._tokens[firstLineIndex].delete(range.startColumn - 1, range.endColumn - 1);
+			return;
+		}
+
+		const firstLine = this._tokens[firstLineIndex];
+		firstLine.deleteEnding(range.startColumn - 1);
+
+		const lastLineIndex = range.endLineNumber - 1;
+		let lastLineTokens: ArrayBuffer = null;
+		if (lastLineIndex < this._tokens.length) {
+			const lastLine = this._tokens[lastLineIndex];
+			lastLine.deleteBeginning(range.endColumn - 1);
+			lastLineTokens = lastLine._lineTokens;
+		}
+
+		// Take remaining text on last line and append it to remaining text on first line
+		firstLine.append(lastLineTokens);
+
+		// Delete middle lines
+		this._tokens.splice(range.startLineNumber, range.endLineNumber - range.startLineNumber);
+	}
+
+	private _acceptInsertText(position: Position, eolCount: number, firstLineLength: number): void {
+
+		if (eolCount === 0 && firstLineLength === 0) {
+			// Nothing to insert
+			return;
+		}
+
+		const lineIndex = position.lineNumber - 1;
+		if (lineIndex >= this._tokens.length) {
+			return;
+		}
+
+		if (eolCount === 0) {
+			// Inserting text on one line
+			this._tokens[lineIndex].insert(position.column - 1, firstLineLength);
+			return;
+		}
+
+		const line = this._tokens[lineIndex];
+		line.deleteEnding(position.column - 1);
+		line.insert(position.column - 1, firstLineLength);
+
+		let insert: ModelLineTokens[] = new Array<ModelLineTokens>(eolCount);
+		for (let i = eolCount - 1; i >= 0; i--) {
+			insert[i] = new ModelLineTokens(null);
+		}
+		this._tokens = arrays.arrayInsert(this._tokens, position.lineNumber, insert);
+	}
+
+	//#endregion
+
+	//#region Tokenization
+
+	public _tokenizeOneLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number {
+		if (!this.hasLinesToTokenize(buffer)) {
+			return buffer.getLineCount() + 1;
+		}
+		const lineNumber = this._invalidLineStartIndex + 1;
+		this._updateTokensUntilLine(buffer, eventBuilder, lineNumber);
+		return lineNumber;
+	}
+
+	public _updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
+		if (!this.tokenizationSupport) {
+			this._invalidLineStartIndex = buffer.getLineCount();
+			return;
+		}
+
+		const linesLength = buffer.getLineCount();
+		const endLineIndex = lineNumber - 1;
+
+		// Validate all states up to and including endLineIndex
+		for (let lineIndex = this._invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) {
+			const endStateIndex = lineIndex + 1;
+			let r: TokenizationResult2 = null;
+			const text = buffer.getLineContent(lineIndex + 1);
+
+			try {
+				// Tokenize only the first X characters
+				let freshState = this._getState(lineIndex).clone();
+				r = this.tokenizationSupport.tokenize2(text, freshState, 0);
+			} catch (e) {
+				onUnexpectedError(e);
+			}
+
+			if (!r) {
+				r = nullTokenize2(this.languageIdentifier.id, text, this._getState(lineIndex), 0);
+			}
+			this._setTokens(this.languageIdentifier.id, lineIndex, text.length, r.tokens);
+			eventBuilder.registerChangedTokens(lineIndex + 1);
+			this._setIsInvalid(lineIndex, false);
+
+			if (endStateIndex < linesLength) {
+				if (this._getState(endStateIndex) !== null && r.endState.equals(this._getState(endStateIndex))) {
+					// The end state of this line remains the same
+					let nextInvalidLineIndex = lineIndex + 1;
+					while (nextInvalidLineIndex < linesLength) {
+						if (this._isInvalid(nextInvalidLineIndex)) {
+							break;
+						}
+						if (nextInvalidLineIndex + 1 < linesLength) {
+							if (this._getState(nextInvalidLineIndex + 1) === null) {
+								break;
+							}
+						} else {
+							if (this._lastState === null) {
+								break;
+							}
+						}
+						nextInvalidLineIndex++;
+					}
+					this._invalidLineStartIndex = Math.max(this._invalidLineStartIndex, nextInvalidLineIndex);
+					lineIndex = nextInvalidLineIndex - 1; // -1 because the outer loop increments it
+				} else {
+					this._setState(endStateIndex, r.endState);
+				}
+			} else {
+				this._lastState = r.endState;
+			}
+		}
+		this._invalidLineStartIndex = Math.max(this._invalidLineStartIndex, endLineIndex + 1);
+	}
+
+	// #endregion
+}
+
+export class ModelTokensChangedEventBuilder {
+
+	private _ranges: { fromLineNumber: number; toLineNumber: number; }[];
+
+	constructor() {
+		this._ranges = [];
+	}
+
+	public registerChangedTokens(lineNumber: number): void {
+		const ranges = this._ranges;
+		const rangesLength = ranges.length;
+		const previousRange = rangesLength > 0 ? ranges[rangesLength - 1] : null;
+
+		if (previousRange && previousRange.toLineNumber === lineNumber - 1) {
+			// extend previous range
+			previousRange.toLineNumber++;
+		} else {
+			// insert new range
+			ranges[rangesLength] = {
+				fromLineNumber: lineNumber,
+				toLineNumber: lineNumber
+			};
+		}
+	}
+
+	public build(): IModelTokensChangedEvent {
+		if (this._ranges.length === 0) {
+			return null;
+		}
+		return {
+			ranges: this._ranges
+		};
+	}
+}

src/vs/editor/common/model/textModelWithDecorations.ts

@@ -1,624 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import { onUnexpectedError } from 'vs/base/common/errors';
-import { IMarkdownString } from 'vs/base/common/htmlContent';
-import * as strings from 'vs/base/common/strings';
-import { CharCode } from 'vs/base/common/charCode';
-import Event, { Emitter } from 'vs/base/common/event';
-import { Range, IRange } from 'vs/editor/common/core/range';
-import * as editorCommon from 'vs/editor/common/editorCommon';
-import { TextModelWithTokens } from 'vs/editor/common/model/textModelWithTokens';
-import { LanguageIdentifier } from 'vs/editor/common/modes';
-import { ITextSource, IRawTextSource } from 'vs/editor/common/model/textSource';
-import { IModelDecorationsChangedEvent } from 'vs/editor/common/model/textModelEvents';
-import { ThemeColor } from 'vs/platform/theme/common/themeService';
-import { IntervalNode, IntervalTree, recomputeMaxEnd, getNodeIsInOverviewRuler } from 'vs/editor/common/model/intervalTree';
-import { Disposable } from 'vs/base/common/lifecycle';
-
-let _INSTANCE_COUNT = 0;
-/**
- * Produces 'a'-'z', followed by 'A'-'Z'... followed by 'a'-'z', etc.
- */
-function nextInstanceId(): string {
-	const LETTERS_CNT = (CharCode.Z - CharCode.A + 1);
-
-	let result = _INSTANCE_COUNT++;
-	result = result % (2 * LETTERS_CNT);
-
-	if (result < LETTERS_CNT) {
-		return String.fromCharCode(CharCode.a + result);
-	}
-
-	return String.fromCharCode(CharCode.A + result - LETTERS_CNT);
-}
-
-export class TextModelWithDecorations extends TextModelWithTokens implements editorCommon.ITextModelWithDecorations {
-
-	protected readonly _onDidChangeDecorations: DidChangeDecorationsEmitter = this._register(new DidChangeDecorationsEmitter());
-	public readonly onDidChangeDecorations: Event<IModelDecorationsChangedEvent> = this._onDidChangeDecorations.event;
-
-	/**
-	 * Used to workaround broken clients that might attempt using a decoration id generated by a different model.
-	 * It is not globally unique in order to limit it to one character.
-	 */
-	private readonly _instanceId: string;
-	private _lastDecorationId: number;
-	private _decorations: { [decorationId: string]: IntervalNode; };
-	private _decorationsTree: DecorationsTrees;
-
-	constructor(rawTextSource: IRawTextSource, creationOptions: editorCommon.ITextModelCreationOptions, languageIdentifier: LanguageIdentifier) {
-		super(rawTextSource, creationOptions, languageIdentifier);
-
-		this._instanceId = nextInstanceId();
-		this._lastDecorationId = 0;
-		this._decorations = Object.create(null);
-		this._decorationsTree = new DecorationsTrees();
-	}
-
-	public dispose(): void {
-		this._decorations = null;
-		this._decorationsTree = null;
-
-		super.dispose();
-	}
-
-	protected _resetValue(newValue: ITextSource): void {
-		super._resetValue(newValue);
-
-		// Destroy all my decorations
-		this._decorations = Object.create(null);
-		this._decorationsTree = new DecorationsTrees();
-	}
-
-	// --- END TrackedRanges
-
-	protected _adjustDecorationsForEdit(offset: number, length: number, textLength: number, forceMoveMarkers: boolean): void {
-		this._onDidChangeDecorations.fire();
-		this._decorationsTree.acceptReplace(offset, length, textLength, forceMoveMarkers);
-	}
-
-	protected _onBeforeEOLChange(): void {
-		super._onBeforeEOLChange();
-
-		// Ensure all decorations get their `range` set.
-		const versionId = this.getVersionId();
-		const allDecorations = this._decorationsTree.search(0, false, false, versionId);
-		this._ensureNodesHaveRanges(allDecorations);
-	}
-
-	protected _onAfterEOLChange(): void {
-		super._onAfterEOLChange();
-
-		// Transform back `range` to offsets
-		const versionId = this.getVersionId();
-		const allDecorations = this._decorationsTree.collectNodesPostOrder();
-		for (let i = 0, len = allDecorations.length; i < len; i++) {
-			const node = allDecorations[i];
-
-			const delta = node.cachedAbsoluteStart - node.start;
-
-			const startOffset = this._lineStarts.getAccumulatedValue(node.range.startLineNumber - 2) + node.range.startColumn - 1;
-			const endOffset = this._lineStarts.getAccumulatedValue(node.range.endLineNumber - 2) + node.range.endColumn - 1;
-
-			node.cachedAbsoluteStart = startOffset;
-			node.cachedAbsoluteEnd = endOffset;
-			node.cachedVersionId = versionId;
-
-			node.start = startOffset - delta;
-			node.end = endOffset - delta;
-
-			recomputeMaxEnd(node);
-		}
-	}
-
-	public changeDecorations<T>(callback: (changeAccessor: editorCommon.IModelDecorationsChangeAccessor) => T, ownerId: number = 0): T {
-		this._assertNotDisposed();
-
-		try {
-			this._onDidChangeDecorations.beginDeferredEmit();
-			return this._changeDecorations(ownerId, callback);
-		} finally {
-			this._onDidChangeDecorations.endDeferredEmit();
-		}
-	}
-
-	private _changeDecorations<T>(ownerId: number, callback: (changeAccessor: editorCommon.IModelDecorationsChangeAccessor) => T): T {
-		let changeAccessor: editorCommon.IModelDecorationsChangeAccessor = {
-			addDecoration: (range: IRange, options: editorCommon.IModelDecorationOptions): string => {
-				this._onDidChangeDecorations.fire();
-				return this._deltaDecorationsImpl(ownerId, [], [{ range: range, options: options }])[0];
-			},
-			changeDecoration: (id: string, newRange: IRange): void => {
-				this._onDidChangeDecorations.fire();
-				this._changeDecorationImpl(id, newRange);
-			},
-			changeDecorationOptions: (id: string, options: editorCommon.IModelDecorationOptions) => {
-				this._onDidChangeDecorations.fire();
-				this._changeDecorationOptionsImpl(id, _normalizeOptions(options));
-			},
-			removeDecoration: (id: string): void => {
-				this._onDidChangeDecorations.fire();
-				this._deltaDecorationsImpl(ownerId, [id], []);
-			},
-			deltaDecorations: (oldDecorations: string[], newDecorations: editorCommon.IModelDeltaDecoration[]): string[] => {
-				if (oldDecorations.length === 0 && newDecorations.length === 0) {
-					// nothing to do
-					return [];
-				}
-				this._onDidChangeDecorations.fire();
-				return this._deltaDecorationsImpl(ownerId, oldDecorations, newDecorations);
-			}
-		};
-		let result: T = null;
-		try {
-			result = callback(changeAccessor);
-		} catch (e) {
-			onUnexpectedError(e);
-		}
-		// Invalidate change accessor
-		changeAccessor.addDecoration = null;
-		changeAccessor.changeDecoration = null;
-		changeAccessor.removeDecoration = null;
-		changeAccessor.deltaDecorations = null;
-		return result;
-	}
-
-	public deltaDecorations(oldDecorations: string[], newDecorations: editorCommon.IModelDeltaDecoration[], ownerId: number = 0): string[] {
-		this._assertNotDisposed();
-		if (!oldDecorations) {
-			oldDecorations = [];
-		}
-		if (oldDecorations.length === 0 && newDecorations.length === 0) {
-			// nothing to do
-			return [];
-		}
-
-		try {
-			this._onDidChangeDecorations.beginDeferredEmit();
-			this._onDidChangeDecorations.fire();
-			return this._deltaDecorationsImpl(ownerId, oldDecorations, newDecorations);
-		} finally {
-			this._onDidChangeDecorations.endDeferredEmit();
-		}
-	}
-
-	_getTrackedRange(id: string): Range {
-		return this.getDecorationRange(id);
-	}
-
-	_setTrackedRange(id: string, newRange: Range, newStickiness: editorCommon.TrackedRangeStickiness): string {
-		const node = (id ? this._decorations[id] : null);
-
-		if (!node) {
-			if (!newRange) {
-				// node doesn't exist, the request is to delete => nothing to do
-				return null;
-			}
-			// node doesn't exist, the request is to set => add the tracked range
-			return this._deltaDecorationsImpl(0, [], [{ range: newRange, options: TRACKED_RANGE_OPTIONS[newStickiness] }])[0];
-		}
-
-		if (!newRange) {
-			// node exists, the request is to delete => delete node
-			this._decorationsTree.delete(node);
-			delete this._decorations[node.id];
-			return null;
-		}
-
-		// node exists, the request is to set => change the tracked range and its options
-		const range = this._validateRangeRelaxedNoAllocations(newRange);
-		const startOffset = this._lineStarts.getAccumulatedValue(range.startLineNumber - 2) + range.startColumn - 1;
-		const endOffset = this._lineStarts.getAccumulatedValue(range.endLineNumber - 2) + range.endColumn - 1;
-		this._decorationsTree.delete(node);
-		node.reset(this.getVersionId(), startOffset, endOffset, range);
-		node.setOptions(TRACKED_RANGE_OPTIONS[newStickiness]);
-		this._decorationsTree.insert(node);
-		return node.id;
-	}
-
-	public removeAllDecorationsWithOwnerId(ownerId: number): void {
-		if (this._isDisposed) {
-			return;
-		}
-		const nodes = this._decorationsTree.collectNodesFromOwner(ownerId);
-		for (let i = 0, len = nodes.length; i < len; i++) {
-			const node = nodes[i];
-
-			this._decorationsTree.delete(node);
-			delete this._decorations[node.id];
-		}
-	}
-
-	public getDecorationOptions(decorationId: string): editorCommon.IModelDecorationOptions {
-		const node = this._decorations[decorationId];
-		if (!node) {
-			return null;
-		}
-		return node.options;
-	}
-
-	public getDecorationRange(decorationId: string): Range {
-		const node = this._decorations[decorationId];
-		if (!node) {
-			return null;
-		}
-		const versionId = this.getVersionId();
-		if (node.cachedVersionId !== versionId) {
-			this._decorationsTree.resolveNode(node, versionId);
-		}
-		if (node.range === null) {
-			node.range = this._getRangeAt(node.cachedAbsoluteStart, node.cachedAbsoluteEnd);
-		}
-		return node.range;
-	}
-
-	public getLineDecorations(lineNumber: number, ownerId: number = 0, filterOutValidation: boolean = false): editorCommon.IModelDecoration[] {
-		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
-			return [];
-		}
-
-		return this.getLinesDecorations(lineNumber, lineNumber, ownerId, filterOutValidation);
-	}
-
-	public getLinesDecorations(_startLineNumber: number, _endLineNumber: number, ownerId: number = 0, filterOutValidation: boolean = false): editorCommon.IModelDecoration[] {
-		let lineCount = this.getLineCount();
-		let startLineNumber = Math.min(lineCount, Math.max(1, _startLineNumber));
-		let endLineNumber = Math.min(lineCount, Math.max(1, _endLineNumber));
-		let endColumn = this.getLineMaxColumn(endLineNumber);
-		return this._getDecorationsInRange(new Range(startLineNumber, 1, endLineNumber, endColumn), ownerId, filterOutValidation);
-	}
-
-	public getDecorationsInRange(range: IRange, ownerId: number = 0, filterOutValidation: boolean = false): editorCommon.IModelDecoration[] {
-		let validatedRange = this.validateRange(range);
-		return this._getDecorationsInRange(validatedRange, ownerId, filterOutValidation);
-	}
-
-	public getOverviewRulerDecorations(ownerId: number = 0, filterOutValidation: boolean = false): editorCommon.IModelDecoration[] {
-		const versionId = this.getVersionId();
-		const result = this._decorationsTree.search(ownerId, filterOutValidation, true, versionId);
-		return this._ensureNodesHaveRanges(result);
-	}
-
-	public getAllDecorations(ownerId: number = 0, filterOutValidation: boolean = false): editorCommon.IModelDecoration[] {
-		const versionId = this.getVersionId();
-		const result = this._decorationsTree.search(ownerId, filterOutValidation, false, versionId);
-		return this._ensureNodesHaveRanges(result);
-	}
-
-	private _getDecorationsInRange(filterRange: Range, filterOwnerId: number, filterOutValidation: boolean): IntervalNode[] {
-		const startOffset = this._lineStarts.getAccumulatedValue(filterRange.startLineNumber - 2) + filterRange.startColumn - 1;
-		const endOffset = this._lineStarts.getAccumulatedValue(filterRange.endLineNumber - 2) + filterRange.endColumn - 1;
-
-		const versionId = this.getVersionId();
-		const result = this._decorationsTree.intervalSearch(startOffset, endOffset, filterOwnerId, filterOutValidation, versionId);
-
-		return this._ensureNodesHaveRanges(result);
-	}
-
-	private _ensureNodesHaveRanges(nodes: IntervalNode[]): IntervalNode[] {
-		for (let i = 0, len = nodes.length; i < len; i++) {
-			const node = nodes[i];
-			if (node.range === null) {
-				node.range = this._getRangeAt(node.cachedAbsoluteStart, node.cachedAbsoluteEnd);
-			}
-		}
-		return nodes;
-	}
-
-	private _getRangeAt(start: number, end: number): Range {
-		const startResult = this._lineStarts.getIndexOf(start);
-		const startLineLength = this._lines[startResult.index].text.length;
-		const startColumn = Math.min(startResult.remainder + 1, startLineLength + 1);
-
-		const endResult = this._lineStarts.getIndexOf(end);
-		const endLineLength = this._lines[endResult.index].text.length;
-		const endColumn = Math.min(endResult.remainder + 1, endLineLength + 1);
-
-		return new Range(startResult.index + 1, startColumn, endResult.index + 1, endColumn);
-	}
-
-	private _changeDecorationImpl(decorationId: string, _range: IRange): void {
-		const node = this._decorations[decorationId];
-		if (!node) {
-			return;
-		}
-		const range = this._validateRangeRelaxedNoAllocations(_range);
-		const startOffset = this._lineStarts.getAccumulatedValue(range.startLineNumber - 2) + range.startColumn - 1;
-		const endOffset = this._lineStarts.getAccumulatedValue(range.endLineNumber - 2) + range.endColumn - 1;
-
-		this._decorationsTree.delete(node);
-		node.reset(this.getVersionId(), startOffset, endOffset, range);
-		this._decorationsTree.insert(node);
-	}
-
-	private _changeDecorationOptionsImpl(decorationId: string, options: ModelDecorationOptions): void {
-		const node = this._decorations[decorationId];
-		if (!node) {
-			return;
-		}
-
-		const nodeWasInOverviewRuler = (node.options.overviewRuler.color ? true : false);
-		const nodeIsInOverviewRuler = (options.overviewRuler.color ? true : false);
-
-		if (nodeWasInOverviewRuler !== nodeIsInOverviewRuler) {
-			// Delete + Insert due to an overview ruler status change
-			this._decorationsTree.delete(node);
-			node.setOptions(options);
-			this._decorationsTree.insert(node);
-		} else {
-			node.setOptions(options);
-		}
-	}
-
-	private _deltaDecorationsImpl(ownerId: number, oldDecorationsIds: string[], newDecorations: editorCommon.IModelDeltaDecoration[]): string[] {
-		const versionId = this.getVersionId();
-
-		const oldDecorationsLen = oldDecorationsIds.length;
-		let oldDecorationIndex = 0;
-
-		const newDecorationsLen = newDecorations.length;
-		let newDecorationIndex = 0;
-
-		let result = new Array<string>(newDecorationsLen);
-		while (oldDecorationIndex < oldDecorationsLen || newDecorationIndex < newDecorationsLen) {
-
-			let node: IntervalNode = null;
-
-			if (oldDecorationIndex < oldDecorationsLen) {
-				// (1) get ourselves an old node
-				do {
-					node = this._decorations[oldDecorationsIds[oldDecorationIndex++]];
-				} while (!node && oldDecorationIndex < oldDecorationsLen);
-
-				// (2) remove the node from the tree (if it exists)
-				if (node) {
-					this._decorationsTree.delete(node);
-				}
-			}
-
-			if (newDecorationIndex < newDecorationsLen) {
-				// (3) create a new node if necessary
-				if (!node) {
-					const internalDecorationId = (++this._lastDecorationId);
-					const decorationId = `${this._instanceId};${internalDecorationId}`;
-					node = new IntervalNode(decorationId, 0, 0);
-					this._decorations[decorationId] = node;
-				}
-
-				// (4) initialize node
-				const newDecoration = newDecorations[newDecorationIndex];
-				const range = this._validateRangeRelaxedNoAllocations(newDecoration.range);
-				const options = _normalizeOptions(newDecoration.options);
-				const startOffset = this._lineStarts.getAccumulatedValue(range.startLineNumber - 2) + range.startColumn - 1;
-				const endOffset = this._lineStarts.getAccumulatedValue(range.endLineNumber - 2) + range.endColumn - 1;
-
-				node.ownerId = ownerId;
-				node.reset(versionId, startOffset, endOffset, range);
-				node.setOptions(options);
-
-				this._decorationsTree.insert(node);
-
-				result[newDecorationIndex] = node.id;
-
-				newDecorationIndex++;
-			} else {
-				if (node) {
-					delete this._decorations[node.id];
-				}
-			}
-		}
-
-		return result;
-	}
-}
-
-class DecorationsTrees {
-
-	/**
-	 * This tree holds decorations that do not show up in the overview ruler.
-	 */
-	private _decorationsTree0: IntervalTree;
-
-	/**
-	 * This tree holds decorations that show up in the overview ruler.
-	 */
-	private _decorationsTree1: IntervalTree;
-
-	constructor() {
-		this._decorationsTree0 = new IntervalTree();
-		this._decorationsTree1 = new IntervalTree();
-	}
-
-	public intervalSearch(start: number, end: number, filterOwnerId: number, filterOutValidation: boolean, cachedVersionId: number): IntervalNode[] {
-		const r0 = this._decorationsTree0.intervalSearch(start, end, filterOwnerId, filterOutValidation, cachedVersionId);
-		const r1 = this._decorationsTree1.intervalSearch(start, end, filterOwnerId, filterOutValidation, cachedVersionId);
-		return r0.concat(r1);
-	}
-
-	public search(filterOwnerId: number, filterOutValidation: boolean, overviewRulerOnly: boolean, cachedVersionId: number): IntervalNode[] {
-		if (overviewRulerOnly) {
-			return this._decorationsTree1.search(filterOwnerId, filterOutValidation, cachedVersionId);
-		} else {
-			const r0 = this._decorationsTree0.search(filterOwnerId, filterOutValidation, cachedVersionId);
-			const r1 = this._decorationsTree1.search(filterOwnerId, filterOutValidation, cachedVersionId);
-			return r0.concat(r1);
-		}
-	}
-
-	public collectNodesFromOwner(ownerId: number): IntervalNode[] {
-		const r0 = this._decorationsTree0.collectNodesFromOwner(ownerId);
-		const r1 = this._decorationsTree1.collectNodesFromOwner(ownerId);
-		return r0.concat(r1);
-	}
-
-	public collectNodesPostOrder(): IntervalNode[] {
-		const r0 = this._decorationsTree0.collectNodesPostOrder();
-		const r1 = this._decorationsTree1.collectNodesPostOrder();
-		return r0.concat(r1);
-	}
-
-	public insert(node: IntervalNode): void {
-		if (getNodeIsInOverviewRuler(node)) {
-			this._decorationsTree1.insert(node);
-		} else {
-			this._decorationsTree0.insert(node);
-		}
-	}
-
-	public delete(node: IntervalNode): void {
-		if (getNodeIsInOverviewRuler(node)) {
-			this._decorationsTree1.delete(node);
-		} else {
-			this._decorationsTree0.delete(node);
-		}
-	}
-
-	public resolveNode(node: IntervalNode, cachedVersionId: number): void {
-		if (getNodeIsInOverviewRuler(node)) {
-			this._decorationsTree1.resolveNode(node, cachedVersionId);
-		} else {
-			this._decorationsTree0.resolveNode(node, cachedVersionId);
-		}
-	}
-
-	public acceptReplace(offset: number, length: number, textLength: number, forceMoveMarkers: boolean): void {
-		this._decorationsTree0.acceptReplace(offset, length, textLength, forceMoveMarkers);
-		this._decorationsTree1.acceptReplace(offset, length, textLength, forceMoveMarkers);
-	}
-}
-
-function cleanClassName(className: string): string {
-	return className.replace(/[^a-z0-9\-]/gi, ' ');
-}
-
-export class ModelDecorationOverviewRulerOptions implements editorCommon.IModelDecorationOverviewRulerOptions {
-	readonly color: string | ThemeColor;
-	readonly darkColor: string | ThemeColor;
-	readonly hcColor: string | ThemeColor;
-	readonly position: editorCommon.OverviewRulerLane;
-	_resolvedColor: string;
-
-	constructor(options: editorCommon.IModelDecorationOverviewRulerOptions) {
-		this.color = strings.empty;
-		this.darkColor = strings.empty;
-		this.hcColor = strings.empty;
-		this.position = editorCommon.OverviewRulerLane.Center;
-		this._resolvedColor = null;
-
-		if (options && options.color) {
-			this.color = options.color;
-		}
-		if (options && options.darkColor) {
-			this.darkColor = options.darkColor;
-			this.hcColor = options.darkColor;
-		}
-		if (options && options.hcColor) {
-			this.hcColor = options.hcColor;
-		}
-		if (options && options.hasOwnProperty('position')) {
-			this.position = options.position;
-		}
-	}
-}
-
-let lastStaticId = 0;
-
-export class ModelDecorationOptions implements editorCommon.IModelDecorationOptions {
-
-	public static EMPTY: ModelDecorationOptions;
-
-	public static register(options: editorCommon.IModelDecorationOptions): ModelDecorationOptions {
-		return new ModelDecorationOptions(++lastStaticId, options);
-	}
-
-	public static createDynamic(options: editorCommon.IModelDecorationOptions): ModelDecorationOptions {
-		return new ModelDecorationOptions(0, options);
-	}
-
-	readonly staticId: number;
-	readonly stickiness: editorCommon.TrackedRangeStickiness;
-	readonly className: string;
-	readonly hoverMessage: IMarkdownString | IMarkdownString[];
-	readonly glyphMarginHoverMessage: IMarkdownString | IMarkdownString[];
-	readonly isWholeLine: boolean;
-	readonly showIfCollapsed: boolean;
-	readonly overviewRuler: ModelDecorationOverviewRulerOptions;
-	readonly glyphMarginClassName: string;
-	readonly linesDecorationsClassName: string;
-	readonly marginClassName: string;
-	readonly inlineClassName: string;
-	readonly beforeContentClassName: string;
-	readonly afterContentClassName: string;
-
-	private constructor(staticId: number, options: editorCommon.IModelDecorationOptions) {
-		this.staticId = staticId;
-		this.stickiness = options.stickiness || editorCommon.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges;
-		this.className = options.className ? cleanClassName(options.className) : strings.empty;
-		this.hoverMessage = options.hoverMessage || [];
-		this.glyphMarginHoverMessage = options.glyphMarginHoverMessage || [];
-		this.isWholeLine = options.isWholeLine || false;
-		this.showIfCollapsed = options.showIfCollapsed || false;
-		this.overviewRuler = new ModelDecorationOverviewRulerOptions(options.overviewRuler);
-		this.glyphMarginClassName = options.glyphMarginClassName ? cleanClassName(options.glyphMarginClassName) : strings.empty;
-		this.linesDecorationsClassName = options.linesDecorationsClassName ? cleanClassName(options.linesDecorationsClassName) : strings.empty;
-		this.marginClassName = options.marginClassName ? cleanClassName(options.marginClassName) : strings.empty;
-		this.inlineClassName = options.inlineClassName ? cleanClassName(options.inlineClassName) : strings.empty;
-		this.beforeContentClassName = options.beforeContentClassName ? cleanClassName(options.beforeContentClassName) : strings.empty;
-		this.afterContentClassName = options.afterContentClassName ? cleanClassName(options.afterContentClassName) : strings.empty;
-	}
-}
-ModelDecorationOptions.EMPTY = ModelDecorationOptions.register({});
-
-/**
- * The order carefully matches the values of the enum.
- */
-const TRACKED_RANGE_OPTIONS = [
-	ModelDecorationOptions.register({ stickiness: editorCommon.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges }),
-	ModelDecorationOptions.register({ stickiness: editorCommon.TrackedRangeStickiness.NeverGrowsWhenTypingAtEdges }),
-	ModelDecorationOptions.register({ stickiness: editorCommon.TrackedRangeStickiness.GrowsOnlyWhenTypingBefore }),
-	ModelDecorationOptions.register({ stickiness: editorCommon.TrackedRangeStickiness.GrowsOnlyWhenTypingAfter }),
-];
-
-function _normalizeOptions(options: editorCommon.IModelDecorationOptions): ModelDecorationOptions {
-	if (options instanceof ModelDecorationOptions) {
-		return options;
-	}
-	return ModelDecorationOptions.createDynamic(options);
-}
-
-export class DidChangeDecorationsEmitter extends Disposable {
-
-	private readonly _actual: Emitter<IModelDecorationsChangedEvent> = this._register(new Emitter<IModelDecorationsChangedEvent>());
-	public readonly event: Event<IModelDecorationsChangedEvent> = this._actual.event;
-
-	private _deferredCnt: number;
-	private _shouldFire: boolean;
-
-	constructor() {
-		super();
-		this._deferredCnt = 0;
-		this._shouldFire = false;
-	}
-
-	public beginDeferredEmit(): void {
-		this._deferredCnt++;
-	}
-
-	public endDeferredEmit(): void {
-		this._deferredCnt--;
-		if (this._deferredCnt === 0) {
-			if (this._shouldFire) {
-				this._shouldFire = false;
-				this._actual.fire({});
-			}
-		}
-	}
-
-	public fire(): void {
-		this._shouldFire = true;
-	}
-}

src/vs/editor/common/model/textModelWithTokens.ts

@@ -1,921 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import * as nls from 'vs/nls';
-import { onUnexpectedError } from 'vs/base/common/errors';
-import { IDisposable } from 'vs/base/common/lifecycle';
-import { StopWatch } from 'vs/base/common/stopwatch';
-import Event, { Emitter } from 'vs/base/common/event';
-import { Range } from 'vs/editor/common/core/range';
-import * as editorCommon from 'vs/editor/common/editorCommon';
-import { TextModel } from 'vs/editor/common/model/textModel';
-import { ITokenizationSupport, IState, TokenizationRegistry, LanguageId, LanguageIdentifier } from 'vs/editor/common/modes';
-import { NULL_LANGUAGE_IDENTIFIER, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
-import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
-import { BracketsUtils, RichEditBrackets, RichEditBracket } from 'vs/editor/common/modes/supports/richEditBrackets';
-import { Position, IPosition } from 'vs/editor/common/core/position';
-import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
-import { LineTokens, LineToken } from 'vs/editor/common/core/lineTokens';
-import { getWordAtText } from 'vs/editor/common/model/wordHelper';
-import { TokenizationResult2 } from 'vs/editor/common/core/token';
-import { ITextSource, IRawTextSource } from 'vs/editor/common/model/textSource';
-import { IModelTokensChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent } from 'vs/editor/common/model/textModelEvents';
-import { computeIndentLevel } from 'vs/editor/common/model/modelLine';
-
-class ModelTokensChangedEventBuilder {
-
-	private _ranges: { fromLineNumber: number; toLineNumber: number; }[];
-
-	constructor() {
-		this._ranges = [];
-	}
-
-	public registerChangedTokens(lineNumber: number): void {
-		const ranges = this._ranges;
-		const rangesLength = ranges.length;
-		const previousRange = rangesLength > 0 ? ranges[rangesLength - 1] : null;
-
-		if (previousRange && previousRange.toLineNumber === lineNumber - 1) {
-			// extend previous range
-			previousRange.toLineNumber++;
-		} else {
-			// insert new range
-			ranges[rangesLength] = {
-				fromLineNumber: lineNumber,
-				toLineNumber: lineNumber
-			};
-		}
-	}
-
-	public build(): IModelTokensChangedEvent {
-		if (this._ranges.length === 0) {
-			return null;
-		}
-		return {
-			ranges: this._ranges
-		};
-	}
-}
-
-export class TextModelWithTokens extends TextModel implements editorCommon.ITokenizedModel {
-
-	private static readonly MODE_TOKENIZATION_FAILED_MSG = nls.localize('mode.tokenizationSupportFailed', "The mode has failed while tokenizing the input.");
-
-	private readonly _onDidChangeLanguage: Emitter<IModelLanguageChangedEvent> = this._register(new Emitter<IModelLanguageChangedEvent>());
-	public readonly onDidChangeLanguage: Event<IModelLanguageChangedEvent> = this._onDidChangeLanguage.event;
-
-	private readonly _onDidChangeLanguageConfiguration: Emitter<IModelLanguageConfigurationChangedEvent> = this._register(new Emitter<IModelLanguageConfigurationChangedEvent>());
-	public readonly onDidChangeLanguageConfiguration: Event<IModelLanguageConfigurationChangedEvent> = this._onDidChangeLanguageConfiguration.event;
-
-	private readonly _onDidChangeTokens: Emitter<IModelTokensChangedEvent> = this._register(new Emitter<IModelTokensChangedEvent>());
-	public readonly onDidChangeTokens: Event<IModelTokensChangedEvent> = this._onDidChangeTokens.event;
-
-	private _languageIdentifier: LanguageIdentifier;
-	private _tokenizationListener: IDisposable;
-	private _tokenizationSupport: ITokenizationSupport;
-
-	private _invalidLineStartIndex: number;
-	private _lastState: IState;
-
-	private _languageRegistryListener: IDisposable;
-
-	private _revalidateTokensTimeout: number;
-
-	constructor(rawTextSource: IRawTextSource, creationOptions: editorCommon.ITextModelCreationOptions, languageIdentifier: LanguageIdentifier) {
-		super(rawTextSource, creationOptions);
-
-		this._languageIdentifier = languageIdentifier || NULL_LANGUAGE_IDENTIFIER;
-		this._tokenizationListener = TokenizationRegistry.onDidChange((e) => {
-			if (e.changedLanguages.indexOf(this._languageIdentifier.language) === -1) {
-				return;
-			}
-
-			this._resetTokenizationState();
-			this.emitModelTokensChangedEvent({
-				ranges: [{
-					fromLineNumber: 1,
-					toLineNumber: this.getLineCount()
-				}]
-			});
-
-			if (this._shouldAutoTokenize()) {
-				this._warmUpTokens();
-			}
-		});
-
-		this._revalidateTokensTimeout = -1;
-
-		this._languageRegistryListener = LanguageConfigurationRegistry.onDidChange((e) => {
-			if (e.languageIdentifier.id === this._languageIdentifier.id) {
-				this._onDidChangeLanguageConfiguration.fire({});
-			}
-		});
-
-		this._resetTokenizationState();
-	}
-
-	public dispose(): void {
-		this._tokenizationListener.dispose();
-		this._languageRegistryListener.dispose();
-		this._clearTimers();
-		this._lastState = null;
-
-		super.dispose();
-	}
-
-	protected _shouldAutoTokenize(): boolean {
-		return false;
-	}
-
-	protected _resetValue(newValue: ITextSource): void {
-		super._resetValue(newValue);
-		// Cancel tokenization, clear all tokens and begin tokenizing
-		this._resetTokenizationState();
-	}
-
-	protected _resetTokenizationState(): void {
-		this._clearTimers();
-		for (let i = 0; i < this._lines.length; i++) {
-			this._lines[i].resetTokenizationState();
-		}
-
-		this._tokenizationSupport = null;
-		if (!this._isTooLargeForTokenization) {
-			this._tokenizationSupport = TokenizationRegistry.get(this._languageIdentifier.language);
-		}
-
-		if (this._tokenizationSupport) {
-			let initialState: IState = null;
-			try {
-				initialState = this._tokenizationSupport.getInitialState();
-			} catch (e) {
-				e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG;
-				onUnexpectedError(e);
-				this._tokenizationSupport = null;
-			}
-
-			if (initialState) {
-				this._lines[0].setState(initialState);
-			}
-		}
-
-		this._lastState = null;
-		this._invalidLineStartIndex = 0;
-		this._beginBackgroundTokenization();
-	}
-
-	private _clearTimers(): void {
-		if (this._revalidateTokensTimeout !== -1) {
-			clearTimeout(this._revalidateTokensTimeout);
-			this._revalidateTokensTimeout = -1;
-		}
-	}
-
-	public forceTokenization(lineNumber: number): void {
-		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
-			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
-		}
-
-		const eventBuilder = new ModelTokensChangedEventBuilder();
-
-		this._updateTokensUntilLine(eventBuilder, lineNumber);
-
-		const e = eventBuilder.build();
-		if (e) {
-			this._onDidChangeTokens.fire(e);
-		}
-	}
-
-	public isCheapToTokenize(lineNumber: number): boolean {
-		const firstInvalidLineNumber = this._invalidLineStartIndex + 1;
-		return (firstInvalidLineNumber >= lineNumber);
-	}
-
-	public tokenizeIfCheap(lineNumber: number): void {
-		if (this.isCheapToTokenize(lineNumber)) {
-			this.forceTokenization(lineNumber);
-		}
-	}
-
-	public getLineTokens(lineNumber: number): LineTokens {
-		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
-			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
-		}
-
-		return this._getLineTokens(lineNumber);
-	}
-
-	private _getLineTokens(lineNumber: number): LineTokens {
-		return this._lines[lineNumber - 1].getTokens(this._languageIdentifier.id);
-	}
-
-	public getLanguageIdentifier(): LanguageIdentifier {
-		return this._languageIdentifier;
-	}
-
-	public getModeId(): string {
-		return this._languageIdentifier.language;
-	}
-
-	public setMode(languageIdentifier: LanguageIdentifier): void {
-		if (this._languageIdentifier.id === languageIdentifier.id) {
-			// There's nothing to do
-			return;
-		}
-
-		let e: IModelLanguageChangedEvent = {
-			oldLanguage: this._languageIdentifier.language,
-			newLanguage: languageIdentifier.language
-		};
-
-		this._languageIdentifier = languageIdentifier;
-
-		// Cancel tokenization, clear all tokens and begin tokenizing
-		this._resetTokenizationState();
-
-		this.emitModelTokensChangedEvent({
-			ranges: [{
-				fromLineNumber: 1,
-				toLineNumber: this.getLineCount()
-			}]
-		});
-		this._onDidChangeLanguage.fire(e);
-		this._onDidChangeLanguageConfiguration.fire({});
-	}
-
-	public getLanguageIdAtPosition(_lineNumber: number, _column: number): LanguageId {
-		if (!this._tokenizationSupport) {
-			return this._languageIdentifier.id;
-		}
-		let { lineNumber, column } = this.validatePosition({ lineNumber: _lineNumber, column: _column });
-
-		let lineTokens = this._getLineTokens(lineNumber);
-		let token = lineTokens.findTokenAtOffset(column - 1);
-		return token.languageId;
-	}
-
-	protected _invalidateLine(lineIndex: number): void {
-		this._lines[lineIndex].setIsInvalid(true);
-		if (lineIndex < this._invalidLineStartIndex) {
-			if (this._invalidLineStartIndex < this._lines.length) {
-				this._lines[this._invalidLineStartIndex].setIsInvalid(true);
-			}
-			this._invalidLineStartIndex = lineIndex;
-			this._beginBackgroundTokenization();
-		}
-	}
-
-	private _beginBackgroundTokenization(): void {
-		if (this._shouldAutoTokenize() && this._revalidateTokensTimeout === -1) {
-			this._revalidateTokensTimeout = setTimeout(() => {
-				this._revalidateTokensTimeout = -1;
-				this._revalidateTokensNow();
-			}, 0);
-		}
-	}
-
-	_warmUpTokens(): void {
-		// Warm up first 100 lines (if it takes less than 50ms)
-		var maxLineNumber = Math.min(100, this.getLineCount());
-		this._revalidateTokensNow(maxLineNumber);
-
-		if (this._invalidLineStartIndex < this._lines.length) {
-			this._beginBackgroundTokenization();
-		}
-	}
-
-	private _revalidateTokensNow(toLineNumber: number = this._invalidLineStartIndex + 1000000): void {
-
-		const eventBuilder = new ModelTokensChangedEventBuilder();
-
-		toLineNumber = Math.min(this._lines.length, toLineNumber);
-
-		var MAX_ALLOWED_TIME = 20,
-			fromLineNumber = this._invalidLineStartIndex + 1,
-			tokenizedChars = 0,
-			currentCharsToTokenize = 0,
-			currentEstimatedTimeToTokenize = 0,
-			sw = StopWatch.create(false),
-			elapsedTime: number;
-
-		// Tokenize at most 1000 lines. Estimate the tokenization speed per character and stop when:
-		// - MAX_ALLOWED_TIME is reached
-		// - tokenizing the next line would go above MAX_ALLOWED_TIME
-
-		for (var lineNumber = fromLineNumber; lineNumber <= toLineNumber; lineNumber++) {
-			elapsedTime = sw.elapsed();
-			if (elapsedTime > MAX_ALLOWED_TIME) {
-				// Stop if MAX_ALLOWED_TIME is reached
-				toLineNumber = lineNumber - 1;
-				break;
-			}
-
-			// Compute how many characters will be tokenized for this line
-			currentCharsToTokenize = this._lines[lineNumber - 1].text.length;
-
-			if (tokenizedChars > 0) {
-				// If we have enough history, estimate how long tokenizing this line would take
-				currentEstimatedTimeToTokenize = (elapsedTime / tokenizedChars) * currentCharsToTokenize;
-				if (elapsedTime + currentEstimatedTimeToTokenize > MAX_ALLOWED_TIME) {
-					// Tokenizing this line will go above MAX_ALLOWED_TIME
-					toLineNumber = lineNumber - 1;
-					break;
-				}
-			}
-
-			this._updateTokensUntilLine(eventBuilder, lineNumber);
-			tokenizedChars += currentCharsToTokenize;
-
-			// Skip the lines that got tokenized
-			lineNumber = Math.max(lineNumber, this._invalidLineStartIndex + 1);
-		}
-
-		elapsedTime = sw.elapsed();
-
-		if (this._invalidLineStartIndex < this._lines.length) {
-			this._beginBackgroundTokenization();
-		}
-
-		const e = eventBuilder.build();
-		if (e) {
-			this._onDidChangeTokens.fire(e);
-		}
-	}
-
-	private _updateTokensUntilLine(eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
-		if (!this._tokenizationSupport) {
-			this._invalidLineStartIndex = this._lines.length;
-			return;
-		}
-
-		const linesLength = this._lines.length;
-		const endLineIndex = lineNumber - 1;
-
-		// Validate all states up to and including endLineIndex
-		for (let lineIndex = this._invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) {
-			const endStateIndex = lineIndex + 1;
-			let r: TokenizationResult2 = null;
-			const text = this._lines[lineIndex].text;
-
-			try {
-				// Tokenize only the first X characters
-				let freshState = this._lines[lineIndex].getState().clone();
-				r = this._tokenizationSupport.tokenize2(this._lines[lineIndex].text, freshState, 0);
-			} catch (e) {
-				e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG;
-				onUnexpectedError(e);
-			}
-
-			if (!r) {
-				r = nullTokenize2(this._languageIdentifier.id, text, this._lines[lineIndex].getState(), 0);
-			}
-			this._lines[lineIndex].setTokens(this._languageIdentifier.id, r.tokens);
-			eventBuilder.registerChangedTokens(lineIndex + 1);
-			this._lines[lineIndex].setIsInvalid(false);
-
-			if (endStateIndex < linesLength) {
-				if (this._lines[endStateIndex].getState() !== null && r.endState.equals(this._lines[endStateIndex].getState())) {
-					// The end state of this line remains the same
-					let nextInvalidLineIndex = lineIndex + 1;
-					while (nextInvalidLineIndex < linesLength) {
-						if (this._lines[nextInvalidLineIndex].isInvalid()) {
-							break;
-						}
-						if (nextInvalidLineIndex + 1 < linesLength) {
-							if (this._lines[nextInvalidLineIndex + 1].getState() === null) {
-								break;
-							}
-						} else {
-							if (this._lastState === null) {
-								break;
-							}
-						}
-						nextInvalidLineIndex++;
-					}
-					this._invalidLineStartIndex = Math.max(this._invalidLineStartIndex, nextInvalidLineIndex);
-					lineIndex = nextInvalidLineIndex - 1; // -1 because the outer loop increments it
-				} else {
-					this._lines[endStateIndex].setState(r.endState);
-				}
-			} else {
-				this._lastState = r.endState;
-			}
-		}
-		this._invalidLineStartIndex = Math.max(this._invalidLineStartIndex, endLineIndex + 1);
-	}
-
-	private emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {
-		if (!this._isDisposing) {
-			this._onDidChangeTokens.fire(e);
-		}
-	}
-
-	// Having tokens allows implementing additional helper methods
-
-	public getWordAtPosition(_position: IPosition): editorCommon.IWordAtPosition {
-		this._assertNotDisposed();
-		const position = this.validatePosition(_position);
-		const lineContent = this.getLineContent(position.lineNumber);
-
-		if (this._invalidLineStartIndex <= position.lineNumber - 1) {
-			// this line is not tokenized
-			return getWordAtText(
-				position.column,
-				LanguageConfigurationRegistry.getWordDefinition(this._languageIdentifier.id),
-				lineContent,
-				0
-			);
-		}
-
-		const lineTokens = this._getLineTokens(position.lineNumber);
-		const offset = position.column - 1;
-		const token = lineTokens.findTokenAtOffset(offset);
-		const languageId = token.languageId;
-
-		// go left until a different language is hit
-		let startOffset: number;
-		for (let leftToken = token.clone(); leftToken !== null && leftToken.languageId === languageId; leftToken = leftToken.prev()) {
-			startOffset = leftToken.startOffset;
-		}
-
-		// go right until a different language is hit
-		let endOffset: number;
-		for (let rightToken = token.clone(); rightToken !== null && rightToken.languageId === languageId; rightToken = rightToken.next()) {
-			endOffset = rightToken.endOffset;
-		}
-
-		return getWordAtText(
-			position.column,
-			LanguageConfigurationRegistry.getWordDefinition(languageId),
-			lineContent.substring(startOffset, endOffset),
-			startOffset
-		);
-	}
-
-	public getWordUntilPosition(position: IPosition): editorCommon.IWordAtPosition {
-		var wordAtPosition = this.getWordAtPosition(position);
-		if (!wordAtPosition) {
-			return {
-				word: '',
-				startColumn: position.column,
-				endColumn: position.column
-			};
-		}
-		return {
-			word: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),
-			startColumn: wordAtPosition.startColumn,
-			endColumn: position.column
-		};
-	}
-
-	public findMatchingBracketUp(_bracket: string, _position: IPosition): Range {
-		let bracket = _bracket.toLowerCase();
-		let position = this.validatePosition(_position);
-
-		let lineTokens = this._getLineTokens(position.lineNumber);
-		let token = lineTokens.findTokenAtOffset(position.column - 1);
-		let bracketsSupport = LanguageConfigurationRegistry.getBracketsSupport(token.languageId);
-
-		if (!bracketsSupport) {
-			return null;
-		}
-
-		let data = bracketsSupport.textIsBracket[bracket];
-
-		if (!data) {
-			return null;
-		}
-
-		return this._findMatchingBracketUp(data, position);
-	}
-
-	public matchBracket(position: IPosition): [Range, Range] {
-		return this._matchBracket(this.validatePosition(position));
-	}
-
-	private _matchBracket(position: Position): [Range, Range] {
-		const lineNumber = position.lineNumber;
-		let lineTokens = this._getLineTokens(lineNumber);
-		const lineText = this._lines[lineNumber - 1].text;
-
-		let currentToken = lineTokens.findTokenAtOffset(position.column - 1);
-		if (!currentToken) {
-			return null;
-		}
-		const currentModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentToken.languageId);
-
-		// check that the token is not to be ignored
-		if (currentModeBrackets && !ignoreBracketsInToken(currentToken.tokenType)) {
-			// limit search to not go before `maxBracketLength`
-			let searchStartOffset = Math.max(currentToken.startOffset, position.column - 1 - currentModeBrackets.maxBracketLength);
-			// limit search to not go after `maxBracketLength`
-			const searchEndOffset = Math.min(currentToken.endOffset, position.column - 1 + currentModeBrackets.maxBracketLength);
-
-			// first, check if there is a bracket to the right of `position`
-			let foundBracket = BracketsUtils.findNextBracketInToken(currentModeBrackets.forwardRegex, lineNumber, lineText, position.column - 1, searchEndOffset);
-			if (foundBracket && foundBracket.startColumn === position.column) {
-				let foundBracketText = lineText.substring(foundBracket.startColumn - 1, foundBracket.endColumn - 1);
-				foundBracketText = foundBracketText.toLowerCase();
-
-				let r = this._matchFoundBracket(foundBracket, currentModeBrackets.textIsBracket[foundBracketText], currentModeBrackets.textIsOpenBracket[foundBracketText]);
-
-				// check that we can actually match this bracket
-				if (r) {
-					return r;
-				}
-			}
-
-			// it might still be the case that [currentTokenStart -> currentTokenEnd] contains multiple brackets
-			while (true) {
-				let foundBracket = BracketsUtils.findNextBracketInToken(currentModeBrackets.forwardRegex, lineNumber, lineText, searchStartOffset, searchEndOffset);
-				if (!foundBracket) {
-					// there are no brackets in this text
-					break;
-				}
-
-				// check that we didn't hit a bracket too far away from position
-				if (foundBracket.startColumn <= position.column && position.column <= foundBracket.endColumn) {
-					let foundBracketText = lineText.substring(foundBracket.startColumn - 1, foundBracket.endColumn - 1);
-					foundBracketText = foundBracketText.toLowerCase();
-
-					let r = this._matchFoundBracket(foundBracket, currentModeBrackets.textIsBracket[foundBracketText], currentModeBrackets.textIsOpenBracket[foundBracketText]);
-
-					// check that we can actually match this bracket
-					if (r) {
-						return r;
-					}
-				}
-
-				searchStartOffset = foundBracket.endColumn - 1;
-			}
-		}
-
-		// If position is in between two tokens, try also looking in the previous token
-		if (currentToken.hasPrev && currentToken.startOffset === position.column - 1) {
-			const searchEndOffset = currentToken.startOffset;
-			currentToken = currentToken.prev();
-			const prevModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentToken.languageId);
-
-			// check that previous token is not to be ignored
-			if (prevModeBrackets && !ignoreBracketsInToken(currentToken.tokenType)) {
-				// limit search in case previous token is very large, there's no need to go beyond `maxBracketLength`
-				const searchStartOffset = Math.max(currentToken.startOffset, position.column - 1 - prevModeBrackets.maxBracketLength);
-				const foundBracket = BracketsUtils.findPrevBracketInToken(prevModeBrackets.reversedRegex, lineNumber, lineText, searchStartOffset, searchEndOffset);
-
-				// check that we didn't hit a bracket too far away from position
-				if (foundBracket && foundBracket.startColumn <= position.column && position.column <= foundBracket.endColumn) {
-					let foundBracketText = lineText.substring(foundBracket.startColumn - 1, foundBracket.endColumn - 1);
-					foundBracketText = foundBracketText.toLowerCase();
-
-					let r = this._matchFoundBracket(foundBracket, prevModeBrackets.textIsBracket[foundBracketText], prevModeBrackets.textIsOpenBracket[foundBracketText]);
-
-					// check that we can actually match this bracket
-					if (r) {
-						return r;
-					}
-				}
-			}
-		}
-
-		return null;
-	}
-
-	private _matchFoundBracket(foundBracket: Range, data: RichEditBracket, isOpen: boolean): [Range, Range] {
-		if (isOpen) {
-			let matched = this._findMatchingBracketDown(data, foundBracket.getEndPosition());
-			if (matched) {
-				return [foundBracket, matched];
-			}
-		} else {
-			let matched = this._findMatchingBracketUp(data, foundBracket.getStartPosition());
-			if (matched) {
-				return [foundBracket, matched];
-			}
-		}
-
-		return null;
-	}
-
-	private _findMatchingBracketUp(bracket: RichEditBracket, position: Position): Range {
-		// console.log('_findMatchingBracketUp: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
-
-		const languageId = bracket.languageIdentifier.id;
-		const reversedBracketRegex = bracket.reversedRegex;
-		let count = -1;
-
-		for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) {
-			const lineTokens = this._getLineTokens(lineNumber);
-			const lineText = this._lines[lineNumber - 1].text;
-
-			let currentToken: LineToken;
-			let searchStopOffset: number;
-			if (lineNumber === position.lineNumber) {
-				currentToken = lineTokens.findTokenAtOffset(position.column - 1);
-				searchStopOffset = position.column - 1;
-			} else {
-				currentToken = lineTokens.lastToken();
-				if (currentToken) {
-					searchStopOffset = currentToken.endOffset;
-				}
-			}
-
-			while (currentToken) {
-				if (currentToken.languageId === languageId && !ignoreBracketsInToken(currentToken.tokenType)) {
-
-					while (true) {
-						let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, lineNumber, lineText, currentToken.startOffset, searchStopOffset);
-						if (!r) {
-							break;
-						}
-
-						let hitText = lineText.substring(r.startColumn - 1, r.endColumn - 1);
-						hitText = hitText.toLowerCase();
-
-						if (hitText === bracket.open) {
-							count++;
-						} else if (hitText === bracket.close) {
-							count--;
-						}
-
-						if (count === 0) {
-							return r;
-						}
-
-						searchStopOffset = r.startColumn - 1;
-					}
-				}
-
-				currentToken = currentToken.prev();
-				if (currentToken) {
-					searchStopOffset = currentToken.endOffset;
-				}
-			}
-		}
-
-		return null;
-	}
-
-	private _findMatchingBracketDown(bracket: RichEditBracket, position: Position): Range {
-		// console.log('_findMatchingBracketDown: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
-
-		const languageId = bracket.languageIdentifier.id;
-		const bracketRegex = bracket.forwardRegex;
-		let count = 1;
-
-		for (let lineNumber = position.lineNumber, lineCount = this.getLineCount(); lineNumber <= lineCount; lineNumber++) {
-			const lineTokens = this._getLineTokens(lineNumber);
-			const lineText = this._lines[lineNumber - 1].text;
-
-			let currentToken: LineToken;
-			let searchStartOffset: number;
-			if (lineNumber === position.lineNumber) {
-				currentToken = lineTokens.findTokenAtOffset(position.column - 1);
-				searchStartOffset = position.column - 1;
-			} else {
-				currentToken = lineTokens.firstToken();
-				if (currentToken) {
-					searchStartOffset = currentToken.startOffset;
-				}
-			}
-
-			while (currentToken) {
-				if (currentToken.languageId === languageId && !ignoreBracketsInToken(currentToken.tokenType)) {
-					while (true) {
-						let r = BracketsUtils.findNextBracketInToken(bracketRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
-						if (!r) {
-							break;
-						}
-
-						let hitText = lineText.substring(r.startColumn - 1, r.endColumn - 1);
-						hitText = hitText.toLowerCase();
-
-						if (hitText === bracket.open) {
-							count++;
-						} else if (hitText === bracket.close) {
-							count--;
-						}
-
-						if (count === 0) {
-							return r;
-						}
-
-						searchStartOffset = r.endColumn - 1;
-					}
-				}
-
-				currentToken = currentToken.next();
-				if (currentToken) {
-					searchStartOffset = currentToken.startOffset;
-				}
-			}
-		}
-
-		return null;
-	}
-
-	public findPrevBracket(_position: IPosition): editorCommon.IFoundBracket {
-		const position = this.validatePosition(_position);
-
-		let languageId: LanguageId = -1;
-		let modeBrackets: RichEditBrackets = null;
-		for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) {
-			const lineTokens = this._getLineTokens(lineNumber);
-			const lineText = this._lines[lineNumber - 1].text;
-
-			let currentToken: LineToken;
-			let searchStopOffset: number;
-			if (lineNumber === position.lineNumber) {
-				currentToken = lineTokens.findTokenAtOffset(position.column - 1);
-				searchStopOffset = position.column - 1;
-			} else {
-				currentToken = lineTokens.lastToken();
-				if (currentToken) {
-					searchStopOffset = currentToken.endOffset;
-				}
-			}
-
-			while (currentToken) {
-				if (languageId !== currentToken.languageId) {
-					languageId = currentToken.languageId;
-					modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(languageId);
-				}
-				if (modeBrackets && !ignoreBracketsInToken(currentToken.tokenType)) {
-					let r = BracketsUtils.findPrevBracketInToken(modeBrackets.reversedRegex, lineNumber, lineText, currentToken.startOffset, searchStopOffset);
-					if (r) {
-						return this._toFoundBracket(modeBrackets, r);
-					}
-				}
-
-				currentToken = currentToken.prev();
-				if (currentToken) {
-					searchStopOffset = currentToken.endOffset;
-				}
-			}
-		}
-
-		return null;
-	}
-
-	public findNextBracket(_position: IPosition): editorCommon.IFoundBracket {
-		const position = this.validatePosition(_position);
-
-		let languageId: LanguageId = -1;
-		let modeBrackets: RichEditBrackets = null;
-		for (let lineNumber = position.lineNumber, lineCount = this.getLineCount(); lineNumber <= lineCount; lineNumber++) {
-			const lineTokens = this._getLineTokens(lineNumber);
-			const lineText = this._lines[lineNumber - 1].text;
-
-			let currentToken: LineToken;
-			let searchStartOffset: number;
-			if (lineNumber === position.lineNumber) {
-				currentToken = lineTokens.findTokenAtOffset(position.column - 1);
-				searchStartOffset = position.column - 1;
-			} else {
-				currentToken = lineTokens.firstToken();
-				if (currentToken) {
-					searchStartOffset = currentToken.startOffset;
-				}
-			}
-
-			while (currentToken) {
-				if (languageId !== currentToken.languageId) {
-					languageId = currentToken.languageId;
-					modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(languageId);
-				}
-				if (modeBrackets && !ignoreBracketsInToken(currentToken.tokenType)) {
-					let r = BracketsUtils.findNextBracketInToken(modeBrackets.forwardRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
-					if (r) {
-						return this._toFoundBracket(modeBrackets, r);
-					}
-				}
-
-				currentToken = currentToken.next();
-				if (currentToken) {
-					searchStartOffset = currentToken.startOffset;
-				}
-			}
-		}
-
-		return null;
-	}
-
-	private _toFoundBracket(modeBrackets: RichEditBrackets, r: Range): editorCommon.IFoundBracket {
-		if (!r) {
-			return null;
-		}
-
-		let text = this.getValueInRange(r);
-		text = text.toLowerCase();
-
-		let data = modeBrackets.textIsBracket[text];
-		if (!data) {
-			return null;
-		}
-
-		return {
-			range: r,
-			open: data.open,
-			close: data.close,
-			isOpen: modeBrackets.textIsOpenBracket[text]
-		};
-	}
-
-	private _computeIndentLevel(lineIndex: number): number {
-		return computeIndentLevel(this._lines[lineIndex].text, this._options.tabSize);
-	}
-
-	public getLinesIndentGuides(startLineNumber: number, endLineNumber: number): number[] {
-		this._assertNotDisposed();
-		const lineCount = this.getLineCount();
-
-		if (startLineNumber < 1 || startLineNumber > lineCount) {
-			throw new Error('Illegal value ' + startLineNumber + ' for `startLineNumber`');
-		}
-		if (endLineNumber < 1 || endLineNumber > lineCount) {
-			throw new Error('Illegal value ' + endLineNumber + ' for `endLineNumber`');
-		}
-
-		const foldingRules = LanguageConfigurationRegistry.getFoldingRules(this._languageIdentifier.id);
-		const offSide = foldingRules && foldingRules.offSide;
-
-		let result: number[] = new Array<number>(endLineNumber - startLineNumber + 1);
-
-		let aboveContentLineIndex = -2; /* -2 is a marker for not having computed it */
-		let aboveContentLineIndent = -1;
-
-		let belowContentLineIndex = -2; /* -2 is a marker for not having computed it */
-		let belowContentLineIndent = -1;
-
-		for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) {
-			let resultIndex = lineNumber - startLineNumber;
-
-			const currentIndent = this._computeIndentLevel(lineNumber - 1);
-			if (currentIndent >= 0) {
-				// This line has content (besides whitespace)
-				// Use the line's indent
-				aboveContentLineIndex = lineNumber - 1;
-				aboveContentLineIndent = currentIndent;
-				result[resultIndex] = Math.ceil(currentIndent / this._options.tabSize);
-				continue;
-			}
-
-			if (aboveContentLineIndex === -2) {
-				aboveContentLineIndex = -1;
-				aboveContentLineIndent = -1;
-
-				// must find previous line with content
-				for (let lineIndex = lineNumber - 2; lineIndex >= 0; lineIndex--) {
-					let indent = this._computeIndentLevel(lineIndex);
-					if (indent >= 0) {
-						aboveContentLineIndex = lineIndex;
-						aboveContentLineIndent = indent;
-						break;
-					}
-				}
-			}
-
-			if (belowContentLineIndex !== -1 && (belowContentLineIndex === -2 || belowContentLineIndex < lineNumber - 1)) {
-				belowContentLineIndex = -1;
-				belowContentLineIndent = -1;
-
-				// must find next line with content
-				for (let lineIndex = lineNumber; lineIndex < lineCount; lineIndex++) {
-					let indent = this._computeIndentLevel(lineIndex);
-					if (indent >= 0) {
-						belowContentLineIndex = lineIndex;
-						belowContentLineIndent = indent;
-						break;
-					}
-				}
-			}
-
-			if (aboveContentLineIndent === -1 || belowContentLineIndent === -1) {
-				// At the top or bottom of the file
-				result[resultIndex] = 0;
-
-			} else if (aboveContentLineIndent < belowContentLineIndent) {
-				// we are inside the region above
-				result[resultIndex] = (1 + Math.floor(aboveContentLineIndent / this._options.tabSize));
-
-			} else if (aboveContentLineIndent === belowContentLineIndent) {
-				// we are in between two regions
-				result[resultIndex] = Math.ceil(belowContentLineIndent / this._options.tabSize);
-
-			} else {
-
-				if (offSide) {
-					// same level as region below
-					result[resultIndex] = Math.ceil(belowContentLineIndent / this._options.tabSize);
-				} else {
-					// we are inside the region that ends below
-					result[resultIndex] = (1 + Math.floor(belowContentLineIndent / this._options.tabSize));
-				}
-
-			}
-		}
-		return result;
-	}
-}

src/vs/editor/common/model/textSource.ts

@@ -1,149 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import * as strings from 'vs/base/common/strings';
-import { DefaultEndOfLine } from 'vs/editor/common/editorCommon';
-
-/**
- * A processed string ready to be turned into an editor model.
- */
-export interface IRawTextSource {
-	/**
-	 * The entire text length.
-	 */
-	readonly length: number;
-	/**
-	 * The text split into lines.
-	 */
-	readonly lines: string[];
-	/**
-	 * The BOM (leading character sequence of the file).
-	 */
-	readonly BOM: string;
-	/**
-	 * The number of lines ending with '\r\n'
-	 */
-	readonly totalCRCount: number;
-	/**
-	 * The text contains Unicode characters classified as "R" or "AL".
-	 */
-	readonly containsRTL: boolean;
-	/**
-	 * The text contains only characters inside the ASCII range 32-126 or \t \r \n
-	 */
-	readonly isBasicASCII: boolean;
-}
-
-export class RawTextSource {
-
-	public static fromString(rawText: string): IRawTextSource {
-		// Count the number of lines that end with \r\n
-		let carriageReturnCnt = 0;
-		let lastCarriageReturnIndex = -1;
-		while ((lastCarriageReturnIndex = rawText.indexOf('\r', lastCarriageReturnIndex + 1)) !== -1) {
-			carriageReturnCnt++;
-		}
-
-		const containsRTL = strings.containsRTL(rawText);
-		const isBasicASCII = (containsRTL ? false : strings.isBasicASCII(rawText));
-
-		// Split the text into lines
-		const lines = rawText.split(/\r\n|\r|\n/);
-
-		// Remove the BOM (if present)
-		let BOM = '';
-		if (strings.startsWithUTF8BOM(lines[0])) {
-			BOM = strings.UTF8_BOM_CHARACTER;
-			lines[0] = lines[0].substr(1);
-		}
-
-		return {
-			BOM: BOM,
-			lines: lines,
-			length: rawText.length,
-			containsRTL: containsRTL,
-			isBasicASCII: isBasicASCII,
-			totalCRCount: carriageReturnCnt
-		};
-	}
-
-}
-
-/**
- * A processed string with its EOL resolved ready to be turned into an editor model.
- */
-export interface ITextSource {
-	/**
-	 * The entire text length.
-	 */
-	readonly length: number;
-	/**
-	 * The text split into lines.
-	 */
-	readonly lines: string[];
-	/**
-	 * The BOM (leading character sequence of the file).
-	 */
-	readonly BOM: string;
-	/**
-	 * The end of line sequence.
-	 */
-	readonly EOL: string;
-	/**
-	 * The text contains Unicode characters classified as "R" or "AL".
-	 */
-	readonly containsRTL: boolean;
-	/**
-	 * The text contains only characters inside the ASCII range 32-126 or \t \r \n
-	 */
-	readonly isBasicASCII: boolean;
-}
-
-export class TextSource {
-
-	/**
-	 * if text source is empty or with precisely one line, returns null. No end of line is detected.
-	 * if text source contains more lines ending with '\r\n', returns '\r\n'.
-	 * Otherwise returns '\n'. More lines end with '\n'.
-	 */
-	private static _getEOL(rawTextSource: IRawTextSource, defaultEOL: DefaultEndOfLine): '\r\n' | '\n' {
-		const lineFeedCnt = rawTextSource.lines.length - 1;
-		if (lineFeedCnt === 0) {
-			// This is an empty file or a file with precisely one line
-			return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n');
-		}
-		if (rawTextSource.totalCRCount > lineFeedCnt / 2) {
-			// More than half of the file contains \r\n ending lines
-			return '\r\n';
-		}
-		// At least one line more ends in \n
-		return '\n';
-	}
-
-	public static fromRawTextSource(rawTextSource: IRawTextSource, defaultEOL: DefaultEndOfLine): ITextSource {
-		return {
-			length: rawTextSource.length,
-			lines: rawTextSource.lines,
-			BOM: rawTextSource.BOM,
-			EOL: this._getEOL(rawTextSource, defaultEOL),
-			containsRTL: rawTextSource.containsRTL,
-			isBasicASCII: rawTextSource.isBasicASCII,
-		};
-	}
-
-	public static fromString(text: string, defaultEOL: DefaultEndOfLine): ITextSource {
-		return this.fromRawTextSource(RawTextSource.fromString(text), defaultEOL);
-	}
-
-	public static create(source: string | IRawTextSource, defaultEOL: DefaultEndOfLine): ITextSource {
-		if (typeof source === 'string') {
-			return this.fromString(source, defaultEOL);
-		}
-
-		return this.fromRawTextSource(source, defaultEOL);
-	}
-
-}

src/vs/editor/common/model/tokensBinaryEncoding.ts

@@ -1,65 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import { ColorId, FontStyle, StandardTokenType, MetadataConsts, LanguageId } from 'vs/editor/common/modes';
-
-export class TokenMetadata {
-
-	public static getLanguageId(metadata: number): LanguageId {
-		return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET;
-	}
-
-	public static getTokenType(metadata: number): StandardTokenType {
-		return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET;
-	}
-
-	public static getFontStyle(metadata: number): FontStyle {
-		return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET;
-	}
-
-	public static getForeground(metadata: number): ColorId {
-		return (metadata & MetadataConsts.FOREGROUND_MASK) >>> MetadataConsts.FOREGROUND_OFFSET;
-	}
-
-	public static getBackground(metadata: number): ColorId {
-		return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET;
-	}
-
-	public static getClassNameFromMetadata(metadata: number): string {
-		let foreground = this.getForeground(metadata);
-		let className = 'mtk' + foreground;
-
-		let fontStyle = this.getFontStyle(metadata);
-		if (fontStyle & FontStyle.Italic) {
-			className += ' mtki';
-		}
-		if (fontStyle & FontStyle.Bold) {
-			className += ' mtkb';
-		}
-		if (fontStyle & FontStyle.Underline) {
-			className += ' mtku';
-		}
-
-		return className;
-	}
-
-	public static getInlineStyleFromMetadata(metadata: number, colorMap: string[]): string {
-		const foreground = this.getForeground(metadata);
-		const fontStyle = this.getFontStyle(metadata);
-
-		let result = `color: ${colorMap[foreground]};`;
-		if (fontStyle & FontStyle.Italic) {
-			result += 'font-style: italic;';
-		}
-		if (fontStyle & FontStyle.Bold) {
-			result += 'font-weight: bold;';
-		}
-		if (fontStyle & FontStyle.Underline) {
-			result += 'text-decoration: underline;';
-		}
-		return result;
-	}
-}

src/vs/editor/common/model/wordHelper.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import { IWordAtPosition } from 'vs/editor/common/editorCommon';
+import { IWordAtPosition } from 'vs/editor/common/model';
 
 export const USUAL_WORD_SEPARATORS = '`~!@#$%^&*()-=+[{]}\\|;:\'",.<>/?';
 
@@ -16,13 +16,12 @@ export const USUAL_WORD_SEPARATORS = '`~!@#$%^&*()-=+[{]}\\|;:\'",.<>/?';
  * /(-?\d*\.\d\w*)|([^\`\~\!\@\#\$\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g
  */
 function createWordRegExp(allowInWords: string = ''): RegExp {
-	var usualSeparators = USUAL_WORD_SEPARATORS;
-	var source = '(-?\\d*\\.\\d\\w*)|([^';
-	for (var i = 0; i < usualSeparators.length; i++) {
-		if (allowInWords.indexOf(usualSeparators[i]) >= 0) {
+	let source = '(-?\\d*\\.\\d\\w*)|([^';
+	for (let i = 0; i < USUAL_WORD_SEPARATORS.length; i++) {
+		if (allowInWords.indexOf(USUAL_WORD_SEPARATORS[i]) >= 0) {
 			continue;
 		}
-		source += '\\' + usualSeparators[i];
+		source += '\\' + USUAL_WORD_SEPARATORS[i];
 	}
 	source += '\\s]+)';
 	return new RegExp(source, 'g');
@@ -32,11 +31,11 @@ function createWordRegExp(allowInWords: string = ''): RegExp {
 export const DEFAULT_WORD_REGEXP = createWordRegExp();
 
 export function ensureValidWordDefinition(wordDefinition?: RegExp): RegExp {
-	var result: RegExp = DEFAULT_WORD_REGEXP;
+	let result: RegExp = DEFAULT_WORD_REGEXP;
 
 	if (wordDefinition && (wordDefinition instanceof RegExp)) {
 		if (!wordDefinition.global) {
-			var flags = 'g';
+			let flags = 'g';
 			if (wordDefinition.ignoreCase) {
 				flags += 'i';
 			}