SQL Operations Studio Public Preview 1 (0.23) release source code

src/vs/editor/common/model/editStack.ts

@@ -0,0 +1,148 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { onUnexpectedError } from 'vs/base/common/errors';
+import { ICursorStateComputer, IEditableTextModel, IIdentifiedSingleEditOperation } from 'vs/editor/common/editorCommon';
+import { Selection } from 'vs/editor/common/core/selection';
+
+interface IEditOperation {
+	operations: IIdentifiedSingleEditOperation[];
+}
+
+interface IStackElement {
+	beforeVersionId: number;
+	beforeCursorState: Selection[];
+
+	editOperations: IEditOperation[];
+
+	afterCursorState: Selection[];
+	afterVersionId: number;
+}
+
+export interface IUndoRedoResult {
+	selections: Selection[];
+	recordedVersionId: number;
+}
+
+export class EditStack {
+
+	private model: IEditableTextModel;
+	private currentOpenStackElement: IStackElement;
+	private past: IStackElement[];
+	private future: IStackElement[];
+
+	constructor(model: IEditableTextModel) {
+		this.model = model;
+		this.currentOpenStackElement = null;
+		this.past = [];
+		this.future = [];
+	}
+
+	public pushStackElement(): void {
+		if (this.currentOpenStackElement !== null) {
+			this.past.push(this.currentOpenStackElement);
+			this.currentOpenStackElement = null;
+		}
+	}
+
+	public clear(): void {
+		this.currentOpenStackElement = null;
+		this.past = [];
+		this.future = [];
+	}
+
+	public pushEditOperation(beforeCursorState: Selection[], editOperations: IIdentifiedSingleEditOperation[], cursorStateComputer: ICursorStateComputer): Selection[] {
+		// No support for parallel universes :(
+		this.future = [];
+
+		if (!this.currentOpenStackElement) {
+			this.currentOpenStackElement = {
+				beforeVersionId: this.model.getAlternativeVersionId(),
+				beforeCursorState: beforeCursorState,
+				editOperations: [],
+				afterCursorState: null,
+				afterVersionId: -1
+			};
+		}
+
+		var inverseEditOperation: IEditOperation = {
+			operations: this.model.applyEdits(editOperations)
+		};
+
+		this.currentOpenStackElement.editOperations.push(inverseEditOperation);
+		try {
+			this.currentOpenStackElement.afterCursorState = cursorStateComputer ? cursorStateComputer(inverseEditOperation.operations) : null;
+		} catch (e) {
+			onUnexpectedError(e);
+			this.currentOpenStackElement.afterCursorState = null;
+		}
+
+		this.currentOpenStackElement.afterVersionId = this.model.getVersionId();
+		return this.currentOpenStackElement.afterCursorState;
+	}
+
+	public undo(): IUndoRedoResult {
+
+		this.pushStackElement();
+
+		if (this.past.length > 0) {
+			var pastStackElement = this.past.pop();
+
+			try {
+				// Apply all operations in reverse order
+				for (var i = pastStackElement.editOperations.length - 1; i >= 0; i--) {
+					pastStackElement.editOperations[i] = {
+						operations: this.model.applyEdits(pastStackElement.editOperations[i].operations)
+					};
+				}
+			} catch (e) {
+				this.clear();
+				return null;
+			}
+
+			this.future.push(pastStackElement);
+
+			return {
+				selections: pastStackElement.beforeCursorState,
+				recordedVersionId: pastStackElement.beforeVersionId
+			};
+		}
+
+		return null;
+	}
+
+	public redo(): IUndoRedoResult {
+
+		if (this.future.length > 0) {
+			if (this.currentOpenStackElement) {
+				throw new Error('How is this possible?');
+			}
+
+			var futureStackElement = this.future.pop();
+
+			try {
+				// Apply all operations
+				for (var i = 0; i < futureStackElement.editOperations.length; i++) {
+					futureStackElement.editOperations[i] = {
+						operations: this.model.applyEdits(futureStackElement.editOperations[i].operations)
+					};
+				}
+			} catch (e) {
+				this.clear();
+				return null;
+			}
+
+			this.past.push(futureStackElement);
+
+			return {
+				selections: futureStackElement.afterCursorState,
+				recordedVersionId: futureStackElement.afterVersionId
+			};
+		}
+
+		return null;
+	}
+}

src/vs/editor/common/model/editableTextModel.ts

@@ -0,0 +1,837 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { Range, IRange } from 'vs/editor/common/core/range';
+import * as editorCommon from 'vs/editor/common/editorCommon';
+import { EditStack } from 'vs/editor/common/model/editStack';
+import { ILineEdit, LineMarker, MarkersTracker, IModelLine } from 'vs/editor/common/model/modelLine';
+import { TextModelWithDecorations, ModelDecorationOptions } from 'vs/editor/common/model/textModelWithDecorations';
+import * as strings from 'vs/base/common/strings';
+import * as arrays from 'vs/base/common/arrays';
+import { Selection } from 'vs/editor/common/core/selection';
+import { Position } from 'vs/editor/common/core/position';
+import { IDisposable } from 'vs/base/common/lifecycle';
+import { LanguageIdentifier } from 'vs/editor/common/modes';
+import { ITextSource, IRawTextSource, RawTextSource } from 'vs/editor/common/model/textSource';
+import { TextModel } from 'vs/editor/common/model/textModel';
+import * as textModelEvents from 'vs/editor/common/model/textModelEvents';
+
+export interface IValidatedEditOperation {
+	sortIndex: number;
+	identifier: editorCommon.ISingleEditOperationIdentifier;
+	range: Range;
+	rangeLength: number;
+	lines: string[];
+	forceMoveMarkers: boolean;
+	isAutoWhitespaceEdit: boolean;
+}
+
+interface IIdentifiedLineEdit extends ILineEdit {
+	lineNumber: number;
+}
+
+export class EditableTextModel extends TextModelWithDecorations implements editorCommon.IEditableTextModel {
+
+	public static createFromString(text: string, options: editorCommon.ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageIdentifier: LanguageIdentifier = null): EditableTextModel {
+		return new EditableTextModel(RawTextSource.fromString(text), options, languageIdentifier);
+	}
+
+	public onDidChangeRawContent(listener: (e: textModelEvents.ModelRawContentChangedEvent) => void): IDisposable {
+		return this._eventEmitter.addListener(textModelEvents.TextModelEventType.ModelRawContentChanged2, listener);
+	}
+	public onDidChangeContent(listener: (e: textModelEvents.IModelContentChangedEvent) => void): IDisposable {
+		return this._eventEmitter.addListener(textModelEvents.TextModelEventType.ModelContentChanged, listener);
+	}
+
+	private _commandManager: EditStack;
+
+	// for extra details about change events:
+	private _isUndoing: boolean;
+	private _isRedoing: boolean;
+
+	// editable range
+	private _hasEditableRange: boolean;
+	private _editableRangeId: string;
+
+	private _trimAutoWhitespaceLines: number[];
+
+	constructor(rawTextSource: IRawTextSource, creationOptions: editorCommon.ITextModelCreationOptions, languageIdentifier: LanguageIdentifier) {
+		super(rawTextSource, creationOptions, languageIdentifier);
+
+		this._commandManager = new EditStack(this);
+
+		this._isUndoing = false;
+		this._isRedoing = false;
+
+		this._hasEditableRange = false;
+		this._editableRangeId = null;
+		this._trimAutoWhitespaceLines = null;
+	}
+
+	public dispose(): void {
+		this._commandManager = null;
+		super.dispose();
+	}
+
+	protected _resetValue(newValue: ITextSource): void {
+		super._resetValue(newValue);
+
+		// Destroy my edit history and settings
+		this._commandManager = new EditStack(this);
+		this._hasEditableRange = false;
+		this._editableRangeId = null;
+		this._trimAutoWhitespaceLines = null;
+	}
+
+	public pushStackElement(): void {
+		this._commandManager.pushStackElement();
+	}
+
+	public pushEditOperations(beforeCursorState: Selection[], editOperations: editorCommon.IIdentifiedSingleEditOperation[], cursorStateComputer: editorCommon.ICursorStateComputer): Selection[] {
+		try {
+			this._eventEmitter.beginDeferredEmit();
+			return this._pushEditOperations(beforeCursorState, editOperations, cursorStateComputer);
+		} finally {
+			this._eventEmitter.endDeferredEmit();
+		}
+	}
+
+	private _pushEditOperations(beforeCursorState: Selection[], editOperations: editorCommon.IIdentifiedSingleEditOperation[], cursorStateComputer: editorCommon.ICursorStateComputer): Selection[] {
+		if (this._options.trimAutoWhitespace && this._trimAutoWhitespaceLines) {
+			// Go through each saved line number and insert a trim whitespace edit
+			// if it is safe to do so (no conflicts with other edits).
+
+			let incomingEdits = editOperations.map((op) => {
+				return {
+					range: this.validateRange(op.range),
+					text: op.text
+				};
+			});
+
+			// Sometimes, auto-formatters change ranges automatically which can cause undesired auto whitespace trimming near the cursor
+			// We'll use the following heuristic: if the edits occur near the cursor, then it's ok to trim auto whitespace
+			let editsAreNearCursors = true;
+			for (let i = 0, len = beforeCursorState.length; i < len; i++) {
+				let sel = beforeCursorState[i];
+				let foundEditNearSel = false;
+				for (let j = 0, lenJ = incomingEdits.length; j < lenJ; j++) {
+					let editRange = incomingEdits[j].range;
+					let selIsAbove = editRange.startLineNumber > sel.endLineNumber;
+					let selIsBelow = sel.startLineNumber > editRange.endLineNumber;
+					if (!selIsAbove && !selIsBelow) {
+						foundEditNearSel = true;
+						break;
+					}
+				}
+				if (!foundEditNearSel) {
+					editsAreNearCursors = false;
+					break;
+				}
+			}
+
+			if (editsAreNearCursors) {
+				for (let i = 0, len = this._trimAutoWhitespaceLines.length; i < len; i++) {
+					let trimLineNumber = this._trimAutoWhitespaceLines[i];
+					let maxLineColumn = this.getLineMaxColumn(trimLineNumber);
+
+					let allowTrimLine = true;
+					for (let j = 0, lenJ = incomingEdits.length; j < lenJ; j++) {
+						let editRange = incomingEdits[j].range;
+						let editText = incomingEdits[j].text;
+
+						if (trimLineNumber < editRange.startLineNumber || trimLineNumber > editRange.endLineNumber) {
+							// `trimLine` is completely outside this edit
+							continue;
+						}
+
+						// At this point:
+						//   editRange.startLineNumber <= trimLine <= editRange.endLineNumber
+
+						if (
+							trimLineNumber === editRange.startLineNumber && editRange.startColumn === maxLineColumn
+							&& editRange.isEmpty() && editText && editText.length > 0 && editText.charAt(0) === '\n'
+						) {
+							// This edit inserts a new line (and maybe other text) after `trimLine`
+							continue;
+						}
+
+						// Looks like we can't trim this line as it would interfere with an incoming edit
+						allowTrimLine = false;
+						break;
+					}
+
+					if (allowTrimLine) {
+						editOperations.push({
+							identifier: null,
+							range: new Range(trimLineNumber, 1, trimLineNumber, maxLineColumn),
+							text: null,
+							forceMoveMarkers: false,
+							isAutoWhitespaceEdit: false
+						});
+					}
+
+				}
+			}
+
+			this._trimAutoWhitespaceLines = null;
+		}
+		return this._commandManager.pushEditOperation(beforeCursorState, editOperations, cursorStateComputer);
+	}
+
+	/**
+	 * Transform operations such that they represent the same logic edit,
+	 * but that they also do not cause OOM crashes.
+	 */
+	private _reduceOperations(operations: IValidatedEditOperation[]): IValidatedEditOperation[] {
+		if (operations.length < 1000) {
+			// We know from empirical testing that a thousand edits work fine regardless of their shape.
+			return operations;
+		}
+
+		// At one point, due to how events are emitted and how each operation is handled,
+		// some operations can trigger a high ammount of temporary string allocations,
+		// that will immediately get edited again.
+		// e.g. a formatter inserting ridiculous ammounts of \n on a model with a single line
+		// Therefore, the strategy is to collapse all the operations into a huge single edit operation
+		return [this._toSingleEditOperation(operations)];
+	}
+
+	_toSingleEditOperation(operations: IValidatedEditOperation[]): IValidatedEditOperation {
+		let forceMoveMarkers = false,
+			firstEditRange = operations[0].range,
+			lastEditRange = operations[operations.length - 1].range,
+			entireEditRange = new Range(firstEditRange.startLineNumber, firstEditRange.startColumn, lastEditRange.endLineNumber, lastEditRange.endColumn),
+			lastEndLineNumber = firstEditRange.startLineNumber,
+			lastEndColumn = firstEditRange.startColumn,
+			result: string[] = [];
+
+		for (let i = 0, len = operations.length; i < len; i++) {
+			let operation = operations[i],
+				range = operation.range;
+
+			forceMoveMarkers = forceMoveMarkers || operation.forceMoveMarkers;
+
+			// (1) -- Push old text
+			for (let lineNumber = lastEndLineNumber; lineNumber < range.startLineNumber; lineNumber++) {
+				if (lineNumber === lastEndLineNumber) {
+					result.push(this._lines[lineNumber - 1].text.substring(lastEndColumn - 1));
+				} else {
+					result.push('\n');
+					result.push(this._lines[lineNumber - 1].text);
+				}
+			}
+
+			if (range.startLineNumber === lastEndLineNumber) {
+				result.push(this._lines[range.startLineNumber - 1].text.substring(lastEndColumn - 1, range.startColumn - 1));
+			} else {
+				result.push('\n');
+				result.push(this._lines[range.startLineNumber - 1].text.substring(0, range.startColumn - 1));
+			}
+
+			// (2) -- Push new text
+			if (operation.lines) {
+				for (let j = 0, lenJ = operation.lines.length; j < lenJ; j++) {
+					if (j !== 0) {
+						result.push('\n');
+					}
+					result.push(operation.lines[j]);
+				}
+			}
+
+			lastEndLineNumber = operation.range.endLineNumber;
+			lastEndColumn = operation.range.endColumn;
+		}
+
+		return {
+			sortIndex: 0,
+			identifier: operations[0].identifier,
+			range: entireEditRange,
+			rangeLength: this.getValueLengthInRange(entireEditRange),
+			lines: result.join('').split('\n'),
+			forceMoveMarkers: forceMoveMarkers,
+			isAutoWhitespaceEdit: false
+		};
+	}
+
+	private static _sortOpsAscending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
+		let r = Range.compareRangesUsingEnds(a.range, b.range);
+		if (r === 0) {
+			return a.sortIndex - b.sortIndex;
+		}
+		return r;
+	}
+
+	private static _sortOpsDescending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
+		let r = Range.compareRangesUsingEnds(a.range, b.range);
+		if (r === 0) {
+			return b.sortIndex - a.sortIndex;
+		}
+		return -r;
+	}
+
+	public applyEdits(rawOperations: editorCommon.IIdentifiedSingleEditOperation[]): editorCommon.IIdentifiedSingleEditOperation[] {
+		try {
+			this._eventEmitter.beginDeferredEmit();
+			let markersTracker = this._acquireMarkersTracker();
+			return this._applyEdits(markersTracker, rawOperations);
+		} finally {
+			this._releaseMarkersTracker();
+			this._eventEmitter.endDeferredEmit();
+		}
+	}
+
+	private _applyEdits(markersTracker: MarkersTracker, rawOperations: editorCommon.IIdentifiedSingleEditOperation[]): editorCommon.IIdentifiedSingleEditOperation[] {
+		if (rawOperations.length === 0) {
+			return [];
+		}
+
+		let mightContainRTL = this._mightContainRTL;
+		let mightContainNonBasicASCII = this._mightContainNonBasicASCII;
+		let canReduceOperations = true;
+
+		let operations: IValidatedEditOperation[] = [];
+		for (let i = 0; i < rawOperations.length; i++) {
+			let op = rawOperations[i];
+			if (canReduceOperations && op._isTracked) {
+				canReduceOperations = false;
+			}
+			let validatedRange = this.validateRange(op.range);
+			if (!mightContainRTL && op.text) {
+				// check if the new inserted text contains RTL
+				mightContainRTL = strings.containsRTL(op.text);
+			}
+			if (!mightContainNonBasicASCII && op.text) {
+				mightContainNonBasicASCII = !strings.isBasicASCII(op.text);
+			}
+			operations[i] = {
+				sortIndex: i,
+				identifier: op.identifier,
+				range: validatedRange,
+				rangeLength: this.getValueLengthInRange(validatedRange),
+				lines: op.text ? op.text.split(/\r\n|\r|\n/) : null,
+				forceMoveMarkers: op.forceMoveMarkers,
+				isAutoWhitespaceEdit: op.isAutoWhitespaceEdit || false
+			};
+		}
+
+		// Sort operations ascending
+		operations.sort(EditableTextModel._sortOpsAscending);
+
+		for (let i = 0, count = operations.length - 1; i < count; i++) {
+			let rangeEnd = operations[i].range.getEndPosition();
+			let nextRangeStart = operations[i + 1].range.getStartPosition();
+
+			if (nextRangeStart.isBefore(rangeEnd)) {
+				// overlapping ranges
+				throw new Error('Overlapping ranges are not allowed!');
+			}
+		}
+
+		if (canReduceOperations) {
+			operations = this._reduceOperations(operations);
+		}
+
+		let editableRange = this.getEditableRange();
+		let editableRangeStart = editableRange.getStartPosition();
+		let editableRangeEnd = editableRange.getEndPosition();
+		for (let i = 0; i < operations.length; i++) {
+			let operationRange = operations[i].range;
+			if (!editableRangeStart.isBeforeOrEqual(operationRange.getStartPosition()) || !operationRange.getEndPosition().isBeforeOrEqual(editableRangeEnd)) {
+				throw new Error('Editing outside of editable range not allowed!');
+			}
+		}
+
+		// Delta encode operations
+		let reverseRanges = EditableTextModel._getInverseEditRanges(operations);
+		let reverseOperations: editorCommon.IIdentifiedSingleEditOperation[] = [];
+
+		let newTrimAutoWhitespaceCandidates: { lineNumber: number, oldContent: string }[] = [];
+
+		for (let i = 0; i < operations.length; i++) {
+			let op = operations[i];
+			let reverseRange = reverseRanges[i];
+
+			reverseOperations[i] = {
+				identifier: op.identifier,
+				range: reverseRange,
+				text: this.getValueInRange(op.range),
+				forceMoveMarkers: op.forceMoveMarkers
+			};
+
+			if (this._options.trimAutoWhitespace && op.isAutoWhitespaceEdit && op.range.isEmpty()) {
+				// Record already the future line numbers that might be auto whitespace removal candidates on next edit
+				for (let lineNumber = reverseRange.startLineNumber; lineNumber <= reverseRange.endLineNumber; lineNumber++) {
+					let currentLineContent = '';
+					if (lineNumber === reverseRange.startLineNumber) {
+						currentLineContent = this.getLineContent(op.range.startLineNumber);
+						if (strings.firstNonWhitespaceIndex(currentLineContent) !== -1) {
+							continue;
+						}
+					}
+					newTrimAutoWhitespaceCandidates.push({ lineNumber: lineNumber, oldContent: currentLineContent });
+				}
+			}
+		}
+
+		this._mightContainRTL = mightContainRTL;
+		this._mightContainNonBasicASCII = mightContainNonBasicASCII;
+		this._doApplyEdits(markersTracker, operations);
+
+		this._trimAutoWhitespaceLines = null;
+		if (this._options.trimAutoWhitespace && newTrimAutoWhitespaceCandidates.length > 0) {
+			// sort line numbers auto whitespace removal candidates for next edit descending
+			newTrimAutoWhitespaceCandidates.sort((a, b) => b.lineNumber - a.lineNumber);
+
+			this._trimAutoWhitespaceLines = [];
+			for (let i = 0, len = newTrimAutoWhitespaceCandidates.length; i < len; i++) {
+				let lineNumber = newTrimAutoWhitespaceCandidates[i].lineNumber;
+				if (i > 0 && newTrimAutoWhitespaceCandidates[i - 1].lineNumber === lineNumber) {
+					// Do not have the same line number twice
+					continue;
+				}
+
+				let prevContent = newTrimAutoWhitespaceCandidates[i].oldContent;
+				let lineContent = this.getLineContent(lineNumber);
+
+				if (lineContent.length === 0 || lineContent === prevContent || strings.firstNonWhitespaceIndex(lineContent) !== -1) {
+					continue;
+				}
+
+				this._trimAutoWhitespaceLines.push(lineNumber);
+			}
+		}
+
+		return reverseOperations;
+	}
+
+	/**
+	 * Assumes `operations` are validated and sorted ascending
+	 */
+	public static _getInverseEditRanges(operations: IValidatedEditOperation[]): Range[] {
+		let result: Range[] = [];
+
+		let prevOpEndLineNumber: number;
+		let prevOpEndColumn: number;
+		let prevOp: IValidatedEditOperation = null;
+		for (let i = 0, len = operations.length; i < len; i++) {
+			let op = operations[i];
+
+			let startLineNumber: number;
+			let startColumn: number;
+
+			if (prevOp) {
+				if (prevOp.range.endLineNumber === op.range.startLineNumber) {
+					startLineNumber = prevOpEndLineNumber;
+					startColumn = prevOpEndColumn + (op.range.startColumn - prevOp.range.endColumn);
+				} else {
+					startLineNumber = prevOpEndLineNumber + (op.range.startLineNumber - prevOp.range.endLineNumber);
+					startColumn = op.range.startColumn;
+				}
+			} else {
+				startLineNumber = op.range.startLineNumber;
+				startColumn = op.range.startColumn;
+			}
+
+			let resultRange: Range;
+
+			if (op.lines && op.lines.length > 0) {
+				// the operation inserts something
+				let lineCount = op.lines.length;
+				let firstLine = op.lines[0];
+				let lastLine = op.lines[lineCount - 1];
+
+				if (lineCount === 1) {
+					// single line insert
+					resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn + firstLine.length);
+				} else {
+					// multi line insert
+					resultRange = new Range(startLineNumber, startColumn, startLineNumber + lineCount - 1, lastLine.length + 1);
+				}
+			} else {
+				// There is nothing to insert
+				resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn);
+			}
+
+			prevOpEndLineNumber = resultRange.endLineNumber;
+			prevOpEndColumn = resultRange.endColumn;
+
+			result.push(resultRange);
+			prevOp = op;
+		}
+
+		return result;
+	}
+
+	private _doApplyEdits(markersTracker: MarkersTracker, operations: IValidatedEditOperation[]): void {
+
+		const tabSize = this._options.tabSize;
+
+		// Sort operations descending
+		operations.sort(EditableTextModel._sortOpsDescending);
+
+		let rawContentChanges: textModelEvents.ModelRawChange[] = [];
+		let contentChanges: textModelEvents.IModelContentChange[] = [];
+		let lineEditsQueue: IIdentifiedLineEdit[] = [];
+
+		const queueLineEdit = (lineEdit: IIdentifiedLineEdit) => {
+			if (lineEdit.startColumn === lineEdit.endColumn && lineEdit.text.length === 0) {
+				// empty edit => ignore it
+				return;
+			}
+			lineEditsQueue.push(lineEdit);
+		};
+
+		const flushLineEdits = () => {
+			if (lineEditsQueue.length === 0) {
+				return;
+			}
+
+			lineEditsQueue.reverse();
+
+			// `lineEditsQueue` now contains edits from smaller (line number,column) to larger (line number,column)
+			let currentLineNumber = lineEditsQueue[0].lineNumber;
+			let currentLineNumberStart = 0;
+
+			for (let i = 1, len = lineEditsQueue.length; i < len; i++) {
+				const lineNumber = lineEditsQueue[i].lineNumber;
+
+				if (lineNumber === currentLineNumber) {
+					continue;
+				}
+
+				this._invalidateLine(currentLineNumber - 1);
+				this._lines[currentLineNumber - 1].applyEdits(markersTracker, lineEditsQueue.slice(currentLineNumberStart, i), tabSize);
+				if (this._lineStarts) {
+					// update prefix sum
+					this._lineStarts.changeValue(currentLineNumber - 1, this._lines[currentLineNumber - 1].text.length + this._EOL.length);
+				}
+				rawContentChanges.push(
+					new textModelEvents.ModelRawLineChanged(currentLineNumber, this._lines[currentLineNumber - 1].text)
+				);
+
+				currentLineNumber = lineNumber;
+				currentLineNumberStart = i;
+			}
+
+			this._invalidateLine(currentLineNumber - 1);
+			this._lines[currentLineNumber - 1].applyEdits(markersTracker, lineEditsQueue.slice(currentLineNumberStart, lineEditsQueue.length), tabSize);
+			if (this._lineStarts) {
+				// update prefix sum
+				this._lineStarts.changeValue(currentLineNumber - 1, this._lines[currentLineNumber - 1].text.length + this._EOL.length);
+			}
+			rawContentChanges.push(
+				new textModelEvents.ModelRawLineChanged(currentLineNumber, this._lines[currentLineNumber - 1].text)
+			);
+
+			lineEditsQueue = [];
+		};
+
+		let minTouchedLineNumber = operations[operations.length - 1].range.startLineNumber;
+		let maxTouchedLineNumber = operations[0].range.endLineNumber + 1;
+		let totalLinesCountDelta = 0;
+
+		for (let i = 0, len = operations.length; i < len; i++) {
+			const op = operations[i];
+
+			// console.log();
+			// console.log('-------------------');
+			// console.log('OPERATION #' + (i));
+			// console.log('op: ', op);
+			// console.log('<<<\n' + this._lines.map(l => l.text).join('\n') + '\n>>>');
+
+			const startLineNumber = op.range.startLineNumber;
+			const startColumn = op.range.startColumn;
+			const endLineNumber = op.range.endLineNumber;
+			const endColumn = op.range.endColumn;
+
+			if (startLineNumber === endLineNumber && startColumn === endColumn && (!op.lines || op.lines.length === 0)) {
+				// no-op
+				continue;
+			}
+
+			const deletingLinesCnt = endLineNumber - startLineNumber;
+			const insertingLinesCnt = (op.lines ? op.lines.length - 1 : 0);
+			const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt);
+
+			totalLinesCountDelta += (insertingLinesCnt - deletingLinesCnt);
+
+			// Iterating descending to overlap with previous op
+			// in case there are common lines being edited in both
+			for (let j = editingLinesCnt; j >= 0; j--) {
+				const editLineNumber = startLineNumber + j;
+
+				queueLineEdit({
+					lineNumber: editLineNumber,
+					startColumn: (editLineNumber === startLineNumber ? startColumn : 1),
+					endColumn: (editLineNumber === endLineNumber ? endColumn : this.getLineMaxColumn(editLineNumber)),
+					text: (op.lines ? op.lines[j] : ''),
+					forceMoveMarkers: op.forceMoveMarkers
+				});
+			}
+
+			if (editingLinesCnt < deletingLinesCnt) {
+				// Must delete some lines
+
+				// Flush any pending line edits
+				flushLineEdits();
+
+				const spliceStartLineNumber = startLineNumber + editingLinesCnt;
+				const spliceStartColumn = this.getLineMaxColumn(spliceStartLineNumber);
+
+				const endLineRemains = this._lines[endLineNumber - 1].split(markersTracker, endColumn, false, tabSize);
+				this._invalidateLine(spliceStartLineNumber - 1);
+
+				const spliceCnt = endLineNumber - spliceStartLineNumber;
+
+				// Collect all these markers
+				let markersOnDeletedLines: LineMarker[] = [];
+				for (let j = 0; j < spliceCnt; j++) {
+					const deleteLineIndex = spliceStartLineNumber + j;
+					const deleteLineMarkers = this._lines[deleteLineIndex].getMarkers();
+					if (deleteLineMarkers) {
+						markersOnDeletedLines = markersOnDeletedLines.concat(deleteLineMarkers);
+					}
+				}
+
+				this._lines.splice(spliceStartLineNumber, spliceCnt);
+				if (this._lineStarts) {
+					// update prefix sum
+					this._lineStarts.removeValues(spliceStartLineNumber, spliceCnt);
+				}
+
+				// Reconstruct first line
+				this._lines[spliceStartLineNumber - 1].append(markersTracker, spliceStartLineNumber, endLineRemains, tabSize);
+				if (this._lineStarts) {
+					// update prefix sum
+					this._lineStarts.changeValue(spliceStartLineNumber - 1, this._lines[spliceStartLineNumber - 1].text.length + this._EOL.length);
+				}
+
+				// Update deleted markers
+				const deletedMarkersPosition = new Position(spliceStartLineNumber, spliceStartColumn);
+				for (let j = 0, lenJ = markersOnDeletedLines.length; j < lenJ; j++) {
+					markersOnDeletedLines[j].updatePosition(markersTracker, deletedMarkersPosition);
+				}
+
+				this._lines[spliceStartLineNumber - 1].addMarkers(markersOnDeletedLines);
+				rawContentChanges.push(
+					new textModelEvents.ModelRawLineChanged(spliceStartLineNumber, this._lines[spliceStartLineNumber - 1].text)
+				);
+
+				rawContentChanges.push(
+					new textModelEvents.ModelRawLinesDeleted(spliceStartLineNumber + 1, spliceStartLineNumber + spliceCnt)
+				);
+			}
+
+			if (editingLinesCnt < insertingLinesCnt) {
+				// Must insert some lines
+
+				// Flush any pending line edits
+				flushLineEdits();
+
+				const spliceLineNumber = startLineNumber + editingLinesCnt;
+				let spliceColumn = (spliceLineNumber === startLineNumber ? startColumn : 1);
+				if (op.lines) {
+					spliceColumn += op.lines[editingLinesCnt].length;
+				}
+
+				// Split last line
+				let leftoverLine = this._lines[spliceLineNumber - 1].split(markersTracker, spliceColumn, op.forceMoveMarkers, tabSize);
+				if (this._lineStarts) {
+					// update prefix sum
+					this._lineStarts.changeValue(spliceLineNumber - 1, this._lines[spliceLineNumber - 1].text.length + this._EOL.length);
+				}
+				rawContentChanges.push(
+					new textModelEvents.ModelRawLineChanged(spliceLineNumber, this._lines[spliceLineNumber - 1].text)
+				);
+				this._invalidateLine(spliceLineNumber - 1);
+
+				// Lines in the middle
+				let newLines: IModelLine[] = [];
+				let newLinesContent: string[] = [];
+				let newLinesLengths = new Uint32Array(insertingLinesCnt - editingLinesCnt);
+				for (let j = editingLinesCnt + 1; j <= insertingLinesCnt; j++) {
+					newLines.push(this._createModelLine(op.lines[j], tabSize));
+					newLinesContent.push(op.lines[j]);
+					newLinesLengths[j - editingLinesCnt - 1] = op.lines[j].length + this._EOL.length;
+				}
+				this._lines = arrays.arrayInsert(this._lines, startLineNumber + editingLinesCnt, newLines);
+				newLinesContent[newLinesContent.length - 1] += leftoverLine.text;
+				if (this._lineStarts) {
+					// update prefix sum
+					this._lineStarts.insertValues(startLineNumber + editingLinesCnt, newLinesLengths);
+				}
+
+				// Last line
+				this._lines[startLineNumber + insertingLinesCnt - 1].append(markersTracker, startLineNumber + insertingLinesCnt, leftoverLine, tabSize);
+				if (this._lineStarts) {
+					// update prefix sum
+					this._lineStarts.changeValue(startLineNumber + insertingLinesCnt - 1, this._lines[startLineNumber + insertingLinesCnt - 1].text.length + this._EOL.length);
+				}
+				rawContentChanges.push(
+					new textModelEvents.ModelRawLinesInserted(spliceLineNumber + 1, startLineNumber + insertingLinesCnt, newLinesContent.join('\n'))
+				);
+			}
+
+			contentChanges.push({
+				range: new Range(startLineNumber, startColumn, endLineNumber, endColumn),
+				rangeLength: op.rangeLength,
+				text: op.lines ? op.lines.join(this.getEOL()) : ''
+			});
+
+			// console.log('AFTER:');
+			// console.log('<<<\n' + this._lines.map(l => l.text).join('\n') + '\n>>>');
+		}
+
+		flushLineEdits();
+
+		maxTouchedLineNumber = Math.max(1, Math.min(this.getLineCount(), maxTouchedLineNumber + totalLinesCountDelta));
+		if (totalLinesCountDelta !== 0) {
+			// must update line numbers all the way to the bottom
+			maxTouchedLineNumber = this.getLineCount();
+		}
+
+		for (let lineNumber = minTouchedLineNumber; lineNumber <= maxTouchedLineNumber; lineNumber++) {
+			this._lines[lineNumber - 1].updateLineNumber(markersTracker, lineNumber);
+		}
+
+		if (rawContentChanges.length !== 0 || contentChanges.length !== 0) {
+			this._increaseVersionId();
+
+			this._emitModelRawContentChangedEvent(new textModelEvents.ModelRawContentChangedEvent(
+				rawContentChanges,
+				this.getVersionId(),
+				this._isUndoing,
+				this._isRedoing
+			));
+
+			const e: textModelEvents.IModelContentChangedEvent = {
+				changes: contentChanges,
+				eol: this._EOL,
+				versionId: this.getVersionId(),
+				isUndoing: this._isUndoing,
+				isRedoing: this._isRedoing,
+				isFlush: false
+			};
+			this._eventEmitter.emit(textModelEvents.TextModelEventType.ModelContentChanged, e);
+		}
+
+		// this._assertLineNumbersOK();
+		this._resetIndentRanges();
+	}
+
+	public _assertLineNumbersOK(): void {
+		let foundMarkersCnt = 0;
+		for (let i = 0, len = this._lines.length; i < len; i++) {
+			let line = this._lines[i];
+			let lineNumber = i + 1;
+
+			let markers = line.getMarkers();
+			if (markers !== null) {
+				for (let j = 0, lenJ = markers.length; j < lenJ; j++) {
+					foundMarkersCnt++;
+					let markerId = markers[j].id;
+					let marker = this._markerIdToMarker[markerId];
+					if (marker.position.lineNumber !== lineNumber) {
+						throw new Error('Misplaced marker with id ' + markerId);
+					}
+				}
+			}
+		}
+
+		let totalMarkersCnt = Object.keys(this._markerIdToMarker).length;
+		if (totalMarkersCnt !== foundMarkersCnt) {
+			throw new Error('There are misplaced markers!');
+		}
+	}
+
+	private _undo(): Selection[] {
+		this._isUndoing = true;
+		let r = this._commandManager.undo();
+		this._isUndoing = false;
+
+		if (!r) {
+			return null;
+		}
+
+		this._overwriteAlternativeVersionId(r.recordedVersionId);
+
+		return r.selections;
+	}
+
+	public undo(): Selection[] {
+		try {
+			this._eventEmitter.beginDeferredEmit();
+			this._acquireMarkersTracker();
+			return this._undo();
+		} finally {
+			this._releaseMarkersTracker();
+			this._eventEmitter.endDeferredEmit();
+		}
+	}
+
+	private _redo(): Selection[] {
+		this._isRedoing = true;
+		let r = this._commandManager.redo();
+		this._isRedoing = false;
+
+		if (!r) {
+			return null;
+		}
+
+		this._overwriteAlternativeVersionId(r.recordedVersionId);
+
+		return r.selections;
+	}
+
+	public redo(): Selection[] {
+		try {
+			this._eventEmitter.beginDeferredEmit();
+			this._acquireMarkersTracker();
+			return this._redo();
+		} finally {
+			this._releaseMarkersTracker();
+			this._eventEmitter.endDeferredEmit();
+		}
+	}
+
+	public setEditableRange(range: IRange): void {
+		this._commandManager.clear();
+
+		if (!this._hasEditableRange && !range) {
+			// Nothing to do
+			return;
+		}
+
+		this.changeDecorations((changeAccessor) => {
+			if (this._hasEditableRange) {
+				changeAccessor.removeDecoration(this._editableRangeId);
+				this._editableRangeId = null;
+				this._hasEditableRange = false;
+			}
+
+			if (range) {
+				this._hasEditableRange = true;
+				this._editableRangeId = changeAccessor.addDecoration(range, EditableTextModel._DECORATION_OPTION);
+			}
+		});
+	}
+
+	private static _DECORATION_OPTION = ModelDecorationOptions.register({
+		stickiness: editorCommon.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges
+	});
+
+	public hasEditableRange(): boolean {
+		return this._hasEditableRange;
+	}
+
+	public getEditableRange(): Range {
+		if (this._hasEditableRange) {
+			return this.getDecorationRange(this._editableRangeId);
+		} else {
+			return this.getFullModelRange();
+		}
+	}
+}

src/vs/editor/common/model/indentRanges.ts

@@ -0,0 +1,69 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { ITextModel } from 'vs/editor/common/editorCommon';
+
+export class IndentRange {
+	_indentRangeBrand: void;
+	startLineNumber: number;
+	endLineNumber: number;
+	indent: number;
+
+	constructor(startLineNumber: number, endLineNumber: number, indent: number) {
+		this.startLineNumber = startLineNumber;
+		this.endLineNumber = endLineNumber;
+		this.indent = indent;
+	}
+
+	public static deepCloneArr(indentRanges: IndentRange[]): IndentRange[] {
+		let result: IndentRange[] = [];
+		for (let i = 0, len = indentRanges.length; i < len; i++) {
+			let r = indentRanges[i];
+			result[i] = new IndentRange(r.startLineNumber, r.endLineNumber, r.indent);
+		}
+		return result;
+	}
+}
+
+export function computeRanges(model: ITextModel, minimumRangeSize: number = 1): IndentRange[] {
+
+	let result: IndentRange[] = [];
+
+	let previousRegions: { indent: number, line: number }[] = [];
+	previousRegions.push({ indent: -1, line: model.getLineCount() + 1 }); // sentinel, to make sure there's at least one entry
+
+	for (let line = model.getLineCount(); line > 0; line--) {
+		let indent = model.getIndentLevel(line);
+		if (indent === -1) {
+			continue; // only whitespace
+		}
+
+		let previous = previousRegions[previousRegions.length - 1];
+
+		if (previous.indent > indent) {
+			// discard all regions with larger indent
+			do {
+				previousRegions.pop();
+				previous = previousRegions[previousRegions.length - 1];
+			} while (previous.indent > indent);
+
+			// new folding range
+			let endLineNumber = previous.line - 1;
+			if (endLineNumber - line >= minimumRangeSize) {
+				result.push(new IndentRange(line, endLineNumber, indent));
+			}
+		}
+		if (previous.indent === indent) {
+			previous.line = line;
+		} else { // previous.indent < indent
+			// new region with a bigger indent
+			previousRegions.push({ indent, line });
+		}
+	}
+
+	return result.reverse();
+}

src/vs/editor/common/model/indentationGuesser.ts

@@ -0,0 +1,174 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { CharCode } from 'vs/base/common/charCode';
+
+/**
+ * Compute the diff in spaces between two line's indentation.
+ */
+function spacesDiff(a: string, aLength: number, b: string, bLength: number): number {
+
+	// This can go both ways (e.g.):
+	//  - a: "\t"
+	//  - b: "\t    "
+	//  => This should count 1 tab and 4 spaces
+
+	let i: number;
+
+	for (i = 0; i < aLength && i < bLength; i++) {
+		let aCharCode = a.charCodeAt(i);
+		let bCharCode = b.charCodeAt(i);
+
+		if (aCharCode !== bCharCode) {
+			break;
+		}
+	}
+
+	let aSpacesCnt = 0, aTabsCount = 0;
+	for (let j = i; j < aLength; j++) {
+		let aCharCode = a.charCodeAt(j);
+		if (aCharCode === CharCode.Space) {
+			aSpacesCnt++;
+		} else {
+			aTabsCount++;
+		}
+	}
+
+	let bSpacesCnt = 0, bTabsCount = 0;
+	for (let j = i; j < bLength; j++) {
+		let bCharCode = b.charCodeAt(j);
+		if (bCharCode === CharCode.Space) {
+			bSpacesCnt++;
+		} else {
+			bTabsCount++;
+		}
+	}
+
+	if (aSpacesCnt > 0 && aTabsCount > 0) {
+		return 0;
+	}
+	if (bSpacesCnt > 0 && bTabsCount > 0) {
+		return 0;
+	}
+
+	let tabsDiff = Math.abs(aTabsCount - bTabsCount);
+	let spacesDiff = Math.abs(aSpacesCnt - bSpacesCnt);
+
+	if (tabsDiff === 0) {
+		return spacesDiff;
+	}
+	if (spacesDiff % tabsDiff === 0) {
+		return spacesDiff / tabsDiff;
+	}
+	return 0;
+}
+
+/**
+ * Result for a guessIndentation
+ */
+export interface IGuessedIndentation {
+	/**
+	 * If indentation is based on spaces (`insertSpaces` = true), then what is the number of spaces that make an indent?
+	 */
+	tabSize: number;
+	/**
+	 * Is indentation based on spaces?
+	 */
+	insertSpaces: boolean;
+}
+
+export function guessIndentation(lines: string[], defaultTabSize: number, defaultInsertSpaces: boolean): IGuessedIndentation {
+	// Look at most at the first 10k lines
+	const linesLen = Math.min(lines.length, 10000);
+
+	let linesIndentedWithTabsCount = 0;				// number of lines that contain at least one tab in indentation
+	let linesIndentedWithSpacesCount = 0;			// number of lines that contain only spaces in indentation
+
+	let previousLineText = '';						// content of latest line that contained non-whitespace chars
+	let previousLineIndentation = 0;				// index at which latest line contained the first non-whitespace char
+
+	const ALLOWED_TAB_SIZE_GUESSES = [2, 4, 6, 8];	// limit guesses for `tabSize` to 2, 4, 6 or 8.
+	const MAX_ALLOWED_TAB_SIZE_GUESS = 8;			// max(2,4,6,8) = 8
+
+	let spacesDiffCount = [0, 0, 0, 0, 0, 0, 0, 0, 0];		// `tabSize` scores
+
+	for (let i = 0; i < linesLen; i++) {
+		let currentLineText = lines[i];
+
+		let currentLineHasContent = false;			// does `currentLineText` contain non-whitespace chars
+		let currentLineIndentation = 0;				// index at which `currentLineText` contains the first non-whitespace char
+		let currentLineSpacesCount = 0;				// count of spaces found in `currentLineText` indentation
+		let currentLineTabsCount = 0;				// count of tabs found in `currentLineText` indentation
+		for (let j = 0, lenJ = currentLineText.length; j < lenJ; j++) {
+			let charCode = currentLineText.charCodeAt(j);
+
+			if (charCode === CharCode.Tab) {
+				currentLineTabsCount++;
+			} else if (charCode === CharCode.Space) {
+				currentLineSpacesCount++;
+			} else {
+				// Hit non whitespace character on this line
+				currentLineHasContent = true;
+				currentLineIndentation = j;
+				break;
+			}
+		}
+
+		// Ignore empty or only whitespace lines
+		if (!currentLineHasContent) {
+			continue;
+		}
+
+		if (currentLineTabsCount > 0) {
+			linesIndentedWithTabsCount++;
+		} else if (currentLineSpacesCount > 1) {
+			linesIndentedWithSpacesCount++;
+		}
+
+		let currentSpacesDiff = spacesDiff(previousLineText, previousLineIndentation, currentLineText, currentLineIndentation);
+		if (currentSpacesDiff <= MAX_ALLOWED_TAB_SIZE_GUESS) {
+			spacesDiffCount[currentSpacesDiff]++;
+		}
+
+		previousLineText = currentLineText;
+		previousLineIndentation = currentLineIndentation;
+	}
+
+	// Take into account the last line as well
+	let deltaSpacesCount = spacesDiff(previousLineText, previousLineIndentation, '', 0);
+	if (deltaSpacesCount <= MAX_ALLOWED_TAB_SIZE_GUESS) {
+		spacesDiffCount[deltaSpacesCount]++;
+	}
+
+	let insertSpaces = defaultInsertSpaces;
+	if (linesIndentedWithTabsCount !== linesIndentedWithSpacesCount) {
+		insertSpaces = (linesIndentedWithTabsCount < linesIndentedWithSpacesCount);
+	}
+
+	let tabSize = defaultTabSize;
+	let tabSizeScore = (insertSpaces ? 0 : 0.1 * linesLen);
+
+	// console.log("score threshold: " + tabSizeScore);
+
+	ALLOWED_TAB_SIZE_GUESSES.forEach((possibleTabSize) => {
+		let possibleTabSizeScore = spacesDiffCount[possibleTabSize];
+		if (possibleTabSizeScore > tabSizeScore) {
+			tabSizeScore = possibleTabSizeScore;
+			tabSize = possibleTabSize;
+		}
+	});
+
+
+	// console.log('--------------------------');
+	// console.log('linesIndentedWithTabsCount: ' + linesIndentedWithTabsCount + ', linesIndentedWithSpacesCount: ' + linesIndentedWithSpacesCount);
+	// console.log('spacesDiffCount: ' + spacesDiffCount);
+	// console.log('tabSize: ' + tabSize + ', tabSizeScore: ' + tabSizeScore);
+
+	return {
+		insertSpaces: insertSpaces,
+		tabSize: tabSize
+	};
+}

src/vs/editor/common/model/mirrorModel.ts

@@ -0,0 +1,164 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import URI from 'vs/base/common/uri';
+import { IRange } from 'vs/editor/common/core/range';
+import { PrefixSumComputer } from 'vs/editor/common/viewModel/prefixSumComputer';
+import { IModelContentChange } from 'vs/editor/common/model/textModelEvents';
+import { IPosition } from 'vs/editor/common/core/position';
+
+export interface IModelChangedEvent {
+	/**
+	 * The actual changes.
+	 */
+	readonly changes: IModelContentChange[];
+	/**
+	 * The (new) end-of-line character.
+	 */
+	readonly eol: string;
+	/**
+	 * The new version id the model has transitioned to.
+	 */
+	readonly versionId: number;
+}
+
+export class MirrorModel {
+
+	protected _uri: URI;
+	protected _lines: string[];
+	protected _eol: string;
+	protected _versionId: number;
+	protected _lineStarts: PrefixSumComputer;
+
+	constructor(uri: URI, lines: string[], eol: string, versionId: number) {
+		this._uri = uri;
+		this._lines = lines;
+		this._eol = eol;
+		this._versionId = versionId;
+	}
+
+	dispose(): void {
+		this._lines.length = 0;
+	}
+
+	get version(): number {
+		return this._versionId;
+	}
+
+	getText(): string {
+		return this._lines.join(this._eol);
+	}
+
+	onEvents(e: IModelChangedEvent): void {
+		if (e.eol && e.eol !== this._eol) {
+			this._eol = e.eol;
+			this._lineStarts = null;
+		}
+
+		// Update my lines
+		const changes = e.changes;
+		for (let i = 0, len = changes.length; i < len; i++) {
+			const change = changes[i];
+			this._acceptDeleteRange(change.range);
+			this._acceptInsertText({
+				lineNumber: change.range.startLineNumber,
+				column: change.range.startColumn
+			}, change.text);
+		}
+
+		this._versionId = e.versionId;
+	}
+
+	protected _ensureLineStarts(): void {
+		if (!this._lineStarts) {
+			const eolLength = this._eol.length;
+			const linesLength = this._lines.length;
+			const lineStartValues = new Uint32Array(linesLength);
+			for (let i = 0; i < linesLength; i++) {
+				lineStartValues[i] = this._lines[i].length + eolLength;
+			}
+			this._lineStarts = new PrefixSumComputer(lineStartValues);
+		}
+	}
+
+	/**
+	 * All changes to a line's text go through this method
+	 */
+	private _setLineText(lineIndex: number, newValue: string): void {
+		this._lines[lineIndex] = newValue;
+		if (this._lineStarts) {
+			// update prefix sum
+			this._lineStarts.changeValue(lineIndex, this._lines[lineIndex].length + this._eol.length);
+		}
+	}
+
+	private _acceptDeleteRange(range: IRange): void {
+
+		if (range.startLineNumber === range.endLineNumber) {
+			if (range.startColumn === range.endColumn) {
+				// Nothing to delete
+				return;
+			}
+			// Delete text on the affected line
+			this._setLineText(range.startLineNumber - 1,
+				this._lines[range.startLineNumber - 1].substring(0, range.startColumn - 1)
+				+ this._lines[range.startLineNumber - 1].substring(range.endColumn - 1)
+			);
+			return;
+		}
+
+		// Take remaining text on last line and append it to remaining text on first line
+		this._setLineText(range.startLineNumber - 1,
+			this._lines[range.startLineNumber - 1].substring(0, range.startColumn - 1)
+			+ this._lines[range.endLineNumber - 1].substring(range.endColumn - 1)
+		);
+
+		// Delete middle lines
+		this._lines.splice(range.startLineNumber, range.endLineNumber - range.startLineNumber);
+		if (this._lineStarts) {
+			// update prefix sum
+			this._lineStarts.removeValues(range.startLineNumber, range.endLineNumber - range.startLineNumber);
+		}
+	}
+
+	private _acceptInsertText(position: IPosition, insertText: string): void {
+		if (insertText.length === 0) {
+			// Nothing to insert
+			return;
+		}
+		let insertLines = insertText.split(/\r\n|\r|\n/);
+		if (insertLines.length === 1) {
+			// Inserting text on one line
+			this._setLineText(position.lineNumber - 1,
+				this._lines[position.lineNumber - 1].substring(0, position.column - 1)
+				+ insertLines[0]
+				+ this._lines[position.lineNumber - 1].substring(position.column - 1)
+			);
+			return;
+		}
+
+		// Append overflowing text from first line to the end of text to insert
+		insertLines[insertLines.length - 1] += this._lines[position.lineNumber - 1].substring(position.column - 1);
+
+		// Delete overflowing text from first line and insert text on first line
+		this._setLineText(position.lineNumber - 1,
+			this._lines[position.lineNumber - 1].substring(0, position.column - 1)
+			+ insertLines[0]
+		);
+
+		// Insert new lines & store lengths
+		let newLengths = new Uint32Array(insertLines.length - 1);
+		for (let i = 1; i < insertLines.length; i++) {
+			this._lines.splice(position.lineNumber + i - 1, 0, insertLines[i]);
+			newLengths[i - 1] = insertLines[i].length + this._eol.length;
+		}
+
+		if (this._lineStarts) {
+			// update prefix sum
+			this._lineStarts.insertValues(position.lineNumber, newLengths);
+		}
+	}
+}

src/vs/editor/common/model/model.ts

@@ -0,0 +1,95 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import URI from 'vs/base/common/uri';
+import {
+	IModel, ITextModelCreationOptions
+} from 'vs/editor/common/editorCommon';
+import { EditableTextModel } from 'vs/editor/common/model/editableTextModel';
+import { TextModel } from 'vs/editor/common/model/textModel';
+import { IDisposable } from 'vs/base/common/lifecycle';
+import { LanguageIdentifier } from 'vs/editor/common/modes';
+import { IRawTextSource, RawTextSource } from 'vs/editor/common/model/textSource';
+import * as textModelEvents from 'vs/editor/common/model/textModelEvents';
+
+// The hierarchy is:
+// Model -> EditableTextModel -> TextModelWithDecorations -> TextModelWithTrackedRanges -> TextModelWithMarkers -> TextModelWithTokens -> TextModel
+
+var MODEL_ID = 0;
+
+export class Model extends EditableTextModel implements IModel {
+
+	public onDidChangeDecorations(listener: (e: textModelEvents.IModelDecorationsChangedEvent) => void): IDisposable {
+		return this._eventEmitter.addListener(textModelEvents.TextModelEventType.ModelDecorationsChanged, listener);
+	}
+	public onDidChangeOptions(listener: (e: textModelEvents.IModelOptionsChangedEvent) => void): IDisposable {
+		return this._eventEmitter.addListener(textModelEvents.TextModelEventType.ModelOptionsChanged, listener);
+	}
+	public onWillDispose(listener: () => void): IDisposable {
+		return this._eventEmitter.addListener(textModelEvents.TextModelEventType.ModelDispose, listener);
+	}
+	public onDidChangeLanguage(listener: (e: textModelEvents.IModelLanguageChangedEvent) => void): IDisposable {
+		return this._eventEmitter.addListener(textModelEvents.TextModelEventType.ModelLanguageChanged, listener);
+	}
+
+	public static createFromString(text: string, options: ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageIdentifier: LanguageIdentifier = null, uri: URI = null): Model {
+		return new Model(RawTextSource.fromString(text), options, languageIdentifier, uri);
+	}
+
+	public readonly id: string;
+
+	private readonly _associatedResource: URI;
+	private _attachedEditorCount: number;
+
+	constructor(rawTextSource: IRawTextSource, creationOptions: ITextModelCreationOptions, languageIdentifier: LanguageIdentifier, associatedResource: URI = null) {
+		super(rawTextSource, creationOptions, languageIdentifier);
+
+		// Generate a new unique model id
+		MODEL_ID++;
+		this.id = '$model' + MODEL_ID;
+
+		if (typeof associatedResource === 'undefined' || associatedResource === null) {
+			this._associatedResource = URI.parse('inmemory://model/' + MODEL_ID);
+		} else {
+			this._associatedResource = associatedResource;
+		}
+
+		this._attachedEditorCount = 0;
+	}
+
+	public destroy(): void {
+		this.dispose();
+	}
+
+	public dispose(): void {
+		this._isDisposing = true;
+		this._eventEmitter.emit(textModelEvents.TextModelEventType.ModelDispose);
+		super.dispose();
+		this._isDisposing = false;
+	}
+
+	public onBeforeAttached(): void {
+		this._attachedEditorCount++;
+		// Warm up tokens for the editor
+		this._warmUpTokens();
+	}
+
+	public onBeforeDetached(): void {
+		this._attachedEditorCount--;
+	}
+
+	protected _shouldAutoTokenize(): boolean {
+		return this.isAttachedToEditor();
+	}
+
+	public isAttachedToEditor(): boolean {
+		return this._attachedEditorCount > 0;
+	}
+
+	public get uri(): URI {
+		return this._associatedResource;
+	}
+}

src/vs/editor/common/model/modelLine.ts

@@ -0,0 +1,893 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { IState, FontStyle, StandardTokenType, MetadataConsts, ColorId, LanguageId } from 'vs/editor/common/modes';
+import { CharCode } from 'vs/base/common/charCode';
+import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { Position } from 'vs/editor/common/core/position';
+import { Constants } from 'vs/editor/common/core/uint';
+
+export interface ILineEdit {
+	startColumn: number;
+	endColumn: number;
+	text: string;
+	forceMoveMarkers: boolean;
+}
+
+export class LineMarker {
+	_lineMarkerBrand: void;
+
+	public readonly id: string;
+	public readonly internalDecorationId: number;
+
+	public stickToPreviousCharacter: boolean;
+	public position: Position;
+
+	constructor(id: string, internalDecorationId: number, position: Position, stickToPreviousCharacter: boolean) {
+		this.id = id;
+		this.internalDecorationId = internalDecorationId;
+		this.position = position;
+		this.stickToPreviousCharacter = stickToPreviousCharacter;
+	}
+
+	public toString(): string {
+		return '{\'' + this.id + '\';' + this.position.toString() + ',' + this.stickToPreviousCharacter + '}';
+	}
+
+	public updateLineNumber(markersTracker: MarkersTracker, lineNumber: number): void {
+		if (this.position.lineNumber === lineNumber) {
+			return;
+		}
+		markersTracker.addChangedMarker(this);
+		this.position = new Position(lineNumber, this.position.column);
+	}
+
+	public updateColumn(markersTracker: MarkersTracker, column: number): void {
+		if (this.position.column === column) {
+			return;
+		}
+		markersTracker.addChangedMarker(this);
+		this.position = new Position(this.position.lineNumber, column);
+	}
+
+	public updatePosition(markersTracker: MarkersTracker, position: Position): void {
+		if (this.position.lineNumber === position.lineNumber && this.position.column === position.column) {
+			return;
+		}
+		markersTracker.addChangedMarker(this);
+		this.position = position;
+	}
+
+	public setPosition(position: Position) {
+		this.position = position;
+	}
+
+
+	public static compareMarkers(a: LineMarker, b: LineMarker): number {
+		if (a.position.column === b.position.column) {
+			return (a.stickToPreviousCharacter ? 0 : 1) - (b.stickToPreviousCharacter ? 0 : 1);
+		}
+		return a.position.column - b.position.column;
+	}
+}
+
+export class MarkersTracker {
+	_changedDecorationsBrand: void;
+
+	private _changedDecorations: number[];
+	private _changedDecorationsLen: number;
+
+	constructor() {
+		this._changedDecorations = [];
+		this._changedDecorationsLen = 0;
+	}
+
+	public addChangedMarker(marker: LineMarker): void {
+		let internalDecorationId = marker.internalDecorationId;
+		if (internalDecorationId !== 0) {
+			this._changedDecorations[this._changedDecorationsLen++] = internalDecorationId;
+		}
+	}
+
+	public getDecorationIds(): number[] {
+		return this._changedDecorations;
+	}
+}
+
+export interface ITokensAdjuster {
+	adjust(toColumn: number, delta: number, minimumAllowedColumn: number): void;
+	finish(delta: number, lineTextLength: number): void;
+}
+
+interface IMarkersAdjuster {
+	adjustDelta(toColumn: number, delta: number, minimumAllowedColumn: number, moveSemantics: MarkerMoveSemantics): void;
+	adjustSet(toColumn: number, newColumn: number, moveSemantics: MarkerMoveSemantics): void;
+	finish(delta: number, lineTextLength: number): void;
+}
+
+var NO_OP_TOKENS_ADJUSTER: ITokensAdjuster = {
+	adjust: () => { },
+	finish: () => { }
+};
+var NO_OP_MARKERS_ADJUSTER: IMarkersAdjuster = {
+	adjustDelta: () => { },
+	adjustSet: () => { },
+	finish: () => { }
+};
+
+const enum MarkerMoveSemantics {
+	MarkerDefined = 0,
+	ForceMove = 1,
+	ForceStay = 2
+}
+
+/**
+ * Returns:
+ *  - 0 => the line consists of whitespace
+ *  - otherwise => the indent level is returned value - 1
+ */
+function computePlusOneIndentLevel(line: string, tabSize: number): number {
+	let indent = 0;
+	let i = 0;
+	let len = line.length;
+
+	while (i < len) {
+		let chCode = line.charCodeAt(i);
+		if (chCode === CharCode.Space) {
+			indent++;
+		} else if (chCode === CharCode.Tab) {
+			indent = indent - indent % tabSize + tabSize;
+		} else {
+			break;
+		}
+		i++;
+	}
+
+	if (i === len) {
+		return 0; // line only consists of whitespace
+	}
+
+	return indent + 1;
+}
+
+export interface IModelLine {
+	readonly text: string;
+
+	// --- markers
+	addMarker(marker: LineMarker): void;
+	addMarkers(markers: LineMarker[]): void;
+	removeMarker(marker: LineMarker): void;
+	removeMarkers(deleteMarkers: { [markerId: string]: boolean; }): void;
+	getMarkers(): LineMarker[];
+
+	// --- tokenization
+	resetTokenizationState(): void;
+	isInvalid(): boolean;
+	setIsInvalid(isInvalid: boolean): void;
+	getState(): IState;
+	setState(state: IState): void;
+	getTokens(topLevelLanguageId: LanguageId): LineTokens;
+	setTokens(topLevelLanguageId: LanguageId, tokens: Uint32Array): void;
+
+	// --- indentation
+	updateTabSize(tabSize: number): void;
+	getIndentLevel(): number;
+
+	// --- editing
+	updateLineNumber(markersTracker: MarkersTracker, newLineNumber: number): void;
+	applyEdits(markersTracker: MarkersTracker, edits: ILineEdit[], tabSize: number): number;
+	append(markersTracker: MarkersTracker, myLineNumber: number, other: IModelLine, tabSize: number): void;
+	split(markersTracker: MarkersTracker, splitColumn: number, forceMoveMarkers: boolean, tabSize: number): IModelLine;
+}
+
+export abstract class AbstractModelLine {
+
+	private _markers: LineMarker[];
+
+	constructor(initializeMarkers: boolean) {
+		if (initializeMarkers) {
+			this._markers = null;
+		}
+	}
+
+	///
+
+	public abstract get text(): string;
+	protected abstract _setText(text: string, tabSize: number);
+	protected abstract _createTokensAdjuster(): ITokensAdjuster;
+	protected abstract _createModelLine(text: string, tabSize: number): IModelLine;
+
+	///
+
+	// private _printMarkers(): string {
+	// 	if (!this._markers) {
+	// 		return '[]';
+	// 	}
+	// 	if (this._markers.length === 0) {
+	// 		return '[]';
+	// 	}
+
+	// 	var markers = this._markers;
+
+	// 	var printMarker = (m:LineMarker) => {
+	// 		if (m.stickToPreviousCharacter) {
+	// 			return '|' + m.position.column;
+	// 		}
+	// 		return m.position.column + '|';
+	// 	};
+	// 	return '[' + markers.map(printMarker).join(', ') + ']';
+	// }
+
+	private _createMarkersAdjuster(markersTracker: MarkersTracker): IMarkersAdjuster {
+		if (!this._markers) {
+			return NO_OP_MARKERS_ADJUSTER;
+		}
+		if (this._markers.length === 0) {
+			return NO_OP_MARKERS_ADJUSTER;
+		}
+
+		this._markers.sort(LineMarker.compareMarkers);
+
+		var markers = this._markers;
+		var markersLength = markers.length;
+		var markersIndex = 0;
+		var marker = markers[markersIndex];
+
+		// console.log('------------- INITIAL MARKERS: ' + this._printMarkers());
+
+		let adjustMarkerBeforeColumn = (toColumn: number, moveSemantics: MarkerMoveSemantics) => {
+			if (marker.position.column < toColumn) {
+				return true;
+			}
+			if (marker.position.column > toColumn) {
+				return false;
+			}
+			if (moveSemantics === MarkerMoveSemantics.ForceMove) {
+				return false;
+			}
+			if (moveSemantics === MarkerMoveSemantics.ForceStay) {
+				return true;
+			}
+			return marker.stickToPreviousCharacter;
+		};
+
+		let adjustDelta = (toColumn: number, delta: number, minimumAllowedColumn: number, moveSemantics: MarkerMoveSemantics) => {
+			// console.log('------------------------------');
+			// console.log('adjustDelta called: toColumn: ' + toColumn + ', delta: ' + delta + ', minimumAllowedColumn: ' + minimumAllowedColumn + ', moveSemantics: ' + MarkerMoveSemantics[moveSemantics]);
+			// console.log('BEFORE::: markersIndex: ' + markersIndex + ' : ' + this._printMarkers());
+
+			while (markersIndex < markersLength && adjustMarkerBeforeColumn(toColumn, moveSemantics)) {
+				if (delta !== 0) {
+					let newColumn = Math.max(minimumAllowedColumn, marker.position.column + delta);
+					marker.updateColumn(markersTracker, newColumn);
+				}
+
+				markersIndex++;
+				if (markersIndex < markersLength) {
+					marker = markers[markersIndex];
+				}
+			}
+
+			// console.log('AFTER::: markersIndex: ' + markersIndex + ' : ' + this._printMarkers());
+		};
+
+		let adjustSet = (toColumn: number, newColumn: number, moveSemantics: MarkerMoveSemantics) => {
+			// console.log('------------------------------');
+			// console.log('adjustSet called: toColumn: ' + toColumn + ', newColumn: ' + newColumn + ', moveSemantics: ' + MarkerMoveSemantics[moveSemantics]);
+			// console.log('BEFORE::: markersIndex: ' + markersIndex + ' : ' + this._printMarkers());
+
+			while (markersIndex < markersLength && adjustMarkerBeforeColumn(toColumn, moveSemantics)) {
+				marker.updateColumn(markersTracker, newColumn);
+
+				markersIndex++;
+				if (markersIndex < markersLength) {
+					marker = markers[markersIndex];
+				}
+			}
+
+			// console.log('AFTER::: markersIndex: ' + markersIndex + ' : ' + this._printMarkers());
+		};
+
+		let finish = (delta: number, lineTextLength: number) => {
+			adjustDelta(Constants.MAX_SAFE_SMALL_INTEGER, delta, 1, MarkerMoveSemantics.MarkerDefined);
+
+			// console.log('------------- FINAL MARKERS: ' + this._printMarkers());
+		};
+
+		return {
+			adjustDelta: adjustDelta,
+			adjustSet: adjustSet,
+			finish: finish
+		};
+	}
+
+	public applyEdits(markersTracker: MarkersTracker, edits: ILineEdit[], tabSize: number): number {
+		let deltaColumn = 0;
+		let resultText = this.text;
+
+		let tokensAdjuster = this._createTokensAdjuster();
+		let markersAdjuster = this._createMarkersAdjuster(markersTracker);
+
+		for (let i = 0, len = edits.length; i < len; i++) {
+			let edit = edits[i];
+
+			// console.log();
+			// console.log('=============================');
+			// console.log('EDIT #' + i + ' [ ' + edit.startColumn + ' -> ' + edit.endColumn + ' ] : <<<' + edit.text + '>>>, forceMoveMarkers: ' + edit.forceMoveMarkers);
+			// console.log('deltaColumn: ' + deltaColumn);
+
+			let startColumn = deltaColumn + edit.startColumn;
+			let endColumn = deltaColumn + edit.endColumn;
+			let deletingCnt = endColumn - startColumn;
+			let insertingCnt = edit.text.length;
+
+			// Adjust tokens & markers before this edit
+			// console.log('Adjust tokens & markers before this edit');
+			tokensAdjuster.adjust(edit.startColumn - 1, deltaColumn, 1);
+			markersAdjuster.adjustDelta(edit.startColumn, deltaColumn, 1, edit.forceMoveMarkers ? MarkerMoveSemantics.ForceMove : (deletingCnt > 0 ? MarkerMoveSemantics.ForceStay : MarkerMoveSemantics.MarkerDefined));
+
+			// Adjust tokens & markers for the common part of this edit
+			let commonLength = Math.min(deletingCnt, insertingCnt);
+			if (commonLength > 0) {
+				// console.log('Adjust tokens & markers for the common part of this edit');
+				tokensAdjuster.adjust(edit.startColumn - 1 + commonLength, deltaColumn, startColumn);
+
+				if (!edit.forceMoveMarkers) {
+					markersAdjuster.adjustDelta(edit.startColumn + commonLength, deltaColumn, startColumn, edit.forceMoveMarkers ? MarkerMoveSemantics.ForceMove : (deletingCnt > insertingCnt ? MarkerMoveSemantics.ForceStay : MarkerMoveSemantics.MarkerDefined));
+				}
+			}
+
+			// Perform the edit & update `deltaColumn`
+			resultText = resultText.substring(0, startColumn - 1) + edit.text + resultText.substring(endColumn - 1);
+			deltaColumn += insertingCnt - deletingCnt;
+
+			// Adjust tokens & markers inside this edit
+			// console.log('Adjust tokens & markers inside this edit');
+			tokensAdjuster.adjust(edit.endColumn, deltaColumn, startColumn);
+			markersAdjuster.adjustSet(edit.endColumn, startColumn + insertingCnt, edit.forceMoveMarkers ? MarkerMoveSemantics.ForceMove : MarkerMoveSemantics.MarkerDefined);
+		}
+
+		// Wrap up tokens & markers; adjust remaining if needed
+		tokensAdjuster.finish(deltaColumn, resultText.length);
+		markersAdjuster.finish(deltaColumn, resultText.length);
+
+		// Save the resulting text
+		this._setText(resultText, tabSize);
+
+		return deltaColumn;
+	}
+
+	public split(markersTracker: MarkersTracker, splitColumn: number, forceMoveMarkers: boolean, tabSize: number): IModelLine {
+		// console.log('--> split @ ' + splitColumn + '::: ' + this._printMarkers());
+		var myText = this.text.substring(0, splitColumn - 1);
+		var otherText = this.text.substring(splitColumn - 1);
+
+		var otherMarkers: LineMarker[] = null;
+
+		if (this._markers) {
+			this._markers.sort(LineMarker.compareMarkers);
+			for (let i = 0, len = this._markers.length; i < len; i++) {
+				let marker = this._markers[i];
+
+				if (
+					marker.position.column > splitColumn
+					|| (
+						marker.position.column === splitColumn
+						&& (
+							forceMoveMarkers
+							|| !marker.stickToPreviousCharacter
+						)
+					)
+				) {
+					let myMarkers = this._markers.slice(0, i);
+					otherMarkers = this._markers.slice(i);
+					this._markers = myMarkers;
+					break;
+				}
+			}
+
+			if (otherMarkers) {
+				for (let i = 0, len = otherMarkers.length; i < len; i++) {
+					let marker = otherMarkers[i];
+
+					marker.updateColumn(markersTracker, marker.position.column - (splitColumn - 1));
+				}
+			}
+		}
+
+		this._setText(myText, tabSize);
+
+		var otherLine = this._createModelLine(otherText, tabSize);
+		if (otherMarkers) {
+			otherLine.addMarkers(otherMarkers);
+		}
+		return otherLine;
+	}
+
+	public append(markersTracker: MarkersTracker, myLineNumber: number, other: IModelLine, tabSize: number): void {
+		// console.log('--> append: THIS :: ' + this._printMarkers());
+		// console.log('--> append: OTHER :: ' + this._printMarkers());
+		let thisTextLength = this.text.length;
+		this._setText(this.text + other.text, tabSize);
+
+		if (other instanceof AbstractModelLine) {
+			if (other._markers) {
+				// Other has markers
+				let otherMarkers = other._markers;
+
+				// Adjust other markers
+				for (let i = 0, len = otherMarkers.length; i < len; i++) {
+					let marker = otherMarkers[i];
+
+					marker.updatePosition(markersTracker, new Position(myLineNumber, marker.position.column + thisTextLength));
+				}
+
+				this.addMarkers(otherMarkers);
+			}
+		}
+	}
+
+	public addMarker(marker: LineMarker): void {
+		if (!this._markers) {
+			this._markers = [marker];
+		} else {
+			this._markers.push(marker);
+		}
+	}
+
+	public addMarkers(markers: LineMarker[]): void {
+		if (markers.length === 0) {
+			return;
+		}
+
+		if (!this._markers) {
+			this._markers = markers.slice(0);
+		} else {
+			this._markers = this._markers.concat(markers);
+		}
+	}
+
+	public removeMarker(marker: LineMarker): void {
+		if (!this._markers) {
+			return;
+		}
+
+		let index = this._indexOfMarkerId(marker.id);
+		if (index < 0) {
+			return;
+		}
+
+		if (this._markers.length === 1) {
+			// was last marker on line
+			this._markers = null;
+		} else {
+			this._markers.splice(index, 1);
+		}
+	}
+
+	public removeMarkers(deleteMarkers: { [markerId: string]: boolean; }): void {
+		if (!this._markers) {
+			return;
+		}
+		for (let i = 0, len = this._markers.length; i < len; i++) {
+			let marker = this._markers[i];
+
+			if (deleteMarkers[marker.id]) {
+				this._markers.splice(i, 1);
+				len--;
+				i--;
+			}
+		}
+		if (this._markers.length === 0) {
+			this._markers = null;
+		}
+	}
+
+	public getMarkers(): LineMarker[] {
+		if (!this._markers) {
+			return null;
+		}
+		return this._markers;
+	}
+
+	public updateLineNumber(markersTracker: MarkersTracker, newLineNumber: number): void {
+		if (this._markers) {
+			let markers = this._markers;
+			for (let i = 0, len = markers.length; i < len; i++) {
+				let marker = markers[i];
+				marker.updateLineNumber(markersTracker, newLineNumber);
+			}
+		}
+	}
+
+	private _indexOfMarkerId(markerId: string): number {
+		let markers = this._markers;
+		for (let i = 0, len = markers.length; i < len; i++) {
+			if (markers[i].id === markerId) {
+				return i;
+			}
+		}
+		return undefined;
+	}
+}
+
+export class ModelLine extends AbstractModelLine implements IModelLine {
+
+	private _text: string;
+	public get text(): string { return this._text; }
+
+	/**
+	 * bits 31 - 1 => indentLevel
+	 * bit 0 => isInvalid
+	 */
+	private _metadata: number;
+
+	public isInvalid(): boolean {
+		return (this._metadata & 0x00000001) ? true : false;
+	}
+
+	public setIsInvalid(isInvalid: boolean): void {
+		this._metadata = (this._metadata & 0xfffffffe) | (isInvalid ? 1 : 0);
+	}
+
+	/**
+	 * Returns:
+	 *  - -1 => the line consists of whitespace
+	 *  - otherwise => the indent level is returned value
+	 */
+	public getIndentLevel(): number {
+		return ((this._metadata & 0xfffffffe) >> 1) - 1;
+	}
+
+	private _setPlusOneIndentLevel(value: number): void {
+		this._metadata = (this._metadata & 0x00000001) | ((value & 0xefffffff) << 1);
+	}
+
+	public updateTabSize(tabSize: number): void {
+		if (tabSize === 0) {
+			// don't care mark
+			this._metadata = this._metadata & 0x00000001;
+		} else {
+			this._setPlusOneIndentLevel(computePlusOneIndentLevel(this._text, tabSize));
+		}
+	}
+
+	private _state: IState;
+	private _lineTokens: ArrayBuffer;
+
+	constructor(text: string, tabSize: number) {
+		super(true);
+		this._metadata = 0;
+		this._setText(text, tabSize);
+		this._state = null;
+		this._lineTokens = null;
+	}
+
+	protected _createModelLine(text: string, tabSize: number): IModelLine {
+		return new ModelLine(text, tabSize);
+	}
+
+	public split(markersTracker: MarkersTracker, splitColumn: number, forceMoveMarkers: boolean, tabSize: number): IModelLine {
+		let result = super.split(markersTracker, splitColumn, forceMoveMarkers, tabSize);
+
+		// Mark overflowing tokens for deletion & delete marked tokens
+		this._deleteMarkedTokens(this._markOverflowingTokensForDeletion(0, this.text.length));
+
+		return result;
+	}
+
+	public append(markersTracker: MarkersTracker, myLineNumber: number, other: IModelLine, tabSize: number): void {
+		let thisTextLength = this.text.length;
+
+		super.append(markersTracker, myLineNumber, other, tabSize);
+
+		if (other instanceof ModelLine) {
+			let otherRawTokens = other._lineTokens;
+			if (otherRawTokens) {
+				// Other has real tokens
+
+				let otherTokens = new Uint32Array(otherRawTokens);
+
+				// Adjust other tokens
+				if (thisTextLength > 0) {
+					for (let i = 0, len = (otherTokens.length >>> 1); i < len; i++) {
+						otherTokens[(i << 1)] = otherTokens[(i << 1)] + thisTextLength;
+					}
+				}
+
+				// Append other tokens
+				let myRawTokens = this._lineTokens;
+				if (myRawTokens) {
+					// I have real tokens
+					let myTokens = new Uint32Array(myRawTokens);
+					let result = new Uint32Array(myTokens.length + otherTokens.length);
+					result.set(myTokens, 0);
+					result.set(otherTokens, myTokens.length);
+					this._lineTokens = result.buffer;
+				} else {
+					// I don't have real tokens
+					this._lineTokens = otherTokens.buffer;
+				}
+			}
+		}
+	}
+
+	// --- BEGIN STATE
+
+	public resetTokenizationState(): void {
+		this._state = null;
+		this._lineTokens = null;
+	}
+
+	public setState(state: IState): void {
+		this._state = state;
+	}
+
+	public getState(): IState {
+		return this._state || null;
+	}
+
+	// --- END STATE
+
+	// --- BEGIN TOKENS
+
+	public setTokens(topLevelLanguageId: LanguageId, tokens: Uint32Array): void {
+		if (!tokens || tokens.length === 0) {
+			this._lineTokens = null;
+			return;
+		}
+		if (tokens.length === 2) {
+			// there is one token
+			if (tokens[0] === 0 && tokens[1] === getDefaultMetadata(topLevelLanguageId)) {
+				this._lineTokens = null;
+				return;
+			}
+		}
+		this._lineTokens = tokens.buffer;
+	}
+
+	public getTokens(topLevelLanguageId: LanguageId): LineTokens {
+		let rawLineTokens = this._lineTokens;
+		if (rawLineTokens) {
+			return new LineTokens(new Uint32Array(rawLineTokens), this._text);
+		}
+
+		let lineTokens = new Uint32Array(2);
+		lineTokens[0] = 0;
+		lineTokens[1] = getDefaultMetadata(topLevelLanguageId);
+		return new LineTokens(lineTokens, this._text);
+	}
+
+	// --- END TOKENS
+
+	protected _createTokensAdjuster(): ITokensAdjuster {
+		if (!this._lineTokens) {
+			// This line does not have real tokens, so there is nothing to adjust
+			return NO_OP_TOKENS_ADJUSTER;
+		}
+
+		let lineTokens = new Uint32Array(this._lineTokens);
+		let tokensLength = (lineTokens.length >>> 1);
+		let tokenIndex = 0;
+		let tokenStartOffset = 0;
+		let removeTokensCount = 0;
+
+		let adjust = (toColumn: number, delta: number, minimumAllowedColumn: number) => {
+			// console.log(`------------------------------------------------------------------`);
+			// console.log(`before call: tokenIndex: ${tokenIndex}: ${lineTokens}`);
+			// console.log(`adjustTokens: ${toColumn} with delta: ${delta} and [${minimumAllowedColumn}]`);
+			// console.log(`tokenStartOffset: ${tokenStartOffset}`);
+			let minimumAllowedIndex = minimumAllowedColumn - 1;
+
+			while (tokenStartOffset < toColumn && tokenIndex < tokensLength) {
+
+				if (tokenStartOffset > 0 && delta !== 0) {
+					// adjust token's `startIndex` by `delta`
+					let newTokenStartOffset = Math.max(minimumAllowedIndex, tokenStartOffset + delta);
+					lineTokens[(tokenIndex << 1)] = newTokenStartOffset;
+
+					// console.log(` * adjusted token start offset for token at ${tokenIndex}: ${newTokenStartOffset}`);
+
+					if (delta < 0) {
+						let tmpTokenIndex = tokenIndex;
+						while (tmpTokenIndex > 0) {
+							let prevTokenStartOffset = lineTokens[((tmpTokenIndex - 1) << 1)];
+							if (prevTokenStartOffset >= newTokenStartOffset) {
+								if (prevTokenStartOffset !== Constants.MAX_UINT_32) {
+									// console.log(` * marking for deletion token at ${tmpTokenIndex - 1}`);
+									lineTokens[((tmpTokenIndex - 1) << 1)] = Constants.MAX_UINT_32;
+									removeTokensCount++;
+								}
+								tmpTokenIndex--;
+							} else {
+								break;
+							}
+						}
+					}
+				}
+
+				tokenIndex++;
+				if (tokenIndex < tokensLength) {
+					tokenStartOffset = lineTokens[(tokenIndex << 1)];
+				}
+			}
+			// console.log(`after call: tokenIndex: ${tokenIndex}: ${lineTokens}`);
+		};
+
+		let finish = (delta: number, lineTextLength: number) => {
+			adjust(Constants.MAX_SAFE_SMALL_INTEGER, delta, 1);
+
+			// Mark overflowing tokens for deletion & delete marked tokens
+			this._deleteMarkedTokens(this._markOverflowingTokensForDeletion(removeTokensCount, lineTextLength));
+		};
+
+		return {
+			adjust: adjust,
+			finish: finish
+		};
+	}
+
+	private _markOverflowingTokensForDeletion(removeTokensCount: number, lineTextLength: number): number {
+		if (!this._lineTokens) {
+			return removeTokensCount;
+		}
+
+		let lineTokens = new Uint32Array(this._lineTokens);
+		let tokensLength = (lineTokens.length >>> 1);
+
+		if (removeTokensCount + 1 === tokensLength) {
+			// no more removing, cannot end up without any tokens for mode transition reasons
+			return removeTokensCount;
+		}
+
+		for (let tokenIndex = tokensLength - 1; tokenIndex > 0; tokenIndex--) {
+			let tokenStartOffset = lineTokens[(tokenIndex << 1)];
+			if (tokenStartOffset < lineTextLength) {
+				// valid token => stop iterating
+				return removeTokensCount;
+			}
+
+			// this token now overflows the text => mark it for removal
+			if (tokenStartOffset !== Constants.MAX_UINT_32) {
+				// console.log(` * marking for deletion token at ${tokenIndex}`);
+				lineTokens[(tokenIndex << 1)] = Constants.MAX_UINT_32;
+				removeTokensCount++;
+
+				if (removeTokensCount + 1 === tokensLength) {
+					// no more removing, cannot end up without any tokens for mode transition reasons
+					return removeTokensCount;
+				}
+			}
+		}
+
+		return removeTokensCount;
+	}
+
+	private _deleteMarkedTokens(removeTokensCount: number): void {
+		if (removeTokensCount === 0) {
+			return;
+		}
+
+		let lineTokens = new Uint32Array(this._lineTokens);
+		let tokensLength = (lineTokens.length >>> 1);
+		let newTokens = new Uint32Array(((tokensLength - removeTokensCount) << 1)), newTokenIdx = 0;
+		for (let i = 0; i < tokensLength; i++) {
+			let startOffset = lineTokens[(i << 1)];
+			if (startOffset === Constants.MAX_UINT_32) {
+				// marked for deletion
+				continue;
+			}
+			let metadata = lineTokens[(i << 1) + 1];
+			newTokens[newTokenIdx++] = startOffset;
+			newTokens[newTokenIdx++] = metadata;
+		}
+		this._lineTokens = newTokens.buffer;
+	}
+
+	protected _setText(text: string, tabSize: number): void {
+		this._text = text;
+		if (tabSize === 0) {
+			// don't care mark
+			this._metadata = this._metadata & 0x00000001;
+		} else {
+			this._setPlusOneIndentLevel(computePlusOneIndentLevel(text, tabSize));
+		}
+	}
+
+}
+
+/**
+ * A model line that cannot store any tokenization state, nor does it compute indentation levels.
+ * It has no fields except the text.
+ */
+export class MinimalModelLine extends AbstractModelLine implements IModelLine {
+
+	private _text: string;
+	public get text(): string { return this._text; }
+
+	public isInvalid(): boolean {
+		return false;
+	}
+
+	public setIsInvalid(isInvalid: boolean): void {
+	}
+
+	/**
+	 * Returns:
+	 *  - -1 => the line consists of whitespace
+	 *  - otherwise => the indent level is returned value
+	 */
+	public getIndentLevel(): number {
+		return 0;
+	}
+
+	public updateTabSize(tabSize: number): void {
+	}
+
+	constructor(text: string, tabSize: number) {
+		super(false);
+		this._setText(text, tabSize);
+	}
+
+	protected _createModelLine(text: string, tabSize: number): IModelLine {
+		return new MinimalModelLine(text, tabSize);
+	}
+
+	public split(markersTracker: MarkersTracker, splitColumn: number, forceMoveMarkers: boolean, tabSize: number): IModelLine {
+		return super.split(markersTracker, splitColumn, forceMoveMarkers, tabSize);
+	}
+
+	public append(markersTracker: MarkersTracker, myLineNumber: number, other: IModelLine, tabSize: number): void {
+		super.append(markersTracker, myLineNumber, other, tabSize);
+	}
+
+	// --- BEGIN STATE
+
+	public resetTokenizationState(): void {
+	}
+
+	public setState(state: IState): void {
+	}
+
+	public getState(): IState {
+		return null;
+	}
+
+	// --- END STATE
+
+	// --- BEGIN TOKENS
+
+	public setTokens(topLevelLanguageId: LanguageId, tokens: Uint32Array): void {
+	}
+
+	public getTokens(topLevelLanguageId: LanguageId): LineTokens {
+		let lineTokens = new Uint32Array(2);
+		lineTokens[0] = 0;
+		lineTokens[1] = getDefaultMetadata(topLevelLanguageId);
+		return new LineTokens(lineTokens, this._text);
+	}
+
+	// --- END TOKENS
+
+	protected _createTokensAdjuster(): ITokensAdjuster {
+		// This line does not have real tokens, so there is nothing to adjust
+		return NO_OP_TOKENS_ADJUSTER;
+	}
+
+	protected _setText(text: string, tabSize: number): void {
+		this._text = text;
+	}
+}
+
+function getDefaultMetadata(topLevelLanguageId: LanguageId): number {
+	return (
+		(topLevelLanguageId << MetadataConsts.LANGUAGEID_OFFSET)
+		| (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET)
+		| (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET)
+		| (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET)
+		| (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET)
+	) >>> 0;
+}

src/vs/editor/common/model/textModel.ts

@@ -0,0 +1,827 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { OrderGuaranteeEventEmitter, BulkListenerCallback } from 'vs/base/common/eventEmitter';
+import * as strings from 'vs/base/common/strings';
+import { Position, IPosition } from 'vs/editor/common/core/position';
+import { Range, IRange } from 'vs/editor/common/core/range';
+import * as editorCommon from 'vs/editor/common/editorCommon';
+import { ModelLine, IModelLine, MinimalModelLine } from 'vs/editor/common/model/modelLine';
+import { guessIndentation } from 'vs/editor/common/model/indentationGuesser';
+import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/config/editorOptions';
+import { PrefixSumComputer } from 'vs/editor/common/viewModel/prefixSumComputer';
+import { IndentRange, computeRanges } from 'vs/editor/common/model/indentRanges';
+import { TextModelSearch, SearchParams } from 'vs/editor/common/model/textModelSearch';
+import { TextSource, ITextSource, IRawTextSource, RawTextSource } from 'vs/editor/common/model/textSource';
+import { IDisposable } from 'vs/base/common/lifecycle';
+import * as textModelEvents from 'vs/editor/common/model/textModelEvents';
+
+const USE_MIMINAL_MODEL_LINE = true;
+
+const LIMIT_FIND_COUNT = 999;
+export const LONG_LINE_BOUNDARY = 10000;
+
+export interface ITextModelCreationData {
+	readonly text: ITextSource;
+	readonly options: editorCommon.TextModelResolvedOptions;
+}
+
+export class TextModel implements editorCommon.ITextModel {
+	private static MODEL_SYNC_LIMIT = 50 * 1024 * 1024; // 50 MB
+	private static MODEL_TOKENIZATION_LIMIT = 20 * 1024 * 1024; // 20 MB
+	private static MANY_MANY_LINES = 300 * 1000; // 300K lines
+
+	public static DEFAULT_CREATION_OPTIONS: editorCommon.ITextModelCreationOptions = {
+		tabSize: EDITOR_MODEL_DEFAULTS.tabSize,
+		insertSpaces: EDITOR_MODEL_DEFAULTS.insertSpaces,
+		detectIndentation: false,
+		defaultEOL: editorCommon.DefaultEndOfLine.LF,
+		trimAutoWhitespace: EDITOR_MODEL_DEFAULTS.trimAutoWhitespace,
+	};
+
+	public static createFromString(text: string, options: editorCommon.ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS): TextModel {
+		return new TextModel(RawTextSource.fromString(text), options);
+	}
+
+	public static resolveCreationData(rawTextSource: IRawTextSource, options: editorCommon.ITextModelCreationOptions): ITextModelCreationData {
+		const textSource = TextSource.fromRawTextSource(rawTextSource, options.defaultEOL);
+
+		let resolvedOpts: editorCommon.TextModelResolvedOptions;
+		if (options.detectIndentation) {
+			const guessedIndentation = guessIndentation(textSource.lines, options.tabSize, options.insertSpaces);
+			resolvedOpts = new editorCommon.TextModelResolvedOptions({
+				tabSize: guessedIndentation.tabSize,
+				insertSpaces: guessedIndentation.insertSpaces,
+				trimAutoWhitespace: options.trimAutoWhitespace,
+				defaultEOL: options.defaultEOL
+			});
+		} else {
+			resolvedOpts = new editorCommon.TextModelResolvedOptions({
+				tabSize: options.tabSize,
+				insertSpaces: options.insertSpaces,
+				trimAutoWhitespace: options.trimAutoWhitespace,
+				defaultEOL: options.defaultEOL
+			});
+		}
+
+		return {
+			text: textSource,
+			options: resolvedOpts
+		};
+	}
+
+	public addBulkListener(listener: BulkListenerCallback): IDisposable {
+		return this._eventEmitter.addBulkListener(listener);
+	}
+
+	protected readonly _eventEmitter: OrderGuaranteeEventEmitter;
+
+	/*protected*/ _lines: IModelLine[];
+	protected _EOL: string;
+	protected _isDisposed: boolean;
+	protected _isDisposing: boolean;
+	protected _options: editorCommon.TextModelResolvedOptions;
+	protected _lineStarts: PrefixSumComputer;
+	private _indentRanges: IndentRange[];
+
+	private _versionId: number;
+	/**
+	 * Unlike, versionId, this can go down (via undo) or go to previous values (via redo)
+	 */
+	private _alternativeVersionId: number;
+	private _BOM: string;
+	protected _mightContainRTL: boolean;
+	protected _mightContainNonBasicASCII: boolean;
+
+	private readonly _shouldSimplifyMode: boolean;
+	protected readonly _isTooLargeForTokenization: boolean;
+
+	constructor(rawTextSource: IRawTextSource, creationOptions: editorCommon.ITextModelCreationOptions) {
+		this._eventEmitter = new OrderGuaranteeEventEmitter();
+
+		const textModelData = TextModel.resolveCreationData(rawTextSource, creationOptions);
+
+		// !!! Make a decision in the ctor and permanently respect this decision !!!
+		// If a model is too large at construction time, it will never get tokenized,
+		// under no circumstances.
+		this._isTooLargeForTokenization = (
+			(textModelData.text.length > TextModel.MODEL_TOKENIZATION_LIMIT)
+			|| (textModelData.text.lines.length > TextModel.MANY_MANY_LINES)
+		);
+
+		this._shouldSimplifyMode = (
+			this._isTooLargeForTokenization
+			|| (textModelData.text.length > TextModel.MODEL_SYNC_LIMIT)
+		);
+
+		this._options = new editorCommon.TextModelResolvedOptions(textModelData.options);
+		this._constructLines(textModelData.text);
+		this._setVersionId(1);
+		this._isDisposed = false;
+		this._isDisposing = false;
+	}
+
+	protected _createModelLine(text: string, tabSize: number): IModelLine {
+		if (USE_MIMINAL_MODEL_LINE && this._isTooLargeForTokenization) {
+			return new MinimalModelLine(text, tabSize);
+		}
+		return new ModelLine(text, tabSize);
+	}
+
+	protected _assertNotDisposed(): void {
+		if (this._isDisposed) {
+			throw new Error('Model is disposed!');
+		}
+	}
+
+	public isTooLargeForHavingARichMode(): boolean {
+		return this._shouldSimplifyMode;
+	}
+
+	public isTooLargeForTokenization(): boolean {
+		return this._isTooLargeForTokenization;
+	}
+
+	public getOptions(): editorCommon.TextModelResolvedOptions {
+		this._assertNotDisposed();
+		return this._options;
+	}
+
+	public updateOptions(_newOpts: editorCommon.ITextModelUpdateOptions): void {
+		this._assertNotDisposed();
+		let tabSize = (typeof _newOpts.tabSize !== 'undefined') ? _newOpts.tabSize : this._options.tabSize;
+		let insertSpaces = (typeof _newOpts.insertSpaces !== 'undefined') ? _newOpts.insertSpaces : this._options.insertSpaces;
+		let trimAutoWhitespace = (typeof _newOpts.trimAutoWhitespace !== 'undefined') ? _newOpts.trimAutoWhitespace : this._options.trimAutoWhitespace;
+
+		let newOpts = new editorCommon.TextModelResolvedOptions({
+			tabSize: tabSize,
+			insertSpaces: insertSpaces,
+			defaultEOL: this._options.defaultEOL,
+			trimAutoWhitespace: trimAutoWhitespace
+		});
+
+		if (this._options.equals(newOpts)) {
+			return;
+		}
+
+		let e = this._options.createChangeEvent(newOpts);
+		this._options = newOpts;
+
+		if (e.tabSize) {
+			let newTabSize = this._options.tabSize;
+			for (let i = 0, len = this._lines.length; i < len; i++) {
+				this._lines[i].updateTabSize(newTabSize);
+			}
+		}
+
+		this._eventEmitter.emit(textModelEvents.TextModelEventType.ModelOptionsChanged, e);
+	}
+
+	public detectIndentation(defaultInsertSpaces: boolean, defaultTabSize: number): void {
+		this._assertNotDisposed();
+		let lines = this._lines.map(line => line.text);
+		let guessedIndentation = guessIndentation(lines, defaultTabSize, defaultInsertSpaces);
+		this.updateOptions({
+			insertSpaces: guessedIndentation.insertSpaces,
+			tabSize: guessedIndentation.tabSize
+		});
+	}
+
+	private static _normalizeIndentationFromWhitespace(str: string, tabSize: number, insertSpaces: boolean): string {
+		let spacesCnt = 0;
+		for (let i = 0; i < str.length; i++) {
+			if (str.charAt(i) === '\t') {
+				spacesCnt += tabSize;
+			} else {
+				spacesCnt++;
+			}
+		}
+
+		let result = '';
+		if (!insertSpaces) {
+			let tabsCnt = Math.floor(spacesCnt / tabSize);
+			spacesCnt = spacesCnt % tabSize;
+			for (let i = 0; i < tabsCnt; i++) {
+				result += '\t';
+			}
+		}
+
+		for (let i = 0; i < spacesCnt; i++) {
+			result += ' ';
+		}
+
+		return result;
+	}
+
+	public static normalizeIndentation(str: string, tabSize: number, insertSpaces: boolean): string {
+		let firstNonWhitespaceIndex = strings.firstNonWhitespaceIndex(str);
+		if (firstNonWhitespaceIndex === -1) {
+			firstNonWhitespaceIndex = str.length;
+		}
+		return TextModel._normalizeIndentationFromWhitespace(str.substring(0, firstNonWhitespaceIndex), tabSize, insertSpaces) + str.substring(firstNonWhitespaceIndex);
+	}
+
+	public normalizeIndentation(str: string): string {
+		this._assertNotDisposed();
+		return TextModel.normalizeIndentation(str, this._options.tabSize, this._options.insertSpaces);
+	}
+
+	public getOneIndent(): string {
+		this._assertNotDisposed();
+		let tabSize = this._options.tabSize;
+		let insertSpaces = this._options.insertSpaces;
+
+		if (insertSpaces) {
+			let result = '';
+			for (let i = 0; i < tabSize; i++) {
+				result += ' ';
+			}
+			return result;
+		} else {
+			return '\t';
+		}
+	}
+
+	public getVersionId(): number {
+		this._assertNotDisposed();
+		return this._versionId;
+	}
+
+	public mightContainRTL(): boolean {
+		return this._mightContainRTL;
+	}
+
+	public mightContainNonBasicASCII(): boolean {
+		return this._mightContainNonBasicASCII;
+	}
+
+	public getAlternativeVersionId(): number {
+		this._assertNotDisposed();
+		return this._alternativeVersionId;
+	}
+
+	private _ensureLineStarts(): void {
+		if (!this._lineStarts) {
+			const eolLength = this._EOL.length;
+			const linesLength = this._lines.length;
+			const lineStartValues = new Uint32Array(linesLength);
+			for (let i = 0; i < linesLength; i++) {
+				lineStartValues[i] = this._lines[i].text.length + eolLength;
+			}
+			this._lineStarts = new PrefixSumComputer(lineStartValues);
+		}
+	}
+
+	public getOffsetAt(rawPosition: IPosition): number {
+		this._assertNotDisposed();
+		let position = this._validatePosition(rawPosition.lineNumber, rawPosition.column, false);
+		this._ensureLineStarts();
+		return this._lineStarts.getAccumulatedValue(position.lineNumber - 2) + position.column - 1;
+	}
+
+	public getPositionAt(offset: number): Position {
+		this._assertNotDisposed();
+		offset = Math.floor(offset);
+		offset = Math.max(0, offset);
+
+		this._ensureLineStarts();
+		let out = this._lineStarts.getIndexOf(offset);
+
+		let lineLength = this._lines[out.index].text.length;
+
+		// Ensure we return a valid position
+		return new Position(out.index + 1, Math.min(out.remainder + 1, lineLength + 1));
+	}
+
+	protected _increaseVersionId(): void {
+		this._setVersionId(this._versionId + 1);
+	}
+
+	private _setVersionId(newVersionId: number): void {
+		this._versionId = newVersionId;
+		this._alternativeVersionId = this._versionId;
+	}
+
+	protected _overwriteAlternativeVersionId(newAlternativeVersionId: number): void {
+		this._alternativeVersionId = newAlternativeVersionId;
+	}
+
+	public isDisposed(): boolean {
+		return this._isDisposed;
+	}
+
+	public dispose(): void {
+		this._isDisposed = true;
+		// Null out members, such that any use of a disposed model will throw exceptions sooner rather than later
+		this._lines = null;
+		this._EOL = null;
+		this._BOM = null;
+
+		this._eventEmitter.dispose();
+	}
+
+	private _emitContentChanged2(startLineNumber: number, startColumn: number, endLineNumber: number, endColumn: number, rangeLength: number, text: string, isUndoing: boolean, isRedoing: boolean, isFlush: boolean): void {
+		const e: textModelEvents.IModelContentChangedEvent = {
+			changes: [{
+				range: new Range(startLineNumber, startColumn, endLineNumber, endColumn),
+				rangeLength: rangeLength,
+				text: text,
+			}],
+			eol: this._EOL,
+			versionId: this.getVersionId(),
+			isUndoing: isUndoing,
+			isRedoing: isRedoing,
+			isFlush: isFlush
+		};
+		if (!this._isDisposing) {
+			this._eventEmitter.emit(textModelEvents.TextModelEventType.ModelContentChanged, e);
+		}
+	}
+
+	protected _resetValue(newValue: ITextSource): void {
+		this._constructLines(newValue);
+		this._increaseVersionId();
+	}
+
+	public equals(other: ITextSource): boolean {
+		this._assertNotDisposed();
+		if (this._BOM !== other.BOM) {
+			return false;
+		}
+		if (this._EOL !== other.EOL) {
+			return false;
+		}
+		if (this._lines.length !== other.lines.length) {
+			return false;
+		}
+		for (let i = 0, len = this._lines.length; i < len; i++) {
+			if (this._lines[i].text !== other.lines[i]) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	public setValue(value: string): void {
+		this._assertNotDisposed();
+		if (value === null) {
+			// There's nothing to do
+			return;
+		}
+		const textSource = TextSource.fromString(value, this._options.defaultEOL);
+		this.setValueFromTextSource(textSource);
+	}
+
+	public setValueFromTextSource(newValue: ITextSource): void {
+		this._assertNotDisposed();
+		if (newValue === null) {
+			// There's nothing to do
+			return;
+		}
+		var oldFullModelRange = this.getFullModelRange();
+		var oldModelValueLength = this.getValueLengthInRange(oldFullModelRange);
+		var endLineNumber = this.getLineCount();
+		var endColumn = this.getLineMaxColumn(endLineNumber);
+
+		this._resetValue(newValue);
+
+		this._emitModelRawContentChangedEvent(
+			new textModelEvents.ModelRawContentChangedEvent(
+				[
+					new textModelEvents.ModelRawFlush()
+				],
+				this._versionId,
+				false,
+				false
+			)
+		);
+
+		this._emitContentChanged2(1, 1, endLineNumber, endColumn, oldModelValueLength, this.getValue(), false, false, true);
+	}
+
+	public getValue(eol?: editorCommon.EndOfLinePreference, preserveBOM: boolean = false): string {
+		this._assertNotDisposed();
+		var fullModelRange = this.getFullModelRange();
+		var fullModelValue = this.getValueInRange(fullModelRange, eol);
+
+		if (preserveBOM) {
+			return this._BOM + fullModelValue;
+		}
+
+		return fullModelValue;
+	}
+
+	public getValueLength(eol?: editorCommon.EndOfLinePreference, preserveBOM: boolean = false): number {
+		this._assertNotDisposed();
+		var fullModelRange = this.getFullModelRange();
+		var fullModelValue = this.getValueLengthInRange(fullModelRange, eol);
+
+		if (preserveBOM) {
+			return this._BOM.length + fullModelValue;
+		}
+
+		return fullModelValue;
+	}
+
+	public getValueInRange(rawRange: IRange, eol: editorCommon.EndOfLinePreference = editorCommon.EndOfLinePreference.TextDefined): string {
+		this._assertNotDisposed();
+		var range = this.validateRange(rawRange);
+
+		if (range.isEmpty()) {
+			return '';
+		}
+
+		if (range.startLineNumber === range.endLineNumber) {
+			return this._lines[range.startLineNumber - 1].text.substring(range.startColumn - 1, range.endColumn - 1);
+		}
+
+		var lineEnding = this._getEndOfLine(eol),
+			startLineIndex = range.startLineNumber - 1,
+			endLineIndex = range.endLineNumber - 1,
+			resultLines: string[] = [];
+
+		resultLines.push(this._lines[startLineIndex].text.substring(range.startColumn - 1));
+		for (var i = startLineIndex + 1; i < endLineIndex; i++) {
+			resultLines.push(this._lines[i].text);
+		}
+		resultLines.push(this._lines[endLineIndex].text.substring(0, range.endColumn - 1));
+
+		return resultLines.join(lineEnding);
+	}
+
+	public getValueLengthInRange(rawRange: IRange, eol: editorCommon.EndOfLinePreference = editorCommon.EndOfLinePreference.TextDefined): number {
+		this._assertNotDisposed();
+		var range = this.validateRange(rawRange);
+
+		if (range.isEmpty()) {
+			return 0;
+		}
+
+		if (range.startLineNumber === range.endLineNumber) {
+			return (range.endColumn - range.startColumn);
+		}
+
+		let startOffset = this.getOffsetAt(new Position(range.startLineNumber, range.startColumn));
+		let endOffset = this.getOffsetAt(new Position(range.endLineNumber, range.endColumn));
+		return endOffset - startOffset;
+	}
+
+	public isDominatedByLongLines(): boolean {
+		this._assertNotDisposed();
+		var smallLineCharCount = 0,
+			longLineCharCount = 0,
+			i: number,
+			len: number,
+			lines = this._lines,
+			lineLength: number;
+
+		for (i = 0, len = this._lines.length; i < len; i++) {
+			lineLength = lines[i].text.length;
+			if (lineLength >= LONG_LINE_BOUNDARY) {
+				longLineCharCount += lineLength;
+			} else {
+				smallLineCharCount += lineLength;
+			}
+		}
+
+		return (longLineCharCount > smallLineCharCount);
+	}
+
+	public getLineCount(): number {
+		this._assertNotDisposed();
+		return this._lines.length;
+	}
+
+	public getLineContent(lineNumber: number): string {
+		this._assertNotDisposed();
+		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
+			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
+		}
+
+		return this._lines[lineNumber - 1].text;
+	}
+
+	public getIndentLevel(lineNumber: number): number {
+		this._assertNotDisposed();
+		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
+			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
+		}
+
+		return this._lines[lineNumber - 1].getIndentLevel();
+	}
+
+	protected _resetIndentRanges(): void {
+		this._indentRanges = null;
+	}
+
+	private _getIndentRanges(): IndentRange[] {
+		if (!this._indentRanges) {
+			this._indentRanges = computeRanges(this);
+		}
+		return this._indentRanges;
+	}
+
+	public getIndentRanges(): IndentRange[] {
+		this._assertNotDisposed();
+		let indentRanges = this._getIndentRanges();
+		return IndentRange.deepCloneArr(indentRanges);
+	}
+
+	private _toValidLineIndentGuide(lineNumber: number, indentGuide: number): number {
+		let lineIndentLevel = this._lines[lineNumber - 1].getIndentLevel();
+		if (lineIndentLevel === -1) {
+			return indentGuide;
+		}
+		let maxIndentGuide = Math.ceil(lineIndentLevel / this._options.tabSize);
+		return Math.min(maxIndentGuide, indentGuide);
+	}
+
+	public getLineIndentGuide(lineNumber: number): number {
+		this._assertNotDisposed();
+		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
+			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
+		}
+
+		let indentRanges = this._getIndentRanges();
+
+		for (let i = indentRanges.length - 1; i >= 0; i--) {
+			let rng = indentRanges[i];
+
+			if (rng.startLineNumber === lineNumber) {
+				return this._toValidLineIndentGuide(lineNumber, Math.ceil(rng.indent / this._options.tabSize));
+			}
+			if (rng.startLineNumber < lineNumber && lineNumber <= rng.endLineNumber) {
+				return this._toValidLineIndentGuide(lineNumber, 1 + Math.floor(rng.indent / this._options.tabSize));
+			}
+			if (rng.endLineNumber + 1 === lineNumber) {
+				let bestIndent = rng.indent;
+				while (i > 0) {
+					i--;
+					rng = indentRanges[i];
+					if (rng.endLineNumber + 1 === lineNumber) {
+						bestIndent = rng.indent;
+					}
+				}
+				return this._toValidLineIndentGuide(lineNumber, Math.ceil(bestIndent / this._options.tabSize));
+			}
+		}
+
+		return 0;
+	}
+
+	public getLinesContent(): string[] {
+		this._assertNotDisposed();
+		var r: string[] = [];
+		for (var i = 0, len = this._lines.length; i < len; i++) {
+			r[i] = this._lines[i].text;
+		}
+		return r;
+	}
+
+	public getEOL(): string {
+		this._assertNotDisposed();
+		return this._EOL;
+	}
+
+	public setEOL(eol: editorCommon.EndOfLineSequence): void {
+		this._assertNotDisposed();
+		const newEOL = (eol === editorCommon.EndOfLineSequence.CRLF ? '\r\n' : '\n');
+		if (this._EOL === newEOL) {
+			// Nothing to do
+			return;
+		}
+
+		const oldFullModelRange = this.getFullModelRange();
+		const oldModelValueLength = this.getValueLengthInRange(oldFullModelRange);
+		const endLineNumber = this.getLineCount();
+		const endColumn = this.getLineMaxColumn(endLineNumber);
+
+		this._EOL = newEOL;
+		this._lineStarts = null;
+		this._increaseVersionId();
+
+		this._emitModelRawContentChangedEvent(
+			new textModelEvents.ModelRawContentChangedEvent(
+				[
+					new textModelEvents.ModelRawEOLChanged()
+				],
+				this._versionId,
+				false,
+				false
+			)
+		);
+
+		this._emitContentChanged2(1, 1, endLineNumber, endColumn, oldModelValueLength, this.getValue(), false, false, false);
+	}
+
+	public getLineMinColumn(lineNumber: number): number {
+		this._assertNotDisposed();
+		return 1;
+	}
+
+	public getLineMaxColumn(lineNumber: number): number {
+		this._assertNotDisposed();
+		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
+			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
+		}
+
+		return this._lines[lineNumber - 1].text.length + 1;
+	}
+
+	public getLineFirstNonWhitespaceColumn(lineNumber: number): number {
+		this._assertNotDisposed();
+		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
+			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
+		}
+
+		var result = strings.firstNonWhitespaceIndex(this._lines[lineNumber - 1].text);
+		if (result === -1) {
+			return 0;
+		}
+		return result + 1;
+	}
+
+	public getLineLastNonWhitespaceColumn(lineNumber: number): number {
+		this._assertNotDisposed();
+		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
+			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
+		}
+
+		var result = strings.lastNonWhitespaceIndex(this._lines[lineNumber - 1].text);
+		if (result === -1) {
+			return 0;
+		}
+		return result + 2;
+	}
+
+	public validateLineNumber(lineNumber: number): number {
+		this._assertNotDisposed();
+		if (lineNumber < 1) {
+			lineNumber = 1;
+		}
+		if (lineNumber > this._lines.length) {
+			lineNumber = this._lines.length;
+		}
+		return lineNumber;
+	}
+
+	/**
+	 * @param strict Do NOT allow a position inside a high-low surrogate pair
+	 */
+	private _validatePosition(_lineNumber: number, _column: number, strict: boolean): Position {
+		const lineNumber = Math.floor(typeof _lineNumber === 'number' ? _lineNumber : 1);
+		const column = Math.floor(typeof _column === 'number' ? _column : 1);
+
+		if (lineNumber < 1) {
+			return new Position(1, 1);
+		}
+
+		if (lineNumber > this._lines.length) {
+			return new Position(this._lines.length, this.getLineMaxColumn(this._lines.length));
+		}
+
+		if (column <= 1) {
+			return new Position(lineNumber, 1);
+		}
+
+		const maxColumn = this.getLineMaxColumn(lineNumber);
+		if (column >= maxColumn) {
+			return new Position(lineNumber, maxColumn);
+		}
+
+		if (strict) {
+			// If the position would end up in the middle of a high-low surrogate pair,
+			// we move it to before the pair
+			// !!At this point, column > 1
+			const charCodeBefore = this._lines[lineNumber - 1].text.charCodeAt(column - 2);
+			if (strings.isHighSurrogate(charCodeBefore)) {
+				return new Position(lineNumber, column - 1);
+			}
+		}
+
+		return new Position(lineNumber, column);
+	}
+
+	public validatePosition(position: IPosition): Position {
+		this._assertNotDisposed();
+		return this._validatePosition(position.lineNumber, position.column, true);
+	}
+
+	public validateRange(_range: IRange): Range {
+		this._assertNotDisposed();
+		const start = this._validatePosition(_range.startLineNumber, _range.startColumn, false);
+		const end = this._validatePosition(_range.endLineNumber, _range.endColumn, false);
+
+		const startLineNumber = start.lineNumber;
+		const startColumn = start.column;
+		const endLineNumber = end.lineNumber;
+		const endColumn = end.column;
+
+		const startLineText = this._lines[startLineNumber - 1].text;
+		const endLineText = this._lines[endLineNumber - 1].text;
+
+		const charCodeBeforeStart = (startColumn > 1 ? startLineText.charCodeAt(startColumn - 2) : 0);
+		const charCodeBeforeEnd = (endColumn > 1 && endColumn <= endLineText.length ? endLineText.charCodeAt(endColumn - 2) : 0);
+
+		const startInsideSurrogatePair = strings.isHighSurrogate(charCodeBeforeStart);
+		const endInsideSurrogatePair = strings.isHighSurrogate(charCodeBeforeEnd);
+
+		if (!startInsideSurrogatePair && !endInsideSurrogatePair) {
+			return new Range(startLineNumber, startColumn, endLineNumber, endColumn);
+		}
+
+		if (startLineNumber === endLineNumber && startColumn === endColumn) {
+			// do not expand a collapsed range, simply move it to a valid location
+			return new Range(startLineNumber, startColumn - 1, endLineNumber, endColumn - 1);
+		}
+
+		if (startInsideSurrogatePair && endInsideSurrogatePair) {
+			// expand range at both ends
+			return new Range(startLineNumber, startColumn - 1, endLineNumber, endColumn + 1);
+		}
+
+		if (startInsideSurrogatePair) {
+			// only expand range at the start
+			return new Range(startLineNumber, startColumn - 1, endLineNumber, endColumn);
+		}
+
+		// only expand range at the end
+		return new Range(startLineNumber, startColumn, endLineNumber, endColumn + 1);
+	}
+
+	public modifyPosition(rawPosition: IPosition, offset: number): Position {
+		this._assertNotDisposed();
+		return this.getPositionAt(this.getOffsetAt(rawPosition) + offset);
+	}
+
+	public getFullModelRange(): Range {
+		this._assertNotDisposed();
+		var lineCount = this.getLineCount();
+		return new Range(1, 1, lineCount, this.getLineMaxColumn(lineCount));
+	}
+
+	protected _emitModelRawContentChangedEvent(e: textModelEvents.ModelRawContentChangedEvent): void {
+		if (this._isDisposing) {
+			// Do not confuse listeners by emitting any event after disposing
+			return;
+		}
+		this._eventEmitter.emit(textModelEvents.TextModelEventType.ModelRawContentChanged2, e);
+	}
+
+	private _constructLines(textSource: ITextSource): void {
+		const tabSize = this._options.tabSize;
+		let rawLines = textSource.lines;
+		let modelLines: IModelLine[] = new Array<IModelLine>(rawLines.length);
+
+		for (let i = 0, len = rawLines.length; i < len; i++) {
+			modelLines[i] = this._createModelLine(rawLines[i], tabSize);
+		}
+		this._BOM = textSource.BOM;
+		this._mightContainRTL = textSource.containsRTL;
+		this._mightContainNonBasicASCII = !textSource.isBasicASCII;
+		this._EOL = textSource.EOL;
+		this._lines = modelLines;
+		this._lineStarts = null;
+		this._resetIndentRanges();
+	}
+
+	private _getEndOfLine(eol: editorCommon.EndOfLinePreference): string {
+		switch (eol) {
+			case editorCommon.EndOfLinePreference.LF:
+				return '\n';
+			case editorCommon.EndOfLinePreference.CRLF:
+				return '\r\n';
+			case editorCommon.EndOfLinePreference.TextDefined:
+				return this.getEOL();
+		}
+		throw new Error('Unknown EOL preference');
+	}
+
+	public findMatches(searchString: string, rawSearchScope: any, isRegex: boolean, matchCase: boolean, wordSeparators: string, captureMatches: boolean, limitResultCount: number = LIMIT_FIND_COUNT): editorCommon.FindMatch[] {
+		this._assertNotDisposed();
+
+		let searchRange: Range;
+		if (Range.isIRange(rawSearchScope)) {
+			searchRange = this.validateRange(rawSearchScope);
+		} else {
+			searchRange = this.getFullModelRange();
+		}
+
+		return TextModelSearch.findMatches(this, new SearchParams(searchString, isRegex, matchCase, wordSeparators), searchRange, captureMatches, limitResultCount);
+	}
+
+	public findNextMatch(searchString: string, rawSearchStart: IPosition, isRegex: boolean, matchCase: boolean, wordSeparators: string, captureMatches: boolean): editorCommon.FindMatch {
+		this._assertNotDisposed();
+		const searchStart = this.validatePosition(rawSearchStart);
+		return TextModelSearch.findNextMatch(this, new SearchParams(searchString, isRegex, matchCase, wordSeparators), searchStart, captureMatches);
+	}
+
+	public findPreviousMatch(searchString: string, rawSearchStart: IPosition, isRegex: boolean, matchCase: boolean, wordSeparators: string, captureMatches: boolean): editorCommon.FindMatch {
+		this._assertNotDisposed();
+		const searchStart = this.validatePosition(rawSearchStart);
+		return TextModelSearch.findPreviousMatch(this, new SearchParams(searchString, isRegex, matchCase, wordSeparators), searchStart, captureMatches);
+	}
+}

src/vs/editor/common/model/textModelEvents.ts

@@ -0,0 +1,256 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { IRange } from 'vs/editor/common/core/range';
+
+/**
+ * @internal
+ */
+export const TextModelEventType = {
+	ModelDispose: 'modelDispose',
+	ModelTokensChanged: 'modelTokensChanged',
+	ModelLanguageChanged: 'modelLanguageChanged',
+	ModelOptionsChanged: 'modelOptionsChanged',
+	ModelContentChanged: 'contentChanged',
+	ModelRawContentChanged2: 'rawContentChanged2',
+	ModelDecorationsChanged: 'decorationsChanged',
+};
+
+/**
+ * An event describing that the current mode associated with a model has changed.
+ */
+export interface IModelLanguageChangedEvent {
+	/**
+	 * Previous language
+	 */
+	readonly oldLanguage: string;
+	/**
+	 * New language
+	 */
+	readonly newLanguage: string;
+}
+
+export interface IModelContentChange {
+	/**
+	 * The range that got replaced.
+	 */
+	readonly range: IRange;
+	/**
+	 * The length of the range that got replaced.
+	 */
+	readonly rangeLength: number;
+	/**
+	 * The new text for the range.
+	 */
+	readonly text: string;
+}
+
+/**
+ * An event describing a change in the text of a model.
+ */
+export interface IModelContentChangedEvent {
+	readonly changes: IModelContentChange[];
+	/**
+	 * The (new) end-of-line character.
+	 */
+	readonly eol: string;
+	/**
+	 * The new version id the model has transitioned to.
+	 */
+	readonly versionId: number;
+	/**
+	 * Flag that indicates that this event was generated while undoing.
+	 */
+	readonly isUndoing: boolean;
+	/**
+	 * Flag that indicates that this event was generated while redoing.
+	 */
+	readonly isRedoing: boolean;
+	/**
+	 * Flag that indicates that all decorations were lost with this edit.
+	 * The model has been reset to a new value.
+	 */
+	readonly isFlush: boolean;
+}
+
+/**
+ * An event describing that model decorations have changed.
+ */
+export interface IModelDecorationsChangedEvent {
+	/**
+	 * Lists of ids for added decorations.
+	 */
+	readonly addedDecorations: string[];
+	/**
+	 * Lists of ids for changed decorations.
+	 */
+	readonly changedDecorations: string[];
+	/**
+	 * List of ids for removed decorations.
+	 */
+	readonly removedDecorations: string[];
+}
+
+/**
+ * An event describing that some ranges of lines have been tokenized (their tokens have changed).
+ */
+export interface IModelTokensChangedEvent {
+	readonly ranges: {
+		/**
+		 * The start of the range (inclusive)
+		 */
+		readonly fromLineNumber: number;
+		/**
+		 * The end of the range (inclusive)
+		 */
+		readonly toLineNumber: number;
+	}[];
+}
+
+export interface IModelOptionsChangedEvent {
+	readonly tabSize: boolean;
+	readonly insertSpaces: boolean;
+	readonly trimAutoWhitespace: boolean;
+}
+
+/**
+ * @internal
+ */
+export const enum RawContentChangedType {
+	Flush = 1,
+	LineChanged = 2,
+	LinesDeleted = 3,
+	LinesInserted = 4,
+	EOLChanged = 5
+}
+
+/**
+ * An event describing that a model has been reset to a new value.
+ * @internal
+ */
+export class ModelRawFlush {
+	public readonly changeType = RawContentChangedType.Flush;
+}
+
+/**
+ * An event describing that a line has changed in a model.
+ * @internal
+ */
+export class ModelRawLineChanged {
+	public readonly changeType = RawContentChangedType.LineChanged;
+	/**
+	 * The line that has changed.
+	 */
+	public readonly lineNumber: number;
+	/**
+	 * The new value of the line.
+	 */
+	public readonly detail: string;
+
+	constructor(lineNumber: number, detail: string) {
+		this.lineNumber = lineNumber;
+		this.detail = detail;
+	}
+}
+
+/**
+ * An event describing that line(s) have been deleted in a model.
+ * @internal
+ */
+export class ModelRawLinesDeleted {
+	public readonly changeType = RawContentChangedType.LinesDeleted;
+	/**
+	 * At what line the deletion began (inclusive).
+	 */
+	public readonly fromLineNumber: number;
+	/**
+	 * At what line the deletion stopped (inclusive).
+	 */
+	public readonly toLineNumber: number;
+
+	constructor(fromLineNumber: number, toLineNumber: number) {
+		this.fromLineNumber = fromLineNumber;
+		this.toLineNumber = toLineNumber;
+	}
+}
+
+/**
+ * An event describing that line(s) have been inserted in a model.
+ * @internal
+ */
+export class ModelRawLinesInserted {
+	public readonly changeType = RawContentChangedType.LinesInserted;
+	/**
+	 * Before what line did the insertion begin
+	 */
+	public readonly fromLineNumber: number;
+	/**
+	 * `toLineNumber` - `fromLineNumber` + 1 denotes the number of lines that were inserted
+	 */
+	public readonly toLineNumber: number;
+	/**
+	 * The text that was inserted
+	 */
+	public readonly detail: string;
+
+	constructor(fromLineNumber: number, toLineNumber: number, detail: string) {
+		this.fromLineNumber = fromLineNumber;
+		this.toLineNumber = toLineNumber;
+		this.detail = detail;
+	}
+}
+
+/**
+ * An event describing that a model has had its EOL changed.
+ * @internal
+ */
+export class ModelRawEOLChanged {
+	public readonly changeType = RawContentChangedType.EOLChanged;
+}
+
+/**
+ * @internal
+ */
+export type ModelRawChange = ModelRawFlush | ModelRawLineChanged | ModelRawLinesDeleted | ModelRawLinesInserted | ModelRawEOLChanged;
+
+/**
+ * An event describing a change in the text of a model.
+ * @internal
+ */
+export class ModelRawContentChangedEvent {
+
+	public readonly changes: ModelRawChange[];
+	/**
+	 * The new version id the model has transitioned to.
+	 */
+	public readonly versionId: number;
+	/**
+	 * Flag that indicates that this event was generated while undoing.
+	 */
+	public readonly isUndoing: boolean;
+	/**
+	 * Flag that indicates that this event was generated while redoing.
+	 */
+	public readonly isRedoing: boolean;
+
+	constructor(changes: ModelRawChange[], versionId: number, isUndoing: boolean, isRedoing: boolean) {
+		this.changes = changes;
+		this.versionId = versionId;
+		this.isUndoing = isUndoing;
+		this.isRedoing = isRedoing;
+	}
+
+	public containsEvent(type: RawContentChangedType): boolean {
+		for (let i = 0, len = this.changes.length; i < len; i++) {
+			const change = this.changes[i];
+			if (change.changeType === type) {
+				return true;
+			}
+		}
+		return false;
+	}
+}

src/vs/editor/common/model/textModelSearch.ts

@@ -0,0 +1,512 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as strings from 'vs/base/common/strings';
+import { Position } from 'vs/editor/common/core/position';
+import { Range } from 'vs/editor/common/core/range';
+import { FindMatch, EndOfLinePreference } from 'vs/editor/common/editorCommon';
+import { CharCode } from 'vs/base/common/charCode';
+import { TextModel } from 'vs/editor/common/model/textModel';
+import { getMapForWordSeparators, WordCharacterClassifier, WordCharacterClass } from 'vs/editor/common/controller/wordCharacterClassifier';
+
+const LIMIT_FIND_COUNT = 999;
+
+export class SearchParams {
+	public readonly searchString: string;
+	public readonly isRegex: boolean;
+	public readonly matchCase: boolean;
+	public readonly wordSeparators: string;
+
+	constructor(searchString: string, isRegex: boolean, matchCase: boolean, wordSeparators: string) {
+		this.searchString = searchString;
+		this.isRegex = isRegex;
+		this.matchCase = matchCase;
+		this.wordSeparators = wordSeparators;
+	}
+
+	private static _isMultilineRegexSource(searchString: string): boolean {
+		if (!searchString || searchString.length === 0) {
+			return false;
+		}
+
+		for (let i = 0, len = searchString.length; i < len; i++) {
+			const chCode = searchString.charCodeAt(i);
+
+			if (chCode === CharCode.Backslash) {
+
+				// move to next char
+				i++;
+
+				if (i >= len) {
+					// string ends with a \
+					break;
+				}
+
+				const nextChCode = searchString.charCodeAt(i);
+				if (nextChCode === CharCode.n || nextChCode === CharCode.r) {
+					return true;
+				}
+			}
+		}
+
+		return false;
+	}
+
+	public parseSearchRequest(): SearchData {
+		if (this.searchString === '') {
+			return null;
+		}
+
+		// Try to create a RegExp out of the params
+		let multiline: boolean;
+		if (this.isRegex) {
+			multiline = SearchParams._isMultilineRegexSource(this.searchString);
+		} else {
+			multiline = (this.searchString.indexOf('\n') >= 0);
+		}
+
+		let regex: RegExp = null;
+		try {
+			regex = strings.createRegExp(this.searchString, this.isRegex, {
+				matchCase: this.matchCase,
+				wholeWord: false,
+				multiline: multiline,
+				global: true
+			});
+		} catch (err) {
+			return null;
+		}
+
+		if (!regex) {
+			return null;
+		}
+
+		let canUseSimpleSearch = (!this.isRegex && !multiline);
+		if (canUseSimpleSearch && this.searchString.toLowerCase() !== this.searchString.toUpperCase()) {
+			// casing might make a difference
+			canUseSimpleSearch = this.matchCase;
+		}
+
+		return new SearchData(regex, this.wordSeparators ? getMapForWordSeparators(this.wordSeparators) : null, canUseSimpleSearch ? this.searchString : null);
+	}
+}
+
+export class SearchData {
+
+	/**
+	 * The regex to search for. Always defined.
+	 */
+	public readonly regex: RegExp;
+	/**
+	 * The word separator classifier.
+	 */
+	public readonly wordSeparators: WordCharacterClassifier;
+	/**
+	 * The simple string to search for (if possible).
+	 */
+	public readonly simpleSearch: string;
+
+	constructor(regex: RegExp, wordSeparators: WordCharacterClassifier, simpleSearch: string) {
+		this.regex = regex;
+		this.wordSeparators = wordSeparators;
+		this.simpleSearch = simpleSearch;
+	}
+}
+
+function createFindMatch(range: Range, rawMatches: RegExpExecArray, captureMatches: boolean): FindMatch {
+	if (!captureMatches) {
+		return new FindMatch(range, null);
+	}
+	let matches: string[] = [];
+	for (let i = 0, len = rawMatches.length; i < len; i++) {
+		matches[i] = rawMatches[i];
+	}
+	return new FindMatch(range, matches);
+}
+
+export class TextModelSearch {
+
+	public static findMatches(model: TextModel, searchParams: SearchParams, searchRange: Range, captureMatches: boolean, limitResultCount: number): FindMatch[] {
+		const searchData = searchParams.parseSearchRequest();
+		if (!searchData) {
+			return [];
+		}
+
+		if (searchData.regex.multiline) {
+			return this._doFindMatchesMultiline(model, searchRange, new Searcher(searchData.wordSeparators, searchData.regex), captureMatches, limitResultCount);
+		}
+		return this._doFindMatchesLineByLine(model, searchRange, searchData, captureMatches, limitResultCount);
+	}
+
+	/**
+	 * Multiline search always executes on the lines concatenated with \n.
+	 * We must therefore compensate for the count of \n in case the model is CRLF
+	 */
+	private static _getMultilineMatchRange(model: TextModel, deltaOffset: number, text: string, matchIndex: number, match0: string): Range {
+		let startOffset: number;
+		if (model.getEOL() === '\r\n') {
+			let lineFeedCountBeforeMatch = 0;
+			for (let i = 0; i < matchIndex; i++) {
+				let chCode = text.charCodeAt(i);
+				if (chCode === CharCode.LineFeed) {
+					lineFeedCountBeforeMatch++;
+				}
+			}
+			startOffset = deltaOffset + matchIndex + lineFeedCountBeforeMatch /* add as many \r as there were \n */;
+		} else {
+			startOffset = deltaOffset + matchIndex;
+		}
+
+		let endOffset: number;
+		if (model.getEOL() === '\r\n') {
+			let lineFeedCountInMatch = 0;
+			for (let i = 0, len = match0.length; i < len; i++) {
+				let chCode = text.charCodeAt(i + matchIndex);
+				if (chCode === CharCode.LineFeed) {
+					lineFeedCountInMatch++;
+				}
+			}
+			endOffset = startOffset + match0.length + lineFeedCountInMatch /* add as many \r as there were \n */;
+		} else {
+			endOffset = startOffset + match0.length;
+		}
+
+		const startPosition = model.getPositionAt(startOffset);
+		const endPosition = model.getPositionAt(endOffset);
+		return new Range(startPosition.lineNumber, startPosition.column, endPosition.lineNumber, endPosition.column);
+	}
+
+	private static _doFindMatchesMultiline(model: TextModel, searchRange: Range, searcher: Searcher, captureMatches: boolean, limitResultCount: number): FindMatch[] {
+		const deltaOffset = model.getOffsetAt(searchRange.getStartPosition());
+		// We always execute multiline search over the lines joined with \n
+		// This makes it that \n will match the EOL for both CRLF and LF models
+		// We compensate for offset errors in `_getMultilineMatchRange`
+		const text = model.getValueInRange(searchRange, EndOfLinePreference.LF);
+
+		const result: FindMatch[] = [];
+		let counter = 0;
+
+		let m: RegExpExecArray;
+		searcher.reset(0);
+		while ((m = searcher.next(text))) {
+			result[counter++] = createFindMatch(this._getMultilineMatchRange(model, deltaOffset, text, m.index, m[0]), m, captureMatches);
+			if (counter >= limitResultCount) {
+				return result;
+			}
+		}
+
+		return result;
+	}
+
+	private static _doFindMatchesLineByLine(model: TextModel, searchRange: Range, searchData: SearchData, captureMatches: boolean, limitResultCount: number): FindMatch[] {
+		const result: FindMatch[] = [];
+		let resultLen = 0;
+
+		// Early case for a search range that starts & stops on the same line number
+		if (searchRange.startLineNumber === searchRange.endLineNumber) {
+			const text = model.getLineContent(searchRange.startLineNumber).substring(searchRange.startColumn - 1, searchRange.endColumn - 1);
+			resultLen = this._findMatchesInLine(searchData, text, searchRange.startLineNumber, searchRange.startColumn - 1, resultLen, result, captureMatches, limitResultCount);
+			return result;
+		}
+
+		// Collect results from first line
+		const text = model.getLineContent(searchRange.startLineNumber).substring(searchRange.startColumn - 1);
+		resultLen = this._findMatchesInLine(searchData, text, searchRange.startLineNumber, searchRange.startColumn - 1, resultLen, result, captureMatches, limitResultCount);
+
+		// Collect results from middle lines
+		for (let lineNumber = searchRange.startLineNumber + 1; lineNumber < searchRange.endLineNumber && resultLen < limitResultCount; lineNumber++) {
+			resultLen = this._findMatchesInLine(searchData, model.getLineContent(lineNumber), lineNumber, 0, resultLen, result, captureMatches, limitResultCount);
+		}
+
+		// Collect results from last line
+		if (resultLen < limitResultCount) {
+			const text = model.getLineContent(searchRange.endLineNumber).substring(0, searchRange.endColumn - 1);
+			resultLen = this._findMatchesInLine(searchData, text, searchRange.endLineNumber, 0, resultLen, result, captureMatches, limitResultCount);
+		}
+
+		return result;
+	}
+
+	private static _findMatchesInLine(searchData: SearchData, text: string, lineNumber: number, deltaOffset: number, resultLen: number, result: FindMatch[], captureMatches: boolean, limitResultCount: number): number {
+		const wordSeparators = searchData.wordSeparators;
+		if (!captureMatches && searchData.simpleSearch) {
+			const searchString = searchData.simpleSearch;
+			const searchStringLen = searchString.length;
+			const textLength = text.length;
+
+			let lastMatchIndex = -searchStringLen;
+			while ((lastMatchIndex = text.indexOf(searchString, lastMatchIndex + searchStringLen)) !== -1) {
+				if (!wordSeparators || isValidMatch(wordSeparators, text, textLength, lastMatchIndex, searchStringLen)) {
+					result[resultLen++] = new FindMatch(new Range(lineNumber, lastMatchIndex + 1 + deltaOffset, lineNumber, lastMatchIndex + 1 + searchStringLen + deltaOffset), null);
+					if (resultLen >= limitResultCount) {
+						return resultLen;
+					}
+				}
+			}
+			return resultLen;
+		}
+
+		const searcher = new Searcher(searchData.wordSeparators, searchData.regex);
+		let m: RegExpExecArray;
+		// Reset regex to search from the beginning
+		searcher.reset(0);
+		do {
+			m = searcher.next(text);
+			if (m) {
+				result[resultLen++] = createFindMatch(new Range(lineNumber, m.index + 1 + deltaOffset, lineNumber, m.index + 1 + m[0].length + deltaOffset), m, captureMatches);
+				if (resultLen >= limitResultCount) {
+					return resultLen;
+				}
+			}
+		} while (m);
+		return resultLen;
+	}
+
+	public static findNextMatch(model: TextModel, searchParams: SearchParams, searchStart: Position, captureMatches: boolean): FindMatch {
+		const searchData = searchParams.parseSearchRequest();
+		if (!searchData) {
+			return null;
+		}
+
+		const searcher = new Searcher(searchData.wordSeparators, searchData.regex);
+
+		if (searchData.regex.multiline) {
+			return this._doFindNextMatchMultiline(model, searchStart, searcher, captureMatches);
+		}
+		return this._doFindNextMatchLineByLine(model, searchStart, searcher, captureMatches);
+	}
+
+	private static _doFindNextMatchMultiline(model: TextModel, searchStart: Position, searcher: Searcher, captureMatches: boolean): FindMatch {
+		const searchTextStart = new Position(searchStart.lineNumber, 1);
+		const deltaOffset = model.getOffsetAt(searchTextStart);
+		const lineCount = model.getLineCount();
+		// We always execute multiline search over the lines joined with \n
+		// This makes it that \n will match the EOL for both CRLF and LF models
+		// We compensate for offset errors in `_getMultilineMatchRange`
+		const text = model.getValueInRange(new Range(searchTextStart.lineNumber, searchTextStart.column, lineCount, model.getLineMaxColumn(lineCount)), EndOfLinePreference.LF);
+		searcher.reset(searchStart.column - 1);
+		let m = searcher.next(text);
+		if (m) {
+			return createFindMatch(
+				this._getMultilineMatchRange(model, deltaOffset, text, m.index, m[0]),
+				m,
+				captureMatches
+			);
+		}
+
+		if (searchStart.lineNumber !== 1 || searchStart.column !== 1) {
+			// Try again from the top
+			return this._doFindNextMatchMultiline(model, new Position(1, 1), searcher, captureMatches);
+		}
+
+		return null;
+	}
+
+	private static _doFindNextMatchLineByLine(model: TextModel, searchStart: Position, searcher: Searcher, captureMatches: boolean): FindMatch {
+		const lineCount = model.getLineCount();
+		const startLineNumber = searchStart.lineNumber;
+
+		// Look in first line
+		const text = model.getLineContent(startLineNumber);
+		const r = this._findFirstMatchInLine(searcher, text, startLineNumber, searchStart.column, captureMatches);
+		if (r) {
+			return r;
+		}
+
+		for (let i = 1; i <= lineCount; i++) {
+			const lineIndex = (startLineNumber + i - 1) % lineCount;
+			const text = model.getLineContent(lineIndex + 1);
+			const r = this._findFirstMatchInLine(searcher, text, lineIndex + 1, 1, captureMatches);
+			if (r) {
+				return r;
+			}
+		}
+
+		return null;
+	}
+
+	private static _findFirstMatchInLine(searcher: Searcher, text: string, lineNumber: number, fromColumn: number, captureMatches: boolean): FindMatch {
+		// Set regex to search from column
+		searcher.reset(fromColumn - 1);
+		const m: RegExpExecArray = searcher.next(text);
+		if (m) {
+			return createFindMatch(
+				new Range(lineNumber, m.index + 1, lineNumber, m.index + 1 + m[0].length),
+				m,
+				captureMatches
+			);
+		}
+		return null;
+	}
+
+	public static findPreviousMatch(model: TextModel, searchParams: SearchParams, searchStart: Position, captureMatches: boolean): FindMatch {
+		const searchData = searchParams.parseSearchRequest();
+		if (!searchData) {
+			return null;
+		}
+
+		const searcher = new Searcher(searchData.wordSeparators, searchData.regex);
+
+		if (searchData.regex.multiline) {
+			return this._doFindPreviousMatchMultiline(model, searchStart, searcher, captureMatches);
+		}
+		return this._doFindPreviousMatchLineByLine(model, searchStart, searcher, captureMatches);
+	}
+
+	private static _doFindPreviousMatchMultiline(model: TextModel, searchStart: Position, searcher: Searcher, captureMatches: boolean): FindMatch {
+		const matches = this._doFindMatchesMultiline(model, new Range(1, 1, searchStart.lineNumber, searchStart.column), searcher, captureMatches, 10 * LIMIT_FIND_COUNT);
+		if (matches.length > 0) {
+			return matches[matches.length - 1];
+		}
+
+		const lineCount = model.getLineCount();
+		if (searchStart.lineNumber !== lineCount || searchStart.column !== model.getLineMaxColumn(lineCount)) {
+			// Try again with all content
+			return this._doFindPreviousMatchMultiline(model, new Position(lineCount, model.getLineMaxColumn(lineCount)), searcher, captureMatches);
+		}
+
+		return null;
+	}
+
+	private static _doFindPreviousMatchLineByLine(model: TextModel, searchStart: Position, searcher: Searcher, captureMatches: boolean): FindMatch {
+		const lineCount = model.getLineCount();
+		const startLineNumber = searchStart.lineNumber;
+
+		// Look in first line
+		const text = model.getLineContent(startLineNumber).substring(0, searchStart.column - 1);
+		const r = this._findLastMatchInLine(searcher, text, startLineNumber, captureMatches);
+		if (r) {
+			return r;
+		}
+
+		for (let i = 1; i <= lineCount; i++) {
+			const lineIndex = (lineCount + startLineNumber - i - 1) % lineCount;
+			const text = model.getLineContent(lineIndex + 1);
+			const r = this._findLastMatchInLine(searcher, text, lineIndex + 1, captureMatches);
+			if (r) {
+				return r;
+			}
+		}
+
+		return null;
+	}
+
+	private static _findLastMatchInLine(searcher: Searcher, text: string, lineNumber: number, captureMatches: boolean): FindMatch {
+		let bestResult: FindMatch = null;
+		let m: RegExpExecArray;
+		searcher.reset(0);
+		while ((m = searcher.next(text))) {
+			bestResult = createFindMatch(new Range(lineNumber, m.index + 1, lineNumber, m.index + 1 + m[0].length), m, captureMatches);
+		}
+		return bestResult;
+	}
+}
+
+function leftIsWordBounday(wordSeparators: WordCharacterClassifier, text: string, textLength: number, matchStartIndex: number, matchLength: number): boolean {
+	if (matchStartIndex === 0) {
+		// Match starts at start of string
+		return true;
+	}
+
+	const charBefore = text.charCodeAt(matchStartIndex - 1);
+	if (wordSeparators.get(charBefore) !== WordCharacterClass.Regular) {
+		// The character before the match is a word separator
+		return true;
+	}
+
+	if (matchLength > 0) {
+		const firstCharInMatch = text.charCodeAt(matchStartIndex);
+		if (wordSeparators.get(firstCharInMatch) !== WordCharacterClass.Regular) {
+			// The first character inside the match is a word separator
+			return true;
+		}
+	}
+
+	return false;
+}
+
+function rightIsWordBounday(wordSeparators: WordCharacterClassifier, text: string, textLength: number, matchStartIndex: number, matchLength: number): boolean {
+	if (matchStartIndex + matchLength === textLength) {
+		// Match ends at end of string
+		return true;
+	}
+
+	const charAfter = text.charCodeAt(matchStartIndex + matchLength);
+	if (wordSeparators.get(charAfter) !== WordCharacterClass.Regular) {
+		// The character after the match is a word separator
+		return true;
+	}
+
+	if (matchLength > 0) {
+		const lastCharInMatch = text.charCodeAt(matchStartIndex + matchLength - 1);
+		if (wordSeparators.get(lastCharInMatch) !== WordCharacterClass.Regular) {
+			// The last character in the match is a word separator
+			return true;
+		}
+	}
+
+	return false;
+}
+
+function isValidMatch(wordSeparators: WordCharacterClassifier, text: string, textLength: number, matchStartIndex: number, matchLength: number): boolean {
+	return (
+		leftIsWordBounday(wordSeparators, text, textLength, matchStartIndex, matchLength)
+		&& rightIsWordBounday(wordSeparators, text, textLength, matchStartIndex, matchLength)
+	);
+}
+
+class Searcher {
+	private _wordSeparators: WordCharacterClassifier;
+	private _searchRegex: RegExp;
+	private _prevMatchStartIndex: number;
+	private _prevMatchLength: number;
+
+	constructor(wordSeparators: WordCharacterClassifier, searchRegex: RegExp, ) {
+		this._wordSeparators = wordSeparators;
+		this._searchRegex = searchRegex;
+		this._prevMatchStartIndex = -1;
+		this._prevMatchLength = 0;
+	}
+
+	public reset(lastIndex: number): void {
+		this._searchRegex.lastIndex = lastIndex;
+		this._prevMatchStartIndex = -1;
+		this._prevMatchLength = 0;
+	}
+
+	public next(text: string): RegExpExecArray {
+		const textLength = text.length;
+
+		let m: RegExpExecArray;
+		do {
+			if (this._prevMatchStartIndex + this._prevMatchLength === textLength) {
+				// Reached the end of the line
+				return null;
+			}
+
+			m = this._searchRegex.exec(text);
+			if (!m) {
+				return null;
+			}
+
+			const matchStartIndex = m.index;
+			const matchLength = m[0].length;
+			if (matchStartIndex === this._prevMatchStartIndex && matchLength === this._prevMatchLength) {
+				// Exit early if the regex matches the same range twice
+				return null;
+			}
+			this._prevMatchStartIndex = matchStartIndex;
+			this._prevMatchLength = matchLength;
+
+			if (!this._wordSeparators || isValidMatch(this._wordSeparators, text, textLength, matchStartIndex, matchLength)) {
+				return m;
+			}
+
+		} while (m);
+
+		return null;
+	}
+}

src/vs/editor/common/model/textModelWithDecorations.ts

@@ -0,0 +1,969 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { onUnexpectedError } from 'vs/base/common/errors';
+import { IMarkdownString, markedStringsEquals } from 'vs/base/common/htmlContent';
+import * as strings from 'vs/base/common/strings';
+import { CharCode } from 'vs/base/common/charCode';
+import { Range, IRange } from 'vs/editor/common/core/range';
+import * as editorCommon from 'vs/editor/common/editorCommon';
+import { MarkersTracker, LineMarker } from 'vs/editor/common/model/modelLine';
+import { Position } from 'vs/editor/common/core/position';
+import { INewMarker, TextModelWithMarkers } from 'vs/editor/common/model/textModelWithMarkers';
+import { LanguageIdentifier } from 'vs/editor/common/modes';
+import { ITextSource, IRawTextSource } from 'vs/editor/common/model/textSource';
+import * as textModelEvents from 'vs/editor/common/model/textModelEvents';
+import { ThemeColor } from 'vs/platform/theme/common/themeService';
+
+export const ClassName = {
+	EditorWarningDecoration: 'greensquiggly',
+	EditorErrorDecoration: 'redsquiggly'
+};
+
+class DecorationsTracker {
+
+	public addedDecorations: string[];
+	public addedDecorationsLen: number;
+	public changedDecorations: string[];
+	public changedDecorationsLen: number;
+	public removedDecorations: string[];
+	public removedDecorationsLen: number;
+
+	constructor() {
+		this.addedDecorations = [];
+		this.addedDecorationsLen = 0;
+		this.changedDecorations = [];
+		this.changedDecorationsLen = 0;
+		this.removedDecorations = [];
+		this.removedDecorationsLen = 0;
+	}
+
+	// --- Build decoration events
+
+	public addNewDecoration(id: string): void {
+		this.addedDecorations[this.addedDecorationsLen++] = id;
+	}
+
+	public addRemovedDecoration(id: string): void {
+		this.removedDecorations[this.removedDecorationsLen++] = id;
+	}
+
+	public addMovedDecoration(id: string): void {
+		this.changedDecorations[this.changedDecorationsLen++] = id;
+	}
+
+	public addUpdatedDecoration(id: string): void {
+		this.changedDecorations[this.changedDecorationsLen++] = id;
+	}
+}
+
+export class InternalDecoration implements editorCommon.IModelDecoration {
+	_internalDecorationBrand: void;
+
+	public readonly id: string;
+	public readonly internalId: number;
+	public readonly ownerId: number;
+	public readonly startMarker: LineMarker;
+	public readonly endMarker: LineMarker;
+	public options: ModelDecorationOptions;
+	public isForValidation: boolean;
+	public range: Range;
+
+	constructor(id: string, internalId: number, ownerId: number, range: Range, startMarker: LineMarker, endMarker: LineMarker, options: ModelDecorationOptions) {
+		this.id = id;
+		this.internalId = internalId;
+		this.ownerId = ownerId;
+		this.range = range;
+		this.startMarker = startMarker;
+		this.endMarker = endMarker;
+		this.setOptions(options);
+	}
+
+	public setOptions(options: ModelDecorationOptions) {
+		this.options = options;
+		this.isForValidation = (
+			this.options.className === ClassName.EditorErrorDecoration
+			|| this.options.className === ClassName.EditorWarningDecoration
+		);
+	}
+
+	public setRange(multiLineDecorationsMap: { [key: string]: InternalDecoration; }, range: Range): void {
+		if (this.range.equalsRange(range)) {
+			return;
+		}
+
+		let rangeWasMultiLine = (this.range.startLineNumber !== this.range.endLineNumber);
+		this.range = range;
+		let rangeIsMultiline = (this.range.startLineNumber !== this.range.endLineNumber);
+
+		if (rangeWasMultiLine === rangeIsMultiline) {
+			return;
+		}
+
+		if (rangeIsMultiline) {
+			multiLineDecorationsMap[this.id] = this;
+		} else {
+			delete multiLineDecorationsMap[this.id];
+		}
+	}
+}
+
+let _INSTANCE_COUNT = 0;
+/**
+ * Produces 'a'-'z', followed by 'A'-'Z'... followed by 'a'-'z', etc.
+ */
+function nextInstanceId(): string {
+	const LETTERS_CNT = (CharCode.Z - CharCode.A + 1);
+
+	let result = _INSTANCE_COUNT++;
+	result = result % (2 * LETTERS_CNT);
+
+	if (result < LETTERS_CNT) {
+		return String.fromCharCode(CharCode.a + result);
+	}
+
+	return String.fromCharCode(CharCode.A + result - LETTERS_CNT);
+}
+
+export class TextModelWithDecorations extends TextModelWithMarkers implements editorCommon.ITextModelWithDecorations {
+
+	/**
+	 * Used to workaround broken clients that might attempt using a decoration id generated by a different model.
+	 * It is not globally unique in order to limit it to one character.
+	 */
+	private readonly _instanceId: string;
+	private _lastDecorationId: number;
+
+	private _currentDecorationsTracker: DecorationsTracker;
+	private _currentDecorationsTrackerCnt: number;
+
+	private _currentMarkersTracker: MarkersTracker;
+	private _currentMarkersTrackerCnt: number;
+
+	private _decorations: { [decorationId: string]: InternalDecoration; };
+	private _internalDecorations: { [internalDecorationId: number]: InternalDecoration; };
+	private _multiLineDecorationsMap: { [key: string]: InternalDecoration; };
+
+	constructor(rawTextSource: IRawTextSource, creationOptions: editorCommon.ITextModelCreationOptions, languageIdentifier: LanguageIdentifier) {
+		super(rawTextSource, creationOptions, languageIdentifier);
+
+		this._instanceId = nextInstanceId();
+		this._lastDecorationId = 0;
+
+		// Initialize decorations
+		this._currentDecorationsTracker = null;
+		this._currentDecorationsTrackerCnt = 0;
+
+		this._currentMarkersTracker = null;
+		this._currentMarkersTrackerCnt = 0;
+
+		this._decorations = Object.create(null);
+		this._internalDecorations = Object.create(null);
+		this._multiLineDecorationsMap = Object.create(null);
+	}
+
+	public dispose(): void {
+		this._decorations = null;
+		this._internalDecorations = null;
+		this._multiLineDecorationsMap = null;
+		super.dispose();
+	}
+
+	protected _resetValue(newValue: ITextSource): void {
+		super._resetValue(newValue);
+
+		// Destroy all my decorations
+		this._decorations = Object.create(null);
+		this._internalDecorations = Object.create(null);
+		this._multiLineDecorationsMap = Object.create(null);
+	}
+
+	private static _shouldStartMarkerSticksToPreviousCharacter(stickiness: editorCommon.TrackedRangeStickiness): boolean {
+		if (stickiness === editorCommon.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges || stickiness === editorCommon.TrackedRangeStickiness.GrowsOnlyWhenTypingBefore) {
+			return true;
+		}
+		return false;
+	}
+
+	private static _shouldEndMarkerSticksToPreviousCharacter(stickiness: editorCommon.TrackedRangeStickiness): boolean {
+		if (stickiness === editorCommon.TrackedRangeStickiness.NeverGrowsWhenTypingAtEdges || stickiness === editorCommon.TrackedRangeStickiness.GrowsOnlyWhenTypingBefore) {
+			return true;
+		}
+		return false;
+	}
+
+	_getTrackedRangesCount(): number {
+		return Object.keys(this._decorations).length;
+	}
+
+	// --- END TrackedRanges
+
+	public changeDecorations<T>(callback: (changeAccessor: editorCommon.IModelDecorationsChangeAccessor) => T, ownerId: number = 0): T {
+		this._assertNotDisposed();
+
+		try {
+			this._eventEmitter.beginDeferredEmit();
+			let decorationsTracker = this._acquireDecorationsTracker();
+			return this._changeDecorations(decorationsTracker, ownerId, callback);
+		} finally {
+			this._releaseDecorationsTracker();
+			this._eventEmitter.endDeferredEmit();
+		}
+	}
+
+	private _changeDecorations<T>(decorationsTracker: DecorationsTracker, ownerId: number, callback: (changeAccessor: editorCommon.IModelDecorationsChangeAccessor) => T): T {
+		let changeAccessor: editorCommon.IModelDecorationsChangeAccessor = {
+			addDecoration: (range: IRange, options: editorCommon.IModelDecorationOptions): string => {
+				return this._addDecorationImpl(decorationsTracker, ownerId, this.validateRange(range), _normalizeOptions(options));
+			},
+			changeDecoration: (id: string, newRange: IRange): void => {
+				this._changeDecorationImpl(decorationsTracker, id, this.validateRange(newRange));
+			},
+			changeDecorationOptions: (id: string, options: editorCommon.IModelDecorationOptions) => {
+				this._changeDecorationOptionsImpl(decorationsTracker, id, _normalizeOptions(options));
+			},
+			removeDecoration: (id: string): void => {
+				this._removeDecorationImpl(decorationsTracker, id);
+			},
+			deltaDecorations: (oldDecorations: string[], newDecorations: editorCommon.IModelDeltaDecoration[]): string[] => {
+				return this._deltaDecorationsImpl(decorationsTracker, ownerId, oldDecorations, this._normalizeDeltaDecorations(newDecorations));
+			}
+		};
+		let result: T = null;
+		try {
+			result = callback(changeAccessor);
+		} catch (e) {
+			onUnexpectedError(e);
+		}
+		// Invalidate change accessor
+		changeAccessor.addDecoration = null;
+		changeAccessor.changeDecoration = null;
+		changeAccessor.removeDecoration = null;
+		changeAccessor.deltaDecorations = null;
+		return result;
+	}
+
+	public deltaDecorations(oldDecorations: string[], newDecorations: editorCommon.IModelDeltaDecoration[], ownerId: number = 0): string[] {
+		this._assertNotDisposed();
+		if (!oldDecorations) {
+			oldDecorations = [];
+		}
+		return this.changeDecorations((changeAccessor) => {
+			return changeAccessor.deltaDecorations(oldDecorations, newDecorations);
+		}, ownerId);
+	}
+
+	public removeAllDecorationsWithOwnerId(ownerId: number): void {
+		let toRemove: string[] = [];
+
+		for (let decorationId in this._decorations) {
+			// No `hasOwnProperty` call due to using Object.create(null)
+
+			let decoration = this._decorations[decorationId];
+
+			if (decoration.ownerId === ownerId) {
+				toRemove.push(decoration.id);
+			}
+		}
+
+		this._removeDecorationsImpl(null, toRemove);
+	}
+
+	public getDecorationOptions(decorationId: string): editorCommon.IModelDecorationOptions {
+		let decoration = this._decorations[decorationId];
+		if (!decoration) {
+			return null;
+		}
+		return decoration.options;
+	}
+
+	public getDecorationRange(decorationId: string): Range {
+		let decoration = this._decorations[decorationId];
+		if (!decoration) {
+			return null;
+		}
+		return decoration.range;
+	}
+
+	public getLineDecorations(lineNumber: number, ownerId: number = 0, filterOutValidation: boolean = false): editorCommon.IModelDecoration[] {
+		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
+			return [];
+		}
+
+		return this.getLinesDecorations(lineNumber, lineNumber, ownerId, filterOutValidation);
+	}
+
+	/**
+	 * Fetch only multi-line decorations that intersect with the given line number range
+	 */
+	private _getMultiLineDecorations(filterRange: Range, filterOwnerId: number, filterOutValidation: boolean): InternalDecoration[] {
+		const filterStartLineNumber = filterRange.startLineNumber;
+		const filterStartColumn = filterRange.startColumn;
+		const filterEndLineNumber = filterRange.endLineNumber;
+		const filterEndColumn = filterRange.endColumn;
+
+		let result: InternalDecoration[] = [], resultLen = 0;
+
+		for (let decorationId in this._multiLineDecorationsMap) {
+			// No `hasOwnProperty` call due to using Object.create(null)
+			let decoration = this._multiLineDecorationsMap[decorationId];
+
+			if (filterOwnerId && decoration.ownerId && decoration.ownerId !== filterOwnerId) {
+				continue;
+			}
+
+			if (filterOutValidation && decoration.isForValidation) {
+				continue;
+			}
+
+			let range = decoration.range;
+
+			if (range.startLineNumber > filterEndLineNumber) {
+				continue;
+			}
+			if (range.startLineNumber === filterEndLineNumber && range.startColumn > filterEndColumn) {
+				continue;
+			}
+			if (range.endLineNumber < filterStartLineNumber) {
+				continue;
+			}
+			if (range.endLineNumber === filterStartLineNumber && range.endColumn < filterStartColumn) {
+				continue;
+			}
+
+			result[resultLen++] = decoration;
+		}
+
+		return result;
+	}
+
+	private _getDecorationsInRange(filterRange: Range, filterOwnerId: number, filterOutValidation: boolean): InternalDecoration[] {
+		const filterStartLineNumber = filterRange.startLineNumber;
+		const filterStartColumn = filterRange.startColumn;
+		const filterEndLineNumber = filterRange.endLineNumber;
+		const filterEndColumn = filterRange.endColumn;
+
+		let result = this._getMultiLineDecorations(filterRange, filterOwnerId, filterOutValidation);
+		let resultLen = result.length;
+		let resultMap: { [decorationId: string]: boolean; } = {};
+
+		for (let i = 0, len = resultLen; i < len; i++) {
+			resultMap[result[i].id] = true;
+		}
+
+		for (let lineNumber = filterStartLineNumber; lineNumber <= filterEndLineNumber; lineNumber++) {
+			let lineMarkers = this._lines[lineNumber - 1].getMarkers();
+			if (lineMarkers === null) {
+				continue;
+			}
+			for (let i = 0, len = lineMarkers.length; i < len; i++) {
+				let lineMarker = lineMarkers[i];
+				let internalDecorationId = lineMarker.internalDecorationId;
+
+				if (internalDecorationId === 0) {
+					// marker does not belong to any decoration
+					continue;
+				}
+
+				let decoration = this._internalDecorations[internalDecorationId];
+
+				if (resultMap.hasOwnProperty(decoration.id)) {
+					// decoration already in result
+					continue;
+				}
+
+				if (filterOwnerId && decoration.ownerId && decoration.ownerId !== filterOwnerId) {
+					continue;
+				}
+
+				if (filterOutValidation && decoration.isForValidation) {
+					continue;
+				}
+
+				let range = decoration.range;
+
+				if (range.startLineNumber > filterEndLineNumber) {
+					continue;
+				}
+				if (range.startLineNumber === filterEndLineNumber && range.startColumn > filterEndColumn) {
+					continue;
+				}
+				if (range.endLineNumber < filterStartLineNumber) {
+					continue;
+				}
+				if (range.endLineNumber === filterStartLineNumber && range.endColumn < filterStartColumn) {
+					continue;
+				}
+
+				result[resultLen++] = decoration;
+				resultMap[decoration.id] = true;
+			}
+		}
+
+		return result;
+	}
+
+	public getLinesDecorations(_startLineNumber: number, _endLineNumber: number, ownerId: number = 0, filterOutValidation: boolean = false): editorCommon.IModelDecoration[] {
+		let lineCount = this.getLineCount();
+		let startLineNumber = Math.min(lineCount, Math.max(1, _startLineNumber));
+		let endLineNumber = Math.min(lineCount, Math.max(1, _endLineNumber));
+		let endColumn = this.getLineMaxColumn(endLineNumber);
+		return this._getDecorationsInRange(new Range(startLineNumber, 1, endLineNumber, endColumn), ownerId, filterOutValidation);
+	}
+
+	public getDecorationsInRange(range: IRange, ownerId?: number, filterOutValidation?: boolean): editorCommon.IModelDecoration[] {
+		let validatedRange = this.validateRange(range);
+		return this._getDecorationsInRange(validatedRange, ownerId, filterOutValidation);
+	}
+
+	public getAllDecorations(ownerId: number = 0, filterOutValidation: boolean = false): editorCommon.IModelDecoration[] {
+		let result: InternalDecoration[] = [], resultLen = 0;
+
+		for (let decorationId in this._decorations) {
+			// No `hasOwnProperty` call due to using Object.create(null)
+			let decoration = this._decorations[decorationId];
+
+			if (ownerId && decoration.ownerId && decoration.ownerId !== ownerId) {
+				continue;
+			}
+
+			if (filterOutValidation && decoration.isForValidation) {
+				continue;
+			}
+
+			result[resultLen++] = decoration;
+		}
+
+		return result;
+	}
+
+	protected _acquireMarkersTracker(): MarkersTracker {
+		if (this._currentMarkersTrackerCnt === 0) {
+			this._currentMarkersTracker = new MarkersTracker();
+		}
+		this._currentMarkersTrackerCnt++;
+		return this._currentMarkersTracker;
+	}
+
+	protected _releaseMarkersTracker(): void {
+		this._currentMarkersTrackerCnt--;
+		if (this._currentMarkersTrackerCnt === 0) {
+			let markersTracker = this._currentMarkersTracker;
+			this._currentMarkersTracker = null;
+			this._handleTrackedMarkers(markersTracker);
+		}
+	}
+
+	/**
+	 * Handle changed markers (i.e. update decorations ranges and return the changed decorations, unique and sorted by id)
+	 */
+	private _handleTrackedMarkers(markersTracker: MarkersTracker): void {
+		let changedInternalDecorationIds = markersTracker.getDecorationIds();
+		if (changedInternalDecorationIds.length === 0) {
+			return;
+		}
+
+		changedInternalDecorationIds.sort();
+
+		let uniqueChangedDecorations: string[] = [], uniqueChangedDecorationsLen = 0;
+		let previousInternalDecorationId: number = 0;
+		for (let i = 0, len = changedInternalDecorationIds.length; i < len; i++) {
+			let internalDecorationId = changedInternalDecorationIds[i];
+			if (internalDecorationId === previousInternalDecorationId) {
+				continue;
+			}
+			previousInternalDecorationId = internalDecorationId;
+
+			let decoration = this._internalDecorations[internalDecorationId];
+			if (!decoration) {
+				// perhaps the decoration was removed in the meantime
+				continue;
+			}
+
+			let startMarker = decoration.startMarker.position;
+			let endMarker = decoration.endMarker.position;
+			let range = TextModelWithDecorations._createRangeFromMarkers(startMarker, endMarker);
+			decoration.setRange(this._multiLineDecorationsMap, range);
+
+			uniqueChangedDecorations[uniqueChangedDecorationsLen++] = decoration.id;
+		}
+
+		if (uniqueChangedDecorations.length > 0) {
+			let e: textModelEvents.IModelDecorationsChangedEvent = {
+				addedDecorations: [],
+				changedDecorations: uniqueChangedDecorations,
+				removedDecorations: []
+			};
+			this.emitModelDecorationsChangedEvent(e);
+		}
+	}
+
+	private static _createRangeFromMarkers(startPosition: Position, endPosition: Position): Range {
+		if (endPosition.isBefore(startPosition)) {
+			// This tracked range has turned in on itself (end marker before start marker)
+			// This can happen in extreme editing conditions where lots of text is removed and lots is added
+
+			// Treat it as a collapsed range
+			return new Range(startPosition.lineNumber, startPosition.column, startPosition.lineNumber, startPosition.column);
+		}
+		return new Range(startPosition.lineNumber, startPosition.column, endPosition.lineNumber, endPosition.column);
+	}
+
+	private _acquireDecorationsTracker(): DecorationsTracker {
+		if (this._currentDecorationsTrackerCnt === 0) {
+			this._currentDecorationsTracker = new DecorationsTracker();
+		}
+		this._currentDecorationsTrackerCnt++;
+		return this._currentDecorationsTracker;
+	}
+
+	private _releaseDecorationsTracker(): void {
+		this._currentDecorationsTrackerCnt--;
+		if (this._currentDecorationsTrackerCnt === 0) {
+			let decorationsTracker = this._currentDecorationsTracker;
+			this._currentDecorationsTracker = null;
+			this._handleTrackedDecorations(decorationsTracker);
+		}
+	}
+
+	private _handleTrackedDecorations(decorationsTracker: DecorationsTracker): void {
+		if (
+			decorationsTracker.addedDecorationsLen === 0
+			&& decorationsTracker.changedDecorationsLen === 0
+			&& decorationsTracker.removedDecorationsLen === 0
+		) {
+			return;
+		}
+
+		let e: textModelEvents.IModelDecorationsChangedEvent = {
+			addedDecorations: decorationsTracker.addedDecorations,
+			changedDecorations: decorationsTracker.changedDecorations,
+			removedDecorations: decorationsTracker.removedDecorations
+		};
+		this.emitModelDecorationsChangedEvent(e);
+	}
+
+	private emitModelDecorationsChangedEvent(e: textModelEvents.IModelDecorationsChangedEvent): void {
+		if (!this._isDisposing) {
+			this._eventEmitter.emit(textModelEvents.TextModelEventType.ModelDecorationsChanged, e);
+		}
+	}
+
+	private _normalizeDeltaDecorations(deltaDecorations: editorCommon.IModelDeltaDecoration[]): ModelDeltaDecoration[] {
+		let result: ModelDeltaDecoration[] = [];
+		for (let i = 0, len = deltaDecorations.length; i < len; i++) {
+			let deltaDecoration = deltaDecorations[i];
+			result.push(new ModelDeltaDecoration(i, this.validateRange(deltaDecoration.range), _normalizeOptions(deltaDecoration.options)));
+		}
+		return result;
+	}
+
+	private _externalDecorationId(internalId: number): string {
+		return `${this._instanceId};${internalId}`;
+	}
+
+	private _addDecorationImpl(decorationsTracker: DecorationsTracker, ownerId: number, _range: Range, options: ModelDecorationOptions): string {
+		let range = this.validateRange(_range);
+
+		let internalDecorationId = (++this._lastDecorationId);
+		let decorationId = this._externalDecorationId(internalDecorationId);
+
+		let markers = this._addMarkers([
+			{
+				internalDecorationId: internalDecorationId,
+				position: new Position(range.startLineNumber, range.startColumn),
+				stickToPreviousCharacter: TextModelWithDecorations._shouldStartMarkerSticksToPreviousCharacter(options.stickiness)
+			},
+			{
+				internalDecorationId: internalDecorationId,
+				position: new Position(range.endLineNumber, range.endColumn),
+				stickToPreviousCharacter: TextModelWithDecorations._shouldEndMarkerSticksToPreviousCharacter(options.stickiness)
+			}
+		]);
+
+		let decoration = new InternalDecoration(decorationId, internalDecorationId, ownerId, range, markers[0], markers[1], options);
+		this._decorations[decorationId] = decoration;
+		this._internalDecorations[internalDecorationId] = decoration;
+		if (range.startLineNumber !== range.endLineNumber) {
+			this._multiLineDecorationsMap[decorationId] = decoration;
+		}
+
+		decorationsTracker.addNewDecoration(decorationId);
+
+		return decorationId;
+	}
+
+	private _addDecorationsImpl(decorationsTracker: DecorationsTracker, ownerId: number, newDecorations: ModelDeltaDecoration[]): string[] {
+		let internalDecorationIds: number[] = [];
+		let decorationIds: string[] = [];
+		let newMarkers: INewMarker[] = [];
+
+		for (let i = 0, len = newDecorations.length; i < len; i++) {
+			let newDecoration = newDecorations[i];
+			let range = newDecoration.range;
+			let stickiness = newDecoration.options.stickiness;
+
+			let internalDecorationId = (++this._lastDecorationId);
+			let decorationId = this._externalDecorationId(internalDecorationId);
+
+			internalDecorationIds[i] = internalDecorationId;
+			decorationIds[i] = decorationId;
+
+			newMarkers[2 * i] = {
+				internalDecorationId: internalDecorationId,
+				position: new Position(range.startLineNumber, range.startColumn),
+				stickToPreviousCharacter: TextModelWithDecorations._shouldStartMarkerSticksToPreviousCharacter(stickiness)
+			};
+
+			newMarkers[2 * i + 1] = {
+				internalDecorationId: internalDecorationId,
+				position: new Position(range.endLineNumber, range.endColumn),
+				stickToPreviousCharacter: TextModelWithDecorations._shouldEndMarkerSticksToPreviousCharacter(stickiness)
+			};
+		}
+
+		let markerIds = this._addMarkers(newMarkers);
+
+		for (let i = 0, len = newDecorations.length; i < len; i++) {
+			let newDecoration = newDecorations[i];
+			let range = newDecoration.range;
+			let internalDecorationId = internalDecorationIds[i];
+			let decorationId = decorationIds[i];
+			let startMarker = markerIds[2 * i];
+			let endMarker = markerIds[2 * i + 1];
+
+			let decoration = new InternalDecoration(decorationId, internalDecorationId, ownerId, range, startMarker, endMarker, newDecoration.options);
+			this._decorations[decorationId] = decoration;
+			this._internalDecorations[internalDecorationId] = decoration;
+			if (range.startLineNumber !== range.endLineNumber) {
+				this._multiLineDecorationsMap[decorationId] = decoration;
+			}
+
+			decorationsTracker.addNewDecoration(decorationId);
+		}
+
+		return decorationIds;
+	}
+
+	private _changeDecorationImpl(decorationsTracker: DecorationsTracker, decorationId: string, newRange: Range): void {
+		let decoration = this._decorations[decorationId];
+		if (!decoration) {
+			return;
+		}
+
+		let startMarker = decoration.startMarker;
+		if (newRange.startLineNumber !== startMarker.position.lineNumber) {
+			// move marker between lines
+			this._lines[startMarker.position.lineNumber - 1].removeMarker(startMarker);
+			this._lines[newRange.startLineNumber - 1].addMarker(startMarker);
+		}
+		startMarker.setPosition(new Position(newRange.startLineNumber, newRange.startColumn));
+
+		let endMarker = decoration.endMarker;
+		if (newRange.endLineNumber !== endMarker.position.lineNumber) {
+			// move marker between lines
+			this._lines[endMarker.position.lineNumber - 1].removeMarker(endMarker);
+			this._lines[newRange.endLineNumber - 1].addMarker(endMarker);
+		}
+		endMarker.setPosition(new Position(newRange.endLineNumber, newRange.endColumn));
+
+		decoration.setRange(this._multiLineDecorationsMap, newRange);
+
+		decorationsTracker.addMovedDecoration(decorationId);
+	}
+
+	private _changeDecorationOptionsImpl(decorationsTracker: DecorationsTracker, decorationId: string, options: ModelDecorationOptions): void {
+		let decoration = this._decorations[decorationId];
+		if (!decoration) {
+			return;
+		}
+
+		if (decoration.options.stickiness !== options.stickiness) {
+			decoration.startMarker.stickToPreviousCharacter = TextModelWithDecorations._shouldStartMarkerSticksToPreviousCharacter(options.stickiness);
+			decoration.endMarker.stickToPreviousCharacter = TextModelWithDecorations._shouldEndMarkerSticksToPreviousCharacter(options.stickiness);
+		}
+
+		decoration.setOptions(options);
+
+		decorationsTracker.addUpdatedDecoration(decorationId);
+	}
+
+	private _removeDecorationImpl(decorationsTracker: DecorationsTracker, decorationId: string): void {
+		let decoration = this._decorations[decorationId];
+		if (!decoration) {
+			return;
+		}
+
+		this._removeMarkers([decoration.startMarker, decoration.endMarker]);
+
+		delete this._multiLineDecorationsMap[decorationId];
+		delete this._decorations[decorationId];
+		delete this._internalDecorations[decoration.internalId];
+
+		if (decorationsTracker) {
+			decorationsTracker.addRemovedDecoration(decorationId);
+		}
+	}
+
+	private _removeDecorationsImpl(decorationsTracker: DecorationsTracker, decorationIds: string[]): void {
+		let removeMarkers: LineMarker[] = [], removeMarkersLen = 0;
+
+		for (let i = 0, len = decorationIds.length; i < len; i++) {
+			let decorationId = decorationIds[i];
+			let decoration = this._decorations[decorationId];
+			if (!decoration) {
+				continue;
+			}
+
+			if (decorationsTracker) {
+				decorationsTracker.addRemovedDecoration(decorationId);
+			}
+
+			removeMarkers[removeMarkersLen++] = decoration.startMarker;
+			removeMarkers[removeMarkersLen++] = decoration.endMarker;
+			delete this._multiLineDecorationsMap[decorationId];
+			delete this._decorations[decorationId];
+			delete this._internalDecorations[decoration.internalId];
+		}
+
+		if (removeMarkers.length > 0) {
+			this._removeMarkers(removeMarkers);
+		}
+	}
+
+	private _resolveOldDecorations(oldDecorations: string[]): InternalDecoration[] {
+		let result: InternalDecoration[] = [];
+		for (let i = 0, len = oldDecorations.length; i < len; i++) {
+			let id = oldDecorations[i];
+			let decoration = this._decorations[id];
+			if (!decoration) {
+				continue;
+			}
+
+			result.push(decoration);
+		}
+		return result;
+	}
+
+	private _deltaDecorationsImpl(decorationsTracker: DecorationsTracker, ownerId: number, oldDecorationsIds: string[], newDecorations: ModelDeltaDecoration[]): string[] {
+
+		if (oldDecorationsIds.length === 0) {
+			// Nothing to remove
+			return this._addDecorationsImpl(decorationsTracker, ownerId, newDecorations);
+		}
+
+		if (newDecorations.length === 0) {
+			// Nothing to add
+			this._removeDecorationsImpl(decorationsTracker, oldDecorationsIds);
+			return [];
+		}
+
+		let oldDecorations = this._resolveOldDecorations(oldDecorationsIds);
+
+		oldDecorations.sort((a, b) => Range.compareRangesUsingStarts(a.range, b.range));
+		newDecorations.sort((a, b) => Range.compareRangesUsingStarts(a.range, b.range));
+
+		let result: string[] = [],
+			oldDecorationsIndex = 0,
+			oldDecorationsLength = oldDecorations.length,
+			newDecorationsIndex = 0,
+			newDecorationsLength = newDecorations.length,
+			decorationsToAdd: ModelDeltaDecoration[] = [],
+			decorationsToRemove: string[] = [];
+
+		while (oldDecorationsIndex < oldDecorationsLength && newDecorationsIndex < newDecorationsLength) {
+			let oldDecoration = oldDecorations[oldDecorationsIndex];
+			let newDecoration = newDecorations[newDecorationsIndex];
+			let comparison = Range.compareRangesUsingStarts(oldDecoration.range, newDecoration.range);
+
+			if (comparison < 0) {
+				// `oldDecoration` is before `newDecoration` => remove `oldDecoration`
+				decorationsToRemove.push(oldDecoration.id);
+				oldDecorationsIndex++;
+				continue;
+			}
+
+			if (comparison > 0) {
+				// `newDecoration` is before `oldDecoration` => add `newDecoration`
+				decorationsToAdd.push(newDecoration);
+				newDecorationsIndex++;
+				continue;
+			}
+
+			// The ranges of `oldDecoration` and `newDecoration` are equal
+
+			if (!oldDecoration.options.equals(newDecoration.options)) {
+				// The options do not match => remove `oldDecoration`
+				decorationsToRemove.push(oldDecoration.id);
+				oldDecorationsIndex++;
+				continue;
+			}
+
+			// Bingo! We can reuse `oldDecoration` for `newDecoration`
+			result[newDecoration.index] = oldDecoration.id;
+			oldDecorationsIndex++;
+			newDecorationsIndex++;
+		}
+
+		while (oldDecorationsIndex < oldDecorationsLength) {
+			// No more new decorations => remove decoration at `oldDecorationsIndex`
+			decorationsToRemove.push(oldDecorations[oldDecorationsIndex].id);
+			oldDecorationsIndex++;
+		}
+
+		while (newDecorationsIndex < newDecorationsLength) {
+			// No more old decorations => add decoration at `newDecorationsIndex`
+			decorationsToAdd.push(newDecorations[newDecorationsIndex]);
+			newDecorationsIndex++;
+		}
+
+		// Remove `decorationsToRemove`
+		if (decorationsToRemove.length > 0) {
+			this._removeDecorationsImpl(decorationsTracker, decorationsToRemove);
+		}
+
+		// Add `decorationsToAdd`
+		if (decorationsToAdd.length > 0) {
+			let newIds = this._addDecorationsImpl(decorationsTracker, ownerId, decorationsToAdd);
+			for (let i = 0, len = decorationsToAdd.length; i < len; i++) {
+				result[decorationsToAdd[i].index] = newIds[i];
+			}
+		}
+
+		return result;
+	}
+}
+
+function cleanClassName(className: string): string {
+	return className.replace(/[^a-z0-9\-]/gi, ' ');
+}
+
+export class ModelDecorationOverviewRulerOptions implements editorCommon.IModelDecorationOverviewRulerOptions {
+	readonly color: string | ThemeColor;
+	readonly darkColor: string | ThemeColor;
+	readonly hcColor: string | ThemeColor;
+	readonly position: editorCommon.OverviewRulerLane;
+
+	constructor(options: editorCommon.IModelDecorationOverviewRulerOptions) {
+		this.color = strings.empty;
+		this.darkColor = strings.empty;
+		this.hcColor = strings.empty;
+		this.position = editorCommon.OverviewRulerLane.Center;
+
+		if (options && options.color) {
+			this.color = options.color;
+		}
+		if (options && options.darkColor) {
+			this.darkColor = options.darkColor;
+			this.hcColor = options.darkColor;
+		}
+		if (options && options.hcColor) {
+			this.hcColor = options.hcColor;
+		}
+		if (options && options.hasOwnProperty('position')) {
+			this.position = options.position;
+		}
+	}
+
+	public equals(other: ModelDecorationOverviewRulerOptions): boolean {
+		return (
+			this.color === other.color
+			&& this.darkColor === other.darkColor
+			&& this.hcColor === other.hcColor
+			&& this.position === other.position
+		);
+	}
+}
+
+let lastStaticId = 0;
+
+export class ModelDecorationOptions implements editorCommon.IModelDecorationOptions {
+
+	public static EMPTY: ModelDecorationOptions;
+
+	public static register(options: editorCommon.IModelDecorationOptions): ModelDecorationOptions {
+		return new ModelDecorationOptions(++lastStaticId, options);
+	}
+
+	public static createDynamic(options: editorCommon.IModelDecorationOptions): ModelDecorationOptions {
+		return new ModelDecorationOptions(0, options);
+	}
+
+	readonly staticId: number;
+	readonly stickiness: editorCommon.TrackedRangeStickiness;
+	readonly className: string;
+	readonly hoverMessage: IMarkdownString | IMarkdownString[];
+	readonly glyphMarginHoverMessage: IMarkdownString | IMarkdownString[];
+	readonly isWholeLine: boolean;
+	readonly showIfCollapsed: boolean;
+	readonly overviewRuler: ModelDecorationOverviewRulerOptions;
+	readonly glyphMarginClassName: string;
+	readonly linesDecorationsClassName: string;
+	readonly marginClassName: string;
+	readonly inlineClassName: string;
+	readonly beforeContentClassName: string;
+	readonly afterContentClassName: string;
+
+	private constructor(staticId: number, options: editorCommon.IModelDecorationOptions) {
+		this.staticId = staticId;
+		this.stickiness = options.stickiness || editorCommon.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges;
+		this.className = options.className ? cleanClassName(options.className) : strings.empty;
+		this.hoverMessage = options.hoverMessage || [];
+		this.glyphMarginHoverMessage = options.glyphMarginHoverMessage || [];
+		this.isWholeLine = options.isWholeLine || false;
+		this.showIfCollapsed = options.showIfCollapsed || false;
+		this.overviewRuler = new ModelDecorationOverviewRulerOptions(options.overviewRuler);
+		this.glyphMarginClassName = options.glyphMarginClassName ? cleanClassName(options.glyphMarginClassName) : strings.empty;
+		this.linesDecorationsClassName = options.linesDecorationsClassName ? cleanClassName(options.linesDecorationsClassName) : strings.empty;
+		this.marginClassName = options.marginClassName ? cleanClassName(options.marginClassName) : strings.empty;
+		this.inlineClassName = options.inlineClassName ? cleanClassName(options.inlineClassName) : strings.empty;
+		this.beforeContentClassName = options.beforeContentClassName ? cleanClassName(options.beforeContentClassName) : strings.empty;
+		this.afterContentClassName = options.afterContentClassName ? cleanClassName(options.afterContentClassName) : strings.empty;
+	}
+
+	public equals(other: ModelDecorationOptions): boolean {
+		if (this.staticId > 0 || other.staticId > 0) {
+			return this.staticId === other.staticId;
+		}
+
+		return (
+			this.stickiness === other.stickiness
+			&& this.className === other.className
+			&& this.isWholeLine === other.isWholeLine
+			&& this.showIfCollapsed === other.showIfCollapsed
+			&& this.glyphMarginClassName === other.glyphMarginClassName
+			&& this.linesDecorationsClassName === other.linesDecorationsClassName
+			&& this.marginClassName === other.marginClassName
+			&& this.inlineClassName === other.inlineClassName
+			&& this.beforeContentClassName === other.beforeContentClassName
+			&& this.afterContentClassName === other.afterContentClassName
+			&& markedStringsEquals(this.hoverMessage, other.hoverMessage)
+			&& markedStringsEquals(this.glyphMarginHoverMessage, other.glyphMarginHoverMessage)
+			&& this.overviewRuler.equals(other.overviewRuler)
+		);
+	}
+}
+ModelDecorationOptions.EMPTY = ModelDecorationOptions.register({});
+
+class ModelDeltaDecoration implements editorCommon.IModelDeltaDecoration {
+
+	index: number;
+	range: Range;
+	options: ModelDecorationOptions;
+
+	constructor(index: number, range: Range, options: ModelDecorationOptions) {
+		this.index = index;
+		this.range = range;
+		this.options = options;
+	}
+}
+
+function _normalizeOptions(options: editorCommon.IModelDecorationOptions): ModelDecorationOptions {
+	if (options instanceof ModelDecorationOptions) {
+		return options;
+	}
+	return ModelDecorationOptions.createDynamic(options);
+}

src/vs/editor/common/model/textModelWithMarkers.ts

@@ -0,0 +1,174 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { IdGenerator } from 'vs/base/common/idGenerator';
+import { Position } from 'vs/editor/common/core/position';
+import { ITextModelWithMarkers, ITextModelCreationOptions } from 'vs/editor/common/editorCommon';
+import { LineMarker } from 'vs/editor/common/model/modelLine';
+import { TextModelWithTokens } from 'vs/editor/common/model/textModelWithTokens';
+import { LanguageIdentifier } from 'vs/editor/common/modes';
+import { ITextSource, IRawTextSource } from 'vs/editor/common/model/textSource';
+
+export interface IMarkerIdToMarkerMap {
+	[key: string]: LineMarker;
+}
+
+export interface INewMarker {
+	internalDecorationId: number;
+	position: Position;
+	stickToPreviousCharacter: boolean;
+}
+
+var _INSTANCE_COUNT = 0;
+
+export class TextModelWithMarkers extends TextModelWithTokens implements ITextModelWithMarkers {
+
+	private _markerIdGenerator: IdGenerator;
+	protected _markerIdToMarker: IMarkerIdToMarkerMap;
+
+	constructor(rawTextSource: IRawTextSource, creationOptions: ITextModelCreationOptions, languageIdentifier: LanguageIdentifier) {
+		super(rawTextSource, creationOptions, languageIdentifier);
+		this._markerIdGenerator = new IdGenerator((++_INSTANCE_COUNT) + ';');
+		this._markerIdToMarker = Object.create(null);
+	}
+
+	public dispose(): void {
+		this._markerIdToMarker = null;
+		super.dispose();
+	}
+
+	protected _resetValue(newValue: ITextSource): void {
+		super._resetValue(newValue);
+
+		// Destroy all my markers
+		this._markerIdToMarker = Object.create(null);
+	}
+
+	_addMarker(internalDecorationId: number, lineNumber: number, column: number, stickToPreviousCharacter: boolean): string {
+		var pos = this.validatePosition(new Position(lineNumber, column));
+
+		var marker = new LineMarker(this._markerIdGenerator.nextId(), internalDecorationId, pos, stickToPreviousCharacter);
+		this._markerIdToMarker[marker.id] = marker;
+
+		this._lines[pos.lineNumber - 1].addMarker(marker);
+
+		return marker.id;
+	}
+
+	protected _addMarkers(newMarkers: INewMarker[]): LineMarker[] {
+		if (newMarkers.length === 0) {
+			return [];
+		}
+
+		let markers: LineMarker[] = [];
+		for (let i = 0, len = newMarkers.length; i < len; i++) {
+			let newMarker = newMarkers[i];
+
+			let marker = new LineMarker(this._markerIdGenerator.nextId(), newMarker.internalDecorationId, newMarker.position, newMarker.stickToPreviousCharacter);
+			this._markerIdToMarker[marker.id] = marker;
+
+			markers[i] = marker;
+		}
+
+		let sortedMarkers = markers.slice(0);
+		sortedMarkers.sort((a, b) => {
+			return a.position.lineNumber - b.position.lineNumber;
+		});
+
+		let currentLineNumber = 0;
+		let currentMarkers: LineMarker[] = [], currentMarkersLen = 0;
+		for (let i = 0, len = sortedMarkers.length; i < len; i++) {
+			let marker = sortedMarkers[i];
+
+			if (marker.position.lineNumber !== currentLineNumber) {
+				if (currentLineNumber !== 0) {
+					this._lines[currentLineNumber - 1].addMarkers(currentMarkers);
+				}
+				currentLineNumber = marker.position.lineNumber;
+				currentMarkers.length = 0;
+				currentMarkersLen = 0;
+			}
+
+			currentMarkers[currentMarkersLen++] = marker;
+		}
+		this._lines[currentLineNumber - 1].addMarkers(currentMarkers);
+
+		return markers;
+	}
+
+	_changeMarker(id: string, lineNumber: number, column: number): void {
+		let marker = this._markerIdToMarker[id];
+		if (!marker) {
+			return;
+		}
+
+		let newPos = this.validatePosition(new Position(lineNumber, column));
+
+		if (newPos.lineNumber !== marker.position.lineNumber) {
+			// Move marker between lines
+			this._lines[marker.position.lineNumber - 1].removeMarker(marker);
+			this._lines[newPos.lineNumber - 1].addMarker(marker);
+		}
+
+		marker.setPosition(newPos);
+	}
+
+	_changeMarkerStickiness(id: string, newStickToPreviousCharacter: boolean): void {
+		let marker = this._markerIdToMarker[id];
+		if (!marker) {
+			return;
+		}
+
+		marker.stickToPreviousCharacter = newStickToPreviousCharacter;
+	}
+
+	_getMarker(id: string): Position {
+		let marker = this._markerIdToMarker[id];
+		if (!marker) {
+			return null;
+		}
+
+		return marker.position;
+	}
+
+	_getMarkersCount(): number {
+		return Object.keys(this._markerIdToMarker).length;
+	}
+
+	_removeMarker(id: string): void {
+		let marker = this._markerIdToMarker[id];
+		if (!marker) {
+			return;
+		}
+
+		this._lines[marker.position.lineNumber - 1].removeMarker(marker);
+		delete this._markerIdToMarker[id];
+	}
+
+	protected _removeMarkers(markers: LineMarker[]): void {
+		markers.sort((a, b) => {
+			return a.position.lineNumber - b.position.lineNumber;
+		});
+
+		let currentLineNumber = 0;
+		let currentMarkers: { [markerId: string]: boolean; } = null;
+		for (let i = 0, len = markers.length; i < len; i++) {
+			let marker = markers[i];
+			delete this._markerIdToMarker[marker.id];
+
+			if (marker.position.lineNumber !== currentLineNumber) {
+				if (currentLineNumber !== 0) {
+					this._lines[currentLineNumber - 1].removeMarkers(currentMarkers);
+				}
+				currentLineNumber = marker.position.lineNumber;
+				currentMarkers = Object.create(null);
+			}
+
+			currentMarkers[marker.id] = true;
+		}
+		this._lines[currentLineNumber - 1].removeMarkers(currentMarkers);
+	}
+}

src/vs/editor/common/model/textModelWithTokens.ts

@@ -0,0 +1,817 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as nls from 'vs/nls';
+import { onUnexpectedError } from 'vs/base/common/errors';
+import { IDisposable } from 'vs/base/common/lifecycle';
+import { StopWatch } from 'vs/base/common/stopwatch';
+import { Range } from 'vs/editor/common/core/range';
+import * as editorCommon from 'vs/editor/common/editorCommon';
+import { TextModel } from 'vs/editor/common/model/textModel';
+import { ITokenizationSupport, IState, TokenizationRegistry, LanguageId, LanguageIdentifier } from 'vs/editor/common/modes';
+import { NULL_LANGUAGE_IDENTIFIER, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
+import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
+import { BracketsUtils, RichEditBrackets, RichEditBracket } from 'vs/editor/common/modes/supports/richEditBrackets';
+import { Position, IPosition } from 'vs/editor/common/core/position';
+import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
+import { LineTokens, LineToken } from 'vs/editor/common/core/lineTokens';
+import { getWordAtText } from 'vs/editor/common/model/wordHelper';
+import { TokenizationResult2 } from 'vs/editor/common/core/token';
+import { ITextSource, IRawTextSource } from 'vs/editor/common/model/textSource';
+import * as textModelEvents from 'vs/editor/common/model/textModelEvents';
+
+class ModelTokensChangedEventBuilder {
+
+	private _ranges: { fromLineNumber: number; toLineNumber: number; }[];
+
+	constructor() {
+		this._ranges = [];
+	}
+
+	public registerChangedTokens(lineNumber: number): void {
+		const ranges = this._ranges;
+		const rangesLength = ranges.length;
+		const previousRange = rangesLength > 0 ? ranges[rangesLength - 1] : null;
+
+		if (previousRange && previousRange.toLineNumber === lineNumber - 1) {
+			// extend previous range
+			previousRange.toLineNumber++;
+		} else {
+			// insert new range
+			ranges[rangesLength] = {
+				fromLineNumber: lineNumber,
+				toLineNumber: lineNumber
+			};
+		}
+	}
+
+	public build(): textModelEvents.IModelTokensChangedEvent {
+		if (this._ranges.length === 0) {
+			return null;
+		}
+		return {
+			ranges: this._ranges
+		};
+	}
+}
+
+export class TextModelWithTokens extends TextModel implements editorCommon.ITokenizedModel {
+
+	private static MODE_TOKENIZATION_FAILED_MSG = nls.localize('mode.tokenizationSupportFailed', "The mode has failed while tokenizing the input.");
+
+	private _languageIdentifier: LanguageIdentifier;
+	private _tokenizationListener: IDisposable;
+	private _tokenizationSupport: ITokenizationSupport;
+
+	private _invalidLineStartIndex: number;
+	private _lastState: IState;
+
+	private _revalidateTokensTimeout: number;
+
+	constructor(rawTextSource: IRawTextSource, creationOptions: editorCommon.ITextModelCreationOptions, languageIdentifier: LanguageIdentifier) {
+		super(rawTextSource, creationOptions);
+
+		this._languageIdentifier = languageIdentifier || NULL_LANGUAGE_IDENTIFIER;
+		this._tokenizationListener = TokenizationRegistry.onDidChange((e) => {
+			if (e.changedLanguages.indexOf(this._languageIdentifier.language) === -1) {
+				return;
+			}
+
+			this._resetTokenizationState();
+			this.emitModelTokensChangedEvent({
+				ranges: [{
+					fromLineNumber: 1,
+					toLineNumber: this.getLineCount()
+				}]
+			});
+
+			if (this._shouldAutoTokenize()) {
+				this._warmUpTokens();
+			}
+		});
+
+		this._revalidateTokensTimeout = -1;
+
+		this._resetTokenizationState();
+	}
+
+	public dispose(): void {
+		this._tokenizationListener.dispose();
+		this._clearTimers();
+		this._lastState = null;
+
+		super.dispose();
+	}
+
+	protected _shouldAutoTokenize(): boolean {
+		return false;
+	}
+
+	protected _resetValue(newValue: ITextSource): void {
+		super._resetValue(newValue);
+		// Cancel tokenization, clear all tokens and begin tokenizing
+		this._resetTokenizationState();
+	}
+
+	protected _resetTokenizationState(): void {
+		this._clearTimers();
+		for (let i = 0; i < this._lines.length; i++) {
+			this._lines[i].resetTokenizationState();
+		}
+
+		this._tokenizationSupport = null;
+		if (!this._isTooLargeForTokenization) {
+			this._tokenizationSupport = TokenizationRegistry.get(this._languageIdentifier.language);
+		}
+
+		if (this._tokenizationSupport) {
+			let initialState: IState = null;
+			try {
+				initialState = this._tokenizationSupport.getInitialState();
+			} catch (e) {
+				e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG;
+				onUnexpectedError(e);
+				this._tokenizationSupport = null;
+			}
+
+			if (initialState) {
+				this._lines[0].setState(initialState);
+			}
+		}
+
+		this._lastState = null;
+		this._invalidLineStartIndex = 0;
+		this._beginBackgroundTokenization();
+	}
+
+	private _clearTimers(): void {
+		if (this._revalidateTokensTimeout !== -1) {
+			clearTimeout(this._revalidateTokensTimeout);
+			this._revalidateTokensTimeout = -1;
+		}
+	}
+
+	private _withModelTokensChangedEventBuilder<T>(callback: (eventBuilder: ModelTokensChangedEventBuilder) => T): T {
+		let eventBuilder = new ModelTokensChangedEventBuilder();
+
+		let result = callback(eventBuilder);
+
+		if (!this._isDisposing) {
+			let e = eventBuilder.build();
+			if (e) {
+				this._eventEmitter.emit(textModelEvents.TextModelEventType.ModelTokensChanged, e);
+			}
+		}
+
+		return result;
+	}
+
+	public forceTokenization(lineNumber: number): void {
+		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
+			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
+		}
+
+		this._withModelTokensChangedEventBuilder((eventBuilder) => {
+			this._updateTokensUntilLine(eventBuilder, lineNumber);
+		});
+	}
+
+	public isCheapToTokenize(lineNumber: number): boolean {
+		const firstInvalidLineNumber = this._invalidLineStartIndex + 1;
+		return (firstInvalidLineNumber >= lineNumber);
+	}
+
+	public tokenizeIfCheap(lineNumber: number): void {
+		if (this.isCheapToTokenize(lineNumber)) {
+			this.forceTokenization(lineNumber);
+		}
+	}
+
+	public getLineTokens(lineNumber: number): LineTokens {
+		if (lineNumber < 1 || lineNumber > this.getLineCount()) {
+			throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
+		}
+
+		return this._getLineTokens(lineNumber);
+	}
+
+	private _getLineTokens(lineNumber: number): LineTokens {
+		return this._lines[lineNumber - 1].getTokens(this._languageIdentifier.id);
+	}
+
+	public getLanguageIdentifier(): LanguageIdentifier {
+		return this._languageIdentifier;
+	}
+
+	public getModeId(): string {
+		return this._languageIdentifier.language;
+	}
+
+	public setMode(languageIdentifier: LanguageIdentifier): void {
+		if (this._languageIdentifier.id === languageIdentifier.id) {
+			// There's nothing to do
+			return;
+		}
+
+		let e: textModelEvents.IModelLanguageChangedEvent = {
+			oldLanguage: this._languageIdentifier.language,
+			newLanguage: languageIdentifier.language
+		};
+
+		this._languageIdentifier = languageIdentifier;
+
+		// Cancel tokenization, clear all tokens and begin tokenizing
+		this._resetTokenizationState();
+
+		this.emitModelTokensChangedEvent({
+			ranges: [{
+				fromLineNumber: 1,
+				toLineNumber: this.getLineCount()
+			}]
+		});
+		this._emitModelModeChangedEvent(e);
+	}
+
+	public getLanguageIdAtPosition(_lineNumber: number, _column: number): LanguageId {
+		if (!this._tokenizationSupport) {
+			return this._languageIdentifier.id;
+		}
+		let { lineNumber, column } = this.validatePosition({ lineNumber: _lineNumber, column: _column });
+
+		let lineTokens = this._getLineTokens(lineNumber);
+		let token = lineTokens.findTokenAtOffset(column - 1);
+		return token.languageId;
+	}
+
+	protected _invalidateLine(lineIndex: number): void {
+		this._lines[lineIndex].setIsInvalid(true);
+		if (lineIndex < this._invalidLineStartIndex) {
+			if (this._invalidLineStartIndex < this._lines.length) {
+				this._lines[this._invalidLineStartIndex].setIsInvalid(true);
+			}
+			this._invalidLineStartIndex = lineIndex;
+			this._beginBackgroundTokenization();
+		}
+	}
+
+	private _beginBackgroundTokenization(): void {
+		if (this._shouldAutoTokenize() && this._revalidateTokensTimeout === -1) {
+			this._revalidateTokensTimeout = setTimeout(() => {
+				this._revalidateTokensTimeout = -1;
+				this._revalidateTokensNow();
+			}, 0);
+		}
+	}
+
+	_warmUpTokens(): void {
+		// Warm up first 100 lines (if it takes less than 50ms)
+		var maxLineNumber = Math.min(100, this.getLineCount());
+		this._revalidateTokensNow(maxLineNumber);
+
+		if (this._invalidLineStartIndex < this._lines.length) {
+			this._beginBackgroundTokenization();
+		}
+	}
+
+	private _revalidateTokensNow(toLineNumber: number = this._invalidLineStartIndex + 1000000): void {
+
+		this._withModelTokensChangedEventBuilder((eventBuilder) => {
+
+			toLineNumber = Math.min(this._lines.length, toLineNumber);
+
+			var MAX_ALLOWED_TIME = 20,
+				fromLineNumber = this._invalidLineStartIndex + 1,
+				tokenizedChars = 0,
+				currentCharsToTokenize = 0,
+				currentEstimatedTimeToTokenize = 0,
+				sw = StopWatch.create(false),
+				elapsedTime: number;
+
+			// Tokenize at most 1000 lines. Estimate the tokenization speed per character and stop when:
+			// - MAX_ALLOWED_TIME is reached
+			// - tokenizing the next line would go above MAX_ALLOWED_TIME
+
+			for (var lineNumber = fromLineNumber; lineNumber <= toLineNumber; lineNumber++) {
+				elapsedTime = sw.elapsed();
+				if (elapsedTime > MAX_ALLOWED_TIME) {
+					// Stop if MAX_ALLOWED_TIME is reached
+					toLineNumber = lineNumber - 1;
+					break;
+				}
+
+				// Compute how many characters will be tokenized for this line
+				currentCharsToTokenize = this._lines[lineNumber - 1].text.length;
+
+				if (tokenizedChars > 0) {
+					// If we have enough history, estimate how long tokenizing this line would take
+					currentEstimatedTimeToTokenize = (elapsedTime / tokenizedChars) * currentCharsToTokenize;
+					if (elapsedTime + currentEstimatedTimeToTokenize > MAX_ALLOWED_TIME) {
+						// Tokenizing this line will go above MAX_ALLOWED_TIME
+						toLineNumber = lineNumber - 1;
+						break;
+					}
+				}
+
+				this._updateTokensUntilLine(eventBuilder, lineNumber);
+				tokenizedChars += currentCharsToTokenize;
+
+				// Skip the lines that got tokenized
+				lineNumber = Math.max(lineNumber, this._invalidLineStartIndex + 1);
+			}
+
+			elapsedTime = sw.elapsed();
+
+			if (this._invalidLineStartIndex < this._lines.length) {
+				this._beginBackgroundTokenization();
+			}
+		});
+	}
+
+	private _updateTokensUntilLine(eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
+		if (!this._tokenizationSupport) {
+			this._invalidLineStartIndex = this._lines.length;
+			return;
+		}
+
+		const linesLength = this._lines.length;
+		const endLineIndex = lineNumber - 1;
+
+		// Validate all states up to and including endLineIndex
+		for (let lineIndex = this._invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) {
+			const endStateIndex = lineIndex + 1;
+			let r: TokenizationResult2 = null;
+			const text = this._lines[lineIndex].text;
+
+			try {
+				// Tokenize only the first X characters
+				let freshState = this._lines[lineIndex].getState().clone();
+				r = this._tokenizationSupport.tokenize2(this._lines[lineIndex].text, freshState, 0);
+			} catch (e) {
+				e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG;
+				onUnexpectedError(e);
+			}
+
+			if (!r) {
+				r = nullTokenize2(this._languageIdentifier.id, text, this._lines[lineIndex].getState(), 0);
+			}
+			this._lines[lineIndex].setTokens(this._languageIdentifier.id, r.tokens);
+			eventBuilder.registerChangedTokens(lineIndex + 1);
+			this._lines[lineIndex].setIsInvalid(false);
+
+			if (endStateIndex < linesLength) {
+				if (this._lines[endStateIndex].getState() !== null && r.endState.equals(this._lines[endStateIndex].getState())) {
+					// The end state of this line remains the same
+					let nextInvalidLineIndex = lineIndex + 1;
+					while (nextInvalidLineIndex < linesLength) {
+						if (this._lines[nextInvalidLineIndex].isInvalid()) {
+							break;
+						}
+						if (nextInvalidLineIndex + 1 < linesLength) {
+							if (this._lines[nextInvalidLineIndex + 1].getState() === null) {
+								break;
+							}
+						} else {
+							if (this._lastState === null) {
+								break;
+							}
+						}
+						nextInvalidLineIndex++;
+					}
+					this._invalidLineStartIndex = Math.max(this._invalidLineStartIndex, nextInvalidLineIndex);
+					lineIndex = nextInvalidLineIndex - 1; // -1 because the outer loop increments it
+				} else {
+					this._lines[endStateIndex].setState(r.endState);
+				}
+			} else {
+				this._lastState = r.endState;
+			}
+		}
+		this._invalidLineStartIndex = Math.max(this._invalidLineStartIndex, endLineIndex + 1);
+	}
+
+	private emitModelTokensChangedEvent(e: textModelEvents.IModelTokensChangedEvent): void {
+		if (!this._isDisposing) {
+			this._eventEmitter.emit(textModelEvents.TextModelEventType.ModelTokensChanged, e);
+		}
+	}
+
+	private _emitModelModeChangedEvent(e: textModelEvents.IModelLanguageChangedEvent): void {
+		if (!this._isDisposing) {
+			this._eventEmitter.emit(textModelEvents.TextModelEventType.ModelLanguageChanged, e);
+		}
+	}
+
+	// Having tokens allows implementing additional helper methods
+
+	public getWordAtPosition(_position: IPosition): editorCommon.IWordAtPosition {
+		this._assertNotDisposed();
+		let position = this.validatePosition(_position);
+		let lineContent = this.getLineContent(position.lineNumber);
+
+		if (this._invalidLineStartIndex <= position.lineNumber) {
+			// this line is not tokenized
+			return getWordAtText(
+				position.column,
+				LanguageConfigurationRegistry.getWordDefinition(this._languageIdentifier.id),
+				lineContent,
+				0
+			);
+		}
+
+		let lineTokens = this._getLineTokens(position.lineNumber);
+		let offset = position.column - 1;
+		let token = lineTokens.findTokenAtOffset(offset);
+
+		let result = getWordAtText(
+			position.column,
+			LanguageConfigurationRegistry.getWordDefinition(token.languageId),
+			lineContent.substring(token.startOffset, token.endOffset),
+			token.startOffset
+		);
+
+		if (!result && token.hasPrev && token.startOffset === offset) {
+			// The position is right at the beginning of `modeIndex`, so try looking at `modeIndex` - 1 too
+
+			let prevToken = token.prev();
+			result = getWordAtText(
+				position.column,
+				LanguageConfigurationRegistry.getWordDefinition(prevToken.languageId),
+				lineContent.substring(prevToken.startOffset, prevToken.endOffset),
+				prevToken.startOffset
+			);
+		}
+
+		return result;
+	}
+
+	public getWordUntilPosition(position: IPosition): editorCommon.IWordAtPosition {
+		var wordAtPosition = this.getWordAtPosition(position);
+		if (!wordAtPosition) {
+			return {
+				word: '',
+				startColumn: position.column,
+				endColumn: position.column
+			};
+		}
+		return {
+			word: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),
+			startColumn: wordAtPosition.startColumn,
+			endColumn: position.column
+		};
+	}
+
+	public findMatchingBracketUp(_bracket: string, _position: IPosition): Range {
+		let bracket = _bracket.toLowerCase();
+		let position = this.validatePosition(_position);
+
+		let lineTokens = this._getLineTokens(position.lineNumber);
+		let token = lineTokens.findTokenAtOffset(position.column - 1);
+		let bracketsSupport = LanguageConfigurationRegistry.getBracketsSupport(token.languageId);
+
+		if (!bracketsSupport) {
+			return null;
+		}
+
+		let data = bracketsSupport.textIsBracket[bracket];
+
+		if (!data) {
+			return null;
+		}
+
+		return this._findMatchingBracketUp(data, position);
+	}
+
+	public matchBracket(position: IPosition): [Range, Range] {
+		return this._matchBracket(this.validatePosition(position));
+	}
+
+	private _matchBracket(position: Position): [Range, Range] {
+		const lineNumber = position.lineNumber;
+		let lineTokens = this._getLineTokens(lineNumber);
+		const lineText = this._lines[lineNumber - 1].text;
+
+		const currentToken = lineTokens.findTokenAtOffset(position.column - 1);
+		if (!currentToken) {
+			return null;
+		}
+		const currentModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentToken.languageId);
+
+		// check that the token is not to be ignored
+		if (currentModeBrackets && !ignoreBracketsInToken(currentToken.tokenType)) {
+			// limit search to not go before `maxBracketLength`
+			let searchStartOffset = Math.max(currentToken.startOffset, position.column - 1 - currentModeBrackets.maxBracketLength);
+			// limit search to not go after `maxBracketLength`
+			const searchEndOffset = Math.min(currentToken.endOffset, position.column - 1 + currentModeBrackets.maxBracketLength);
+
+			// first, check if there is a bracket to the right of `position`
+			let foundBracket = BracketsUtils.findNextBracketInToken(currentModeBrackets.forwardRegex, lineNumber, lineText, position.column - 1, searchEndOffset);
+			if (foundBracket && foundBracket.startColumn === position.column) {
+				let foundBracketText = lineText.substring(foundBracket.startColumn - 1, foundBracket.endColumn - 1);
+				foundBracketText = foundBracketText.toLowerCase();
+
+				let r = this._matchFoundBracket(foundBracket, currentModeBrackets.textIsBracket[foundBracketText], currentModeBrackets.textIsOpenBracket[foundBracketText]);
+
+				// check that we can actually match this bracket
+				if (r) {
+					return r;
+				}
+			}
+
+			// it might still be the case that [currentTokenStart -> currentTokenEnd] contains multiple brackets
+			while (true) {
+				let foundBracket = BracketsUtils.findNextBracketInToken(currentModeBrackets.forwardRegex, lineNumber, lineText, searchStartOffset, searchEndOffset);
+				if (!foundBracket) {
+					// there are no brackets in this text
+					break;
+				}
+
+				// check that we didn't hit a bracket too far away from position
+				if (foundBracket.startColumn <= position.column && position.column <= foundBracket.endColumn) {
+					let foundBracketText = lineText.substring(foundBracket.startColumn - 1, foundBracket.endColumn - 1);
+					foundBracketText = foundBracketText.toLowerCase();
+
+					let r = this._matchFoundBracket(foundBracket, currentModeBrackets.textIsBracket[foundBracketText], currentModeBrackets.textIsOpenBracket[foundBracketText]);
+
+					// check that we can actually match this bracket
+					if (r) {
+						return r;
+					}
+				}
+
+				searchStartOffset = foundBracket.endColumn - 1;
+			}
+		}
+
+		// If position is in between two tokens, try also looking in the previous token
+		if (currentToken.hasPrev && currentToken.startOffset === position.column - 1) {
+			const prevToken = currentToken.prev();
+			const prevModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(prevToken.languageId);
+
+			// check that previous token is not to be ignored
+			if (prevModeBrackets && !ignoreBracketsInToken(prevToken.tokenType)) {
+				// limit search in case previous token is very large, there's no need to go beyond `maxBracketLength`
+				const searchStartOffset = Math.max(prevToken.startOffset, position.column - 1 - prevModeBrackets.maxBracketLength);
+				const searchEndOffset = currentToken.startOffset;
+				const foundBracket = BracketsUtils.findPrevBracketInToken(prevModeBrackets.reversedRegex, lineNumber, lineText, searchStartOffset, searchEndOffset);
+
+				// check that we didn't hit a bracket too far away from position
+				if (foundBracket && foundBracket.startColumn <= position.column && position.column <= foundBracket.endColumn) {
+					let foundBracketText = lineText.substring(foundBracket.startColumn - 1, foundBracket.endColumn - 1);
+					foundBracketText = foundBracketText.toLowerCase();
+
+					let r = this._matchFoundBracket(foundBracket, prevModeBrackets.textIsBracket[foundBracketText], prevModeBrackets.textIsOpenBracket[foundBracketText]);
+
+					// check that we can actually match this bracket
+					if (r) {
+						return r;
+					}
+				}
+			}
+		}
+
+		return null;
+	}
+
+	private _matchFoundBracket(foundBracket: Range, data: RichEditBracket, isOpen: boolean): [Range, Range] {
+		if (isOpen) {
+			let matched = this._findMatchingBracketDown(data, foundBracket.getEndPosition());
+			if (matched) {
+				return [foundBracket, matched];
+			}
+		} else {
+			let matched = this._findMatchingBracketUp(data, foundBracket.getStartPosition());
+			if (matched) {
+				return [foundBracket, matched];
+			}
+		}
+
+		return null;
+	}
+
+	private _findMatchingBracketUp(bracket: RichEditBracket, position: Position): Range {
+		// console.log('_findMatchingBracketUp: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
+
+		const languageId = bracket.languageIdentifier.id;
+		const reversedBracketRegex = bracket.reversedRegex;
+		let count = -1;
+
+		for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) {
+			const lineTokens = this._getLineTokens(lineNumber);
+			const lineText = this._lines[lineNumber - 1].text;
+
+			let currentToken: LineToken;
+			let searchStopOffset: number;
+			if (lineNumber === position.lineNumber) {
+				currentToken = lineTokens.findTokenAtOffset(position.column - 1);
+				searchStopOffset = position.column - 1;
+			} else {
+				currentToken = lineTokens.lastToken();
+				if (currentToken) {
+					searchStopOffset = currentToken.endOffset;
+				}
+			}
+
+			while (currentToken) {
+				if (currentToken.languageId === languageId && !ignoreBracketsInToken(currentToken.tokenType)) {
+
+					while (true) {
+						let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, lineNumber, lineText, currentToken.startOffset, searchStopOffset);
+						if (!r) {
+							break;
+						}
+
+						let hitText = lineText.substring(r.startColumn - 1, r.endColumn - 1);
+						hitText = hitText.toLowerCase();
+
+						if (hitText === bracket.open) {
+							count++;
+						} else if (hitText === bracket.close) {
+							count--;
+						}
+
+						if (count === 0) {
+							return r;
+						}
+
+						searchStopOffset = r.startColumn - 1;
+					}
+				}
+
+				currentToken = currentToken.prev();
+				if (currentToken) {
+					searchStopOffset = currentToken.endOffset;
+				}
+			}
+		}
+
+		return null;
+	}
+
+	private _findMatchingBracketDown(bracket: RichEditBracket, position: Position): Range {
+		// console.log('_findMatchingBracketDown: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
+
+		const languageId = bracket.languageIdentifier.id;
+		const bracketRegex = bracket.forwardRegex;
+		let count = 1;
+
+		for (let lineNumber = position.lineNumber, lineCount = this.getLineCount(); lineNumber <= lineCount; lineNumber++) {
+			const lineTokens = this._getLineTokens(lineNumber);
+			const lineText = this._lines[lineNumber - 1].text;
+
+			let currentToken: LineToken;
+			let searchStartOffset: number;
+			if (lineNumber === position.lineNumber) {
+				currentToken = lineTokens.findTokenAtOffset(position.column - 1);
+				searchStartOffset = position.column - 1;
+			} else {
+				currentToken = lineTokens.firstToken();
+				if (currentToken) {
+					searchStartOffset = currentToken.startOffset;
+				}
+			}
+
+			while (currentToken) {
+				if (currentToken.languageId === languageId && !ignoreBracketsInToken(currentToken.tokenType)) {
+					while (true) {
+						let r = BracketsUtils.findNextBracketInToken(bracketRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
+						if (!r) {
+							break;
+						}
+
+						let hitText = lineText.substring(r.startColumn - 1, r.endColumn - 1);
+						hitText = hitText.toLowerCase();
+
+						if (hitText === bracket.open) {
+							count++;
+						} else if (hitText === bracket.close) {
+							count--;
+						}
+
+						if (count === 0) {
+							return r;
+						}
+
+						searchStartOffset = r.endColumn - 1;
+					}
+				}
+
+				currentToken = currentToken.next();
+				if (currentToken) {
+					searchStartOffset = currentToken.startOffset;
+				}
+			}
+		}
+
+		return null;
+	}
+
+	public findPrevBracket(_position: IPosition): editorCommon.IFoundBracket {
+		const position = this.validatePosition(_position);
+
+		let languageId: LanguageId = -1;
+		let modeBrackets: RichEditBrackets = null;
+		for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) {
+			const lineTokens = this._getLineTokens(lineNumber);
+			const lineText = this._lines[lineNumber - 1].text;
+
+			let currentToken: LineToken;
+			let searchStopOffset: number;
+			if (lineNumber === position.lineNumber) {
+				currentToken = lineTokens.findTokenAtOffset(position.column - 1);
+				searchStopOffset = position.column - 1;
+			} else {
+				currentToken = lineTokens.lastToken();
+				if (currentToken) {
+					searchStopOffset = currentToken.endOffset;
+				}
+			}
+
+			while (currentToken) {
+				if (languageId !== currentToken.languageId) {
+					languageId = currentToken.languageId;
+					modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(languageId);
+				}
+				if (modeBrackets && !ignoreBracketsInToken(currentToken.tokenType)) {
+					let r = BracketsUtils.findPrevBracketInToken(modeBrackets.reversedRegex, lineNumber, lineText, currentToken.startOffset, searchStopOffset);
+					if (r) {
+						return this._toFoundBracket(modeBrackets, r);
+					}
+				}
+
+				currentToken = currentToken.prev();
+				if (currentToken) {
+					searchStopOffset = currentToken.endOffset;
+				}
+			}
+		}
+
+		return null;
+	}
+
+	public findNextBracket(_position: IPosition): editorCommon.IFoundBracket {
+		const position = this.validatePosition(_position);
+
+		let languageId: LanguageId = -1;
+		let modeBrackets: RichEditBrackets = null;
+		for (let lineNumber = position.lineNumber, lineCount = this.getLineCount(); lineNumber <= lineCount; lineNumber++) {
+			const lineTokens = this._getLineTokens(lineNumber);
+			const lineText = this._lines[lineNumber - 1].text;
+
+			let currentToken: LineToken;
+			let searchStartOffset: number;
+			if (lineNumber === position.lineNumber) {
+				currentToken = lineTokens.findTokenAtOffset(position.column - 1);
+				searchStartOffset = position.column - 1;
+			} else {
+				currentToken = lineTokens.firstToken();
+				if (currentToken) {
+					searchStartOffset = currentToken.startOffset;
+				}
+			}
+
+			while (currentToken) {
+				if (languageId !== currentToken.languageId) {
+					languageId = currentToken.languageId;
+					modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(languageId);
+				}
+				if (modeBrackets && !ignoreBracketsInToken(currentToken.tokenType)) {
+					let r = BracketsUtils.findNextBracketInToken(modeBrackets.forwardRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
+					if (r) {
+						return this._toFoundBracket(modeBrackets, r);
+					}
+				}
+
+				currentToken = currentToken.next();
+				if (currentToken) {
+					searchStartOffset = currentToken.startOffset;
+				}
+			}
+		}
+
+		return null;
+	}
+
+	private _toFoundBracket(modeBrackets: RichEditBrackets, r: Range): editorCommon.IFoundBracket {
+		if (!r) {
+			return null;
+		}
+
+		let text = this.getValueInRange(r);
+		text = text.toLowerCase();
+
+		let data = modeBrackets.textIsBracket[text];
+		if (!data) {
+			return null;
+		}
+
+		return {
+			range: r,
+			open: data.open,
+			close: data.close,
+			isOpen: modeBrackets.textIsOpenBracket[text]
+		};
+	}
+}

src/vs/editor/common/model/textSource.ts

@@ -0,0 +1,149 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as strings from 'vs/base/common/strings';
+import { DefaultEndOfLine } from 'vs/editor/common/editorCommon';
+
+/**
+ * A processed string ready to be turned into an editor model.
+ */
+export interface IRawTextSource {
+	/**
+	 * The entire text length.
+	 */
+	readonly length: number;
+	/**
+	 * The text split into lines.
+	 */
+	readonly lines: string[];
+	/**
+	 * The BOM (leading character sequence of the file).
+	 */
+	readonly BOM: string;
+	/**
+	 * The number of lines ending with '\r\n'
+	 */
+	readonly totalCRCount: number;
+	/**
+	 * The text contains Unicode characters classified as "R" or "AL".
+	 */
+	readonly containsRTL: boolean;
+	/**
+	 * The text contains only characters inside the ASCII range 32-126 or \t \r \n
+	 */
+	readonly isBasicASCII: boolean;
+}
+
+export class RawTextSource {
+
+	public static fromString(rawText: string): IRawTextSource {
+		// Count the number of lines that end with \r\n
+		let carriageReturnCnt = 0;
+		let lastCarriageReturnIndex = -1;
+		while ((lastCarriageReturnIndex = rawText.indexOf('\r', lastCarriageReturnIndex + 1)) !== -1) {
+			carriageReturnCnt++;
+		}
+
+		const containsRTL = strings.containsRTL(rawText);
+		const isBasicASCII = (containsRTL ? false : strings.isBasicASCII(rawText));
+
+		// Split the text into lines
+		const lines = rawText.split(/\r\n|\r|\n/);
+
+		// Remove the BOM (if present)
+		let BOM = '';
+		if (strings.startsWithUTF8BOM(lines[0])) {
+			BOM = strings.UTF8_BOM_CHARACTER;
+			lines[0] = lines[0].substr(1);
+		}
+
+		return {
+			BOM: BOM,
+			lines: lines,
+			length: rawText.length,
+			containsRTL: containsRTL,
+			isBasicASCII: isBasicASCII,
+			totalCRCount: carriageReturnCnt
+		};
+	}
+
+}
+
+/**
+ * A processed string with its EOL resolved ready to be turned into an editor model.
+ */
+export interface ITextSource {
+	/**
+	 * The entire text length.
+	 */
+	readonly length: number;
+	/**
+	 * The text split into lines.
+	 */
+	readonly lines: string[];
+	/**
+	 * The BOM (leading character sequence of the file).
+	 */
+	readonly BOM: string;
+	/**
+	 * The end of line sequence.
+	 */
+	readonly EOL: string;
+	/**
+	 * The text contains Unicode characters classified as "R" or "AL".
+	 */
+	readonly containsRTL: boolean;
+	/**
+	 * The text contains only characters inside the ASCII range 32-126 or \t \r \n
+	 */
+	readonly isBasicASCII: boolean;
+}
+
+export class TextSource {
+
+	/**
+	 * if text source is empty or with precisely one line, returns null. No end of line is detected.
+	 * if text source contains more lines ending with '\r\n', returns '\r\n'.
+	 * Otherwise returns '\n'. More lines end with '\n'.
+	 */
+	private static _getEOL(rawTextSource: IRawTextSource, defaultEOL: DefaultEndOfLine): '\r\n' | '\n' {
+		const lineFeedCnt = rawTextSource.lines.length - 1;
+		if (lineFeedCnt === 0) {
+			// This is an empty file or a file with precisely one line
+			return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n');
+		}
+		if (rawTextSource.totalCRCount > lineFeedCnt / 2) {
+			// More than half of the file contains \r\n ending lines
+			return '\r\n';
+		}
+		// At least one line more ends in \n
+		return '\n';
+	}
+
+	public static fromRawTextSource(rawTextSource: IRawTextSource, defaultEOL: DefaultEndOfLine): ITextSource {
+		return {
+			length: rawTextSource.length,
+			lines: rawTextSource.lines,
+			BOM: rawTextSource.BOM,
+			EOL: this._getEOL(rawTextSource, defaultEOL),
+			containsRTL: rawTextSource.containsRTL,
+			isBasicASCII: rawTextSource.isBasicASCII,
+		};
+	}
+
+	public static fromString(text: string, defaultEOL: DefaultEndOfLine): ITextSource {
+		return this.fromRawTextSource(RawTextSource.fromString(text), defaultEOL);
+	}
+
+	public static create(source: string | IRawTextSource, defaultEOL: DefaultEndOfLine): ITextSource {
+		if (typeof source === 'string') {
+			return this.fromString(source, defaultEOL);
+		}
+
+		return this.fromRawTextSource(source, defaultEOL);
+	}
+
+}

src/vs/editor/common/model/tokensBinaryEncoding.ts

@@ -0,0 +1,65 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { ColorId, FontStyle, StandardTokenType, MetadataConsts, LanguageId } from 'vs/editor/common/modes';
+
+export class TokenMetadata {
+
+	public static getLanguageId(metadata: number): LanguageId {
+		return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET;
+	}
+
+	public static getTokenType(metadata: number): StandardTokenType {
+		return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET;
+	}
+
+	public static getFontStyle(metadata: number): FontStyle {
+		return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET;
+	}
+
+	public static getForeground(metadata: number): ColorId {
+		return (metadata & MetadataConsts.FOREGROUND_MASK) >>> MetadataConsts.FOREGROUND_OFFSET;
+	}
+
+	public static getBackground(metadata: number): ColorId {
+		return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET;
+	}
+
+	public static getClassNameFromMetadata(metadata: number): string {
+		let foreground = this.getForeground(metadata);
+		let className = 'mtk' + foreground;
+
+		let fontStyle = this.getFontStyle(metadata);
+		if (fontStyle & FontStyle.Italic) {
+			className += ' mtki';
+		}
+		if (fontStyle & FontStyle.Bold) {
+			className += ' mtkb';
+		}
+		if (fontStyle & FontStyle.Underline) {
+			className += ' mtku';
+		}
+
+		return className;
+	}
+
+	public static getInlineStyleFromMetadata(metadata: number, colorMap: string[]): string {
+		const foreground = this.getForeground(metadata);
+		const fontStyle = this.getFontStyle(metadata);
+
+		let result = `color: ${colorMap[foreground]};`;
+		if (fontStyle & FontStyle.Italic) {
+			result += 'font-style: italic;';
+		}
+		if (fontStyle & FontStyle.Bold) {
+			result += 'font-weight: bold;';
+		}
+		if (fontStyle & FontStyle.Underline) {
+			result += 'text-decoration: underline;';
+		}
+		return result;
+	}
+}

src/vs/editor/common/model/wordHelper.ts

@@ -0,0 +1,133 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { IWordAtPosition } from 'vs/editor/common/editorCommon';
+
+export const USUAL_WORD_SEPARATORS = '`~!@#$%^&*()-=+[{]}\\|;:\'",.<>/?';
+
+/**
+ * Create a word definition regular expression based on default word separators.
+ * Optionally provide allowed separators that should be included in words.
+ *
+ * The default would look like this:
+ * /(-?\d*\.\d\w*)|([^\`\~\!\@\#\$\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g
+ */
+function createWordRegExp(allowInWords: string = ''): RegExp {
+	var usualSeparators = USUAL_WORD_SEPARATORS;
+	var source = '(-?\\d*\\.\\d\\w*)|([^';
+	for (var i = 0; i < usualSeparators.length; i++) {
+		if (allowInWords.indexOf(usualSeparators[i]) >= 0) {
+			continue;
+		}
+		source += '\\' + usualSeparators[i];
+	}
+	source += '\\s]+)';
+	return new RegExp(source, 'g');
+}
+
+// catches numbers (including floating numbers) in the first group, and alphanum in the second
+export const DEFAULT_WORD_REGEXP = createWordRegExp();
+
+export function ensureValidWordDefinition(wordDefinition?: RegExp): RegExp {
+	var result: RegExp = DEFAULT_WORD_REGEXP;
+
+	if (wordDefinition && (wordDefinition instanceof RegExp)) {
+		if (!wordDefinition.global) {
+			var flags = 'g';
+			if (wordDefinition.ignoreCase) {
+				flags += 'i';
+			}
+			if (wordDefinition.multiline) {
+				flags += 'm';
+			}
+			result = new RegExp(wordDefinition.source, flags);
+		} else {
+			result = wordDefinition;
+		}
+	}
+
+	result.lastIndex = 0;
+
+	return result;
+}
+
+function getWordAtPosFast(column: number, wordDefinition: RegExp, text: string, textOffset: number): IWordAtPosition {
+	// find whitespace enclosed text around column and match from there
+
+	let pos = column - 1 - textOffset;
+	let start = text.lastIndexOf(' ', pos - 1) + 1;
+	let end = text.indexOf(' ', pos);
+	if (end === -1) {
+		end = text.length;
+	}
+
+	wordDefinition.lastIndex = start;
+	let match: RegExpMatchArray;
+	while (match = wordDefinition.exec(text)) {
+		if (match.index <= pos && wordDefinition.lastIndex >= pos) {
+			return {
+				word: match[0],
+				startColumn: textOffset + 1 + match.index,
+				endColumn: textOffset + 1 + wordDefinition.lastIndex
+			};
+		}
+	}
+
+	return null;
+}
+
+
+function getWordAtPosSlow(column: number, wordDefinition: RegExp, text: string, textOffset: number): IWordAtPosition {
+	// matches all words starting at the beginning
+	// of the input until it finds a match that encloses
+	// the desired column. slow but correct
+
+	let pos = column - 1 - textOffset;
+	wordDefinition.lastIndex = 0;
+
+	let match: RegExpMatchArray;
+	while (match = wordDefinition.exec(text)) {
+
+		if (match.index > pos) {
+			// |nW -> matched only after the pos
+			return null;
+
+		} else if (wordDefinition.lastIndex >= pos) {
+			// W|W -> match encloses pos
+			return {
+				word: match[0],
+				startColumn: textOffset + 1 + match.index,
+				endColumn: textOffset + 1 + wordDefinition.lastIndex
+			};
+		}
+	}
+
+	return null;
+}
+
+export function getWordAtText(column: number, wordDefinition: RegExp, text: string, textOffset: number): IWordAtPosition {
+
+	// if `words` can contain whitespace character we have to use the slow variant
+	// otherwise we use the fast variant of finding a word
+	wordDefinition.lastIndex = 0;
+	let match = wordDefinition.exec(text);
+	if (!match) {
+		return null;
+	}
+	// todo@joh the `match` could already be the (first) word
+	const ret = match[0].indexOf(' ') >= 0
+		// did match a word which contains a space character -> use slow word find
+		? getWordAtPosSlow(column, wordDefinition, text, textOffset)
+		// sane word definition -> use fast word find
+		: getWordAtPosFast(column, wordDefinition, text, textOffset);
+
+	// both (getWordAtPosFast and getWordAtPosSlow) leave the wordDefinition-RegExp
+	// in an undefined state and to not confuse other users of the wordDefinition
+	// we reset the lastIndex
+	wordDefinition.lastIndex = 0;
+
+	return ret;
+}