SQL Operations Studio Public Preview 1 (0.23) release source code

src/vs/editor/test/common/model/editableTextModelAuto.test.ts

@@ -0,0 +1,300 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { Position } from 'vs/editor/common/core/position';
+import { Range } from 'vs/editor/common/core/range';
+import { IIdentifiedSingleEditOperation } from 'vs/editor/common/editorCommon';
+import { testApplyEditsWithSyncedModels } from 'vs/editor/test/common/model/editableTextModelTestUtils';
+import { CharCode } from 'vs/base/common/charCode';
+
+const GENERATE_TESTS = false;
+
+suite('EditorModel Auto Tests', () => {
+	function editOp(startLineNumber: number, startColumn: number, endLineNumber: number, endColumn: number, text: string[]): IIdentifiedSingleEditOperation {
+		return {
+			identifier: null,
+			range: new Range(startLineNumber, startColumn, endLineNumber, endColumn),
+			text: text.join('\n'),
+			forceMoveMarkers: false
+		};
+	}
+
+	test('auto1', () => {
+		testApplyEditsWithSyncedModels(
+			[
+				'ioe',
+				'',
+				'yjct',
+				'',
+				'',
+			],
+			[
+				editOp(1, 2, 1, 2, ['b', 'r', 'fq']),
+				editOp(1, 4, 2, 1, ['', '']),
+			],
+			[
+				'ib',
+				'r',
+				'fqoe',
+				'',
+				'yjct',
+				'',
+				'',
+			]
+		);
+	});
+
+	test('auto2', () => {
+		testApplyEditsWithSyncedModels(
+			[
+				'f',
+				'littnhskrq',
+				'utxvsizqnk',
+				'lslqz',
+				'jxn',
+				'gmm',
+			],
+			[
+				editOp(1, 2, 1, 2, ['', 'o']),
+				editOp(2, 4, 2, 4, ['zaq', 'avb']),
+				editOp(2, 5, 6, 2, ['jlr', 'zl', 'j']),
+			],
+			[
+				'f',
+				'o',
+				'litzaq',
+				'avbtjlr',
+				'zl',
+				'jmm',
+			]
+		);
+	});
+
+	test('auto3', () => {
+		testApplyEditsWithSyncedModels(
+			[
+				'ofw',
+				'qsxmziuvzw',
+				'rp',
+				'qsnymek',
+				'elth',
+				'wmgzbwudxz',
+				'iwsdkndh',
+				'bujlbwb',
+				'asuouxfv',
+				'xuccnb',
+			],
+			[
+				editOp(4, 3, 4, 3, ['']),
+			],
+			[
+				'ofw',
+				'qsxmziuvzw',
+				'rp',
+				'qsnymek',
+				'elth',
+				'wmgzbwudxz',
+				'iwsdkndh',
+				'bujlbwb',
+				'asuouxfv',
+				'xuccnb',
+			]
+		);
+	});
+
+	test('auto4', () => {
+		testApplyEditsWithSyncedModels(
+			[
+				'fefymj',
+				'qum',
+				'vmiwxxaiqq',
+				'dz',
+				'lnqdgorosf',
+			],
+			[
+				editOp(1, 3, 1, 5, ['hp']),
+				editOp(1, 7, 2, 1, ['kcg', '', 'mpx']),
+				editOp(2, 2, 2, 2, ['', 'aw', '']),
+				editOp(2, 2, 2, 2, ['vqr', 'mo']),
+				editOp(4, 2, 5, 3, ['xyc']),
+			],
+			[
+				'fehpmjkcg',
+				'',
+				'mpxq',
+				'aw',
+				'vqr',
+				'moum',
+				'vmiwxxaiqq',
+				'dxycqdgorosf',
+			]
+		);
+	});
+});
+
+function getRandomInt(min: number, max: number): number {
+	return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+function getRandomString(minLength: number, maxLength: number): string {
+	let length = getRandomInt(minLength, maxLength);
+	let r = '';
+	for (let i = 0; i < length; i++) {
+		r += String.fromCharCode(getRandomInt(CharCode.a, CharCode.z));
+	}
+	return r;
+}
+
+function generateFile(small: boolean): string {
+	let lineCount = getRandomInt(1, small ? 3 : 10);
+	let lines: string[] = [];
+	for (let i = 0; i < lineCount; i++) {
+		lines.push(getRandomString(0, small ? 3 : 10));
+	}
+	return lines.join('\n');
+}
+
+function generateEdits(content: string): ITestModelEdit[] {
+
+	let result: ITestModelEdit[] = [];
+	let cnt = getRandomInt(1, 5);
+
+	let maxOffset = content.length;
+
+	while (cnt > 0 && maxOffset > 0) {
+
+		let offset = getRandomInt(0, maxOffset);
+		let length = getRandomInt(0, maxOffset - offset);
+		let text = generateFile(true);
+
+		result.push({
+			offset: offset,
+			length: length,
+			text: text
+		});
+
+		maxOffset = offset;
+		cnt--;
+	}
+
+	result.reverse();
+
+	return result;
+}
+
+interface ITestModelEdit {
+	offset: number;
+	length: number;
+	text: string;
+}
+
+class TestModel {
+
+	public initialContent: string;
+	public resultingContent: string;
+	public edits: IIdentifiedSingleEditOperation[];
+
+	private static _generateOffsetToPosition(content: string): Position[] {
+		let result: Position[] = [];
+		let lineNumber = 1;
+		let column = 1;
+
+		for (let offset = 0, len = content.length; offset <= len; offset++) {
+			let ch = content.charAt(offset);
+
+			result[offset] = new Position(lineNumber, column);
+
+			if (ch === '\n') {
+				lineNumber++;
+				column = 1;
+			} else {
+				column++;
+			}
+		}
+
+		return result;
+	}
+
+	constructor() {
+		this.initialContent = generateFile(false);
+
+		let edits = generateEdits(this.initialContent);
+
+		let offsetToPosition = TestModel._generateOffsetToPosition(this.initialContent);
+		this.edits = [];
+		for (let i = 0; i < edits.length; i++) {
+			let startPosition = offsetToPosition[edits[i].offset];
+			let endPosition = offsetToPosition[edits[i].offset + edits[i].length];
+			this.edits.push({
+				identifier: null,
+				range: new Range(startPosition.lineNumber, startPosition.column, endPosition.lineNumber, endPosition.column),
+				text: edits[i].text,
+				forceMoveMarkers: false
+			});
+		}
+
+		this.resultingContent = this.initialContent;
+		for (let i = edits.length - 1; i >= 0; i--) {
+			this.resultingContent = (
+				this.resultingContent.substring(0, edits[i].offset) +
+				edits[i].text +
+				this.resultingContent.substring(edits[i].offset + edits[i].length)
+			);
+		}
+	}
+
+	public print(): string {
+		let r: string[] = [];
+		r.push('testApplyEditsWithSyncedModels(');
+		r.push('\t[');
+		let initialLines = this.initialContent.split('\n');
+		r = r.concat(initialLines.map((i) => `\t\t'${i}',`));
+		r.push('\t],');
+		r.push('\t[');
+		r = r.concat(this.edits.map((i) => {
+			let text = `['` + i.text.split('\n').join(`', '`) + `']`;
+			return `\t\teditOp(${i.range.startLineNumber}, ${i.range.startColumn}, ${i.range.endLineNumber}, ${i.range.endColumn}, ${text}),`;
+		}));
+		r.push('\t],');
+		r.push('\t[');
+		let resultLines = this.resultingContent.split('\n');
+		r = r.concat(resultLines.map((i) => `\t\t'${i}',`));
+		r.push('\t]');
+		r.push(');');
+
+		return r.join('\n');
+	}
+}
+
+if (GENERATE_TESTS) {
+	let number = 1;
+	while (true) {
+
+		console.log('------BEGIN NEW TEST: ' + number);
+
+		let testModel = new TestModel();
+
+		// console.log(testModel.print());
+
+		console.log('------END NEW TEST: ' + (number++));
+
+		try {
+			testApplyEditsWithSyncedModels(
+				testModel.initialContent.split('\n'),
+				testModel.edits,
+				testModel.resultingContent.split('\n')
+			);
+			// throw new Error('a');
+		} catch (err) {
+			console.log(err);
+			console.log(testModel.print());
+			break;
+		}
+
+		// break;
+	}
+
+}

src/vs/editor/test/common/model/editableTextModelTestUtils.ts

@@ -0,0 +1,117 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as assert from 'assert';
+import * as editorCommon from 'vs/editor/common/editorCommon';
+import { EditableTextModel } from 'vs/editor/common/model/editableTextModel';
+import { MirrorModel } from 'vs/editor/common/model/mirrorModel';
+import { TextModel } from 'vs/editor/common/model/textModel';
+import { Position } from 'vs/editor/common/core/position';
+import { RawTextSource } from 'vs/editor/common/model/textSource';
+import { IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
+
+export function testApplyEditsWithSyncedModels(original: string[], edits: editorCommon.IIdentifiedSingleEditOperation[], expected: string[], inputEditsAreInvalid: boolean = false): void {
+	var originalStr = original.join('\n');
+	var expectedStr = expected.join('\n');
+
+	assertSyncedModels(originalStr, (model, assertMirrorModels) => {
+		// Apply edits & collect inverse edits
+		var inverseEdits = model.applyEdits(edits);
+
+		// Assert edits produced expected result
+		assert.deepEqual(model.getValue(editorCommon.EndOfLinePreference.LF), expectedStr);
+
+		assertMirrorModels();
+
+		// Apply the inverse edits
+		var inverseInverseEdits = model.applyEdits(inverseEdits);
+
+		// Assert the inverse edits brought back model to original state
+		assert.deepEqual(model.getValue(editorCommon.EndOfLinePreference.LF), originalStr);
+
+		if (!inputEditsAreInvalid) {
+			// Assert the inverse of the inverse edits are the original edits
+			assert.deepEqual(inverseInverseEdits, edits);
+		}
+
+		assertMirrorModels();
+	});
+}
+
+const enum AssertDocumentLineMappingDirection {
+	OffsetToPosition,
+	PositionToOffset
+}
+
+function assertOneDirectionLineMapping(model: TextModel, direction: AssertDocumentLineMappingDirection, msg: string): void {
+	let allText = model.getValue();
+
+	let line = 1, column = 1, previousIsCarriageReturn = false;
+	for (let offset = 0; offset <= allText.length; offset++) {
+		// The position coordinate system cannot express the position between \r and \n
+		let position = new Position(line, column + (previousIsCarriageReturn ? -1 : 0));
+
+		if (direction === AssertDocumentLineMappingDirection.OffsetToPosition) {
+			let actualPosition = model.getPositionAt(offset);
+			assert.equal(actualPosition.toString(), position.toString(), msg + ' - getPositionAt mismatch for offset ' + offset);
+		} else {
+			// The position coordinate system cannot express the position between \r and \n
+			let expectedOffset = offset + (previousIsCarriageReturn ? -1 : 0);
+			let actualOffset = model.getOffsetAt(position);
+			assert.equal(actualOffset, expectedOffset, msg + ' - getOffsetAt mismatch for position ' + position.toString());
+		}
+
+		if (allText.charAt(offset) === '\n') {
+			line++;
+			column = 1;
+		} else {
+			column++;
+		}
+
+		previousIsCarriageReturn = (allText.charAt(offset) === '\r');
+	}
+}
+
+function assertLineMapping(model: TextModel, msg: string): void {
+	assertOneDirectionLineMapping(model, AssertDocumentLineMappingDirection.PositionToOffset, msg);
+	assertOneDirectionLineMapping(model, AssertDocumentLineMappingDirection.OffsetToPosition, msg);
+}
+
+
+export function assertSyncedModels(text: string, callback: (model: EditableTextModel, assertMirrorModels: () => void) => void, setup: (model: EditableTextModel) => void = null): void {
+	var model = new EditableTextModel(RawTextSource.fromString(text), TextModel.DEFAULT_CREATION_OPTIONS, null);
+	model.setEOL(editorCommon.EndOfLineSequence.LF);
+	assertLineMapping(model, 'model');
+
+	if (setup) {
+		setup(model);
+		assertLineMapping(model, 'model');
+	}
+
+	var mirrorModel2 = new MirrorModel(null, model.getLinesContent(), model.getEOL(), model.getVersionId());
+	var mirrorModel2PrevVersionId = model.getVersionId();
+
+	model.onDidChangeContent((e: IModelContentChangedEvent) => {
+		let versionId = e.versionId;
+		if (versionId < mirrorModel2PrevVersionId) {
+			console.warn('Model version id did not advance between edits (2)');
+		}
+		mirrorModel2PrevVersionId = versionId;
+		mirrorModel2.onEvents(e);
+	});
+
+	var assertMirrorModels = () => {
+		assertLineMapping(model, 'model');
+		model._assertLineNumbersOK();
+		assert.equal(mirrorModel2.getText(), model.getValue(), 'mirror model 2 text OK');
+		assert.equal(mirrorModel2.version, model.getVersionId(), 'mirror model 2 version OK');
+	};
+
+	callback(model, assertMirrorModels);
+
+	model.dispose();
+	mirrorModel2.dispose();
+}

src/vs/editor/test/common/model/indentRanges.test.ts

@@ -0,0 +1,122 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as assert from 'assert';
+import { Model } from 'vs/editor/common/model/model';
+import { computeRanges } from 'vs/editor/common/model/indentRanges';
+
+export interface IndentRange {
+	startLineNumber: number;
+	endLineNumber: number;
+	indent: number;
+}
+
+suite('Indentation Folding', () => {
+	function assertRanges(lines: string[], expected: IndentRange[]): void {
+		let model = Model.createFromString(lines.join('\n'));
+		let actual = computeRanges(model);
+		actual.sort((r1, r2) => r1.startLineNumber - r2.startLineNumber);
+		assert.deepEqual(actual, expected);
+		model.dispose();
+	}
+
+	function r(startLineNumber: number, endLineNumber: number, indent: number): IndentRange {
+		return { startLineNumber, endLineNumber, indent };
+	}
+
+	test('Fold one level', () => {
+		assertRanges([
+			'A',
+			'  A',
+			'  A',
+			'  A'
+		], [r(1, 4, 0)]);
+	});
+
+	test('Fold two levels', () => {
+		assertRanges([
+			'A',
+			'  A',
+			'  A',
+			'    A',
+			'    A'
+		], [r(1, 5, 0), r(3, 5, 2)]);
+	});
+
+	test('Fold three levels', () => {
+		assertRanges([
+			'A',
+			'  A',
+			'    A',
+			'      A',
+			'A'
+		], [r(1, 4, 0), r(2, 4, 2), r(3, 4, 4)]);
+	});
+
+	test('Fold decreasing indent', () => {
+		assertRanges([
+			'    A',
+			'  A',
+			'A'
+		], []);
+	});
+
+	test('Fold Java', () => {
+		assertRanges([
+		/* 1*/	'class A {',
+		/* 2*/	'  void foo() {',
+		/* 3*/	'    console.log();',
+		/* 4*/	'    console.log();',
+		/* 5*/	'  }',
+		/* 6*/	'',
+		/* 7*/	'  void bar() {',
+		/* 8*/	'    console.log();',
+		/* 9*/	'  }',
+		/*10*/	'}',
+		/*11*/	'interface B {',
+		/*12*/	'  void bar();',
+		/*13*/	'}',
+		], [r(1, 9, 0), r(2, 4, 2), r(7, 8, 2), r(11, 12, 0)]);
+	});
+
+	test('Fold Javadoc', () => {
+		assertRanges([
+		/* 1*/	'/**',
+		/* 2*/	' * Comment',
+		/* 3*/	' */',
+		/* 4*/	'class A {',
+		/* 5*/	'  void foo() {',
+		/* 6*/	'  }',
+		/* 7*/	'}',
+		], [r(1, 3, 0), r(4, 6, 0)]);
+	});
+	test('Fold Whitespace', () => {
+		assertRanges([
+		/* 1*/	'class A {',
+		/* 2*/	'',
+		/* 3*/	'  void foo() {',
+		/* 4*/	'     ',
+		/* 5*/	'     return 0;',
+		/* 6*/	'  }',
+		/* 7*/	'      ',
+		/* 8*/	'}',
+		], [r(1, 7, 0), r(3, 5, 2)]);
+	});
+
+	test('Fold Tabs', () => {
+		assertRanges([
+		/* 1*/	'class A {',
+		/* 2*/	'\t\t',
+		/* 3*/	'\tvoid foo() {',
+		/* 4*/	'\t \t//hello',
+		/* 5*/	'\t    return 0;',
+		/* 6*/	'  \t}',
+		/* 7*/	'      ',
+		/* 8*/	'}',
+		], [r(1, 7, 0), r(3, 5, 4)]);
+	});
+});

src/vs/editor/test/common/model/model.modes.test.ts

@@ -0,0 +1,290 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as assert from 'assert';
+import { IDisposable } from 'vs/base/common/lifecycle';
+import { EditOperation } from 'vs/editor/common/core/editOperation';
+import { Position } from 'vs/editor/common/core/position';
+import { Range } from 'vs/editor/common/core/range';
+import { Model } from 'vs/editor/common/model/model';
+import * as modes from 'vs/editor/common/modes';
+import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
+import { TokenizationResult2 } from 'vs/editor/common/core/token';
+
+// --------- utils
+
+suite('Editor Model - Model Modes 1', () => {
+
+	let calledFor: string[] = [];
+
+	function checkAndClear(arr: string[]) {
+		assert.deepEqual(calledFor, arr);
+		calledFor = [];
+	}
+
+	const tokenizationSupport: modes.ITokenizationSupport = {
+		getInitialState: () => NULL_STATE,
+		tokenize: undefined,
+		tokenize2: (line: string, state: modes.IState): TokenizationResult2 => {
+			calledFor.push(line.charAt(0));
+			return new TokenizationResult2(null, state);
+		}
+	};
+
+	let thisModel: Model = null;
+	let languageRegistration: IDisposable = null;
+
+	setup(() => {
+		const TEXT =
+			'1\r\n' +
+			'2\n' +
+			'3\n' +
+			'4\r\n' +
+			'5';
+		const LANGUAGE_ID = 'modelModeTest1';
+		calledFor = [];
+		languageRegistration = modes.TokenizationRegistry.register(LANGUAGE_ID, tokenizationSupport);
+		thisModel = Model.createFromString(TEXT, undefined, new modes.LanguageIdentifier(LANGUAGE_ID, 0));
+	});
+
+	teardown(() => {
+		thisModel.dispose();
+		thisModel = null;
+		languageRegistration.dispose();
+		languageRegistration = null;
+		calledFor = [];
+	});
+
+	test('model calls syntax highlighter 1', () => {
+		thisModel.forceTokenization(1);
+		checkAndClear(['1']);
+	});
+
+	test('model calls syntax highlighter 2', () => {
+		thisModel.forceTokenization(2);
+		checkAndClear(['1', '2']);
+
+		thisModel.forceTokenization(2);
+		checkAndClear([]);
+	});
+
+	test('model caches states', () => {
+		thisModel.forceTokenization(1);
+		checkAndClear(['1']);
+
+		thisModel.forceTokenization(2);
+		checkAndClear(['2']);
+
+		thisModel.forceTokenization(3);
+		checkAndClear(['3']);
+
+		thisModel.forceTokenization(4);
+		checkAndClear(['4']);
+
+		thisModel.forceTokenization(5);
+		checkAndClear(['5']);
+
+		thisModel.forceTokenization(5);
+		checkAndClear([]);
+	});
+
+	test('model invalidates states for one line insert', () => {
+		thisModel.forceTokenization(5);
+		checkAndClear(['1', '2', '3', '4', '5']);
+
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '-')]);
+		thisModel.forceTokenization(5);
+		checkAndClear(['-']);
+
+		thisModel.forceTokenization(5);
+		checkAndClear([]);
+	});
+
+	test('model invalidates states for many lines insert', () => {
+		thisModel.forceTokenization(5);
+		checkAndClear(['1', '2', '3', '4', '5']);
+
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '0\n-\n+')]);
+		assert.equal(thisModel.getLineCount(), 7);
+		thisModel.forceTokenization(7);
+		checkAndClear(['0', '-', '+']);
+
+		thisModel.forceTokenization(7);
+		checkAndClear([]);
+	});
+
+	test('model invalidates states for one new line', () => {
+		thisModel.forceTokenization(5);
+		checkAndClear(['1', '2', '3', '4', '5']);
+
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 2), '\n')]);
+		thisModel.applyEdits([EditOperation.insert(new Position(2, 1), 'a')]);
+		thisModel.forceTokenization(6);
+		checkAndClear(['1', 'a']);
+	});
+
+	test('model invalidates states for one line delete', () => {
+		thisModel.forceTokenization(5);
+		checkAndClear(['1', '2', '3', '4', '5']);
+
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 2), '-')]);
+		thisModel.forceTokenization(5);
+		checkAndClear(['1']);
+
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 2))]);
+		thisModel.forceTokenization(5);
+		checkAndClear(['-']);
+
+		thisModel.forceTokenization(5);
+		checkAndClear([]);
+	});
+
+	test('model invalidates states for many lines delete', () => {
+		thisModel.forceTokenization(5);
+		checkAndClear(['1', '2', '3', '4', '5']);
+
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 3, 1))]);
+		thisModel.forceTokenization(3);
+		checkAndClear(['3']);
+
+		thisModel.forceTokenization(3);
+		checkAndClear([]);
+	});
+});
+
+suite('Editor Model - Model Modes 2', () => {
+
+	class ModelState2 implements modes.IState {
+		prevLineContent: string;
+
+		constructor(prevLineContent: string) {
+			this.prevLineContent = prevLineContent;
+		}
+
+		clone(): modes.IState {
+			return new ModelState2(this.prevLineContent);
+		}
+
+		equals(other: modes.IState): boolean {
+			return (other instanceof ModelState2) && other.prevLineContent === this.prevLineContent;
+		}
+	}
+
+	const tokenizationSupport: modes.ITokenizationSupport = {
+		getInitialState: () => new ModelState2(''),
+		tokenize: undefined,
+		tokenize2: (line: string, state: modes.IState): TokenizationResult2 => {
+			(<ModelState2>state).prevLineContent = line;
+			return new TokenizationResult2(null, state);
+		}
+	};
+
+	function invalidEqual(model: Model, expected: number[]): void {
+		let actual: number[] = [];
+		for (let i = 0, len = model.getLineCount(); i < len; i++) {
+			if (model._lines[i].isInvalid()) {
+				actual.push(i);
+			}
+		}
+		assert.deepEqual(actual, expected);
+	}
+
+	function stateEqual(state: modes.IState, content: string): void {
+		assert.equal((<ModelState2>state).prevLineContent, content);
+	}
+
+	function statesEqual(model: Model, states: string[]): void {
+		var i, len = states.length - 1;
+		for (i = 0; i < len; i++) {
+			stateEqual(model._lines[i].getState(), states[i]);
+		}
+		stateEqual((<any>model)._lastState, states[len]);
+	}
+
+	let thisModel: Model = null;
+	let languageRegistration: IDisposable = null;
+
+	setup(() => {
+		const TEXT =
+			'Line1' + '\r\n' +
+			'Line2' + '\n' +
+			'Line3' + '\n' +
+			'Line4' + '\r\n' +
+			'Line5';
+		const LANGUAGE_ID = 'modelModeTest2';
+		languageRegistration = modes.TokenizationRegistry.register(LANGUAGE_ID, tokenizationSupport);
+		thisModel = Model.createFromString(TEXT, undefined, new modes.LanguageIdentifier(LANGUAGE_ID, 0));
+	});
+
+	teardown(() => {
+		thisModel.dispose();
+		thisModel = null;
+		languageRegistration.dispose();
+		languageRegistration = null;
+	});
+
+	test('getTokensForInvalidLines one text insert', () => {
+		thisModel.forceTokenization(5);
+		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 6), '-')]);
+		invalidEqual(thisModel, [0]);
+		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
+		thisModel.forceTokenization(5);
+		statesEqual(thisModel, ['', 'Line1-', 'Line2', 'Line3', 'Line4', 'Line5']);
+	});
+
+	test('getTokensForInvalidLines two text insert', () => {
+		thisModel.forceTokenization(5);
+		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
+		thisModel.applyEdits([
+			EditOperation.insert(new Position(1, 6), '-'),
+			EditOperation.insert(new Position(3, 6), '-')
+		]);
+
+		invalidEqual(thisModel, [0, 2]);
+		thisModel.forceTokenization(5);
+		statesEqual(thisModel, ['', 'Line1-', 'Line2', 'Line3-', 'Line4', 'Line5']);
+	});
+
+	test('getTokensForInvalidLines one multi-line text insert, one small text insert', () => {
+		thisModel.forceTokenization(5);
+		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 6), '\nNew line\nAnother new line')]);
+		thisModel.applyEdits([EditOperation.insert(new Position(5, 6), '-')]);
+		invalidEqual(thisModel, [0, 4]);
+		thisModel.forceTokenization(7);
+		statesEqual(thisModel, ['', 'Line1', 'New line', 'Another new line', 'Line2', 'Line3-', 'Line4', 'Line5']);
+	});
+
+	test('getTokensForInvalidLines one delete text', () => {
+		thisModel.forceTokenization(5);
+		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 5))]);
+		invalidEqual(thisModel, [0]);
+		thisModel.forceTokenization(5);
+		statesEqual(thisModel, ['', '1', 'Line2', 'Line3', 'Line4', 'Line5']);
+	});
+
+	test('getTokensForInvalidLines one line delete text', () => {
+		thisModel.forceTokenization(5);
+		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 2, 1))]);
+		invalidEqual(thisModel, [0]);
+		statesEqual(thisModel, ['', 'Line2', 'Line3', 'Line4', 'Line5']);
+		thisModel.forceTokenization(4);
+		statesEqual(thisModel, ['', 'Line2', 'Line3', 'Line4', 'Line5']);
+	});
+
+	test('getTokensForInvalidLines multiple lines delete text', () => {
+		thisModel.forceTokenization(5);
+		statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']);
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 3, 3))]);
+		invalidEqual(thisModel, [0]);
+		statesEqual(thisModel, ['', 'Line3', 'Line4', 'Line5']);
+		thisModel.forceTokenization(3);
+		statesEqual(thisModel, ['', 'ne3', 'Line4', 'Line5']);
+	});
+});

src/vs/editor/test/common/model/model.test.ts

@@ -0,0 +1,395 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as assert from 'assert';
+import { EditOperation } from 'vs/editor/common/core/editOperation';
+import { Position } from 'vs/editor/common/core/position';
+import { Range } from 'vs/editor/common/core/range';
+import {
+	ModelRawContentChangedEvent, ModelRawFlush, ModelRawLineChanged,
+	ModelRawLinesDeleted, ModelRawLinesInserted
+} from 'vs/editor/common/model/textModelEvents';
+import { Model } from 'vs/editor/common/model/model';
+
+// --------- utils
+
+var LINE1 = 'My First Line';
+var LINE2 = '\t\tMy Second Line';
+var LINE3 = '    Third Line';
+var LINE4 = '';
+var LINE5 = '1';
+
+suite('Editor Model - Model', () => {
+
+	var thisModel: Model;
+
+	setup(() => {
+		var text =
+			LINE1 + '\r\n' +
+			LINE2 + '\n' +
+			LINE3 + '\n' +
+			LINE4 + '\r\n' +
+			LINE5;
+		thisModel = Model.createFromString(text);
+	});
+
+	teardown(() => {
+		thisModel.dispose();
+	});
+
+	// --------- insert text
+
+	test('model getValue', () => {
+		assert.equal(thisModel.getValue(), 'My First Line\n\t\tMy Second Line\n    Third Line\n\n1');
+	});
+
+	test('model insert empty text', () => {
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '')]);
+		assert.equal(thisModel.getLineCount(), 5);
+		assert.equal(thisModel.getLineContent(1), 'My First Line');
+	});
+
+	test('model insert text without newline 1', () => {
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), 'foo ')]);
+		assert.equal(thisModel.getLineCount(), 5);
+		assert.equal(thisModel.getLineContent(1), 'foo My First Line');
+	});
+
+	test('model insert text without newline 2', () => {
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 3), ' foo')]);
+		assert.equal(thisModel.getLineCount(), 5);
+		assert.equal(thisModel.getLineContent(1), 'My foo First Line');
+	});
+
+	test('model insert text with one newline', () => {
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 3), ' new line\nNo longer')]);
+		assert.equal(thisModel.getLineCount(), 6);
+		assert.equal(thisModel.getLineContent(1), 'My new line');
+		assert.equal(thisModel.getLineContent(2), 'No longer First Line');
+	});
+
+	test('model insert text with two newlines', () => {
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 3), ' new line\nOne more line in the middle\nNo longer')]);
+		assert.equal(thisModel.getLineCount(), 7);
+		assert.equal(thisModel.getLineContent(1), 'My new line');
+		assert.equal(thisModel.getLineContent(2), 'One more line in the middle');
+		assert.equal(thisModel.getLineContent(3), 'No longer First Line');
+	});
+
+	test('model insert text with many newlines', () => {
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 3), '\n\n\n\n')]);
+		assert.equal(thisModel.getLineCount(), 9);
+		assert.equal(thisModel.getLineContent(1), 'My');
+		assert.equal(thisModel.getLineContent(2), '');
+		assert.equal(thisModel.getLineContent(3), '');
+		assert.equal(thisModel.getLineContent(4), '');
+		assert.equal(thisModel.getLineContent(5), ' First Line');
+	});
+
+
+	// --------- insert text eventing
+
+	test('model insert empty text does not trigger eventing', () => {
+		thisModel.onDidChangeRawContent((e) => {
+			assert.ok(false, 'was not expecting event');
+		});
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '')]);
+	});
+
+	test('model insert text without newline eventing', () => {
+		let e: ModelRawContentChangedEvent = null;
+		thisModel.onDidChangeRawContent((_e) => {
+			if (e !== null) {
+				assert.fail();
+			}
+			e = _e;
+		});
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), 'foo ')]);
+		assert.deepEqual(e, new ModelRawContentChangedEvent(
+			[
+				new ModelRawLineChanged(1, 'foo My First Line')
+			],
+			2,
+			false,
+			false
+		));
+	});
+
+	test('model insert text with one newline eventing', () => {
+		let e: ModelRawContentChangedEvent = null;
+		thisModel.onDidChangeRawContent((_e) => {
+			if (e !== null) {
+				assert.fail();
+			}
+			e = _e;
+		});
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 3), ' new line\nNo longer')]);
+		assert.deepEqual(e, new ModelRawContentChangedEvent(
+			[
+				new ModelRawLineChanged(1, 'My new line First Line'),
+				new ModelRawLineChanged(1, 'My new line'),
+				new ModelRawLinesInserted(2, 2, 'No longer First Line'),
+			],
+			2,
+			false,
+			false
+		));
+	});
+
+
+	// --------- delete text
+
+	test('model delete empty text', () => {
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 1))]);
+		assert.equal(thisModel.getLineCount(), 5);
+		assert.equal(thisModel.getLineContent(1), 'My First Line');
+	});
+
+	test('model delete text from one line', () => {
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 2))]);
+		assert.equal(thisModel.getLineCount(), 5);
+		assert.equal(thisModel.getLineContent(1), 'y First Line');
+	});
+
+	test('model delete text from one line 2', () => {
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), 'a')]);
+		assert.equal(thisModel.getLineContent(1), 'aMy First Line');
+
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 2, 1, 4))]);
+		assert.equal(thisModel.getLineCount(), 5);
+		assert.equal(thisModel.getLineContent(1), 'a First Line');
+	});
+
+	test('model delete all text from a line', () => {
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 14))]);
+		assert.equal(thisModel.getLineCount(), 5);
+		assert.equal(thisModel.getLineContent(1), '');
+	});
+
+	test('model delete text from two lines', () => {
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 4, 2, 6))]);
+		assert.equal(thisModel.getLineCount(), 4);
+		assert.equal(thisModel.getLineContent(1), 'My Second Line');
+	});
+
+	test('model delete text from many lines', () => {
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 4, 3, 5))]);
+		assert.equal(thisModel.getLineCount(), 3);
+		assert.equal(thisModel.getLineContent(1), 'My Third Line');
+	});
+
+	test('model delete everything', () => {
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 5, 2))]);
+		assert.equal(thisModel.getLineCount(), 1);
+		assert.equal(thisModel.getLineContent(1), '');
+	});
+
+	// --------- delete text eventing
+
+	test('model delete empty text does not trigger eventing', () => {
+		thisModel.onDidChangeRawContent((e) => {
+			assert.ok(false, 'was not expecting event');
+		});
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 1))]);
+	});
+
+	test('model delete text from one line eventing', () => {
+		let e: ModelRawContentChangedEvent = null;
+		thisModel.onDidChangeRawContent((_e) => {
+			if (e !== null) {
+				assert.fail();
+			}
+			e = _e;
+		});
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 2))]);
+		assert.deepEqual(e, new ModelRawContentChangedEvent(
+			[
+				new ModelRawLineChanged(1, 'y First Line'),
+			],
+			2,
+			false,
+			false
+		));
+	});
+
+	test('model delete all text from a line eventing', () => {
+		let e: ModelRawContentChangedEvent = null;
+		thisModel.onDidChangeRawContent((_e) => {
+			if (e !== null) {
+				assert.fail();
+			}
+			e = _e;
+		});
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 14))]);
+		assert.deepEqual(e, new ModelRawContentChangedEvent(
+			[
+				new ModelRawLineChanged(1, ''),
+			],
+			2,
+			false,
+			false
+		));
+	});
+
+	test('model delete text from two lines eventing', () => {
+		let e: ModelRawContentChangedEvent = null;
+		thisModel.onDidChangeRawContent((_e) => {
+			if (e !== null) {
+				assert.fail();
+			}
+			e = _e;
+		});
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 4, 2, 6))]);
+		assert.deepEqual(e, new ModelRawContentChangedEvent(
+			[
+				new ModelRawLineChanged(1, 'My '),
+				new ModelRawLineChanged(1, 'My Second Line'),
+				new ModelRawLinesDeleted(2, 2),
+			],
+			2,
+			false,
+			false
+		));
+	});
+
+	test('model delete text from many lines eventing', () => {
+		let e: ModelRawContentChangedEvent = null;
+		thisModel.onDidChangeRawContent((_e) => {
+			if (e !== null) {
+				assert.fail();
+			}
+			e = _e;
+		});
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 4, 3, 5))]);
+		assert.deepEqual(e, new ModelRawContentChangedEvent(
+			[
+				new ModelRawLineChanged(1, 'My '),
+				new ModelRawLineChanged(1, 'My Third Line'),
+				new ModelRawLinesDeleted(2, 3),
+			],
+			2,
+			false,
+			false
+		));
+	});
+
+	// --------- getValueInRange
+
+	test('getValueInRange', () => {
+		assert.equal(thisModel.getValueInRange(new Range(1, 1, 1, 1)), '');
+		assert.equal(thisModel.getValueInRange(new Range(1, 1, 1, 2)), 'M');
+		assert.equal(thisModel.getValueInRange(new Range(1, 2, 1, 3)), 'y');
+		assert.equal(thisModel.getValueInRange(new Range(1, 1, 1, 14)), 'My First Line');
+		assert.equal(thisModel.getValueInRange(new Range(1, 1, 2, 1)), 'My First Line\n');
+		assert.equal(thisModel.getValueInRange(new Range(1, 1, 2, 2)), 'My First Line\n\t');
+		assert.equal(thisModel.getValueInRange(new Range(1, 1, 2, 3)), 'My First Line\n\t\t');
+		assert.equal(thisModel.getValueInRange(new Range(1, 1, 2, 17)), 'My First Line\n\t\tMy Second Line');
+		assert.equal(thisModel.getValueInRange(new Range(1, 1, 3, 1)), 'My First Line\n\t\tMy Second Line\n');
+		assert.equal(thisModel.getValueInRange(new Range(1, 1, 4, 1)), 'My First Line\n\t\tMy Second Line\n    Third Line\n');
+	});
+
+	// --------- getValueLengthInRange
+
+	test('getValueLengthInRange', () => {
+		assert.equal(thisModel.getValueLengthInRange(new Range(1, 1, 1, 1)), ''.length);
+		assert.equal(thisModel.getValueLengthInRange(new Range(1, 1, 1, 2)), 'M'.length);
+		assert.equal(thisModel.getValueLengthInRange(new Range(1, 2, 1, 3)), 'y'.length);
+		assert.equal(thisModel.getValueLengthInRange(new Range(1, 1, 1, 14)), 'My First Line'.length);
+		assert.equal(thisModel.getValueLengthInRange(new Range(1, 1, 2, 1)), 'My First Line\n'.length);
+		assert.equal(thisModel.getValueLengthInRange(new Range(1, 1, 2, 2)), 'My First Line\n\t'.length);
+		assert.equal(thisModel.getValueLengthInRange(new Range(1, 1, 2, 3)), 'My First Line\n\t\t'.length);
+		assert.equal(thisModel.getValueLengthInRange(new Range(1, 1, 2, 17)), 'My First Line\n\t\tMy Second Line'.length);
+		assert.equal(thisModel.getValueLengthInRange(new Range(1, 1, 3, 1)), 'My First Line\n\t\tMy Second Line\n'.length);
+		assert.equal(thisModel.getValueLengthInRange(new Range(1, 1, 4, 1)), 'My First Line\n\t\tMy Second Line\n    Third Line\n'.length);
+	});
+
+	// --------- setValue
+	test('setValue eventing', () => {
+		let e: ModelRawContentChangedEvent = null;
+		thisModel.onDidChangeRawContent((_e) => {
+			if (e !== null) {
+				assert.fail();
+			}
+			e = _e;
+		});
+		thisModel.setValue('new value');
+		assert.deepEqual(e, new ModelRawContentChangedEvent(
+			[
+				new ModelRawFlush()
+			],
+			2,
+			false,
+			false
+		));
+	});
+});
+
+
+// --------- Special Unicode LINE SEPARATOR character
+suite('Editor Model - Model Line Separators', () => {
+
+	var thisModel: Model;
+
+	setup(() => {
+		var text =
+			LINE1 + '\u2028' +
+			LINE2 + '\n' +
+			LINE3 + '\u2028' +
+			LINE4 + '\r\n' +
+			LINE5;
+		thisModel = Model.createFromString(text);
+	});
+
+	teardown(() => {
+		thisModel.dispose();
+	});
+
+	test('model getValue', () => {
+		assert.equal(thisModel.getValue(), 'My First Line\u2028\t\tMy Second Line\n    Third Line\u2028\n1');
+	});
+
+	test('model lines', () => {
+		assert.equal(thisModel.getLineCount(), 3);
+	});
+
+	test('Bug 13333:Model should line break on lonely CR too', () => {
+		var model = Model.createFromString('Hello\rWorld!\r\nAnother line');
+		assert.equal(model.getLineCount(), 3);
+		assert.equal(model.getValue(), 'Hello\r\nWorld!\r\nAnother line');
+		model.dispose();
+	});
+});
+
+
+// --------- Words
+
+suite('Editor Model - Words', () => {
+
+	var thisModel: Model;
+
+	setup(() => {
+		var text = ['This text has some  words. '];
+		thisModel = Model.createFromString(text.join('\n'));
+	});
+
+	teardown(() => {
+		thisModel.dispose();
+	});
+
+	test('Get word at position', () => {
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 1)), { word: 'This', startColumn: 1, endColumn: 5 });
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 2)), { word: 'This', startColumn: 1, endColumn: 5 });
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 4)), { word: 'This', startColumn: 1, endColumn: 5 });
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 5)), { word: 'This', startColumn: 1, endColumn: 5 });
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 6)), { word: 'text', startColumn: 6, endColumn: 10 });
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 19)), { word: 'some', startColumn: 15, endColumn: 19 });
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 20)), null);
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 21)), { word: 'words', startColumn: 21, endColumn: 26 });
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 26)), { word: 'words', startColumn: 21, endColumn: 26 });
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 27)), null);
+		assert.deepEqual(thisModel.getWordAtPosition(new Position(1, 28)), null);
+	});
+});

src/vs/editor/test/common/model/modelDecorations.test.ts

@@ -0,0 +1,638 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as assert from 'assert';
+import { EditOperation } from 'vs/editor/common/core/editOperation';
+import { Position } from 'vs/editor/common/core/position';
+import { Range } from 'vs/editor/common/core/range';
+import { IModelDeltaDecoration, TrackedRangeStickiness } from 'vs/editor/common/editorCommon';
+import { Model } from 'vs/editor/common/model/model';
+
+// --------- utils
+
+interface ILightWeightDecoration2 {
+	range: Range;
+	className: string;
+}
+
+function modelHasDecorations(model: Model, decorations: ILightWeightDecoration2[]) {
+	let modelDecorations: ILightWeightDecoration2[] = [];
+	let actualDecorations = model.getAllDecorations();
+	for (let i = 0, len = actualDecorations.length; i < len; i++) {
+		modelDecorations.push({
+			range: actualDecorations[i].range,
+			className: actualDecorations[i].options.className
+		});
+	}
+	assert.deepEqual(modelDecorations, decorations, 'Model decorations');
+}
+
+function modelHasDecoration(model: Model, startLineNumber: number, startColumn: number, endLineNumber: number, endColumn: number, className: string) {
+	modelHasDecorations(model, [{
+		range: new Range(startLineNumber, startColumn, endLineNumber, endColumn),
+		className: className
+	}]);
+}
+
+function modelHasNoDecorations(model: Model) {
+	assert.equal(model.getAllDecorations().length, 0, 'Model has no decoration');
+}
+
+function addDecoration(model: Model, startLineNumber: number, startColumn: number, endLineNumber: number, endColumn: number, className: string): string {
+	return model.changeDecorations((changeAccessor) => {
+		return changeAccessor.addDecoration(new Range(startLineNumber, startColumn, endLineNumber, endColumn), {
+			className: className
+		});
+	});
+}
+
+function lineHasDecorations(model: Model, lineNumber: number, decorations: { start: number; end: number; className: string; }[]) {
+	var lineDecorations = [];
+	var decs = model.getLineDecorations(lineNumber);
+	for (var i = 0, len = decs.length; i < len; i++) {
+		lineDecorations.push({
+			start: decs[i].range.startColumn,
+			end: decs[i].range.endColumn,
+			className: decs[i].options.className
+		});
+	}
+	assert.deepEqual(lineDecorations, decorations, 'Line decorations');
+}
+
+function lineHasNoDecorations(model: Model, lineNumber: number) {
+	lineHasDecorations(model, lineNumber, []);
+}
+
+function lineHasDecoration(model: Model, lineNumber: number, start: number, end: number, className: string) {
+	lineHasDecorations(model, lineNumber, [{
+		start: start,
+		end: end,
+		className: className
+	}]);
+}
+
+suite('Editor Model - Model Decorations', () => {
+	var LINE1 = 'My First Line';
+	var LINE2 = '\t\tMy Second Line';
+	var LINE3 = '    Third Line';
+	var LINE4 = '';
+	var LINE5 = '1';
+
+	// --------- Model Decorations
+
+	var thisModel: Model;
+
+	setup(() => {
+		var text =
+			LINE1 + '\r\n' +
+			LINE2 + '\n' +
+			LINE3 + '\n' +
+			LINE4 + '\r\n' +
+			LINE5;
+		thisModel = Model.createFromString(text);
+	});
+
+	teardown(() => {
+		thisModel.dispose();
+		thisModel = null;
+	});
+
+	test('single character decoration', () => {
+		addDecoration(thisModel, 1, 1, 1, 2, 'myType');
+		lineHasDecoration(thisModel, 1, 1, 2, 'myType');
+		lineHasNoDecorations(thisModel, 2);
+		lineHasNoDecorations(thisModel, 3);
+		lineHasNoDecorations(thisModel, 4);
+		lineHasNoDecorations(thisModel, 5);
+	});
+
+	test('line decoration', () => {
+		addDecoration(thisModel, 1, 1, 1, 14, 'myType');
+		lineHasDecoration(thisModel, 1, 1, 14, 'myType');
+		lineHasNoDecorations(thisModel, 2);
+		lineHasNoDecorations(thisModel, 3);
+		lineHasNoDecorations(thisModel, 4);
+		lineHasNoDecorations(thisModel, 5);
+	});
+
+	test('full line decoration', () => {
+		addDecoration(thisModel, 1, 1, 2, 1, 'myType');
+
+		var line1Decorations = thisModel.getLineDecorations(1);
+		assert.equal(line1Decorations.length, 1);
+		assert.equal(line1Decorations[0].options.className, 'myType');
+
+		var line2Decorations = thisModel.getLineDecorations(1);
+		assert.equal(line2Decorations.length, 1);
+		assert.equal(line2Decorations[0].options.className, 'myType');
+
+		lineHasNoDecorations(thisModel, 3);
+		lineHasNoDecorations(thisModel, 4);
+		lineHasNoDecorations(thisModel, 5);
+	});
+
+	test('multiple line decoration', () => {
+		addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+
+		var line1Decorations = thisModel.getLineDecorations(1);
+		assert.equal(line1Decorations.length, 1);
+		assert.equal(line1Decorations[0].options.className, 'myType');
+
+		var line2Decorations = thisModel.getLineDecorations(1);
+		assert.equal(line2Decorations.length, 1);
+		assert.equal(line2Decorations[0].options.className, 'myType');
+
+		var line3Decorations = thisModel.getLineDecorations(1);
+		assert.equal(line3Decorations.length, 1);
+		assert.equal(line3Decorations[0].options.className, 'myType');
+
+		lineHasNoDecorations(thisModel, 4);
+		lineHasNoDecorations(thisModel, 5);
+	});
+
+	// --------- removing, changing decorations
+
+	test('decoration gets removed', () => {
+		var decId = addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.changeDecorations((changeAccessor) => {
+			changeAccessor.removeDecoration(decId);
+		});
+		modelHasNoDecorations(thisModel);
+	});
+
+	test('decorations get removed', () => {
+		var decId1 = addDecoration(thisModel, 1, 2, 3, 2, 'myType1');
+		var decId2 = addDecoration(thisModel, 1, 2, 3, 1, 'myType2');
+		modelHasDecorations(thisModel, [
+			{
+				range: new Range(1, 2, 3, 2),
+				className: 'myType1'
+			},
+			{
+				range: new Range(1, 2, 3, 1),
+				className: 'myType2'
+			}
+		]);
+		thisModel.changeDecorations((changeAccessor) => {
+			changeAccessor.removeDecoration(decId1);
+		});
+		modelHasDecorations(thisModel, [
+			{
+				range: new Range(1, 2, 3, 1),
+				className: 'myType2'
+			}
+		]);
+		thisModel.changeDecorations((changeAccessor) => {
+			changeAccessor.removeDecoration(decId2);
+		});
+		modelHasNoDecorations(thisModel);
+	});
+
+	test('decoration range can be changed', () => {
+		var decId = addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.changeDecorations((changeAccessor) => {
+			changeAccessor.changeDecoration(decId, new Range(1, 1, 1, 2));
+		});
+		modelHasDecoration(thisModel, 1, 1, 1, 2, 'myType');
+	});
+
+	// --------- eventing
+
+	test('decorations emit event on add', () => {
+		let listenerCalled = 0;
+		thisModel.onDidChangeDecorations((e) => {
+			listenerCalled++;
+			assert.equal(e.addedDecorations.length, 1);
+			assert.equal(e.changedDecorations.length, 0);
+			assert.equal(e.removedDecorations.length, 0);
+		});
+		addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		assert.equal(listenerCalled, 1, 'listener called');
+	});
+
+	test('decorations emit event on change', () => {
+		let listenerCalled = 0;
+		let decId = addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.onDidChangeDecorations((e) => {
+			listenerCalled++;
+			assert.equal(e.addedDecorations.length, 0);
+			assert.equal(e.changedDecorations.length, 1);
+			assert.equal(e.changedDecorations[0], decId);
+			assert.equal(e.removedDecorations.length, 0);
+		});
+		thisModel.changeDecorations((changeAccessor) => {
+			changeAccessor.changeDecoration(decId, new Range(1, 1, 1, 2));
+		});
+		assert.equal(listenerCalled, 1, 'listener called');
+	});
+
+	test('decorations emit event on remove', () => {
+		let listenerCalled = 0;
+		let decId = addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.onDidChangeDecorations((e) => {
+			listenerCalled++;
+			assert.equal(e.addedDecorations.length, 0);
+			assert.equal(e.changedDecorations.length, 0);
+			assert.equal(e.removedDecorations.length, 1);
+			assert.equal(e.removedDecorations[0], decId);
+		});
+		thisModel.changeDecorations((changeAccessor) => {
+			changeAccessor.removeDecoration(decId);
+		});
+		assert.equal(listenerCalled, 1, 'listener called');
+	});
+
+	test('decorations emit event when inserting one line text before it', () => {
+		let listenerCalled = 0;
+		let decId = addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+
+		thisModel.onDidChangeDecorations((e) => {
+			listenerCalled++;
+			assert.equal(e.addedDecorations.length, 0);
+			assert.equal(e.changedDecorations.length, 1);
+			assert.equal(e.changedDecorations[0], decId);
+			assert.equal(e.removedDecorations.length, 0);
+		});
+
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), 'Hallo ')]);
+		assert.equal(listenerCalled, 1, 'listener called');
+	});
+
+	// --------- editing text & effects on decorations
+
+	test('decorations are updated when inserting one line text before it', () => {
+		addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), 'Hallo ')]);
+		modelHasDecoration(thisModel, 1, 8, 3, 2, 'myType');
+	});
+
+	test('decorations are updated when inserting one line text before it 2', () => {
+		addDecoration(thisModel, 1, 1, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 1, 1, 3, 2, 'myType');
+		thisModel.applyEdits([EditOperation.replace(new Range(1, 1, 1, 1), 'Hallo ')]);
+		modelHasDecoration(thisModel, 1, 1, 3, 2, 'myType');
+	});
+
+	test('decorations are updated when inserting multiple lines text before it', () => {
+		addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 1), 'Hallo\nI\'m inserting multiple\nlines')]);
+		modelHasDecoration(thisModel, 3, 7, 5, 2, 'myType');
+	});
+
+	test('decorations change when inserting text after them', () => {
+		addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.applyEdits([EditOperation.insert(new Position(3, 2), 'Hallo')]);
+		modelHasDecoration(thisModel, 1, 2, 3, 7, 'myType');
+	});
+
+	test('decorations are updated when inserting text inside', () => {
+		addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 3), 'Hallo ')]);
+		modelHasDecoration(thisModel, 1, 2, 3, 2, 'myType');
+	});
+
+	test('decorations are updated when inserting text inside 2', () => {
+		addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.applyEdits([EditOperation.insert(new Position(3, 1), 'Hallo ')]);
+		modelHasDecoration(thisModel, 1, 2, 3, 8, 'myType');
+	});
+
+	test('decorations are updated when inserting text inside 3', () => {
+		addDecoration(thisModel, 1, 1, 2, 16, 'myType');
+		modelHasDecoration(thisModel, 1, 1, 2, 16, 'myType');
+		thisModel.applyEdits([EditOperation.insert(new Position(2, 2), '\n')]);
+		modelHasDecoration(thisModel, 1, 1, 3, 15, 'myType');
+	});
+
+	test('decorations are updated when inserting multiple lines text inside', () => {
+		addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.applyEdits([EditOperation.insert(new Position(1, 3), 'Hallo\nI\'m inserting multiple\nlines')]);
+		modelHasDecoration(thisModel, 1, 2, 5, 2, 'myType');
+	});
+
+	test('decorations are updated when deleting one line text before it', () => {
+		addDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 3, 2, 'myType');
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 2))]);
+		modelHasDecoration(thisModel, 1, 1, 3, 2, 'myType');
+	});
+
+	test('decorations are updated when deleting multiple lines text before it', () => {
+		addDecoration(thisModel, 2, 2, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 2, 2, 3, 2, 'myType');
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 2, 1))]);
+		modelHasDecoration(thisModel, 1, 2, 2, 2, 'myType');
+	});
+
+	test('decorations are updated when deleting multiple lines text before it 2', () => {
+		addDecoration(thisModel, 2, 3, 3, 2, 'myType');
+		modelHasDecoration(thisModel, 2, 3, 3, 2, 'myType');
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 2, 2))]);
+		modelHasDecoration(thisModel, 1, 2, 2, 2, 'myType');
+	});
+
+	test('decorations are updated when deleting text inside', () => {
+		addDecoration(thisModel, 1, 2, 4, 1, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 4, 1, 'myType');
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 3, 2, 1))]);
+		modelHasDecoration(thisModel, 1, 2, 3, 1, 'myType');
+	});
+
+	test('decorations are updated when deleting text inside 2', () => {
+		addDecoration(thisModel, 1, 2, 4, 1, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 4, 1, 'myType');
+		thisModel.applyEdits([
+			EditOperation.delete(new Range(1, 1, 1, 2)),
+			EditOperation.delete(new Range(4, 1, 4, 1))
+		]);
+		modelHasDecoration(thisModel, 1, 1, 4, 1, 'myType');
+	});
+
+	test('decorations are updated when deleting multiple lines text', () => {
+		addDecoration(thisModel, 1, 2, 4, 1, 'myType');
+		modelHasDecoration(thisModel, 1, 2, 4, 1, 'myType');
+		thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 3, 1))]);
+		modelHasDecoration(thisModel, 1, 1, 2, 1, 'myType');
+	});
+});
+
+interface ILightWeightDecoration {
+	id: string;
+	range: Range;
+}
+
+suite('deltaDecorations', () => {
+
+	function decoration(id: string, startLineNumber: number, startColumn: number, endLineNumber: number, endColum: number): ILightWeightDecoration {
+		return {
+			id: id,
+			range: new Range(startLineNumber, startColumn, endLineNumber, endColum)
+		};
+	}
+
+	function toModelDeltaDecoration(dec: ILightWeightDecoration): IModelDeltaDecoration {
+		return {
+			range: dec.range,
+			options: {
+				className: dec.id
+			}
+		};
+	}
+
+	function strcmp(a: string, b: string): number {
+		if (a === b) {
+			return 0;
+		}
+		if (a < b) {
+			return -1;
+		}
+		return 1;
+	}
+
+	function readModelDecorations(model: Model, ids: string[]): ILightWeightDecoration[] {
+		return ids.map((id) => {
+			return {
+				range: model.getDecorationRange(id),
+				id: model.getDecorationOptions(id).className
+			};
+		});
+	}
+
+	function testDeltaDecorations(text: string[], decorations: ILightWeightDecoration[], newDecorations: ILightWeightDecoration[]): void {
+
+		var model = Model.createFromString(text.join('\n'));
+
+		// Add initial decorations & assert they are added
+		var initialIds = model.deltaDecorations([], decorations.map(toModelDeltaDecoration));
+		var actualDecorations = readModelDecorations(model, initialIds);
+
+		assert.equal(initialIds.length, decorations.length, 'returns expected cnt of ids');
+		assert.equal(initialIds.length, model.getAllDecorations().length, 'does not leak decorations');
+		assert.equal(initialIds.length, model._getTrackedRangesCount(), 'does not leak tracked ranges');
+		assert.equal(2 * initialIds.length, model._getMarkersCount(), 'does not leak markers');
+		actualDecorations.sort((a, b) => strcmp(a.id, b.id));
+		decorations.sort((a, b) => strcmp(a.id, b.id));
+		assert.deepEqual(actualDecorations, decorations);
+
+		var newIds = model.deltaDecorations(initialIds, newDecorations.map(toModelDeltaDecoration));
+		var actualNewDecorations = readModelDecorations(model, newIds);
+
+		assert.equal(newIds.length, newDecorations.length, 'returns expected cnt of ids');
+		assert.equal(newIds.length, model.getAllDecorations().length, 'does not leak decorations');
+		assert.equal(newIds.length, model._getTrackedRangesCount(), 'does not leak tracked ranges');
+		assert.equal(2 * newIds.length, model._getMarkersCount(), 'does not leak markers');
+		actualNewDecorations.sort((a, b) => strcmp(a.id, b.id));
+		newDecorations.sort((a, b) => strcmp(a.id, b.id));
+		assert.deepEqual(actualDecorations, decorations);
+
+		model.dispose();
+	}
+
+	function range(startLineNumber: number, startColumn: number, endLineNumber: number, endColumn: number): Range {
+		return new Range(startLineNumber, startColumn, endLineNumber, endColumn);
+	}
+
+	test('result respects input', () => {
+		var model = Model.createFromString([
+			'Hello world,',
+			'How are you?'
+		].join('\n'));
+
+		var ids = model.deltaDecorations([], [
+			toModelDeltaDecoration(decoration('a', 1, 1, 1, 12)),
+			toModelDeltaDecoration(decoration('b', 2, 1, 2, 13))
+		]);
+
+		assert.deepEqual(model.getDecorationRange(ids[0]), range(1, 1, 1, 12));
+		assert.deepEqual(model.getDecorationRange(ids[1]), range(2, 1, 2, 13));
+
+		model.dispose();
+	});
+
+	test('deltaDecorations 1', () => {
+		testDeltaDecorations(
+			[
+				'This is a text',
+				'That has multiple lines',
+				'And is very friendly',
+				'Towards testing'
+			],
+			[
+				decoration('a', 1, 1, 1, 2),
+				decoration('b', 1, 1, 1, 15),
+				decoration('c', 1, 1, 2, 1),
+				decoration('d', 1, 1, 2, 24),
+				decoration('e', 2, 1, 2, 24),
+				decoration('f', 2, 1, 4, 16)
+			],
+			[
+				decoration('x', 1, 1, 1, 2),
+				decoration('b', 1, 1, 1, 15),
+				decoration('c', 1, 1, 2, 1),
+				decoration('d', 1, 1, 2, 24),
+				decoration('e', 2, 1, 2, 21),
+				decoration('f', 2, 17, 4, 16)
+			]
+		);
+	});
+
+	test('deltaDecorations 2', () => {
+		testDeltaDecorations(
+			[
+				'This is a text',
+				'That has multiple lines',
+				'And is very friendly',
+				'Towards testing'
+			],
+			[
+				decoration('a', 1, 1, 1, 2),
+				decoration('b', 1, 2, 1, 3),
+				decoration('c', 1, 3, 1, 4),
+				decoration('d', 1, 4, 1, 5),
+				decoration('e', 1, 5, 1, 6)
+			],
+			[
+				decoration('a', 1, 2, 1, 3),
+				decoration('b', 1, 3, 1, 4),
+				decoration('c', 1, 4, 1, 5),
+				decoration('d', 1, 5, 1, 6)
+			]
+		);
+	});
+
+	test('deltaDecorations 3', () => {
+		testDeltaDecorations(
+			[
+				'This is a text',
+				'That has multiple lines',
+				'And is very friendly',
+				'Towards testing'
+			],
+			[
+				decoration('a', 1, 1, 1, 2),
+				decoration('b', 1, 2, 1, 3),
+				decoration('c', 1, 3, 1, 4),
+				decoration('d', 1, 4, 1, 5),
+				decoration('e', 1, 5, 1, 6)
+			],
+			[]
+		);
+	});
+
+	test('issue #4317: editor.setDecorations doesn\'t update the hover message', () => {
+
+		let model = Model.createFromString('Hello world!');
+
+		let ids = model.deltaDecorations([], [{
+			range: {
+				startLineNumber: 1,
+				startColumn: 1,
+				endLineNumber: 100,
+				endColumn: 1
+			},
+			options: {
+				hoverMessage: { value: 'hello1' }
+			}
+		}]);
+
+		ids = model.deltaDecorations(ids, [{
+			range: {
+				startLineNumber: 1,
+				startColumn: 1,
+				endLineNumber: 100,
+				endColumn: 1
+			},
+			options: {
+				hoverMessage: { value: 'hello2' }
+			}
+		}]);
+
+		let actualDecoration = model.getDecorationOptions(ids[0]);
+
+		assert.deepEqual(actualDecoration.hoverMessage, { value: 'hello2' });
+
+		model.dispose();
+	});
+
+	test('model doesn\'t get confused with individual tracked ranges', () => {
+		var model = Model.createFromString([
+			'Hello world,',
+			'How are you?'
+		].join('\n'));
+
+		var trackedRangeId = model.changeDecorations((changeAcessor) => {
+			return changeAcessor.addDecoration(
+				{
+					startLineNumber: 1,
+					startColumn: 1,
+					endLineNumber: 1,
+					endColumn: 1
+				}, {
+					stickiness: TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges
+				}
+			);
+		});
+		model.changeDecorations((changeAccessor) => {
+			changeAccessor.removeDecoration(trackedRangeId);
+		});
+
+		var ids = model.deltaDecorations([], [
+			toModelDeltaDecoration(decoration('a', 1, 1, 1, 12)),
+			toModelDeltaDecoration(decoration('b', 2, 1, 2, 13))
+		]);
+
+		assert.deepEqual(model.getDecorationRange(ids[0]), range(1, 1, 1, 12));
+		assert.deepEqual(model.getDecorationRange(ids[1]), range(2, 1, 2, 13));
+
+		ids = model.deltaDecorations(ids, [
+			toModelDeltaDecoration(decoration('a', 1, 1, 1, 12)),
+			toModelDeltaDecoration(decoration('b', 2, 1, 2, 13))
+		]);
+
+		assert.deepEqual(model.getDecorationRange(ids[0]), range(1, 1, 1, 12));
+		assert.deepEqual(model.getDecorationRange(ids[1]), range(2, 1, 2, 13));
+
+		model.dispose();
+	});
+
+	test('issue #16922: Clicking on link doesn\'t seem to do anything', () => {
+		var model = Model.createFromString([
+			'Hello world,',
+			'How are you?',
+			'Fine.',
+			'Good.',
+		].join('\n'));
+
+		model.deltaDecorations([], [
+			{ range: new Range(1, 1, 1, 1), options: { className: '1' } },
+			{ range: new Range(1, 13, 1, 13), options: { className: '2' } },
+			{ range: new Range(2, 1, 2, 1), options: { className: '3' } },
+			{ range: new Range(2, 1, 2, 4), options: { className: '4' } },
+			{ range: new Range(2, 8, 2, 13), options: { className: '5' } },
+			{ range: new Range(3, 1, 4, 6), options: { className: '6' } },
+			{ range: new Range(1, 1, 3, 6), options: { className: 'x1' } },
+			{ range: new Range(2, 5, 2, 8), options: { className: 'x2' } },
+			{ range: new Range(1, 1, 2, 8), options: { className: 'x3' } },
+			{ range: new Range(2, 5, 3, 1), options: { className: 'x4' } },
+		]);
+
+		let inRange = model.getDecorationsInRange(new Range(2, 6, 2, 6));
+
+		let inRangeClassNames = inRange.map(d => d.options.className);
+		inRangeClassNames.sort();
+		assert.deepEqual(inRangeClassNames, ['x1', 'x2', 'x3', 'x4']);
+
+		model.dispose();
+	});
+});

src/vs/editor/test/common/model/modelEditOperation.test.ts

@@ -0,0 +1,178 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as assert from 'assert';
+import { Range } from 'vs/editor/common/core/range';
+import { IIdentifiedSingleEditOperation } from 'vs/editor/common/editorCommon';
+import { Model } from 'vs/editor/common/model/model';
+
+suite('Editor Model - Model Edit Operation', () => {
+	var LINE1 = 'My First Line';
+	var LINE2 = '\t\tMy Second Line';
+	var LINE3 = '    Third Line';
+	var LINE4 = '';
+	var LINE5 = '1';
+
+	var model: Model;
+
+	setup(() => {
+		var text =
+			LINE1 + '\r\n' +
+			LINE2 + '\n' +
+			LINE3 + '\n' +
+			LINE4 + '\r\n' +
+			LINE5;
+		model = Model.createFromString(text);
+	});
+
+	teardown(() => {
+		model.dispose();
+		model = null;
+	});
+
+	function createSingleEditOp(text: string, positionLineNumber: number, positionColumn: number, selectionLineNumber: number = positionLineNumber, selectionColumn: number = positionColumn): IIdentifiedSingleEditOperation {
+		var range = new Range(
+			selectionLineNumber,
+			selectionColumn,
+			positionLineNumber,
+			positionColumn
+		);
+
+		return {
+			identifier: {
+				major: 0,
+				minor: 0
+			},
+			range: range,
+			text: text,
+			forceMoveMarkers: false
+		};
+	}
+
+	function assertSingleEditOp(singleEditOp: IIdentifiedSingleEditOperation, editedLines: string[]) {
+		var editOp = [singleEditOp];
+
+		var inverseEditOp = model.applyEdits(editOp);
+
+		assert.equal(model.getLineCount(), editedLines.length);
+		for (var i = 0; i < editedLines.length; i++) {
+			assert.equal(model.getLineContent(i + 1), editedLines[i]);
+		}
+
+		var originalOp = model.applyEdits(inverseEditOp);
+
+		assert.equal(model.getLineCount(), 5);
+		assert.equal(model.getLineContent(1), LINE1);
+		assert.equal(model.getLineContent(2), LINE2);
+		assert.equal(model.getLineContent(3), LINE3);
+		assert.equal(model.getLineContent(4), LINE4);
+		assert.equal(model.getLineContent(5), LINE5);
+
+		assert.deepEqual(originalOp, editOp);
+	}
+
+	test('Insert inline', () => {
+		assertSingleEditOp(
+			createSingleEditOp('a', 1, 1),
+			[
+				'aMy First Line',
+				LINE2,
+				LINE3,
+				LINE4,
+				LINE5
+			]
+		);
+	});
+
+	test('Replace inline/inline 1', () => {
+		assertSingleEditOp(
+			createSingleEditOp(' incredibly awesome', 1, 3),
+			[
+				'My incredibly awesome First Line',
+				LINE2,
+				LINE3,
+				LINE4,
+				LINE5
+			]
+		);
+	});
+
+	test('Replace inline/inline 2', () => {
+		assertSingleEditOp(
+			createSingleEditOp(' with text at the end.', 1, 14),
+			[
+				'My First Line with text at the end.',
+				LINE2,
+				LINE3,
+				LINE4,
+				LINE5
+			]
+		);
+	});
+
+	test('Replace inline/inline 3', () => {
+		assertSingleEditOp(
+			createSingleEditOp('My new First Line.', 1, 1, 1, 14),
+			[
+				'My new First Line.',
+				LINE2,
+				LINE3,
+				LINE4,
+				LINE5
+			]
+		);
+	});
+
+	test('Replace inline/multi line 1', () => {
+		assertSingleEditOp(
+			createSingleEditOp('My new First Line.', 1, 1, 3, 15),
+			[
+				'My new First Line.',
+				LINE4,
+				LINE5
+			]
+		);
+	});
+
+	test('Replace inline/multi line 2', () => {
+		assertSingleEditOp(
+			createSingleEditOp('My new First Line.', 1, 2, 3, 15),
+			[
+				'MMy new First Line.',
+				LINE4,
+				LINE5
+			]
+		);
+	});
+
+	test('Replace inline/multi line 3', () => {
+		assertSingleEditOp(
+			createSingleEditOp('My new First Line.', 1, 2, 3, 2),
+			[
+				'MMy new First Line.   Third Line',
+				LINE4,
+				LINE5
+			]
+		);
+	});
+
+	test('Replace muli line/multi line', () => {
+		assertSingleEditOp(
+			createSingleEditOp('1\n2\n3\n4\n', 1, 1),
+			[
+				'1',
+				'2',
+				'3',
+				'4',
+				LINE1,
+				LINE2,
+				LINE3,
+				LINE4,
+				LINE5
+			]
+		);
+	});
+});

src/vs/editor/test/common/model/textModel.test.ts

@@ -0,0 +1,997 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as assert from 'assert';
+import { Position } from 'vs/editor/common/core/position';
+import { Range } from 'vs/editor/common/core/range';
+import { TextModel, ITextModelCreationData } from 'vs/editor/common/model/textModel';
+import { DefaultEndOfLine, TextModelResolvedOptions } from 'vs/editor/common/editorCommon';
+import { RawTextSource } from 'vs/editor/common/model/textSource';
+
+function testGuessIndentation(defaultInsertSpaces: boolean, defaultTabSize: number, expectedInsertSpaces: boolean, expectedTabSize: number, text: string[], msg?: string): void {
+	var m = TextModel.createFromString(
+		text.join('\n'),
+		{
+			tabSize: defaultTabSize,
+			insertSpaces: defaultInsertSpaces,
+			detectIndentation: true,
+			defaultEOL: DefaultEndOfLine.LF,
+			trimAutoWhitespace: true
+		}
+	);
+	var r = m.getOptions();
+	m.dispose();
+
+	assert.equal(r.insertSpaces, expectedInsertSpaces, msg);
+	assert.equal(r.tabSize, expectedTabSize, msg);
+}
+
+function assertGuess(expectedInsertSpaces: boolean, expectedTabSize: number, text: string[], msg?: string): void {
+	if (typeof expectedInsertSpaces === 'undefined') {
+		// cannot guess insertSpaces
+		if (typeof expectedTabSize === 'undefined') {
+			// cannot guess tabSize
+			testGuessIndentation(true, 13370, true, 13370, text, msg);
+			testGuessIndentation(false, 13371, false, 13371, text, msg);
+		} else {
+			// can guess tabSize
+			testGuessIndentation(true, 13370, true, expectedTabSize, text, msg);
+			testGuessIndentation(false, 13371, false, expectedTabSize, text, msg);
+		}
+	} else {
+		// can guess insertSpaces
+		if (typeof expectedTabSize === 'undefined') {
+			// cannot guess tabSize
+			testGuessIndentation(true, 13370, expectedInsertSpaces, 13370, text, msg);
+			testGuessIndentation(false, 13371, expectedInsertSpaces, 13371, text, msg);
+		} else {
+			// can guess tabSize
+			testGuessIndentation(true, 13370, expectedInsertSpaces, expectedTabSize, text, msg);
+			testGuessIndentation(false, 13371, expectedInsertSpaces, expectedTabSize, text, msg);
+		}
+	}
+}
+
+suite('TextModelData.fromString', () => {
+
+	function testTextModelDataFromString(text: string, expected: ITextModelCreationData): void {
+		const rawTextSource = RawTextSource.fromString(text);
+		const actual = TextModel.resolveCreationData(rawTextSource, TextModel.DEFAULT_CREATION_OPTIONS);
+		assert.deepEqual(actual, expected);
+	}
+
+	test('one line text', () => {
+		testTextModelDataFromString('Hello world!', {
+			text: {
+				BOM: '',
+				EOL: '\n',
+				length: 12,
+				'lines': [
+					'Hello world!'
+				],
+				containsRTL: false,
+				isBasicASCII: true
+			},
+			options: new TextModelResolvedOptions({
+				defaultEOL: DefaultEndOfLine.LF,
+				insertSpaces: true,
+				tabSize: 4,
+				trimAutoWhitespace: true,
+			})
+		});
+	});
+
+	test('multiline text', () => {
+		testTextModelDataFromString('Hello,\r\ndear friend\nHow\rare\r\nyou?', {
+			text: {
+				BOM: '',
+				EOL: '\r\n',
+				length: 33,
+				'lines': [
+					'Hello,',
+					'dear friend',
+					'How',
+					'are',
+					'you?'
+				],
+				containsRTL: false,
+				isBasicASCII: true
+			},
+			options: new TextModelResolvedOptions({
+				defaultEOL: DefaultEndOfLine.LF,
+				insertSpaces: true,
+				tabSize: 4,
+				trimAutoWhitespace: true,
+			})
+		});
+	});
+
+	test('Non Basic ASCII 1', () => {
+		testTextModelDataFromString('Hello,\nZürich', {
+			text: {
+				BOM: '',
+				EOL: '\n',
+				length: 13,
+				'lines': [
+					'Hello,',
+					'Zürich'
+				],
+				containsRTL: false,
+				isBasicASCII: false
+			},
+			options: new TextModelResolvedOptions({
+				defaultEOL: DefaultEndOfLine.LF,
+				insertSpaces: true,
+				tabSize: 4,
+				trimAutoWhitespace: true,
+			})
+		});
+	});
+
+	test('containsRTL 1', () => {
+		testTextModelDataFromString('Hello,\nזוהי עובדה מבוססת שדעתו', {
+			text: {
+				BOM: '',
+				EOL: '\n',
+				length: 30,
+				'lines': [
+					'Hello,',
+					'זוהי עובדה מבוססת שדעתו'
+				],
+				containsRTL: true,
+				isBasicASCII: false
+			},
+			options: new TextModelResolvedOptions({
+				defaultEOL: DefaultEndOfLine.LF,
+				insertSpaces: true,
+				tabSize: 4,
+				trimAutoWhitespace: true,
+			})
+		});
+	});
+
+	test('containsRTL 2', () => {
+		testTextModelDataFromString('Hello,\nهناك حقيقة مثبتة منذ زمن طويل', {
+			text: {
+				BOM: '',
+				EOL: '\n',
+				length: 36,
+				'lines': [
+					'Hello,',
+					'هناك حقيقة مثبتة منذ زمن طويل'
+				],
+				containsRTL: true,
+				isBasicASCII: false
+			},
+			options: new TextModelResolvedOptions({
+				defaultEOL: DefaultEndOfLine.LF,
+				insertSpaces: true,
+				tabSize: 4,
+				trimAutoWhitespace: true,
+			})
+		});
+	});
+
+});
+
+suite('Editor Model - TextModel', () => {
+
+	test('getValueLengthInRange', () => {
+
+		var m = TextModel.createFromString('My First Line\r\nMy Second Line\r\nMy Third Line');
+		assert.equal(m.getValueLengthInRange(new Range(1, 1, 1, 1)), ''.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 1, 1, 2)), 'M'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 1, 3)), 'y'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 1, 1, 14)), 'My First Line'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 1, 2, 1)), 'My First Line\r\n'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 2, 1)), 'y First Line\r\n'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 2, 2)), 'y First Line\r\nM'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 2, 1000)), 'y First Line\r\nMy Second Line'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 3, 1)), 'y First Line\r\nMy Second Line\r\n'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 3, 1000)), 'y First Line\r\nMy Second Line\r\nMy Third Line'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 1, 1000, 1000)), 'My First Line\r\nMy Second Line\r\nMy Third Line'.length);
+
+		m = TextModel.createFromString('My First Line\nMy Second Line\nMy Third Line');
+		assert.equal(m.getValueLengthInRange(new Range(1, 1, 1, 1)), ''.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 1, 1, 2)), 'M'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 1, 3)), 'y'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 1, 1, 14)), 'My First Line'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 1, 2, 1)), 'My First Line\n'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 2, 1)), 'y First Line\n'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 2, 2)), 'y First Line\nM'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 2, 1000)), 'y First Line\nMy Second Line'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 3, 1)), 'y First Line\nMy Second Line\n'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 2, 3, 1000)), 'y First Line\nMy Second Line\nMy Third Line'.length);
+		assert.equal(m.getValueLengthInRange(new Range(1, 1, 1000, 1000)), 'My First Line\nMy Second Line\nMy Third Line'.length);
+	});
+
+	test('guess indentation 1', () => {
+
+		assertGuess(undefined, undefined, [
+			'x',
+			'x',
+			'x',
+			'x',
+			'x',
+			'x',
+			'x'
+		], 'no clues');
+
+		assertGuess(false, undefined, [
+			'\tx',
+			'x',
+			'x',
+			'x',
+			'x',
+			'x',
+			'x'
+		], 'no spaces, 1xTAB');
+
+		assertGuess(true, 2, [
+			'  x',
+			'x',
+			'x',
+			'x',
+			'x',
+			'x',
+			'x'
+		], '1x2');
+
+		assertGuess(false, undefined, [
+			'\tx',
+			'\tx',
+			'\tx',
+			'\tx',
+			'\tx',
+			'\tx',
+			'\tx'
+		], '7xTAB');
+
+		assertGuess(undefined, 2, [
+			'\tx',
+			'  x',
+			'\tx',
+			'  x',
+			'\tx',
+			'  x',
+			'\tx',
+			'  x',
+		], '4x2, 4xTAB');
+		assertGuess(false, undefined, [
+			'\tx',
+			' x',
+			'\tx',
+			' x',
+			'\tx',
+			' x',
+			'\tx',
+			' x'
+		], '4x1, 4xTAB');
+		assertGuess(false, 2, [
+			'\tx',
+			'\tx',
+			'  x',
+			'\tx',
+			'  x',
+			'\tx',
+			'  x',
+			'\tx',
+			'  x',
+		], '4x2, 5xTAB');
+		assertGuess(false, 2, [
+			'\tx',
+			'\tx',
+			'x',
+			'\tx',
+			'x',
+			'\tx',
+			'x',
+			'\tx',
+			'  x',
+		], '1x2, 5xTAB');
+		assertGuess(false, 4, [
+			'\tx',
+			'\tx',
+			'x',
+			'\tx',
+			'x',
+			'\tx',
+			'x',
+			'\tx',
+			'    x',
+		], '1x4, 5xTAB');
+		assertGuess(false, 2, [
+			'\tx',
+			'\tx',
+			'x',
+			'\tx',
+			'x',
+			'\tx',
+			'  x',
+			'\tx',
+			'    x',
+		], '1x2, 1x4, 5xTAB');
+
+		assertGuess(undefined, undefined, [
+			'x',
+			' x',
+			' x',
+			' x',
+			' x',
+			' x',
+			' x',
+			' x'
+		], '7x1 - 1 space is never guessed as an indentation');
+		assertGuess(true, undefined, [
+			'x',
+			'          x',
+			' x',
+			' x',
+			' x',
+			' x',
+			' x',
+			' x'
+		], '1x10, 6x1');
+		assertGuess(undefined, undefined, [
+			'',
+			'  ',
+			'    ',
+			'      ',
+			'        ',
+			'          ',
+			'            ',
+			'              ',
+		], 'whitespace lines don\'t count');
+		assertGuess(true, 4, [
+			'x',
+			'   x',
+			'   x',
+			'    x',
+			'x',
+			'   x',
+			'   x',
+			'    x',
+			'x',
+			'   x',
+			'   x',
+			'    x',
+		], 'odd number is not allowed: 6x3, 3x4');
+		assertGuess(true, 4, [
+			'x',
+			'     x',
+			'     x',
+			'    x',
+			'x',
+			'     x',
+			'     x',
+			'    x',
+			'x',
+			'     x',
+			'     x',
+			'    x',
+		], 'odd number is not allowed: 6x5, 3x4');
+		assertGuess(true, 4, [
+			'x',
+			'       x',
+			'       x',
+			'    x',
+			'x',
+			'       x',
+			'       x',
+			'    x',
+			'x',
+			'       x',
+			'       x',
+			'    x',
+		], 'odd number is not allowed: 6x7, 3x4');
+		assertGuess(true, 2, [
+			'x',
+			'  x',
+			'  x',
+			'  x',
+			'  x',
+			'x',
+			'  x',
+			'  x',
+			'  x',
+			'  x',
+		], '8x2');
+
+		assertGuess(true, 2, [
+			'x',
+			'  x',
+			'  x',
+			'x',
+			'  x',
+			'  x',
+			'x',
+			'  x',
+			'  x',
+			'x',
+			'  x',
+			'  x',
+		], '8x2');
+		assertGuess(true, 2, [
+			'x',
+			'  x',
+			'    x',
+			'x',
+			'  x',
+			'    x',
+			'x',
+			'  x',
+			'    x',
+			'x',
+			'  x',
+			'    x',
+		], '4x2, 4x4');
+		assertGuess(true, 2, [
+			'x',
+			'  x',
+			'  x',
+			'    x',
+			'x',
+			'  x',
+			'  x',
+			'    x',
+			'x',
+			'  x',
+			'  x',
+			'    x',
+		], '6x2, 3x4');
+		assertGuess(true, 2, [
+			'x',
+			'  x',
+			'  x',
+			'    x',
+			'    x',
+			'x',
+			'  x',
+			'  x',
+			'    x',
+			'    x',
+		], '4x2, 4x4');
+		assertGuess(true, 2, [
+			'x',
+			'  x',
+			'    x',
+			'    x',
+			'x',
+			'  x',
+			'    x',
+			'    x',
+		], '2x2, 4x4');
+		assertGuess(true, 4, [
+			'x',
+			'    x',
+			'    x',
+			'x',
+			'    x',
+			'    x',
+			'x',
+			'    x',
+			'    x',
+			'x',
+			'    x',
+			'    x',
+		], '8x4');
+		assertGuess(true, 2, [
+			'x',
+			'  x',
+			'    x',
+			'    x',
+			'      x',
+			'x',
+			'  x',
+			'    x',
+			'    x',
+			'      x',
+		], '2x2, 4x4, 2x6');
+		assertGuess(true, 2, [
+			'x',
+			'  x',
+			'    x',
+			'    x',
+			'      x',
+			'      x',
+			'        x',
+		], '1x2, 2x4, 2x6, 1x8');
+		assertGuess(true, 4, [
+			'x',
+			'    x',
+			'    x',
+			'    x',
+			'     x',
+			'        x',
+			'x',
+			'    x',
+			'    x',
+			'    x',
+			'     x',
+			'        x',
+		], '6x4, 2x5, 2x8');
+		assertGuess(true, 4, [
+			'x',
+			'    x',
+			'    x',
+			'    x',
+			'     x',
+			'        x',
+			'        x',
+		], '3x4, 1x5, 2x8');
+		assertGuess(true, 4, [
+			'x',
+			'x',
+			'    x',
+			'    x',
+			'     x',
+			'        x',
+			'        x',
+			'x',
+			'x',
+			'    x',
+			'    x',
+			'     x',
+			'        x',
+			'        x',
+		], '6x4, 2x5, 4x8');
+		assertGuess(true, 4, [
+			'x',
+			' x',
+			' x',
+			' x',
+			' x',
+			' x',
+			'x',
+			'   x',
+			'    x',
+			'    x',
+		], '5x1, 2x0, 1x3, 2x4');
+		assertGuess(false, undefined, [
+			'\t x',
+			' \t x',
+			'\tx'
+		], 'mixed whitespace 1');
+		assertGuess(false, 4, [
+			'\tx',
+			'\t    x'
+		], 'mixed whitespace 2');
+	});
+
+	test('validatePosition', () => {
+
+		let m = TextModel.createFromString('line one\nline two');
+
+		assert.deepEqual(m.validatePosition(new Position(0, 0)), new Position(1, 1));
+		assert.deepEqual(m.validatePosition(new Position(0, 1)), new Position(1, 1));
+
+		assert.deepEqual(m.validatePosition(new Position(1, 1)), new Position(1, 1));
+		assert.deepEqual(m.validatePosition(new Position(1, 2)), new Position(1, 2));
+		assert.deepEqual(m.validatePosition(new Position(1, 30)), new Position(1, 9));
+
+		assert.deepEqual(m.validatePosition(new Position(2, 0)), new Position(2, 1));
+		assert.deepEqual(m.validatePosition(new Position(2, 1)), new Position(2, 1));
+		assert.deepEqual(m.validatePosition(new Position(2, 2)), new Position(2, 2));
+		assert.deepEqual(m.validatePosition(new Position(2, 30)), new Position(2, 9));
+
+		assert.deepEqual(m.validatePosition(new Position(3, 0)), new Position(2, 9));
+		assert.deepEqual(m.validatePosition(new Position(3, 1)), new Position(2, 9));
+		assert.deepEqual(m.validatePosition(new Position(3, 30)), new Position(2, 9));
+
+		assert.deepEqual(m.validatePosition(new Position(30, 30)), new Position(2, 9));
+
+		assert.deepEqual(m.validatePosition(new Position(-123.123, -0.5)), new Position(1, 1));
+		assert.deepEqual(m.validatePosition(new Position(Number.MIN_VALUE, Number.MIN_VALUE)), new Position(1, 1));
+
+		assert.deepEqual(m.validatePosition(new Position(Number.MAX_VALUE, Number.MAX_VALUE)), new Position(2, 9));
+		assert.deepEqual(m.validatePosition(new Position(123.23, 47.5)), new Position(2, 9));
+	});
+
+	test('validatePosition around high-low surrogate pairs 1', () => {
+
+		let m = TextModel.createFromString('a📚b');
+
+		assert.deepEqual(m.validatePosition(new Position(0, 0)), new Position(1, 1));
+		assert.deepEqual(m.validatePosition(new Position(0, 1)), new Position(1, 1));
+		assert.deepEqual(m.validatePosition(new Position(0, 7)), new Position(1, 1));
+
+		assert.deepEqual(m.validatePosition(new Position(1, 1)), new Position(1, 1));
+		assert.deepEqual(m.validatePosition(new Position(1, 2)), new Position(1, 2));
+		assert.deepEqual(m.validatePosition(new Position(1, 3)), new Position(1, 2));
+		assert.deepEqual(m.validatePosition(new Position(1, 4)), new Position(1, 4));
+		assert.deepEqual(m.validatePosition(new Position(1, 5)), new Position(1, 5));
+		assert.deepEqual(m.validatePosition(new Position(1, 30)), new Position(1, 5));
+
+		assert.deepEqual(m.validatePosition(new Position(2, 0)), new Position(1, 5));
+		assert.deepEqual(m.validatePosition(new Position(2, 1)), new Position(1, 5));
+		assert.deepEqual(m.validatePosition(new Position(2, 2)), new Position(1, 5));
+		assert.deepEqual(m.validatePosition(new Position(2, 30)), new Position(1, 5));
+
+		assert.deepEqual(m.validatePosition(new Position(-123.123, -0.5)), new Position(1, 1));
+		assert.deepEqual(m.validatePosition(new Position(Number.MIN_VALUE, Number.MIN_VALUE)), new Position(1, 1));
+
+		assert.deepEqual(m.validatePosition(new Position(Number.MAX_VALUE, Number.MAX_VALUE)), new Position(1, 5));
+		assert.deepEqual(m.validatePosition(new Position(123.23, 47.5)), new Position(1, 5));
+	});
+
+	test('validatePosition around high-low surrogate pairs 2', () => {
+
+		let m = TextModel.createFromString('a📚📚b');
+
+		assert.deepEqual(m.validatePosition(new Position(1, 1)), new Position(1, 1));
+		assert.deepEqual(m.validatePosition(new Position(1, 2)), new Position(1, 2));
+		assert.deepEqual(m.validatePosition(new Position(1, 3)), new Position(1, 2));
+		assert.deepEqual(m.validatePosition(new Position(1, 4)), new Position(1, 4));
+		assert.deepEqual(m.validatePosition(new Position(1, 5)), new Position(1, 4));
+		assert.deepEqual(m.validatePosition(new Position(1, 6)), new Position(1, 6));
+		assert.deepEqual(m.validatePosition(new Position(1, 7)), new Position(1, 7));
+
+	});
+
+	test('validateRange around high-low surrogate pairs 1', () => {
+
+		let m = TextModel.createFromString('a📚b');
+
+		assert.deepEqual(m.validateRange(new Range(0, 0, 0, 1)), new Range(1, 1, 1, 1));
+		assert.deepEqual(m.validateRange(new Range(0, 0, 0, 7)), new Range(1, 1, 1, 1));
+
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 1)), new Range(1, 1, 1, 1));
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 2)), new Range(1, 1, 1, 2));
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 3)), new Range(1, 1, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 4)), new Range(1, 1, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 5)), new Range(1, 1, 1, 5));
+
+		assert.deepEqual(m.validateRange(new Range(1, 2, 1, 2)), new Range(1, 2, 1, 2));
+		assert.deepEqual(m.validateRange(new Range(1, 2, 1, 3)), new Range(1, 2, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 2, 1, 4)), new Range(1, 2, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 2, 1, 5)), new Range(1, 2, 1, 5));
+
+		assert.deepEqual(m.validateRange(new Range(1, 3, 1, 3)), new Range(1, 2, 1, 2));
+		assert.deepEqual(m.validateRange(new Range(1, 3, 1, 4)), new Range(1, 2, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 3, 1, 5)), new Range(1, 2, 1, 5));
+
+		assert.deepEqual(m.validateRange(new Range(1, 4, 1, 4)), new Range(1, 4, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 4, 1, 5)), new Range(1, 4, 1, 5));
+
+		assert.deepEqual(m.validateRange(new Range(1, 5, 1, 5)), new Range(1, 5, 1, 5));
+	});
+
+	test('validateRange around high-low surrogate pairs 2', () => {
+
+		let m = TextModel.createFromString('a📚📚b');
+
+		assert.deepEqual(m.validateRange(new Range(0, 0, 0, 1)), new Range(1, 1, 1, 1));
+		assert.deepEqual(m.validateRange(new Range(0, 0, 0, 7)), new Range(1, 1, 1, 1));
+
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 1)), new Range(1, 1, 1, 1));
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 2)), new Range(1, 1, 1, 2));
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 3)), new Range(1, 1, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 4)), new Range(1, 1, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 5)), new Range(1, 1, 1, 6));
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 6)), new Range(1, 1, 1, 6));
+		assert.deepEqual(m.validateRange(new Range(1, 1, 1, 7)), new Range(1, 1, 1, 7));
+
+		assert.deepEqual(m.validateRange(new Range(1, 2, 1, 2)), new Range(1, 2, 1, 2));
+		assert.deepEqual(m.validateRange(new Range(1, 2, 1, 3)), new Range(1, 2, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 2, 1, 4)), new Range(1, 2, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 2, 1, 5)), new Range(1, 2, 1, 6));
+		assert.deepEqual(m.validateRange(new Range(1, 2, 1, 6)), new Range(1, 2, 1, 6));
+		assert.deepEqual(m.validateRange(new Range(1, 2, 1, 7)), new Range(1, 2, 1, 7));
+
+		assert.deepEqual(m.validateRange(new Range(1, 3, 1, 3)), new Range(1, 2, 1, 2));
+		assert.deepEqual(m.validateRange(new Range(1, 3, 1, 4)), new Range(1, 2, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 3, 1, 5)), new Range(1, 2, 1, 6));
+		assert.deepEqual(m.validateRange(new Range(1, 3, 1, 6)), new Range(1, 2, 1, 6));
+		assert.deepEqual(m.validateRange(new Range(1, 3, 1, 7)), new Range(1, 2, 1, 7));
+
+		assert.deepEqual(m.validateRange(new Range(1, 4, 1, 4)), new Range(1, 4, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 4, 1, 5)), new Range(1, 4, 1, 6));
+		assert.deepEqual(m.validateRange(new Range(1, 4, 1, 6)), new Range(1, 4, 1, 6));
+		assert.deepEqual(m.validateRange(new Range(1, 4, 1, 7)), new Range(1, 4, 1, 7));
+
+		assert.deepEqual(m.validateRange(new Range(1, 5, 1, 5)), new Range(1, 4, 1, 4));
+		assert.deepEqual(m.validateRange(new Range(1, 5, 1, 6)), new Range(1, 4, 1, 6));
+		assert.deepEqual(m.validateRange(new Range(1, 5, 1, 7)), new Range(1, 4, 1, 7));
+
+		assert.deepEqual(m.validateRange(new Range(1, 6, 1, 6)), new Range(1, 6, 1, 6));
+		assert.deepEqual(m.validateRange(new Range(1, 6, 1, 7)), new Range(1, 6, 1, 7));
+
+		assert.deepEqual(m.validateRange(new Range(1, 7, 1, 7)), new Range(1, 7, 1, 7));
+	});
+
+	test('modifyPosition', () => {
+
+		var m = TextModel.createFromString('line one\nline two');
+		assert.deepEqual(m.modifyPosition(new Position(1, 1), 0), new Position(1, 1));
+		assert.deepEqual(m.modifyPosition(new Position(0, 0), 0), new Position(1, 1));
+		assert.deepEqual(m.modifyPosition(new Position(30, 1), 0), new Position(2, 9));
+
+		assert.deepEqual(m.modifyPosition(new Position(1, 1), 17), new Position(2, 9));
+		assert.deepEqual(m.modifyPosition(new Position(1, 1), 1), new Position(1, 2));
+		assert.deepEqual(m.modifyPosition(new Position(1, 1), 3), new Position(1, 4));
+		assert.deepEqual(m.modifyPosition(new Position(1, 2), 10), new Position(2, 3));
+		assert.deepEqual(m.modifyPosition(new Position(1, 5), 13), new Position(2, 9));
+		assert.deepEqual(m.modifyPosition(new Position(1, 2), 16), new Position(2, 9));
+
+		assert.deepEqual(m.modifyPosition(new Position(2, 9), -17), new Position(1, 1));
+		assert.deepEqual(m.modifyPosition(new Position(1, 2), -1), new Position(1, 1));
+		assert.deepEqual(m.modifyPosition(new Position(1, 4), -3), new Position(1, 1));
+		assert.deepEqual(m.modifyPosition(new Position(2, 3), -10), new Position(1, 2));
+		assert.deepEqual(m.modifyPosition(new Position(2, 9), -13), new Position(1, 5));
+		assert.deepEqual(m.modifyPosition(new Position(2, 9), -16), new Position(1, 2));
+
+		assert.deepEqual(m.modifyPosition(new Position(1, 2), 17), new Position(2, 9));
+		assert.deepEqual(m.modifyPosition(new Position(1, 2), 100), new Position(2, 9));
+
+		assert.deepEqual(m.modifyPosition(new Position(1, 2), -2), new Position(1, 1));
+		assert.deepEqual(m.modifyPosition(new Position(1, 2), -100), new Position(1, 1));
+		assert.deepEqual(m.modifyPosition(new Position(2, 2), -100), new Position(1, 1));
+		assert.deepEqual(m.modifyPosition(new Position(2, 9), -18), new Position(1, 1));
+	});
+
+	test('normalizeIndentation 1', () => {
+		let model = TextModel.createFromString('',
+			{
+				detectIndentation: false,
+				tabSize: 4,
+				insertSpaces: false,
+				trimAutoWhitespace: true,
+				defaultEOL: DefaultEndOfLine.LF
+			}
+		);
+
+		assert.equal(model.normalizeIndentation('\t'), '\t');
+		assert.equal(model.normalizeIndentation('    '), '\t');
+		assert.equal(model.normalizeIndentation('   '), '   ');
+		assert.equal(model.normalizeIndentation('  '), '  ');
+		assert.equal(model.normalizeIndentation(' '), ' ');
+		assert.equal(model.normalizeIndentation(''), '');
+		assert.equal(model.normalizeIndentation(' \t   '), '\t\t');
+		assert.equal(model.normalizeIndentation(' \t  '), '\t   ');
+		assert.equal(model.normalizeIndentation(' \t '), '\t  ');
+		assert.equal(model.normalizeIndentation(' \t'), '\t ');
+
+		assert.equal(model.normalizeIndentation('\ta'), '\ta');
+		assert.equal(model.normalizeIndentation('    a'), '\ta');
+		assert.equal(model.normalizeIndentation('   a'), '   a');
+		assert.equal(model.normalizeIndentation('  a'), '  a');
+		assert.equal(model.normalizeIndentation(' a'), ' a');
+		assert.equal(model.normalizeIndentation('a'), 'a');
+		assert.equal(model.normalizeIndentation(' \t   a'), '\t\ta');
+		assert.equal(model.normalizeIndentation(' \t  a'), '\t   a');
+		assert.equal(model.normalizeIndentation(' \t a'), '\t  a');
+		assert.equal(model.normalizeIndentation(' \ta'), '\t a');
+
+		model.dispose();
+	});
+
+	test('normalizeIndentation 2', () => {
+		let model = TextModel.createFromString('',
+			{
+				detectIndentation: false,
+				tabSize: 4,
+				insertSpaces: true,
+				trimAutoWhitespace: true,
+				defaultEOL: DefaultEndOfLine.LF
+			}
+		);
+
+		assert.equal(model.normalizeIndentation('\ta'), '    a');
+		assert.equal(model.normalizeIndentation('    a'), '    a');
+		assert.equal(model.normalizeIndentation('   a'), '   a');
+		assert.equal(model.normalizeIndentation('  a'), '  a');
+		assert.equal(model.normalizeIndentation(' a'), ' a');
+		assert.equal(model.normalizeIndentation('a'), 'a');
+		assert.equal(model.normalizeIndentation(' \t   a'), '        a');
+		assert.equal(model.normalizeIndentation(' \t  a'), '       a');
+		assert.equal(model.normalizeIndentation(' \t a'), '      a');
+		assert.equal(model.normalizeIndentation(' \ta'), '     a');
+
+		model.dispose();
+	});
+});
+
+suite('TextModel.mightContainRTL', () => {
+
+	test('nope', () => {
+		let model = TextModel.createFromString('hello world!');
+		assert.equal(model.mightContainRTL(), false);
+	});
+
+	test('yes', () => {
+		let model = TextModel.createFromString('Hello,\nזוהי עובדה מבוססת שדעתו');
+		assert.equal(model.mightContainRTL(), true);
+	});
+
+	test('setValue resets 1', () => {
+		let model = TextModel.createFromString('hello world!');
+		assert.equal(model.mightContainRTL(), false);
+		model.setValue('Hello,\nזוהי עובדה מבוססת שדעתו');
+		assert.equal(model.mightContainRTL(), true);
+	});
+
+	test('setValue resets 2', () => {
+		let model = TextModel.createFromString('Hello,\nهناك حقيقة مثبتة منذ زمن طويل');
+		assert.equal(model.mightContainRTL(), true);
+		model.setValue('hello world!');
+		assert.equal(model.mightContainRTL(), false);
+	});
+
+});
+
+suite('TextModel.getLineIndentGuide', () => {
+	function assertIndentGuides(lines: [number, string][]): void {
+		let text = lines.map(l => l[1]).join('\n');
+		let model = TextModel.createFromString(text);
+
+		let actual: [number, string][] = [];
+		for (let line = 1; line <= model.getLineCount(); line++) {
+			actual[line - 1] = [model.getLineIndentGuide(line), model.getLineContent(line)];
+		}
+
+		// let expected = lines.map(l => l[0]);
+
+		assert.deepEqual(actual, lines);
+
+		model.dispose();
+	}
+
+	test('getLineIndentGuide one level', () => {
+		assertIndentGuides([
+			[0, 'A'],
+			[1, '  A'],
+			[1, '  A'],
+			[1, '  A'],
+		]);
+	});
+
+	test('getLineIndentGuide two levels', () => {
+		assertIndentGuides([
+			[0, 'A'],
+			[1, '  A'],
+			[1, '  A'],
+			[1, '    A'],
+			[1, '    A'],
+		]);
+	});
+
+	test('getLineIndentGuide three levels', () => {
+		assertIndentGuides([
+			[0, 'A'],
+			[1, '  A'],
+			[1, '    A'],
+			[2, '      A'],
+			[0, 'A'],
+		]);
+	});
+
+	test('getLineIndentGuide decreasing indent', () => {
+		assertIndentGuides([
+			[0, '    A'],
+			[0, '  A'],
+			[0, 'A'],
+		]);
+	});
+
+	test('getLineIndentGuide Java', () => {
+		assertIndentGuides([
+			/* 1*/[0, 'class A {'],
+			/* 2*/[1, '  void foo() {'],
+			/* 3*/[1, '    console.log(1);'],
+			/* 4*/[1, '    console.log(2);'],
+			/* 5*/[1, '  }'],
+			/* 6*/[1, ''],
+			/* 7*/[1, '  void bar() {'],
+			/* 8*/[1, '    console.log(3);'],
+			/* 9*/[1, '  }'],
+			/*10*/[0, '}'],
+			/*11*/[0, 'interface B {'],
+			/*12*/[1, '  void bar();'],
+			/*13*/[0, '}'],
+		]);
+	});
+
+	test('getLineIndentGuide Javadoc', () => {
+		assertIndentGuides([
+			[0, '/**'],
+			[1, ' * Comment'],
+			[1, ' */'],
+			[0, 'class A {'],
+			[1, '  void foo() {'],
+			[1, '  }'],
+			[0, '}'],
+		]);
+	});
+
+	test('getLineIndentGuide Whitespace', () => {
+		assertIndentGuides([
+			[0, 'class A {'],
+			[1, ''],
+			[1, '  void foo() {'],
+			[1, '     '],
+			[1, '     return 1;'],
+			[1, '  }'],
+			[1, '      '],
+			[0, '}'],
+		]);
+	});
+
+	test('getLineIndentGuide Tabs', () => {
+		assertIndentGuides([
+			[0, 'class A {'],
+			[1, '\t\t'],
+			[1, '\tvoid foo() {'],
+			[2, '\t \t//hello'],
+			[2, '\t    return 2;'],
+			[1, '  \t}'],
+			[1, '      '],
+			[0, '}'],
+		]);
+	});
+
+	test('getLineIndentGuide checker.ts', () => {
+		assertIndentGuides([
+			/* 1*/[0, '/// <reference path="binder.ts"/>'],
+			/* 2*/[0, ''],
+			/* 3*/[0, '/* @internal */'],
+			/* 4*/[0, 'namespace ts {'],
+			/* 5*/[1, '    let nextSymbolId = 1;'],
+			/* 6*/[1, '    let nextNodeId = 1;'],
+			/* 7*/[1, '    let nextMergeId = 1;'],
+			/* 8*/[1, '    let nextFlowId = 1;'],
+			/* 9*/[1, ''],
+			/*10*/[1, '    export function getNodeId(node: Node): number {'],
+			/*11*/[2, '        if (!node.id) {'],
+			/*12*/[3, '            node.id = nextNodeId;'],
+			/*13*/[3, '            nextNodeId++;'],
+			/*14*/[2, '        }'],
+			/*15*/[2, '        return node.id;'],
+			/*16*/[1, '    }'],
+			/*17*/[0, '}'],
+		]);
+	});
+
+	test('issue #8425 - Missing indentation lines for first level indentation', () => {
+		assertIndentGuides([
+			[1, '\tindent1'],
+			[2, '\t\tindent2'],
+			[2, '\t\tindent2'],
+			[1, '\tindent1'],
+		]);
+	});
+
+	test('issue #8952 - Indentation guide lines going through text on .yml file', () => {
+		assertIndentGuides([
+			[0, 'properties:'],
+			[1, '    emailAddress:'],
+			[2, '        - bla'],
+			[2, '        - length:'],
+			[3, '            max: 255'],
+			[0, 'getters:'],
+		]);
+	});
+
+	test('issue #11892 - Indent guides look funny', () => {
+		assertIndentGuides([
+			[0, 'function test(base) {'],
+			[1, '\tswitch (base) {'],
+			[2, '\t\tcase 1:'],
+			[3, '\t\t\treturn 1;'],
+			[2, '\t\tcase 2:'],
+			[3, '\t\t\treturn 2;'],
+			[1, '\t}'],
+			[0, '}'],
+		]);
+	});
+
+	test('issue #12398 - Problem in indent guidelines', () => {
+		assertIndentGuides([
+			[2, '\t\t.bla'],
+			[3, '\t\t\tlabel(for)'],
+			[0, 'include script'],
+		]);
+	});
+});

src/vs/editor/test/common/model/textModelSearch.test.ts

@@ -0,0 +1,636 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as assert from 'assert';
+import { Position } from 'vs/editor/common/core/position';
+import { FindMatch, EndOfLineSequence } from 'vs/editor/common/editorCommon';
+import { Range } from 'vs/editor/common/core/range';
+import { TextModel } from 'vs/editor/common/model/textModel';
+import { TextModelSearch, SearchParams, SearchData } from 'vs/editor/common/model/textModelSearch';
+import { getMapForWordSeparators } from 'vs/editor/common/controller/wordCharacterClassifier';
+import { USUAL_WORD_SEPARATORS } from 'vs/editor/common/model/wordHelper';
+
+// --------- Find
+suite('TextModelSearch', () => {
+
+	const usualWordSeparators = getMapForWordSeparators(USUAL_WORD_SEPARATORS);
+
+	function assertFindMatch(actual: FindMatch, expectedRange: Range, expectedMatches: string[] = null): void {
+		assert.deepEqual(actual, new FindMatch(expectedRange, expectedMatches));
+	}
+
+	function _assertFindMatches(model: TextModel, searchParams: SearchParams, expectedMatches: FindMatch[]): void {
+		let actual = TextModelSearch.findMatches(model, searchParams, model.getFullModelRange(), false, 1000);
+		assert.deepEqual(actual, expectedMatches, 'findMatches OK');
+
+		// test `findNextMatch`
+		let startPos = new Position(1, 1);
+		let match = TextModelSearch.findNextMatch(model, searchParams, startPos, false);
+		assert.deepEqual(match, expectedMatches[0], `findNextMatch ${startPos}`);
+		for (let i = 0; i < expectedMatches.length; i++) {
+			startPos = expectedMatches[i].range.getStartPosition();
+			match = TextModelSearch.findNextMatch(model, searchParams, startPos, false);
+			assert.deepEqual(match, expectedMatches[i], `findNextMatch ${startPos}`);
+		}
+
+		// test `findPrevMatch`
+		startPos = new Position(model.getLineCount(), model.getLineMaxColumn(model.getLineCount()));
+		match = TextModelSearch.findPreviousMatch(model, searchParams, startPos, false);
+		assert.deepEqual(match, expectedMatches[expectedMatches.length - 1], `findPrevMatch ${startPos}`);
+		for (let i = 0; i < expectedMatches.length; i++) {
+			startPos = expectedMatches[i].range.getEndPosition();
+			match = TextModelSearch.findPreviousMatch(model, searchParams, startPos, false);
+			assert.deepEqual(match, expectedMatches[i], `findPrevMatch ${startPos}`);
+		}
+	}
+
+	function assertFindMatches(text: string, searchString: string, isRegex: boolean, matchCase: boolean, wordSeparators: string, _expected: [number, number, number, number][]): void {
+		let expectedRanges = _expected.map(entry => new Range(entry[0], entry[1], entry[2], entry[3]));
+		let expectedMatches = expectedRanges.map(entry => new FindMatch(entry, null));
+		let searchParams = new SearchParams(searchString, isRegex, matchCase, wordSeparators);
+
+		let model = TextModel.createFromString(text);
+		_assertFindMatches(model, searchParams, expectedMatches);
+		model.dispose();
+
+
+		let model2 = TextModel.createFromString(text);
+		model2.setEOL(EndOfLineSequence.CRLF);
+		_assertFindMatches(model2, searchParams, expectedMatches);
+		model2.dispose();
+	}
+
+	let regularText = [
+		'This is some foo - bar text which contains foo and bar - as in Barcelona.',
+		'Now it begins a word fooBar and now it is caps Foo-isn\'t this great?',
+		'And here\'s a dull line with nothing interesting in it',
+		'It is also interesting if it\'s part of a word like amazingFooBar',
+		'Again nothing interesting here'
+	];
+
+	test('Simple find', () => {
+		assertFindMatches(
+			regularText.join('\n'),
+			'foo', false, false, null,
+			[
+				[1, 14, 1, 17],
+				[1, 44, 1, 47],
+				[2, 22, 2, 25],
+				[2, 48, 2, 51],
+				[4, 59, 4, 62]
+			]
+		);
+	});
+
+	test('Case sensitive find', () => {
+		assertFindMatches(
+			regularText.join('\n'),
+			'foo', false, true, null,
+			[
+				[1, 14, 1, 17],
+				[1, 44, 1, 47],
+				[2, 22, 2, 25]
+			]
+		);
+	});
+
+	test('Whole words find', () => {
+		assertFindMatches(
+			regularText.join('\n'),
+			'foo', false, false, USUAL_WORD_SEPARATORS,
+			[
+				[1, 14, 1, 17],
+				[1, 44, 1, 47],
+				[2, 48, 2, 51]
+			]
+		);
+	});
+
+	test('/^/ find', () => {
+		assertFindMatches(
+			regularText.join('\n'),
+			'^', true, false, null,
+			[
+				[1, 1, 1, 1],
+				[2, 1, 2, 1],
+				[3, 1, 3, 1],
+				[4, 1, 4, 1],
+				[5, 1, 5, 1]
+			]
+		);
+	});
+
+	test('/$/ find', () => {
+		assertFindMatches(
+			regularText.join('\n'),
+			'$', true, false, null,
+			[
+				[1, 74, 1, 74],
+				[2, 69, 2, 69],
+				[3, 54, 3, 54],
+				[4, 65, 4, 65],
+				[5, 31, 5, 31]
+			]
+		);
+	});
+
+	test('/.*/ find', () => {
+		assertFindMatches(
+			regularText.join('\n'),
+			'.*', true, false, null,
+			[
+				[1, 1, 1, 74],
+				[2, 1, 2, 69],
+				[3, 1, 3, 54],
+				[4, 1, 4, 65],
+				[5, 1, 5, 31]
+			]
+		);
+	});
+
+	test('/^$/ find', () => {
+		assertFindMatches(
+			[
+				'This is some foo - bar text which contains foo and bar - as in Barcelona.',
+				'',
+				'And here\'s a dull line with nothing interesting in it',
+				'',
+				'Again nothing interesting here'
+			].join('\n'),
+			'^$', true, false, null,
+			[
+				[2, 1, 2, 1],
+				[4, 1, 4, 1]
+			]
+		);
+	});
+
+	test('multiline find 1', () => {
+		assertFindMatches(
+			[
+				'Just some text text',
+				'Just some text text',
+				'some text again',
+				'again some text'
+			].join('\n'),
+			'text\\n', true, false, null,
+			[
+				[1, 16, 2, 1],
+				[2, 16, 3, 1],
+			]
+		);
+	});
+
+	test('multiline find 2', () => {
+		assertFindMatches(
+			[
+				'Just some text text',
+				'Just some text text',
+				'some text again',
+				'again some text'
+			].join('\n'),
+			'text\\nJust', true, false, null,
+			[
+				[1, 16, 2, 5]
+			]
+		);
+	});
+
+	test('multiline find 3', () => {
+		assertFindMatches(
+			[
+				'Just some text text',
+				'Just some text text',
+				'some text again',
+				'again some text'
+			].join('\n'),
+			'\\nagain', true, false, null,
+			[
+				[3, 16, 4, 6]
+			]
+		);
+	});
+
+	test('multiline find 4', () => {
+		assertFindMatches(
+			[
+				'Just some text text',
+				'Just some text text',
+				'some text again',
+				'again some text'
+			].join('\n'),
+			'.*\\nJust.*\\n', true, false, null,
+			[
+				[1, 1, 3, 1]
+			]
+		);
+	});
+
+	test('multiline find with line beginning regex', () => {
+		assertFindMatches(
+			[
+				'if',
+				'else',
+				'',
+				'if',
+				'else'
+			].join('\n'),
+			'^if\\nelse', true, false, null,
+			[
+				[1, 1, 2, 5],
+				[4, 1, 5, 5]
+			]
+		);
+	});
+
+	test('matching empty lines using boundary expression', () => {
+		assertFindMatches(
+			[
+				'if',
+				'',
+				'else',
+				'  ',
+				'if',
+				' ',
+				'else'
+			].join('\n'),
+			'^\\s*$\\n', true, false, null,
+			[
+				[2, 1, 3, 1],
+				[4, 1, 5, 1],
+				[6, 1, 7, 1]
+			]
+		);
+	});
+
+	test('matching lines starting with A and ending with B', () => {
+		assertFindMatches(
+			[
+				'a if b',
+				'a',
+				'ab',
+				'eb'
+			].join('\n'),
+			'^a.*b$', true, false, null,
+			[
+				[1, 1, 1, 7],
+				[3, 1, 3, 3]
+			]
+		);
+	});
+
+	test('multiline find with line ending regex', () => {
+		assertFindMatches(
+			[
+				'if',
+				'else',
+				'',
+				'if',
+				'elseif',
+				'else'
+			].join('\n'),
+			'if\\nelse$', true, false, null,
+			[
+				[1, 1, 2, 5],
+				[5, 5, 6, 5]
+			]
+		);
+	});
+
+	test('issue #4836 - ^.*$', () => {
+		assertFindMatches(
+			[
+				'Just some text text',
+				'',
+				'some text again',
+				'',
+				'again some text'
+			].join('\n'),
+			'^.*$', true, false, null,
+			[
+				[1, 1, 1, 20],
+				[2, 1, 2, 1],
+				[3, 1, 3, 16],
+				[4, 1, 4, 1],
+				[5, 1, 5, 16],
+			]
+		);
+	});
+
+	test('multiline find for non-regex string', () => {
+		assertFindMatches(
+			[
+				'Just some text text',
+				'some text text',
+				'some text again',
+				'again some text',
+				'but not some'
+			].join('\n'),
+			'text\nsome', false, false, null,
+			[
+				[1, 16, 2, 5],
+				[2, 11, 3, 5],
+			]
+		);
+	});
+
+	test('issue #3623: Match whole word does not work for not latin characters', () => {
+		assertFindMatches(
+			[
+				'я',
+				'компилятор',
+				'обфускация',
+				':я-я'
+			].join('\n'),
+			'я', false, false, USUAL_WORD_SEPARATORS,
+			[
+				[1, 1, 1, 2],
+				[4, 2, 4, 3],
+				[4, 4, 4, 5],
+			]
+		);
+	});
+
+	test('issue #27459: Match whole words regression', () => {
+		assertFindMatches(
+			[
+				'this._register(this._textAreaInput.onKeyDown((e: IKeyboardEvent) => {',
+				'	this._viewController.emitKeyDown(e);',
+				'}));',
+			].join('\n'),
+			'((e: ', false, false, USUAL_WORD_SEPARATORS,
+			[
+				[1, 45, 1, 50]
+			]
+		);
+	});
+
+	test('issue #27594: Search results disappear', () => {
+		assertFindMatches(
+			[
+				'this.server.listen(0);',
+			].join('\n'),
+			'listen(', false, false, USUAL_WORD_SEPARATORS,
+			[
+				[1, 13, 1, 20]
+			]
+		);
+	});
+
+	test('findNextMatch without regex', () => {
+		let model = TextModel.createFromString('line line one\nline two\nthree');
+
+		let searchParams = new SearchParams('line', false, false, null);
+
+		let actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), false);
+		assertFindMatch(actual, new Range(1, 1, 1, 5));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, actual.range.getEndPosition(), false);
+		assertFindMatch(actual, new Range(1, 6, 1, 10));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 3), false);
+		assertFindMatch(actual, new Range(1, 6, 1, 10));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, actual.range.getEndPosition(), false);
+		assertFindMatch(actual, new Range(2, 1, 2, 5));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, actual.range.getEndPosition(), false);
+		assertFindMatch(actual, new Range(1, 1, 1, 5));
+
+		model.dispose();
+	});
+
+	test('findNextMatch with beginning boundary regex', () => {
+		let model = TextModel.createFromString('line one\nline two\nthree');
+
+		let searchParams = new SearchParams('^line', true, false, null);
+
+		let actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), false);
+		assertFindMatch(actual, new Range(1, 1, 1, 5));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, actual.range.getEndPosition(), false);
+		assertFindMatch(actual, new Range(2, 1, 2, 5));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 3), false);
+		assertFindMatch(actual, new Range(2, 1, 2, 5));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, actual.range.getEndPosition(), false);
+		assertFindMatch(actual, new Range(1, 1, 1, 5));
+
+		model.dispose();
+	});
+
+	test('findNextMatch with beginning boundary regex and line has repetitive beginnings', () => {
+		let model = TextModel.createFromString('line line one\nline two\nthree');
+
+		let searchParams = new SearchParams('^line', true, false, null);
+
+		let actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), false);
+		assertFindMatch(actual, new Range(1, 1, 1, 5));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, actual.range.getEndPosition(), false);
+		assertFindMatch(actual, new Range(2, 1, 2, 5));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 3), false);
+		assertFindMatch(actual, new Range(2, 1, 2, 5));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, actual.range.getEndPosition(), false);
+		assertFindMatch(actual, new Range(1, 1, 1, 5));
+
+		model.dispose();
+	});
+
+	test('findNextMatch with beginning boundary multiline regex and line has repetitive beginnings', () => {
+		let model = TextModel.createFromString('line line one\nline two\nline three\nline four');
+
+		let searchParams = new SearchParams('^line.*\\nline', true, false, null);
+
+		let actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), false);
+		assertFindMatch(actual, new Range(1, 1, 2, 5));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, actual.range.getEndPosition(), false);
+		assertFindMatch(actual, new Range(3, 1, 4, 5));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, new Position(2, 1), false);
+		assertFindMatch(actual, new Range(2, 1, 3, 5));
+
+		model.dispose();
+	});
+
+	test('findNextMatch with ending boundary regex', () => {
+		let model = TextModel.createFromString('one line line\ntwo line\nthree');
+
+		let searchParams = new SearchParams('line$', true, false, null);
+
+		let actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), false);
+		assertFindMatch(actual, new Range(1, 10, 1, 14));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 4), false);
+		assertFindMatch(actual, new Range(1, 10, 1, 14));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, actual.range.getEndPosition(), false);
+		assertFindMatch(actual, new Range(2, 5, 2, 9));
+
+		actual = TextModelSearch.findNextMatch(model, searchParams, actual.range.getEndPosition(), false);
+		assertFindMatch(actual, new Range(1, 10, 1, 14));
+
+		model.dispose();
+	});
+
+	test('findMatches with capturing matches', () => {
+		let model = TextModel.createFromString('one line line\ntwo line\nthree');
+
+		let searchParams = new SearchParams('(l(in)e)', true, false, null);
+
+		let actual = TextModelSearch.findMatches(model, searchParams, model.getFullModelRange(), true, 100);
+		assert.deepEqual(actual, [
+			new FindMatch(new Range(1, 5, 1, 9), ['line', 'line', 'in']),
+			new FindMatch(new Range(1, 10, 1, 14), ['line', 'line', 'in']),
+			new FindMatch(new Range(2, 5, 2, 9), ['line', 'line', 'in']),
+		]);
+
+		model.dispose();
+	});
+
+	test('findMatches multiline with capturing matches', () => {
+		let model = TextModel.createFromString('one line line\ntwo line\nthree');
+
+		let searchParams = new SearchParams('(l(in)e)\\n', true, false, null);
+
+		let actual = TextModelSearch.findMatches(model, searchParams, model.getFullModelRange(), true, 100);
+		assert.deepEqual(actual, [
+			new FindMatch(new Range(1, 10, 2, 1), ['line\n', 'line', 'in']),
+			new FindMatch(new Range(2, 5, 3, 1), ['line\n', 'line', 'in']),
+		]);
+
+		model.dispose();
+	});
+
+	test('findNextMatch with capturing matches', () => {
+		let model = TextModel.createFromString('one line line\ntwo line\nthree');
+
+		let searchParams = new SearchParams('(l(in)e)', true, false, null);
+
+		let actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), true);
+		assertFindMatch(actual, new Range(1, 5, 1, 9), ['line', 'line', 'in']);
+
+		model.dispose();
+	});
+
+	test('findNextMatch multiline with capturing matches', () => {
+		let model = TextModel.createFromString('one line line\ntwo line\nthree');
+
+		let searchParams = new SearchParams('(l(in)e)\\n', true, false, null);
+
+		let actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), true);
+		assertFindMatch(actual, new Range(1, 10, 2, 1), ['line\n', 'line', 'in']);
+
+		model.dispose();
+	});
+
+	test('findPreviousMatch with capturing matches', () => {
+		let model = TextModel.createFromString('one line line\ntwo line\nthree');
+
+		let searchParams = new SearchParams('(l(in)e)', true, false, null);
+
+		let actual = TextModelSearch.findPreviousMatch(model, searchParams, new Position(1, 1), true);
+		assertFindMatch(actual, new Range(2, 5, 2, 9), ['line', 'line', 'in']);
+
+		model.dispose();
+	});
+
+	test('findPreviousMatch multiline with capturing matches', () => {
+		let model = TextModel.createFromString('one line line\ntwo line\nthree');
+
+		let searchParams = new SearchParams('(l(in)e)\\n', true, false, null);
+
+		let actual = TextModelSearch.findPreviousMatch(model, searchParams, new Position(1, 1), true);
+		assertFindMatch(actual, new Range(2, 5, 3, 1), ['line\n', 'line', 'in']);
+
+		model.dispose();
+	});
+
+	test('\\n matches \\r\\n', () => {
+		let model = TextModel.createFromString('a\r\nb\r\nc\r\nd\r\ne\r\nf\r\ng\r\nh\r\ni');
+
+		assert.equal(model.getEOL(), '\r\n');
+
+		let searchParams = new SearchParams('h\\n', true, false, null);
+		let actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), true);
+		actual = TextModelSearch.findMatches(model, searchParams, model.getFullModelRange(), true, 1000)[0];
+		assertFindMatch(actual, new Range(8, 1, 9, 1), ['h\n']);
+
+		searchParams = new SearchParams('g\\nh\\n', true, false, null);
+		actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), true);
+		actual = TextModelSearch.findMatches(model, searchParams, model.getFullModelRange(), true, 1000)[0];
+		assertFindMatch(actual, new Range(7, 1, 9, 1), ['g\nh\n']);
+
+		searchParams = new SearchParams('\\ni', true, false, null);
+		actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), true);
+		actual = TextModelSearch.findMatches(model, searchParams, model.getFullModelRange(), true, 1000)[0];
+		assertFindMatch(actual, new Range(8, 2, 9, 2), ['\ni']);
+
+		model.dispose();
+	});
+
+	test('\\r can never be found', () => {
+		let model = TextModel.createFromString('a\r\nb\r\nc\r\nd\r\ne\r\nf\r\ng\r\nh\r\ni');
+
+		assert.equal(model.getEOL(), '\r\n');
+
+		let searchParams = new SearchParams('\\r\\n', true, false, null);
+		let actual = TextModelSearch.findNextMatch(model, searchParams, new Position(1, 1), true);
+		assert.equal(actual, null);
+		assert.deepEqual(TextModelSearch.findMatches(model, searchParams, model.getFullModelRange(), true, 1000), []);
+
+		model.dispose();
+	});
+
+	function assertParseSearchResult(searchString: string, isRegex: boolean, matchCase: boolean, wordSeparators: string, expected: SearchData): void {
+		let searchParams = new SearchParams(searchString, isRegex, matchCase, wordSeparators);
+		let actual = searchParams.parseSearchRequest();
+
+		if (expected === null) {
+			assert.ok(actual === null);
+		} else {
+			assert.deepEqual(actual.regex, expected.regex);
+			assert.deepEqual(actual.simpleSearch, expected.simpleSearch);
+			if (wordSeparators) {
+				assert.ok(actual.wordSeparators !== null);
+			} else {
+				assert.ok(actual.wordSeparators === null);
+			}
+		}
+	}
+
+	test('parseSearchRequest invalid', () => {
+		assertParseSearchResult('', true, true, USUAL_WORD_SEPARATORS, null);
+		assertParseSearchResult(null, true, true, USUAL_WORD_SEPARATORS, null);
+		assertParseSearchResult('(', true, false, null, null);
+	});
+
+	test('parseSearchRequest non regex', () => {
+		assertParseSearchResult('foo', false, false, null, new SearchData(/foo/gi, null, null));
+		assertParseSearchResult('foo', false, false, USUAL_WORD_SEPARATORS, new SearchData(/foo/gi, usualWordSeparators, null));
+		assertParseSearchResult('foo', false, true, null, new SearchData(/foo/g, null, 'foo'));
+		assertParseSearchResult('foo', false, true, USUAL_WORD_SEPARATORS, new SearchData(/foo/g, usualWordSeparators, 'foo'));
+		assertParseSearchResult('foo\\n', false, false, null, new SearchData(/foo\\n/gi, null, null));
+		assertParseSearchResult('foo\\\\n', false, false, null, new SearchData(/foo\\\\n/gi, null, null));
+		assertParseSearchResult('foo\\r', false, false, null, new SearchData(/foo\\r/gi, null, null));
+		assertParseSearchResult('foo\\\\r', false, false, null, new SearchData(/foo\\\\r/gi, null, null));
+	});
+
+	test('parseSearchRequest regex', () => {
+		assertParseSearchResult('foo', true, false, null, new SearchData(/foo/gi, null, null));
+		assertParseSearchResult('foo', true, false, USUAL_WORD_SEPARATORS, new SearchData(/foo/gi, usualWordSeparators, null));
+		assertParseSearchResult('foo', true, true, null, new SearchData(/foo/g, null, null));
+		assertParseSearchResult('foo', true, true, USUAL_WORD_SEPARATORS, new SearchData(/foo/g, usualWordSeparators, null));
+		assertParseSearchResult('foo\\n', true, false, null, new SearchData(/foo\n/gim, null, null));
+		assertParseSearchResult('foo\\\\n', true, false, null, new SearchData(/foo\\n/gi, null, null));
+		assertParseSearchResult('foo\\r', true, false, null, new SearchData(/foo\r/gim, null, null));
+		assertParseSearchResult('foo\\\\r', true, false, null, new SearchData(/foo\\r/gi, null, null));
+	});
+});

src/vs/editor/test/common/model/textModelWithTokens.test.ts

@@ -0,0 +1,361 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as assert from 'assert';
+import { Model } from 'vs/editor/common/model/model';
+import { IDisposable } from 'vs/base/common/lifecycle';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+import { ITokenizationSupport, TokenizationRegistry, LanguageId, LanguageIdentifier, MetadataConsts } from 'vs/editor/common/modes';
+import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration';
+import { Range } from 'vs/editor/common/core/range';
+import { Position } from 'vs/editor/common/core/position';
+import { IFoundBracket } from 'vs/editor/common/editorCommon';
+import { TextModel } from 'vs/editor/common/model/textModel';
+import { TextModelWithTokens } from 'vs/editor/common/model/textModelWithTokens';
+import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
+import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
+import { TokenizationResult2 } from 'vs/editor/common/core/token';
+import { RawTextSource } from 'vs/editor/common/model/textSource';
+
+suite('TextModelWithTokens', () => {
+
+	function testBrackets(contents: string[], brackets: CharacterPair[]): void {
+		function toRelaxedFoundBracket(a: IFoundBracket) {
+			if (!a) {
+				return null;
+			}
+			return {
+				range: a.range.toString(),
+				open: a.open,
+				close: a.close,
+				isOpen: a.isOpen
+			};
+		}
+
+		let charIsBracket: { [char: string]: boolean } = {};
+		let charIsOpenBracket: { [char: string]: boolean } = {};
+		let openForChar: { [char: string]: string } = {};
+		let closeForChar: { [char: string]: string } = {};
+		brackets.forEach((b) => {
+			charIsBracket[b[0]] = true;
+			charIsBracket[b[1]] = true;
+
+			charIsOpenBracket[b[0]] = true;
+			charIsOpenBracket[b[1]] = false;
+
+			openForChar[b[0]] = b[0];
+			closeForChar[b[0]] = b[1];
+
+			openForChar[b[1]] = b[0];
+			closeForChar[b[1]] = b[1];
+		});
+
+		let expectedBrackets: IFoundBracket[] = [];
+		for (let lineIndex = 0; lineIndex < contents.length; lineIndex++) {
+			let lineText = contents[lineIndex];
+
+			for (let charIndex = 0; charIndex < lineText.length; charIndex++) {
+				let ch = lineText.charAt(charIndex);
+				if (charIsBracket[ch]) {
+					expectedBrackets.push({
+						open: openForChar[ch],
+						close: closeForChar[ch],
+						isOpen: charIsOpenBracket[ch],
+						range: new Range(lineIndex + 1, charIndex + 1, lineIndex + 1, charIndex + 2)
+					});
+				}
+			}
+		}
+
+		const languageIdentifier = new LanguageIdentifier('testMode', LanguageId.PlainText);
+
+		let registration = LanguageConfigurationRegistry.register(languageIdentifier, {
+			brackets: brackets
+		});
+
+		let model = new TextModelWithTokens(
+			RawTextSource.fromString(contents.join('\n')),
+			TextModel.DEFAULT_CREATION_OPTIONS,
+			languageIdentifier
+		);
+
+		// findPrevBracket
+		{
+			let expectedBracketIndex = expectedBrackets.length - 1;
+			let currentExpectedBracket = expectedBracketIndex >= 0 ? expectedBrackets[expectedBracketIndex] : null;
+			for (let lineNumber = contents.length; lineNumber >= 1; lineNumber--) {
+				let lineText = contents[lineNumber - 1];
+
+				for (let column = lineText.length + 1; column >= 1; column--) {
+
+					if (currentExpectedBracket) {
+						if (lineNumber === currentExpectedBracket.range.startLineNumber && column < currentExpectedBracket.range.endColumn) {
+							expectedBracketIndex--;
+							currentExpectedBracket = expectedBracketIndex >= 0 ? expectedBrackets[expectedBracketIndex] : null;
+						}
+					}
+
+					let actual = model.findPrevBracket({
+						lineNumber: lineNumber,
+						column: column
+					});
+
+					assert.deepEqual(toRelaxedFoundBracket(actual), toRelaxedFoundBracket(currentExpectedBracket), 'findPrevBracket of ' + lineNumber + ', ' + column);
+				}
+			}
+		}
+
+		// findNextBracket
+		{
+			let expectedBracketIndex = 0;
+			let currentExpectedBracket = expectedBracketIndex < expectedBrackets.length ? expectedBrackets[expectedBracketIndex] : null;
+			for (let lineNumber = 1; lineNumber <= contents.length; lineNumber++) {
+				let lineText = contents[lineNumber - 1];
+
+				for (let column = 1; column <= lineText.length + 1; column++) {
+
+					if (currentExpectedBracket) {
+						if (lineNumber === currentExpectedBracket.range.startLineNumber && column > currentExpectedBracket.range.startColumn) {
+							expectedBracketIndex++;
+							currentExpectedBracket = expectedBracketIndex < expectedBrackets.length ? expectedBrackets[expectedBracketIndex] : null;
+						}
+					}
+
+					let actual = model.findNextBracket({
+						lineNumber: lineNumber,
+						column: column
+					});
+
+					assert.deepEqual(toRelaxedFoundBracket(actual), toRelaxedFoundBracket(currentExpectedBracket), 'findNextBracket of ' + lineNumber + ', ' + column);
+				}
+			}
+		}
+
+		model.dispose();
+		registration.dispose();
+	}
+
+	test('brackets', () => {
+		testBrackets([
+			'if (a == 3) { return (7 * (a + 5)); }'
+		], [
+				['{', '}'],
+				['[', ']'],
+				['(', ')']
+			]);
+	});
+});
+
+suite('TextModelWithTokens - bracket matching', () => {
+
+	function isNotABracket(model: Model, lineNumber: number, column: number) {
+		let match = model.matchBracket(new Position(lineNumber, column));
+		assert.equal(match, null, 'is not matching brackets at ' + lineNumber + ', ' + column);
+	}
+
+	function isBracket2(model: Model, testPosition: Position, expected: [Range, Range]): void {
+		let actual = model.matchBracket(testPosition);
+		assert.deepEqual(actual, expected, 'matches brackets at ' + testPosition);
+	}
+
+	const languageIdentifier = new LanguageIdentifier('bracketMode1', LanguageId.PlainText);
+	let registration: IDisposable = null;
+
+	setup(() => {
+		registration = LanguageConfigurationRegistry.register(languageIdentifier, {
+			brackets: [
+				['{', '}'],
+				['[', ']'],
+				['(', ')'],
+			]
+		});
+	});
+
+	teardown(() => {
+		registration.dispose();
+		registration = null;
+	});
+
+	test('bracket matching 1', () => {
+		let text =
+			')]}{[(' + '\n' +
+			')]}{[(';
+		let model = Model.createFromString(text, undefined, languageIdentifier);
+
+		isNotABracket(model, 1, 1);
+		isNotABracket(model, 1, 2);
+		isNotABracket(model, 1, 3);
+		isBracket2(model, new Position(1, 4), [new Range(1, 4, 1, 5), new Range(2, 3, 2, 4)]);
+		isBracket2(model, new Position(1, 5), [new Range(1, 5, 1, 6), new Range(2, 2, 2, 3)]);
+		isBracket2(model, new Position(1, 6), [new Range(1, 6, 1, 7), new Range(2, 1, 2, 2)]);
+		isBracket2(model, new Position(1, 7), [new Range(1, 6, 1, 7), new Range(2, 1, 2, 2)]);
+
+		isBracket2(model, new Position(2, 1), [new Range(2, 1, 2, 2), new Range(1, 6, 1, 7)]);
+		isBracket2(model, new Position(2, 2), [new Range(2, 2, 2, 3), new Range(1, 5, 1, 6)]);
+		isBracket2(model, new Position(2, 3), [new Range(2, 3, 2, 4), new Range(1, 4, 1, 5)]);
+		isBracket2(model, new Position(2, 4), [new Range(2, 3, 2, 4), new Range(1, 4, 1, 5)]);
+		isNotABracket(model, 2, 5);
+		isNotABracket(model, 2, 6);
+		isNotABracket(model, 2, 7);
+
+		model.dispose();
+	});
+
+	test('bracket matching 2', () => {
+		let text =
+			'var bar = {' + '\n' +
+			'foo: {' + '\n' +
+			'}, bar: {hallo: [{' + '\n' +
+			'}, {' + '\n' +
+			'}]}}';
+		let model = Model.createFromString(text, undefined, languageIdentifier);
+
+		let brackets: [Position, Range, Range][] = [
+			[new Position(1, 11), new Range(1, 11, 1, 12), new Range(5, 4, 5, 5)],
+			[new Position(1, 12), new Range(1, 11, 1, 12), new Range(5, 4, 5, 5)],
+
+			[new Position(2, 6), new Range(2, 6, 2, 7), new Range(3, 1, 3, 2)],
+			[new Position(2, 7), new Range(2, 6, 2, 7), new Range(3, 1, 3, 2)],
+
+			[new Position(3, 1), new Range(3, 1, 3, 2), new Range(2, 6, 2, 7)],
+			[new Position(3, 2), new Range(3, 1, 3, 2), new Range(2, 6, 2, 7)],
+			[new Position(3, 9), new Range(3, 9, 3, 10), new Range(5, 3, 5, 4)],
+			[new Position(3, 10), new Range(3, 9, 3, 10), new Range(5, 3, 5, 4)],
+			[new Position(3, 17), new Range(3, 17, 3, 18), new Range(5, 2, 5, 3)],
+			[new Position(3, 18), new Range(3, 18, 3, 19), new Range(4, 1, 4, 2)],
+			[new Position(3, 19), new Range(3, 18, 3, 19), new Range(4, 1, 4, 2)],
+
+			[new Position(4, 1), new Range(4, 1, 4, 2), new Range(3, 18, 3, 19)],
+			[new Position(4, 2), new Range(4, 1, 4, 2), new Range(3, 18, 3, 19)],
+			[new Position(4, 4), new Range(4, 4, 4, 5), new Range(5, 1, 5, 2)],
+			[new Position(4, 5), new Range(4, 4, 4, 5), new Range(5, 1, 5, 2)],
+
+			[new Position(5, 1), new Range(5, 1, 5, 2), new Range(4, 4, 4, 5)],
+			[new Position(5, 2), new Range(5, 2, 5, 3), new Range(3, 17, 3, 18)],
+			[new Position(5, 3), new Range(5, 3, 5, 4), new Range(3, 9, 3, 10)],
+			[new Position(5, 4), new Range(5, 4, 5, 5), new Range(1, 11, 1, 12)],
+			[new Position(5, 5), new Range(5, 4, 5, 5), new Range(1, 11, 1, 12)],
+		];
+
+		let isABracket: { [lineNumber: number]: { [col: number]: boolean; }; } = { 1: {}, 2: {}, 3: {}, 4: {}, 5: {} };
+		for (let i = 0, len = brackets.length; i < len; i++) {
+			let [testPos, b1, b2] = brackets[i];
+			isBracket2(model, testPos, [b1, b2]);
+			isABracket[testPos.lineNumber][testPos.column] = true;
+		}
+
+		for (let i = 1, len = model.getLineCount(); i <= len; i++) {
+			let line = model.getLineContent(i);
+			for (let j = 1, lenJ = line.length + 1; j <= lenJ; j++) {
+				if (!isABracket[i].hasOwnProperty(<any>j)) {
+					isNotABracket(model, i, j);
+				}
+			}
+		}
+
+		model.dispose();
+	});
+});
+
+
+suite('TextModelWithTokens regression tests', () => {
+
+	test('Microsoft/monaco-editor#122: Unhandled Exception: TypeError: Unable to get property \'replace\' of undefined or null reference', () => {
+		function assertViewLineTokens(model: Model, lineNumber: number, forceTokenization: boolean, expected: ViewLineToken[]): void {
+			if (forceTokenization) {
+				model.forceTokenization(lineNumber);
+			}
+			let actual = model.getLineTokens(lineNumber).inflate();
+			let decode = (token: ViewLineToken) => {
+				return {
+					endIndex: token.endIndex,
+					foreground: token.getForeground()
+				};
+			};
+			assert.deepEqual(actual.map(decode), expected.map(decode));
+		}
+
+		let _tokenId = 10;
+		const LANG_ID1 = 'indicisiveMode1';
+		const LANG_ID2 = 'indicisiveMode2';
+		const languageIdentifier1 = new LanguageIdentifier(LANG_ID1, 3);
+		const languageIdentifier2 = new LanguageIdentifier(LANG_ID2, 4);
+
+		const tokenizationSupport: ITokenizationSupport = {
+			getInitialState: () => NULL_STATE,
+			tokenize: undefined,
+			tokenize2: (line, state) => {
+				let myId = ++_tokenId;
+				let tokens = new Uint32Array(2);
+				tokens[0] = 0;
+				tokens[1] = (
+					myId << MetadataConsts.FOREGROUND_OFFSET
+				) >>> 0;
+				return new TokenizationResult2(tokens, state);
+			}
+		};
+
+		let registration1 = TokenizationRegistry.register(LANG_ID1, tokenizationSupport);
+		let registration2 = TokenizationRegistry.register(LANG_ID2, tokenizationSupport);
+
+		let model = Model.createFromString('A model with\ntwo lines');
+
+		assertViewLineTokens(model, 1, true, [createViewLineToken(12, 1)]);
+		assertViewLineTokens(model, 2, true, [createViewLineToken(9, 1)]);
+
+		model.setMode(languageIdentifier1);
+
+		assertViewLineTokens(model, 1, true, [createViewLineToken(12, 11)]);
+		assertViewLineTokens(model, 2, true, [createViewLineToken(9, 12)]);
+
+		model.setMode(languageIdentifier2);
+
+		assertViewLineTokens(model, 1, false, [createViewLineToken(12, 1)]);
+		assertViewLineTokens(model, 2, false, [createViewLineToken(9, 1)]);
+
+		model.dispose();
+		registration1.dispose();
+		registration2.dispose();
+
+		function createViewLineToken(endIndex: number, foreground: number): ViewLineToken {
+			let metadata = (
+				(foreground << MetadataConsts.FOREGROUND_OFFSET)
+			) >>> 0;
+			return new ViewLineToken(endIndex, metadata);
+		}
+	});
+
+
+	test('Microsoft/monaco-editor#133: Error: Cannot read property \'modeId\' of undefined', () => {
+
+		const languageIdentifier = new LanguageIdentifier('testMode', LanguageId.PlainText);
+
+		let registration = LanguageConfigurationRegistry.register(languageIdentifier, {
+			brackets: [
+				['module', 'end module'],
+				['sub', 'end sub']
+			]
+		});
+
+		let model = Model.createFromString([
+			'Imports System',
+			'Imports System.Collections.Generic',
+			'',
+			'Module m1',
+			'',
+			'\tSub Main()',
+			'\tEnd Sub',
+			'',
+			'End Module',
+		].join('\n'), undefined, languageIdentifier);
+
+		let actual = model.matchBracket(new Position(4, 1));
+		assert.deepEqual(actual, [new Range(4, 1, 4, 7), new Range(9, 1, 9, 11)]);
+
+		model.dispose();
+		registration.dispose();
+	});
+});