Merge VS Code 1.21 source code (#1067)

* Initial VS Code 1.21 file copy with patches

* A few more merges

* Post npm install

* Fix batch of build breaks

* Fix more build breaks

* Fix more build errors

* Fix more build breaks

* Runtime fixes 1

* Get connection dialog working with some todos

* Fix a few packaging issues

* Copy several node_modules to package build to fix loader issues

* Fix breaks from master

* A few more fixes

* Make tests pass

* First pass of license header updates

* Second pass of license header updates

* Fix restore dialog issues

* Remove add additional themes menu items

* fix select box issues where the list doesn't show up

* formatting

* Fix editor dispose issue

* Copy over node modules to correct location on all platforms

src/vs/editor/common/modes/languageConfigurationRegistry.ts

@@ -10,7 +10,7 @@ import { IOnEnterSupportOptions, OnEnterSupport } from 'vs/editor/common/modes/s
 import { IndentRulesSupport, IndentConsts } from 'vs/editor/common/modes/supports/indentRules';
 import { RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
 import Event, { Emitter } from 'vs/base/common/event';
-import { ITokenizedModel } from 'vs/editor/common/editorCommon';
+import { ITextModel } from 'vs/editor/common/model';
 import { onUnexpectedError } from 'vs/base/common/errors';
 import * as strings from 'vs/base/common/strings';
 import { IDisposable } from 'vs/base/common/lifecycle';
@@ -544,7 +544,7 @@ export class LanguageConfigurationRegistryImpl {
 		return null;
 	}
 
-	public getIndentForEnter(model: ITokenizedModel, range: Range, indentConverter: IIndentConverter, autoIndent: boolean): { beforeEnter: string, afterEnter: string } {
+	public getIndentForEnter(model: ITextModel, range: Range, indentConverter: IIndentConverter, autoIndent: boolean): { beforeEnter: string, afterEnter: string } {
 		model.forceTokenization(range.startLineNumber);
 		let lineTokens = model.getLineTokens(range.startLineNumber);
 
@@ -642,7 +642,7 @@ export class LanguageConfigurationRegistryImpl {
 	 * We should always allow intentional indentation. It means, if users change the indentation of `lineNumber` and the content of
 	 * this line doesn't match decreaseIndentPattern, we should not adjust the indentation.
 	 */
-	public getIndentActionForType(model: ITokenizedModel, range: Range, ch: string, indentConverter: IIndentConverter): string {
+	public getIndentActionForType(model: ITextModel, range: Range, ch: string, indentConverter: IIndentConverter): string {
 		let scopedLineTokens = this.getScopedLineTokens(model, range.startLineNumber, range.startColumn);
 		let indentRulesSupport = this.getIndentRulesSupport(scopedLineTokens.languageId);
 		if (!indentRulesSupport) {
@@ -683,7 +683,7 @@ export class LanguageConfigurationRegistryImpl {
 		return null;
 	}
 
-	public getIndentMetadata(model: ITokenizedModel, lineNumber: number): number {
+	public getIndentMetadata(model: ITextModel, lineNumber: number): number {
 		let indentRulesSupport = this.getIndentRulesSupport(model.getLanguageIdentifier().id);
 		if (!indentRulesSupport) {
 			return null;
@@ -708,13 +708,13 @@ export class LanguageConfigurationRegistryImpl {
 		return value.onEnter || null;
 	}
 
-	public getRawEnterActionAtPosition(model: ITokenizedModel, lineNumber: number, column: number): EnterAction {
+	public getRawEnterActionAtPosition(model: ITextModel, lineNumber: number, column: number): EnterAction {
 		let r = this.getEnterAction(model, new Range(lineNumber, column, lineNumber, column));
 
 		return r ? r.enterAction : null;
 	}
 
-	public getEnterAction(model: ITokenizedModel, range: Range): { enterAction: EnterAction; indentation: string; } {
+	public getEnterAction(model: ITextModel, range: Range): { enterAction: EnterAction; indentation: string; } {
 		let indentation = this.getIndentationAtPosition(model, range.startLineNumber, range.startColumn);
 
 		let scopedLineTokens = this.getScopedLineTokens(model, range.startLineNumber, range.startColumn);
@@ -780,7 +780,7 @@ export class LanguageConfigurationRegistryImpl {
 		};
 	}
 
-	public getIndentationAtPosition(model: ITokenizedModel, lineNumber: number, column: number): string {
+	public getIndentationAtPosition(model: ITextModel, lineNumber: number, column: number): string {
 		let lineText = model.getLineContent(lineNumber);
 		let indentation = strings.getLeadingWhitespace(lineText);
 		if (indentation.length > column - 1) {
@@ -790,7 +790,7 @@ export class LanguageConfigurationRegistryImpl {
 		return indentation;
 	}
 
-	private getScopedLineTokens(model: ITokenizedModel, lineNumber: number, columnNumber?: number) {
+	private getScopedLineTokens(model: ITextModel, lineNumber: number, columnNumber?: number) {
 		model.forceTokenization(lineNumber);
 		let lineTokens = model.getLineTokens(lineNumber);
 		let column = isNaN(columnNumber) ? model.getLineMaxColumn(lineNumber) - 1 : columnNumber - 1;

src/vs/editor/common/modes/languageFeatureRegistry.ts

@@ -7,7 +7,7 @@
 
 import Event, { Emitter } from 'vs/base/common/event';
 import { IDisposable } from 'vs/base/common/lifecycle';
-import { IReadOnlyModel } from 'vs/editor/common/editorCommon';
+import { ITextModel } from 'vs/editor/common/model';
 import { LanguageSelector, score } from 'vs/editor/common/modes/languageSelector';
 
 interface Entry<T> {
@@ -58,11 +58,11 @@ export default class LanguageFeatureRegistry<T> {
 		};
 	}
 
-	has(model: IReadOnlyModel): boolean {
+	has(model: ITextModel): boolean {
 		return this.all(model).length > 0;
 	}
 
-	all(model: IReadOnlyModel): T[] {
+	all(model: ITextModel): T[] {
 		if (!model || model.isTooLargeForHavingARichMode()) {
 			return [];
 		}
@@ -80,13 +80,13 @@ export default class LanguageFeatureRegistry<T> {
 		return result;
 	}
 
-	ordered(model: IReadOnlyModel): T[] {
+	ordered(model: ITextModel): T[] {
 		const result: T[] = [];
 		this._orderedForEach(model, entry => result.push(entry.provider));
 		return result;
 	}
 
-	orderedGroups(model: IReadOnlyModel): T[][] {
+	orderedGroups(model: ITextModel): T[][] {
 		const result: T[][] = [];
 		let lastBucket: T[];
 		let lastBucketScore: number;
@@ -104,7 +104,7 @@ export default class LanguageFeatureRegistry<T> {
 		return result;
 	}
 
-	private _orderedForEach(model: IReadOnlyModel, callback: (provider: Entry<T>) => any): void {
+	private _orderedForEach(model: ITextModel, callback: (provider: Entry<T>) => any): void {
 
 		if (!model || model.isTooLargeForHavingARichMode()) {
 			return;
@@ -122,7 +122,7 @@ export default class LanguageFeatureRegistry<T> {
 
 	private _lastCandidate: { uri: string; language: string; };
 
-	private _updateScores(model: IReadOnlyModel): void {
+	private _updateScores(model: ITextModel): void {
 
 		let candidate = {
 			uri: model.uri.toString(),

src/vs/editor/common/modes/modesRegistry.ts

@@ -12,7 +12,7 @@ import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageCo
 import { LanguageIdentifier, LanguageId } from 'vs/editor/common/modes';
 
 // Define extension point ids
-export var Extensions = {
+export const Extensions = {
 	ModesRegistry: 'editor.modesRegistry'
 };
 
@@ -42,7 +42,7 @@ export class EditorModesRegistry {
 	}
 }
 
-export var ModesRegistry = new EditorModesRegistry();
+export const ModesRegistry = new EditorModesRegistry();
 Registry.add(Extensions.ModesRegistry, ModesRegistry);
 
 export const PLAINTEXT_MODE_ID = 'plaintext';

src/vs/editor/common/modes/supports.ts

@@ -8,7 +8,7 @@ import * as modes from 'vs/editor/common/modes';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
 
 export function createScopedLineTokens(context: LineTokens, offset: number): ScopedLineTokens {
-	let tokenCount = context.getTokenCount();
+	let tokenCount = context.getCount();
 	let tokenIndex = context.findTokenIndexAtOffset(offset);
 	let desiredLanguageId = context.getLanguageId(tokenIndex);
 
@@ -27,8 +27,8 @@ export function createScopedLineTokens(context: LineTokens, offset: number): Sco
 		desiredLanguageId,
 		firstTokenIndex,
 		lastTokenIndex + 1,
-		context.getTokenStartOffset(firstTokenIndex),
-		context.getTokenEndOffset(lastTokenIndex)
+		context.getStartOffset(firstTokenIndex),
+		context.getEndOffset(lastTokenIndex)
 	);
 }
 
@@ -59,7 +59,7 @@ export class ScopedLineTokens {
 	}
 
 	public getLineContent(): string {
-		var actualLineContent = this._actual.getLineContent();
+		const actualLineContent = this._actual.getLineContent();
 		return actualLineContent.substring(this.firstCharOffset, this._lastCharOffset);
 	}
 

src/vs/editor/common/modes/supports/characterPair.ts

@@ -45,7 +45,7 @@ export class CharacterPairSupport {
 			}
 		}
 
-		return true;
+		return false;
 	}
 
 	public getSurroundingPairs(): IAutoClosingPair[] {

src/vs/editor/common/modes/supports/electricCharacter.ts

@@ -39,7 +39,7 @@ export class BracketElectricCharacterSupport {
 	}
 
 	public getElectricCharacters(): string[] {
-		var result: string[] = [];
+		let result: string[] = [];
 
 		if (this._richEditBrackets) {
 			for (let i = 0, len = this._richEditBrackets.brackets.length; i < len; i++) {

src/vs/editor/common/modes/supports/inplaceReplaceSupport.ts

@@ -45,9 +45,9 @@ export class BasicInplaceReplace {
 	}
 
 	private numberReplace(value: string, up: boolean): string {
-		var precision = Math.pow(10, value.length - (value.lastIndexOf('.') + 1)),
-			n1 = Number(value),
-			n2 = parseFloat(value);
+		let precision = Math.pow(10, value.length - (value.lastIndexOf('.') + 1));
+		let n1 = Number(value);
+		let n2 = parseFloat(value);
 
 		if (!isNaN(n1) && !isNaN(n2) && n1 === n2) {
 
@@ -77,7 +77,7 @@ export class BasicInplaceReplace {
 	}
 
 	private valueSetsReplace(valueSets: string[][], value: string, up: boolean): string {
-		var result: string = null;
+		let result: string = null;
 		for (let i = 0, len = valueSets.length; result === null && i < len; i++) {
 			result = this.valueSetReplace(valueSets[i], value, up);
 		}
@@ -85,7 +85,7 @@ export class BasicInplaceReplace {
 	}
 
 	private valueSetReplace(valueSet: string[], value: string, up: boolean): string {
-		var idx = valueSet.indexOf(value);
+		let idx = valueSet.indexOf(value);
 		if (idx >= 0) {
 			idx += up ? +1 : -1;
 			if (idx < 0) {

src/vs/editor/common/modes/supports/onEnter.ts

@@ -84,7 +84,7 @@ export class OnEnterSupport {
 	}
 
 	private static _createOpenBracketRegExp(bracket: string): RegExp {
-		var str = strings.escapeRegExpCharacters(bracket);
+		let str = strings.escapeRegExpCharacters(bracket);
 		if (!/\B/.test(str.charAt(0))) {
 			str = '\\b' + str;
 		}
@@ -93,7 +93,7 @@ export class OnEnterSupport {
 	}
 
 	private static _createCloseBracketRegExp(bracket: string): RegExp {
-		var str = strings.escapeRegExpCharacters(bracket);
+		let str = strings.escapeRegExpCharacters(bracket);
 		if (!/\B/.test(str.charAt(str.length - 1))) {
 			str = str + '\\b';
 		}

src/vs/editor/common/modes/supports/richEditBrackets.ts

@@ -82,21 +82,21 @@ function once<T, R>(keyFn: (input: T) => string, computeFn: (input: T) => R): (i
 	};
 }
 
-var getRegexForBracketPair = once<ISimpleInternalBracket, RegExp>(
+const getRegexForBracketPair = once<ISimpleInternalBracket, RegExp>(
 	(input) => `${input.open};${input.close}`,
 	(input) => {
 		return createOrRegex([input.open, input.close]);
 	}
 );
 
-var getReversedRegexForBracketPair = once<ISimpleInternalBracket, RegExp>(
+const getReversedRegexForBracketPair = once<ISimpleInternalBracket, RegExp>(
 	(input) => `${input.open};${input.close}`,
 	(input) => {
 		return createOrRegex([toReversedString(input.open), toReversedString(input.close)]);
 	}
 );
 
-var getRegexForBrackets = once<ISimpleInternalBracket[], RegExp>(
+const getRegexForBrackets = once<ISimpleInternalBracket[], RegExp>(
 	(input) => input.map(b => `${b.open};${b.close}`).join(';'),
 	(input) => {
 		let pieces: string[] = [];
@@ -108,7 +108,7 @@ var getRegexForBrackets = once<ISimpleInternalBracket[], RegExp>(
 	}
 );
 
-var getReversedRegexForBrackets = once<ISimpleInternalBracket[], RegExp>(
+const getReversedRegexForBrackets = once<ISimpleInternalBracket[], RegExp>(
 	(input) => input.map(b => `${b.open};${b.close}`).join(';'),
 	(input) => {
 		let pieces: string[] = [];

src/vs/editor/common/modes/textToHtmlTokenizer.ts

@@ -7,24 +7,22 @@
 import * as strings from 'vs/base/common/strings';
 import { IState, ITokenizationSupport, TokenizationRegistry, LanguageId } from 'vs/editor/common/modes';
 import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { LineTokens, IViewLineTokens } from 'vs/editor/common/core/lineTokens';
 import { CharCode } from 'vs/base/common/charCode';
-import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
 
 export function tokenizeToString(text: string, languageId: string): string {
 	return _tokenizeToString(text, _getSafeTokenizationSupport(languageId));
 }
 
-export function tokenizeLineToHTML(text: string, viewLineTokens: ViewLineToken[], colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string {
+export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens, colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string {
 	let result = `<div>`;
 	let charIndex = startOffset;
 	let tabsCharDelta = 0;
 
-	for (let tokenIndex = 0, lenJ = viewLineTokens.length; tokenIndex < lenJ; tokenIndex++) {
-		const token = viewLineTokens[tokenIndex];
-		const tokenEndIndex = token.endIndex;
+	for (let tokenIndex = 0, tokenCount = viewLineTokens.getCount(); tokenIndex < tokenCount; tokenIndex++) {
+		const tokenEndIndex = viewLineTokens.getEndOffset(tokenIndex);
 
-		if (token.endIndex <= startOffset) {
+		if (tokenEndIndex <= startOffset) {
 			continue;
 		}
 
@@ -74,9 +72,9 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: ViewLineToken[]
 			}
 		}
 
-		result += `<span style="${token.getInlineStyle(colorMap)}">${partContent}</span>`;
+		result += `<span style="${viewLineTokens.getInlineStyle(tokenIndex, colorMap)}">${partContent}</span>`;
 
-		if (token.endIndex > endOffset || charIndex >= endOffset) {
+		if (tokenEndIndex > endOffset || charIndex >= endOffset) {
 			break;
 		}
 	}
@@ -109,14 +107,16 @@ function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSuppo
 		}
 
 		let tokenizationResult = tokenizationSupport.tokenize2(line, currentState, 0);
+		LineTokens.convertToEndOffset(tokenizationResult.tokens, line.length);
 		let lineTokens = new LineTokens(tokenizationResult.tokens, line);
 		let viewLineTokens = lineTokens.inflate();
 
 		let startOffset = 0;
-		for (let j = 0, lenJ = viewLineTokens.length; j < lenJ; j++) {
-			const viewLineToken = viewLineTokens[j];
-			result += `<span class="${viewLineToken.getType()}">${strings.escape(line.substring(startOffset, viewLineToken.endIndex))}</span>`;
-			startOffset = viewLineToken.endIndex;
+		for (let j = 0, lenJ = viewLineTokens.getCount(); j < lenJ; j++) {
+			const type = viewLineTokens.getClassName(j);
+			const endIndex = viewLineTokens.getEndOffset(j);
+			result += `<span class="${type}">${strings.escape(line.substring(startOffset, endIndex))}</span>`;
+			startOffset = endIndex;
 		}
 
 		currentState = tokenizationResult.endState;