Merge VS Code 1.21 source code (#1067)

* Initial VS Code 1.21 file copy with patches

* A few more merges

* Post npm install

* Fix batch of build breaks

* Fix more build breaks

* Fix more build errors

* Fix more build breaks

* Runtime fixes 1

* Get connection dialog working with some todos

* Fix a few packaging issues

* Copy several node_modules to package build to fix loader issues

* Fix breaks from master

* A few more fixes

* Make tests pass

* First pass of license header updates

* Second pass of license header updates

* Fix restore dialog issues

* Remove add additional themes menu items

* fix select box issues where the list doesn't show up

* formatting

* Fix editor dispose issue

* Copy over node modules to correct location on all platforms

src/vs/workbench/services/textMate/electron-browser/TMGrammars.ts

@@ -5,7 +5,7 @@
 'use strict';
 
 import * as nls from 'vs/nls';
-import { IExtensionPoint, ExtensionsRegistry } from 'vs/platform/extensions/common/extensionsRegistry';
+import { IExtensionPoint, ExtensionsRegistry } from 'vs/workbench/services/extensions/common/extensionsRegistry';
 import { languagesExtPoint } from 'vs/workbench/services/mode/common/workbenchModeService';
 
 export interface IEmbeddedLanguagesMap {

src/vs/workbench/services/textMate/electron-browser/TMHelper.ts

@@ -7,11 +7,11 @@
 
 import { IColorTheme, ITokenColorizationSetting } from 'vs/workbench/services/themes/common/workbenchThemeService';
 
-export function findMatchingThemeRule(theme: IColorTheme, scopes: string[]): ThemeRule {
+export function findMatchingThemeRule(theme: IColorTheme, scopes: string[], onlyColorRules: boolean = true): ThemeRule {
 	for (let i = scopes.length - 1; i >= 0; i--) {
 		let parentScopes = scopes.slice(0, i);
 		let scope = scopes[i];
-		let r = findMatchingThemeRule2(theme, scope, parentScopes);
+		let r = findMatchingThemeRule2(theme, scope, parentScopes, onlyColorRules);
 		if (r) {
 			return r;
 		}
@@ -19,13 +19,13 @@ export function findMatchingThemeRule(theme: IColorTheme, scopes: string[]): The
 	return null;
 }
 
-function findMatchingThemeRule2(theme: IColorTheme, scope: string, parentScopes: string[]): ThemeRule {
+function findMatchingThemeRule2(theme: IColorTheme, scope: string, parentScopes: string[], onlyColorRules: boolean): ThemeRule {
 	let result: ThemeRule = null;
 
 	// Loop backwards, to ensure the last most specific rule wins
 	for (let i = theme.tokenColors.length - 1; i >= 0; i--) {
 		let rule = theme.tokenColors[i];
-		if (!rule.settings.foreground) {
+		if (onlyColorRules && !rule.settings.foreground) {
 			continue;
 		}
 

src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts

@@ -11,15 +11,14 @@ import Event, { Emitter } from 'vs/base/common/event';
 import { join, normalize } from 'path';
 import { TPromise } from 'vs/base/common/winjs.base';
 import { onUnexpectedError } from 'vs/base/common/errors';
-import { ExtensionMessageCollector } from 'vs/platform/extensions/common/extensionsRegistry';
-import { ITokenizationSupport, TokenizationRegistry, IState, LanguageId } from 'vs/editor/common/modes';
+import { ExtensionMessageCollector } from 'vs/workbench/services/extensions/common/extensionsRegistry';
+import { ITokenizationSupport, TokenizationRegistry, IState, LanguageId, TokenMetadata } from 'vs/editor/common/modes';
 import { IModeService } from 'vs/editor/common/services/modeService';
-import { INITIAL, StackElement, IGrammar, Registry, IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2 } from 'vscode-textmate';
+import { StackElement, IGrammar, Registry, IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2 } from 'vscode-textmate';
 import { IWorkbenchThemeService, ITokenColorizationRule } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import { ITextMateService } from 'vs/workbench/services/textMate/electron-browser/textMateService';
 import { grammarsExtPoint, IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint } from 'vs/workbench/services/textMate/electron-browser/TMGrammars';
 import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token';
-import { TokenMetadata } from 'vs/editor/common/model/tokensBinaryEncoding';
 import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
 import { Color } from 'vs/base/common/color';
@@ -38,6 +37,16 @@ export class TMScopeRegistry {
 	}
 
 	public register(scopeName: string, filePath: string, embeddedLanguages?: IEmbeddedLanguagesMap): void {
+		if (this._scopeNameToLanguageRegistration[scopeName]) {
+			const existingRegistration = this._scopeNameToLanguageRegistration[scopeName];
+			if (existingRegistration.grammarFilePath !== filePath) {
+				console.warn(
+					`Overwriting grammar scope name to file mapping for scope ${scopeName}.\n` +
+					`Old grammar file: ${existingRegistration.grammarFilePath}.\n` +
+					`New grammar file: ${filePath}`
+				);
+			}
+		}
 		this._scopeNameToLanguageRegistration[scopeName] = new TMLanguageRegistration(scopeName, filePath, embeddedLanguages);
 	}
 
@@ -94,13 +103,14 @@ export class TMLanguageRegistration {
 interface ICreateGrammarResult {
 	languageId: LanguageId;
 	grammar: IGrammar;
+	initialState: StackElement;
 	containsEmbeddedLanguages: boolean;
 }
 
 export class TextMateService implements ITextMateService {
 	public _serviceBrand: any;
 
-	private _grammarRegistry: Registry;
+	private _grammarRegistry: TPromise<[Registry, StackElement]>;
 	private _modeService: IModeService;
 	private _themeService: IWorkbenchThemeService;
 	private _scopeRegistry: TMScopeRegistry;
@@ -128,16 +138,7 @@ export class TextMateService implements ITextMateService {
 		this._injectedEmbeddedLanguages = {};
 		this._languageToScope = new Map<string, string>();
 
-		this._grammarRegistry = new Registry({
-			getFilePath: (scopeName: string) => {
-				return this._scopeRegistry.getFilePath(scopeName);
-			},
-			getInjections: (scopeName: string) => {
-				return this._injections[scopeName];
-			}
-		});
-		this._updateTheme();
-		this._themeService.onDidColorThemeChange((e) => this._updateTheme());
+		this._grammarRegistry = null;
 
 		grammarsExtPoint.setHandler((extensions) => {
 			for (let i = 0; i < extensions.length; i++) {
@@ -148,6 +149,23 @@ export class TextMateService implements ITextMateService {
 			}
 		});
 
+		// Generate some color map until the grammar registry is loaded
+		let colorTheme = this._themeService.getColorTheme();
+		let defaultForeground: Color = Color.transparent;
+		let defaultBackground: Color = Color.transparent;
+		for (let i = 0, len = colorTheme.tokenColors.length; i < len; i++) {
+			let rule = colorTheme.tokenColors[i];
+			if (!rule.scope && rule.settings) {
+				if (rule.settings.foreground) {
+					defaultForeground = Color.fromHex(rule.settings.foreground);
+				}
+				if (rule.settings.background) {
+					defaultBackground = Color.fromHex(rule.settings.background);
+				}
+			}
+		}
+		TokenizationRegistry.setColorMap([null, defaultForeground, defaultBackground]);
+
 		this._modeService.onDidCreateMode((mode) => {
 			let modeId = mode.getId();
 			if (this._languageToScope.has(modeId)) {
@@ -156,6 +174,26 @@ export class TextMateService implements ITextMateService {
 		});
 	}
 
+	private _getOrCreateGrammarRegistry(): TPromise<[Registry, StackElement]> {
+		if (!this._grammarRegistry) {
+			this._grammarRegistry = TPromise.wrap(import('vscode-textmate')).then(({ Registry, INITIAL }) => {
+				const grammarRegistry = new Registry({
+					getFilePath: (scopeName: string) => {
+						return this._scopeRegistry.getFilePath(scopeName);
+					},
+					getInjections: (scopeName: string) => {
+						return this._injections[scopeName];
+					}
+				});
+				this._updateTheme(grammarRegistry);
+				this._themeService.onDidColorThemeChange((e) => this._updateTheme(grammarRegistry));
+				return <[Registry, StackElement]>[grammarRegistry, INITIAL];
+			});
+		}
+
+		return this._grammarRegistry;
+	}
+
 	private static _toColorMap(colorMap: string[]): Color[] {
 		let result: Color[] = [null];
 		for (let i = 1, len = colorMap.length; i < len; i++) {
@@ -164,13 +202,13 @@ export class TextMateService implements ITextMateService {
 		return result;
 	}
 
-	private _updateTheme(): void {
+	private _updateTheme(grammarRegistry: Registry): void {
 		let colorTheme = this._themeService.getColorTheme();
 		if (!this.compareTokenRules(colorTheme.tokenColors)) {
 			return;
 		}
-		this._grammarRegistry.setTheme({ name: colorTheme.label, settings: colorTheme.tokenColors });
-		let colorMap = TextMateService._toColorMap(this._grammarRegistry.getColorMap());
+		grammarRegistry.setTheme({ name: colorTheme.label, settings: colorTheme.tokenColors });
+		let colorMap = TextMateService._toColorMap(grammarRegistry.getColorMap());
 		let cssRules = generateTokensCSSForColorMap(colorMap);
 		this._styleElement.innerHTML = cssRules;
 		TokenizationRegistry.setColorMap(colorMap);
@@ -295,15 +333,19 @@ export class TextMateService implements ITextMateService {
 
 		let languageId = this._modeService.getLanguageIdentifier(modeId).id;
 		let containsEmbeddedLanguages = (Object.keys(embeddedLanguages).length > 0);
-		return new TPromise<ICreateGrammarResult>((c, e, p) => {
-			this._grammarRegistry.loadGrammarWithEmbeddedLanguages(scopeName, languageId, embeddedLanguages, (err, grammar) => {
-				if (err) {
-					return e(err);
-				}
-				c({
-					languageId: languageId,
-					grammar: grammar,
-					containsEmbeddedLanguages: containsEmbeddedLanguages
+		return this._getOrCreateGrammarRegistry().then((_res) => {
+			const [grammarRegistry, initialState] = _res;
+			return new TPromise<ICreateGrammarResult>((c, e, p) => {
+				grammarRegistry.loadGrammarWithEmbeddedLanguages(scopeName, languageId, embeddedLanguages, (err, grammar) => {
+					if (err) {
+						return e(err);
+					}
+					c({
+						languageId: languageId,
+						grammar: grammar,
+						initialState: initialState,
+						containsEmbeddedLanguages: containsEmbeddedLanguages
+					});
 				});
 			});
 		});
@@ -311,7 +353,7 @@ export class TextMateService implements ITextMateService {
 
 	private registerDefinition(modeId: string): void {
 		this._createGrammar(modeId).then((r) => {
-			TokenizationRegistry.register(modeId, new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.containsEmbeddedLanguages));
+			TokenizationRegistry.register(modeId, new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages));
 		}, onUnexpectedError);
 	}
 }
@@ -323,17 +365,19 @@ class TMTokenization implements ITokenizationSupport {
 	private readonly _grammar: IGrammar;
 	private readonly _containsEmbeddedLanguages: boolean;
 	private readonly _seenLanguages: boolean[];
+	private readonly _initialState: StackElement;
 
-	constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, containsEmbeddedLanguages: boolean) {
+	constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean) {
 		this._scopeRegistry = scopeRegistry;
 		this._languageId = languageId;
 		this._grammar = grammar;
+		this._initialState = initialState;
 		this._containsEmbeddedLanguages = containsEmbeddedLanguages;
 		this._seenLanguages = [];
 	}
 
 	public getInitialState(): IState {
-		return INITIAL;
+		return this._initialState;
 	}
 
 	public tokenize(line: string, state: IState, offsetDelta: number): TokenizationResult {