SQL Operations Studio Public Preview 1 (0.23) release source code

src/vs/workbench/services/textMate/electron-browser/OSSREADME.json

@@ -0,0 +1,37 @@
+// Listing in here dependencies that come in at build time from vscode-textmate
+
+[{
+	"name": "lib-oniguruma",
+	"version": "5.9.3",
+	"license": "BSD", // according to wikipedia: https://en.wikipedia.org/wiki/Oniguruma
+	"repositoryURL": "https://github.com/kkos/oniguruma",
+	"licenseDetail": [
+		"Copyright (c) 2002-2007  K.Kosako. All rights reserved.",
+		"",
+		"The BSD License",
+		"",
+		"Redistribution and use in source and binary forms, with or without",
+		"modification, are permitted provided that the following conditions",
+		"are met:",
+		"",
+		"1. Redistributions of source code must retain the above copyright",
+		"   notice, this list of conditions and the following disclaimer.",
+		"",
+		"2. Redistributions in binary form must reproduce the above copyright",
+		"   notice, this list of conditions and the following disclaimer in the",
+		"   documentation and/or other materials provided with the distribution.",
+		"",
+		"THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND",
+		"ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE",
+		"IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR",
+		"PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS",
+		"BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR",
+		"CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF",
+		"SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR",
+		"BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,",
+		"WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE",
+		"OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN",
+		"IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+	],
+	"isProd": true
+}]

src/vs/workbench/services/textMate/electron-browser/TMGrammars.ts

@@ -0,0 +1,57 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as nls from 'vs/nls';
+import { IExtensionPoint, ExtensionsRegistry } from 'vs/platform/extensions/common/extensionsRegistry';
+import { languagesExtPoint } from 'vs/workbench/services/mode/common/workbenchModeService';
+
+export interface IEmbeddedLanguagesMap {
+	[scopeName: string]: string;
+}
+
+export interface ITMSyntaxExtensionPoint {
+	language: string;
+	scopeName: string;
+	path: string;
+	embeddedLanguages: IEmbeddedLanguagesMap;
+	injectTo: string[];
+}
+
+export const grammarsExtPoint: IExtensionPoint<ITMSyntaxExtensionPoint[]> = ExtensionsRegistry.registerExtensionPoint<ITMSyntaxExtensionPoint[]>('grammars', [languagesExtPoint], {
+	description: nls.localize('vscode.extension.contributes.grammars', 'Contributes textmate tokenizers.'),
+	type: 'array',
+	defaultSnippets: [{ body: [{ language: '${1:id}', scopeName: 'source.${2:id}', path: './syntaxes/${3:id}.tmLanguage.' }] }],
+	items: {
+		type: 'object',
+		defaultSnippets: [{ body: { language: '${1:id}', scopeName: 'source.${2:id}', path: './syntaxes/${3:id}.tmLanguage.' } }],
+		properties: {
+			language: {
+				description: nls.localize('vscode.extension.contributes.grammars.language', 'Language identifier for which this syntax is contributed to.'),
+				type: 'string'
+			},
+			scopeName: {
+				description: nls.localize('vscode.extension.contributes.grammars.scopeName', 'Textmate scope name used by the tmLanguage file.'),
+				type: 'string'
+			},
+			path: {
+				description: nls.localize('vscode.extension.contributes.grammars.path', 'Path of the tmLanguage file. The path is relative to the extension folder and typically starts with \'./syntaxes/\'.'),
+				type: 'string'
+			},
+			embeddedLanguages: {
+				description: nls.localize('vscode.extension.contributes.grammars.embeddedLanguages', 'A map of scope name to language id if this grammar contains embedded languages.'),
+				type: 'object'
+			},
+			injectTo: {
+				description: nls.localize('vscode.extension.contributes.grammars.injectTo', 'List of language scope names to which this grammar is injected to.'),
+				type: 'array',
+				items: {
+					type: 'string'
+				}
+			}
+		},
+		required: ['scopeName', 'path']
+	}
+});

src/vs/workbench/services/textMate/electron-browser/TMHelper.ts

@@ -0,0 +1,137 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { IColorTheme, ITokenColorizationSetting } from 'vs/workbench/services/themes/common/workbenchThemeService';
+
+export function findMatchingThemeRule(theme: IColorTheme, scopes: string[]): ThemeRule {
+	for (let i = scopes.length - 1; i >= 0; i--) {
+		let parentScopes = scopes.slice(0, i);
+		let scope = scopes[i];
+		let r = findMatchingThemeRule2(theme, scope, parentScopes);
+		if (r) {
+			return r;
+		}
+	}
+	return null;
+}
+
+function findMatchingThemeRule2(theme: IColorTheme, scope: string, parentScopes: string[]): ThemeRule {
+	let result: ThemeRule = null;
+
+	// Loop backwards, to ensure the last most specific rule wins
+	for (let i = theme.tokenColors.length - 1; i >= 0; i--) {
+		let rule = theme.tokenColors[i];
+		if (!rule.settings.foreground) {
+			continue;
+		}
+
+		let selectors: string[];
+		if (typeof rule.scope === 'string') {
+			selectors = rule.scope.split(/,/).map(scope => scope.trim());
+		} else if (Array.isArray(rule.scope)) {
+			selectors = rule.scope;
+		} else {
+			continue;
+		}
+
+		for (let j = 0, lenJ = selectors.length; j < lenJ; j++) {
+			let rawSelector = selectors[j];
+
+			let themeRule = new ThemeRule(rawSelector, rule.settings);
+			if (themeRule.matches(scope, parentScopes)) {
+				if (themeRule.isMoreSpecific(result)) {
+					result = themeRule;
+				}
+			}
+		}
+	}
+
+	return result;
+}
+
+export class ThemeRule {
+	readonly rawSelector: string;
+	readonly settings: ITokenColorizationSetting;
+	readonly scope: string;
+	readonly parentScopes: string[];
+
+	constructor(rawSelector: string, settings: ITokenColorizationSetting) {
+		this.rawSelector = rawSelector;
+		this.settings = settings;
+		let rawSelectorPieces = this.rawSelector.split(/ /);
+		this.scope = rawSelectorPieces[rawSelectorPieces.length - 1];
+		this.parentScopes = rawSelectorPieces.slice(0, rawSelectorPieces.length - 1);
+	}
+
+	public matches(scope: string, parentScopes: string[]): boolean {
+		return ThemeRule._matches(this.scope, this.parentScopes, scope, parentScopes);
+	}
+
+	private static _cmp(a: ThemeRule, b: ThemeRule): number {
+		if (a === null && b === null) {
+			return 0;
+		}
+		if (a === null) {
+			// b > a
+			return -1;
+		}
+		if (b === null) {
+			// a > b
+			return 1;
+		}
+		if (a.scope.length !== b.scope.length) {
+			// longer scope length > shorter scope length
+			return a.scope.length - b.scope.length;
+		}
+		const aParentScopesLen = a.parentScopes.length;
+		const bParentScopesLen = b.parentScopes.length;
+		if (aParentScopesLen !== bParentScopesLen) {
+			// more parents > less parents
+			return aParentScopesLen - bParentScopesLen;
+		}
+		for (let i = 0; i < aParentScopesLen; i++) {
+			var aLen = a.parentScopes[i].length;
+			var bLen = b.parentScopes[i].length;
+			if (aLen !== bLen) {
+				return aLen - bLen;
+			}
+		}
+		return 0;
+	}
+
+	public isMoreSpecific(other: ThemeRule): boolean {
+		return (ThemeRule._cmp(this, other) > 0);
+	}
+
+	private static _matchesOne(selectorScope: string, scope: string): boolean {
+		let selectorPrefix = selectorScope + '.';
+		if (selectorScope === scope || scope.substring(0, selectorPrefix.length) === selectorPrefix) {
+			return true;
+		}
+		return false;
+	}
+
+	private static _matches(selectorScope: string, selectorParentScopes: string[], scope: string, parentScopes: string[]): boolean {
+		if (!this._matchesOne(selectorScope, scope)) {
+			return false;
+		}
+
+		let selectorParentIndex = selectorParentScopes.length - 1;
+		let parentIndex = parentScopes.length - 1;
+		while (selectorParentIndex >= 0 && parentIndex >= 0) {
+			if (this._matchesOne(selectorParentScopes[selectorParentIndex], parentScopes[parentIndex])) {
+				selectorParentIndex--;
+			}
+			parentIndex--;
+		}
+
+		if (selectorParentIndex === -1) {
+			return true;
+		}
+		return false;
+	}
+}

src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts

@@ -0,0 +1,361 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as nls from 'vs/nls';
+import * as dom from 'vs/base/browser/dom';
+import * as types from 'vs/base/common/types';
+import Event, { Emitter } from 'vs/base/common/event';
+import { join, normalize } from 'path';
+import { TPromise } from 'vs/base/common/winjs.base';
+import { onUnexpectedError } from 'vs/base/common/errors';
+import { ExtensionMessageCollector } from 'vs/platform/extensions/common/extensionsRegistry';
+import { ITokenizationSupport, TokenizationRegistry, IState, LanguageId } from 'vs/editor/common/modes';
+import { IModeService } from 'vs/editor/common/services/modeService';
+import { INITIAL, StackElement, IGrammar, Registry, IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2 } from 'vscode-textmate';
+import { IWorkbenchThemeService, ITokenColorizationRule } from 'vs/workbench/services/themes/common/workbenchThemeService';
+import { ITextMateService } from 'vs/workbench/services/textMate/electron-browser/textMateService';
+import { grammarsExtPoint, IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint } from 'vs/workbench/services/textMate/electron-browser/TMGrammars';
+import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token';
+import { TokenMetadata } from 'vs/editor/common/model/tokensBinaryEncoding';
+import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
+import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
+import { Color } from 'vs/base/common/color';
+
+export class TMScopeRegistry {
+
+	private _scopeNameToLanguageRegistration: { [scopeName: string]: TMLanguageRegistration; };
+	private _encounteredLanguages: boolean[];
+
+	private _onDidEncounterLanguage: Emitter<LanguageId> = new Emitter<LanguageId>();
+	public onDidEncounterLanguage: Event<LanguageId> = this._onDidEncounterLanguage.event;
+
+	constructor() {
+		this._scopeNameToLanguageRegistration = Object.create(null);
+		this._encounteredLanguages = [];
+	}
+
+	public register(scopeName: string, filePath: string, embeddedLanguages?: IEmbeddedLanguagesMap): void {
+		this._scopeNameToLanguageRegistration[scopeName] = new TMLanguageRegistration(scopeName, filePath, embeddedLanguages);
+	}
+
+	public getLanguageRegistration(scopeName: string): TMLanguageRegistration {
+		return this._scopeNameToLanguageRegistration[scopeName] || null;
+	}
+
+	public getFilePath(scopeName: string): string {
+		let data = this.getLanguageRegistration(scopeName);
+		return data ? data.grammarFilePath : null;
+	}
+
+	/**
+	 * To be called when tokenization found/hit an embedded language.
+	 */
+	public onEncounteredLanguage(languageId: LanguageId): void {
+		if (!this._encounteredLanguages[languageId]) {
+			this._encounteredLanguages[languageId] = true;
+			this._onDidEncounterLanguage.fire(languageId);
+		}
+	}
+}
+
+export class TMLanguageRegistration {
+	_topLevelScopeNameDataBrand: void;
+
+	readonly scopeName: string;
+	readonly grammarFilePath: string;
+	readonly embeddedLanguages: IEmbeddedLanguagesMap;
+
+	constructor(scopeName: string, grammarFilePath: string, embeddedLanguages: IEmbeddedLanguagesMap) {
+		this.scopeName = scopeName;
+		this.grammarFilePath = grammarFilePath;
+
+		// embeddedLanguages handling
+		this.embeddedLanguages = Object.create(null);
+
+		if (embeddedLanguages) {
+			// If embeddedLanguages are configured, fill in `this._embeddedLanguages`
+			let scopes = Object.keys(embeddedLanguages);
+			for (let i = 0, len = scopes.length; i < len; i++) {
+				let scope = scopes[i];
+				let language = embeddedLanguages[scope];
+				if (typeof language !== 'string') {
+					// never hurts to be too careful
+					continue;
+				}
+				this.embeddedLanguages[scope] = language;
+			}
+		}
+	}
+}
+
+interface ICreateGrammarResult {
+	languageId: LanguageId;
+	grammar: IGrammar;
+	containsEmbeddedLanguages: boolean;
+}
+
+export class TextMateService implements ITextMateService {
+	public _serviceBrand: any;
+
+	private _grammarRegistry: Registry;
+	private _modeService: IModeService;
+	private _themeService: IWorkbenchThemeService;
+	private _scopeRegistry: TMScopeRegistry;
+	private _injections: { [scopeName: string]: string[]; };
+	private _languageToScope: Map<string, string>;
+	private _styleElement: HTMLStyleElement;
+
+	private _currentTokenColors: ITokenColorizationRule[];
+
+	public onDidEncounterLanguage: Event<LanguageId>;
+
+	constructor(
+		@IModeService modeService: IModeService,
+		@IWorkbenchThemeService themeService: IWorkbenchThemeService
+	) {
+		this._styleElement = dom.createStyleSheet();
+		this._styleElement.className = 'vscode-tokens-styles';
+		this._modeService = modeService;
+		this._themeService = themeService;
+		this._scopeRegistry = new TMScopeRegistry();
+		this.onDidEncounterLanguage = this._scopeRegistry.onDidEncounterLanguage;
+		this._injections = {};
+		this._languageToScope = new Map<string, string>();
+
+		this._grammarRegistry = new Registry({
+			getFilePath: (scopeName: string) => {
+				return this._scopeRegistry.getFilePath(scopeName);
+			},
+			getInjections: (scopeName: string) => {
+				return this._injections[scopeName];
+			}
+		});
+		this._updateTheme();
+		this._themeService.onDidColorThemeChange((e) => this._updateTheme());
+
+		grammarsExtPoint.setHandler((extensions) => {
+			for (let i = 0; i < extensions.length; i++) {
+				let grammars = extensions[i].value;
+				for (let j = 0; j < grammars.length; j++) {
+					this._handleGrammarExtensionPointUser(extensions[i].description.extensionFolderPath, grammars[j], extensions[i].collector);
+				}
+			}
+		});
+
+		this._modeService.onDidCreateMode((mode) => {
+			let modeId = mode.getId();
+			if (this._languageToScope.has(modeId)) {
+				this.registerDefinition(modeId);
+			}
+		});
+	}
+
+	private static _toColorMap(colorMap: string[]): Color[] {
+		let result: Color[] = [null];
+		for (let i = 1, len = colorMap.length; i < len; i++) {
+			result[i] = Color.fromHex(colorMap[i]);
+		}
+		return result;
+	}
+
+	private _updateTheme(): void {
+		let colorTheme = this._themeService.getColorTheme();
+		if (!this.compareTokenRules(colorTheme.tokenColors)) {
+			return;
+		}
+		this._grammarRegistry.setTheme({ name: colorTheme.label, settings: colorTheme.tokenColors });
+		let colorMap = TextMateService._toColorMap(this._grammarRegistry.getColorMap());
+		let cssRules = generateTokensCSSForColorMap(colorMap);
+		this._styleElement.innerHTML = cssRules;
+		TokenizationRegistry.setColorMap(colorMap);
+	}
+
+	private compareTokenRules(newRules: ITokenColorizationRule[]): boolean {
+		let currRules = this._currentTokenColors;
+		this._currentTokenColors = newRules;
+		if (!newRules || !currRules || newRules.length !== currRules.length) {
+			return true;
+		}
+		for (let i = newRules.length - 1; i >= 0; i--) {
+			let r1 = newRules[i];
+			let r2 = currRules[i];
+			if (r1.scope !== r2.scope) {
+				return true;
+			}
+			let s1 = r1.settings;
+			let s2 = r2.settings;
+			if (s1 && s2) {
+				if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground) {
+					return true;
+				}
+			} else if (!s1 || !s2) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+
+	private _handleGrammarExtensionPointUser(extensionFolderPath: string, syntax: ITMSyntaxExtensionPoint, collector: ExtensionMessageCollector): void {
+		if (syntax.language && ((typeof syntax.language !== 'string') || !this._modeService.isRegisteredMode(syntax.language))) {
+			collector.error(nls.localize('invalid.language', "Unknown language in `contributes.{0}.language`. Provided value: {1}", grammarsExtPoint.name, String(syntax.language)));
+			return;
+		}
+		if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
+			collector.error(nls.localize('invalid.scopeName', "Expected string in `contributes.{0}.scopeName`. Provided value: {1}", grammarsExtPoint.name, String(syntax.scopeName)));
+			return;
+		}
+		if (!syntax.path || (typeof syntax.path !== 'string')) {
+			collector.error(nls.localize('invalid.path.0', "Expected string in `contributes.{0}.path`. Provided value: {1}", grammarsExtPoint.name, String(syntax.path)));
+			return;
+		}
+		if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || syntax.injectTo.some(scope => typeof scope !== 'string'))) {
+			collector.error(nls.localize('invalid.injectTo', "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.injectTo)));
+			return;
+		}
+		if (syntax.embeddedLanguages && !types.isObject(syntax.embeddedLanguages)) {
+			collector.error(nls.localize('invalid.embeddedLanguages', "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.embeddedLanguages)));
+			return;
+		}
+
+		let normalizedAbsolutePath = normalize(join(extensionFolderPath, syntax.path));
+
+		if (normalizedAbsolutePath.indexOf(extensionFolderPath) !== 0) {
+			collector.warn(nls.localize('invalid.path.1', "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.", grammarsExtPoint.name, normalizedAbsolutePath, extensionFolderPath));
+		}
+
+		this._scopeRegistry.register(syntax.scopeName, normalizedAbsolutePath, syntax.embeddedLanguages);
+
+		if (syntax.injectTo) {
+			for (let injectScope of syntax.injectTo) {
+				let injections = this._injections[injectScope];
+				if (!injections) {
+					this._injections[injectScope] = injections = [];
+				}
+				injections.push(syntax.scopeName);
+			}
+		}
+
+		let modeId = syntax.language;
+		if (modeId) {
+			this._languageToScope.set(modeId, syntax.scopeName);
+		}
+	}
+
+	private _resolveEmbeddedLanguages(embeddedLanguages: IEmbeddedLanguagesMap): IEmbeddedLanguagesMap2 {
+		let scopes = Object.keys(embeddedLanguages);
+		let result: IEmbeddedLanguagesMap2 = Object.create(null);
+		for (let i = 0, len = scopes.length; i < len; i++) {
+			let scope = scopes[i];
+			let language = embeddedLanguages[scope];
+			let languageIdentifier = this._modeService.getLanguageIdentifier(language);
+			if (languageIdentifier) {
+				result[scope] = languageIdentifier.id;
+			}
+		}
+		return result;
+	}
+
+	public createGrammar(modeId: string): TPromise<IGrammar> {
+		return this._createGrammar(modeId).then(r => r.grammar);
+	}
+
+	private _createGrammar(modeId: string): TPromise<ICreateGrammarResult> {
+		let scopeName = this._languageToScope.get(modeId);
+		let languageRegistration = this._scopeRegistry.getLanguageRegistration(scopeName);
+		if (!languageRegistration) {
+			// No TM grammar defined
+			return TPromise.wrapError<ICreateGrammarResult>(new Error(nls.localize('no-tm-grammar', "No TM Grammar registered for this language.")));
+		}
+		let embeddedLanguages = this._resolveEmbeddedLanguages(languageRegistration.embeddedLanguages);
+		let languageId = this._modeService.getLanguageIdentifier(modeId).id;
+		let containsEmbeddedLanguages = (Object.keys(embeddedLanguages).length > 0);
+		return new TPromise<ICreateGrammarResult>((c, e, p) => {
+			this._grammarRegistry.loadGrammarWithEmbeddedLanguages(scopeName, languageId, embeddedLanguages, (err, grammar) => {
+				if (err) {
+					return e(err);
+				}
+				c({
+					languageId: languageId,
+					grammar: grammar,
+					containsEmbeddedLanguages: containsEmbeddedLanguages
+				});
+			});
+		});
+	}
+
+	private registerDefinition(modeId: string): void {
+		this._createGrammar(modeId).then((r) => {
+			TokenizationRegistry.register(modeId, new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.containsEmbeddedLanguages));
+		}, onUnexpectedError);
+	}
+}
+
+class TMTokenization implements ITokenizationSupport {
+
+	private readonly _scopeRegistry: TMScopeRegistry;
+	private readonly _languageId: LanguageId;
+	private readonly _grammar: IGrammar;
+	private readonly _containsEmbeddedLanguages: boolean;
+	private readonly _seenLanguages: boolean[];
+
+	constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, containsEmbeddedLanguages: boolean) {
+		this._scopeRegistry = scopeRegistry;
+		this._languageId = languageId;
+		this._grammar = grammar;
+		this._containsEmbeddedLanguages = containsEmbeddedLanguages;
+		this._seenLanguages = [];
+	}
+
+	public getInitialState(): IState {
+		return INITIAL;
+	}
+
+	public tokenize(line: string, state: IState, offsetDelta: number): TokenizationResult {
+		throw new Error('Not supported!');
+	}
+
+	public tokenize2(line: string, state: StackElement, offsetDelta: number): TokenizationResult2 {
+		if (offsetDelta !== 0) {
+			throw new Error('Unexpected: offsetDelta should be 0.');
+		}
+
+		// Do not attempt to tokenize if a line has over 20k
+		if (line.length >= 20000) {
+			console.log(`Line (${line.substr(0, 15)}...): longer than 20k characters, tokenization skipped.`);
+			return nullTokenize2(this._languageId, line, state, offsetDelta);
+		}
+
+		let textMateResult = this._grammar.tokenizeLine2(line, state);
+
+		if (this._containsEmbeddedLanguages) {
+			let seenLanguages = this._seenLanguages;
+			let tokens = textMateResult.tokens;
+
+			// Must check if any of the embedded languages was hit
+			for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
+				let metadata = tokens[(i << 1) + 1];
+				let languageId = TokenMetadata.getLanguageId(metadata);
+
+				if (!seenLanguages[languageId]) {
+					seenLanguages[languageId] = true;
+					this._scopeRegistry.onEncounteredLanguage(languageId);
+				}
+			}
+		}
+
+		let endState: StackElement;
+		// try to save an object if possible
+		if (state.equals(textMateResult.ruleStack)) {
+			endState = state;
+		} else {
+			endState = textMateResult.ruleStack;
+
+		}
+
+		return new TokenizationResult2(textMateResult.tokens, endState);
+	}
+}

src/vs/workbench/services/textMate/electron-browser/textMateService.ts

@@ -0,0 +1,21 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { TPromise } from 'vs/base/common/winjs.base';
+import Event from 'vs/base/common/event';
+import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
+import { IGrammar } from 'vscode-textmate';
+import { LanguageId } from 'vs/editor/common/modes';
+
+export var ITextMateService = createDecorator<ITextMateService>('textMateService');
+
+export interface ITextMateService {
+	_serviceBrand: any;
+
+	onDidEncounterLanguage: Event<LanguageId>;
+
+	createGrammar(modeId: string): TPromise<IGrammar>;
+}