SQL Operations Studio Public Preview 1 (0.23) release source code

src/vs/editor/common/modes/abstractMode.ts

@@ -0,0 +1,24 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { IMode, LanguageIdentifier } from 'vs/editor/common/modes';
+
+export class FrankensteinMode implements IMode {
+
+	private _languageIdentifier: LanguageIdentifier;
+
+	constructor(languageIdentifier: LanguageIdentifier) {
+		this._languageIdentifier = languageIdentifier;
+	}
+
+	public getId(): string {
+		return this._languageIdentifier.language;
+	}
+
+	public getLanguageIdentifier(): LanguageIdentifier {
+		return this._languageIdentifier;
+	}
+}

src/vs/editor/common/modes/editorModeContext.ts

@@ -0,0 +1,130 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { Disposable } from 'vs/base/common/lifecycle';
+import { IContextKey, IContextKeyService } from 'vs/platform/contextkey/common/contextkey';
+import * as modes from 'vs/editor/common/modes';
+import { ICommonCodeEditor } from 'vs/editor/common/editorCommon';
+import { EditorContextKeys } from 'vs/editor/common/editorContextKeys';
+import { Schemas } from 'vs/base/common/network';
+
+export class EditorModeContext extends Disposable {
+
+	private _editor: ICommonCodeEditor;
+
+	private _langId: IContextKey<string>;
+	private _hasCompletionItemProvider: IContextKey<boolean>;
+	private _hasCodeActionsProvider: IContextKey<boolean>;
+	private _hasCodeLensProvider: IContextKey<boolean>;
+	private _hasDefinitionProvider: IContextKey<boolean>;
+	private _hasImplementationProvider: IContextKey<boolean>;
+	private _hasTypeDefinitionProvider: IContextKey<boolean>;
+	private _hasHoverProvider: IContextKey<boolean>;
+	private _hasDocumentHighlightProvider: IContextKey<boolean>;
+	private _hasDocumentSymbolProvider: IContextKey<boolean>;
+	private _hasReferenceProvider: IContextKey<boolean>;
+	private _hasRenameProvider: IContextKey<boolean>;
+	private _hasDocumentFormattingProvider: IContextKey<boolean>;
+	private _hasDocumentSelectionFormattingProvider: IContextKey<boolean>;
+	private _hasSignatureHelpProvider: IContextKey<boolean>;
+	private _isInWalkThrough: IContextKey<boolean>;
+
+	constructor(
+		editor: ICommonCodeEditor,
+		contextKeyService: IContextKeyService
+	) {
+		super();
+		this._editor = editor;
+
+		this._langId = EditorContextKeys.languageId.bindTo(contextKeyService);
+		this._hasCompletionItemProvider = EditorContextKeys.hasCompletionItemProvider.bindTo(contextKeyService);
+		this._hasCodeActionsProvider = EditorContextKeys.hasCodeActionsProvider.bindTo(contextKeyService);
+		this._hasCodeLensProvider = EditorContextKeys.hasCodeLensProvider.bindTo(contextKeyService);
+		this._hasDefinitionProvider = EditorContextKeys.hasDefinitionProvider.bindTo(contextKeyService);
+		this._hasImplementationProvider = EditorContextKeys.hasImplementationProvider.bindTo(contextKeyService);
+		this._hasTypeDefinitionProvider = EditorContextKeys.hasTypeDefinitionProvider.bindTo(contextKeyService);
+		this._hasHoverProvider = EditorContextKeys.hasHoverProvider.bindTo(contextKeyService);
+		this._hasDocumentHighlightProvider = EditorContextKeys.hasDocumentHighlightProvider.bindTo(contextKeyService);
+		this._hasDocumentSymbolProvider = EditorContextKeys.hasDocumentSymbolProvider.bindTo(contextKeyService);
+		this._hasReferenceProvider = EditorContextKeys.hasReferenceProvider.bindTo(contextKeyService);
+		this._hasRenameProvider = EditorContextKeys.hasRenameProvider.bindTo(contextKeyService);
+		this._hasDocumentFormattingProvider = EditorContextKeys.hasDocumentFormattingProvider.bindTo(contextKeyService);
+		this._hasDocumentSelectionFormattingProvider = EditorContextKeys.hasDocumentSelectionFormattingProvider.bindTo(contextKeyService);
+		this._hasSignatureHelpProvider = EditorContextKeys.hasSignatureHelpProvider.bindTo(contextKeyService);
+		this._isInWalkThrough = EditorContextKeys.isInEmbeddedEditor.bindTo(contextKeyService);
+
+		const update = () => this._update();
+
+		// update when model/mode changes
+		this._register(editor.onDidChangeModel(update));
+		this._register(editor.onDidChangeModelLanguage(update));
+
+		// update when registries change
+		this._register(modes.SuggestRegistry.onDidChange(update));
+		this._register(modes.CodeActionProviderRegistry.onDidChange(update));
+		this._register(modes.CodeLensProviderRegistry.onDidChange(update));
+		this._register(modes.DefinitionProviderRegistry.onDidChange(update));
+		this._register(modes.ImplementationProviderRegistry.onDidChange(update));
+		this._register(modes.TypeDefinitionProviderRegistry.onDidChange(update));
+		this._register(modes.HoverProviderRegistry.onDidChange(update));
+		this._register(modes.DocumentHighlightProviderRegistry.onDidChange(update));
+		this._register(modes.DocumentSymbolProviderRegistry.onDidChange(update));
+		this._register(modes.ReferenceProviderRegistry.onDidChange(update));
+		this._register(modes.RenameProviderRegistry.onDidChange(update));
+		this._register(modes.DocumentFormattingEditProviderRegistry.onDidChange(update));
+		this._register(modes.DocumentRangeFormattingEditProviderRegistry.onDidChange(update));
+		this._register(modes.SignatureHelpProviderRegistry.onDidChange(update));
+
+		update();
+	}
+
+	dispose() {
+		super.dispose();
+	}
+
+	reset() {
+		this._langId.reset();
+		this._hasCompletionItemProvider.reset();
+		this._hasCodeActionsProvider.reset();
+		this._hasCodeLensProvider.reset();
+		this._hasDefinitionProvider.reset();
+		this._hasImplementationProvider.reset();
+		this._hasTypeDefinitionProvider.reset();
+		this._hasHoverProvider.reset();
+		this._hasDocumentHighlightProvider.reset();
+		this._hasDocumentSymbolProvider.reset();
+		this._hasReferenceProvider.reset();
+		this._hasRenameProvider.reset();
+		this._hasDocumentFormattingProvider.reset();
+		this._hasDocumentSelectionFormattingProvider.reset();
+		this._hasSignatureHelpProvider.reset();
+		this._isInWalkThrough.reset();
+	}
+
+	private _update() {
+		const model = this._editor.getModel();
+		if (!model) {
+			this.reset();
+			return;
+		}
+		this._langId.set(model.getLanguageIdentifier().language);
+		this._hasCompletionItemProvider.set(modes.SuggestRegistry.has(model));
+		this._hasCodeActionsProvider.set(modes.CodeActionProviderRegistry.has(model));
+		this._hasCodeLensProvider.set(modes.CodeLensProviderRegistry.has(model));
+		this._hasDefinitionProvider.set(modes.DefinitionProviderRegistry.has(model));
+		this._hasImplementationProvider.set(modes.ImplementationProviderRegistry.has(model));
+		this._hasTypeDefinitionProvider.set(modes.TypeDefinitionProviderRegistry.has(model));
+		this._hasHoverProvider.set(modes.HoverProviderRegistry.has(model));
+		this._hasDocumentHighlightProvider.set(modes.DocumentHighlightProviderRegistry.has(model));
+		this._hasDocumentSymbolProvider.set(modes.DocumentSymbolProviderRegistry.has(model));
+		this._hasReferenceProvider.set(modes.ReferenceProviderRegistry.has(model));
+		this._hasRenameProvider.set(modes.RenameProviderRegistry.has(model));
+		this._hasSignatureHelpProvider.set(modes.SignatureHelpProviderRegistry.has(model));
+		this._hasDocumentFormattingProvider.set(modes.DocumentFormattingEditProviderRegistry.has(model) || modes.DocumentRangeFormattingEditProviderRegistry.has(model));
+		this._hasDocumentSelectionFormattingProvider.set(modes.DocumentRangeFormattingEditProviderRegistry.has(model));
+		this._isInWalkThrough.set(model.uri.scheme === Schemas.walkThroughSnippet);
+	}
+}

src/vs/editor/common/modes/languageConfiguration.ts

@@ -0,0 +1,229 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { StandardTokenType } from 'vs/editor/common/modes';
+
+/**
+ * Describes how comments for a language work.
+ */
+export interface CommentRule {
+	/**
+	 * The line comment token, like `// this is a comment`
+	 */
+	lineComment?: string;
+	/**
+	 * The block comment character pair, like `/* block comment */`
+	 */
+	blockComment?: CharacterPair;
+}
+
+/**
+ * The language configuration interface defines the contract between extensions and
+ * various editor features, like automatic bracket insertion, automatic indentation etc.
+ */
+export interface LanguageConfiguration {
+	/**
+	 * The language's comment settings.
+	 */
+	comments?: CommentRule;
+	/**
+	 * The language's brackets.
+	 * This configuration implicitly affects pressing Enter around these brackets.
+	 */
+	brackets?: CharacterPair[];
+	/**
+	 * The language's word definition.
+	 * If the language supports Unicode identifiers (e.g. JavaScript), it is preferable
+	 * to provide a word definition that uses exclusion of known separators.
+	 * e.g.: A regex that matches anything except known separators (and dot is allowed to occur in a floating point number):
+	 *   /(-?\d*\.\d\w*)|([^\`\~\!\@\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g
+	 */
+	wordPattern?: RegExp;
+	/**
+	 * The language's indentation settings.
+	 */
+	indentationRules?: IndentationRule;
+	/**
+	 * The language's rules to be evaluated when pressing Enter.
+	 */
+	onEnterRules?: OnEnterRule[];
+	/**
+	 * The language's auto closing pairs. The 'close' character is automatically inserted with the
+	 * 'open' character is typed. If not set, the configured brackets will be used.
+	 */
+	autoClosingPairs?: IAutoClosingPairConditional[];
+	/**
+	 * The language's surrounding pairs. When the 'open' character is typed on a selection, the
+	 * selected string is surrounded by the open and close characters. If not set, the autoclosing pairs
+	 * settings will be used.
+	 */
+	surroundingPairs?: IAutoClosingPair[];
+	/**
+	 * **Deprecated** Do not use.
+	 *
+	 * @deprecated Will be replaced by a better API soon.
+	 */
+	__electricCharacterSupport?: IBracketElectricCharacterContribution;
+}
+
+/**
+ * Describes indentation rules for a language.
+ */
+export interface IndentationRule {
+	/**
+	 * If a line matches this pattern, then all the lines after it should be unindendented once (until another rule matches).
+	 */
+	decreaseIndentPattern: RegExp;
+	/**
+	 * If a line matches this pattern, then all the lines after it should be indented once (until another rule matches).
+	 */
+	increaseIndentPattern: RegExp;
+	/**
+	 * If a line matches this pattern, then **only the next line** after it should be indented once.
+	 */
+	indentNextLinePattern?: RegExp;
+	/**
+	 * If a line matches this pattern, then its indentation should not be changed and it should not be evaluated against the other rules.
+	 */
+	unIndentedLinePattern?: RegExp;
+}
+
+/**
+ * Describes a rule to be evaluated when pressing Enter.
+ */
+export interface OnEnterRule {
+	/**
+	 * This rule will only execute if the text before the cursor matches this regular expression.
+	 */
+	beforeText: RegExp;
+	/**
+	 * This rule will only execute if the text after the cursor matches this regular expression.
+	 */
+	afterText?: RegExp;
+	/**
+	 * The action to execute.
+	 */
+	action: EnterAction;
+}
+
+export interface IBracketElectricCharacterContribution {
+	docComment?: IDocComment;
+}
+
+/**
+ * Definition of documentation comments (e.g. Javadoc/JSdoc)
+ */
+export interface IDocComment {
+	/**
+	 * The string that starts a doc comment (e.g. '/**')
+	 */
+	open: string;
+	/**
+	 * The string that appears on the last line and closes the doc comment (e.g. ' * /').
+	 */
+	close: string;
+}
+
+/**
+ * A tuple of two characters, like a pair of
+ * opening and closing brackets.
+ */
+export type CharacterPair = [string, string];
+
+export interface IAutoClosingPair {
+	open: string;
+	close: string;
+}
+
+export interface IAutoClosingPairConditional extends IAutoClosingPair {
+	notIn?: string[];
+}
+
+/**
+ * Describes what to do with the indentation when pressing Enter.
+ */
+export enum IndentAction {
+	/**
+	 * Insert new line and copy the previous line's indentation.
+	 */
+	None = 0,
+	/**
+	 * Insert new line and indent once (relative to the previous line's indentation).
+	 */
+	Indent = 1,
+	/**
+	 * Insert two new lines:
+	 *  - the first one indented which will hold the cursor
+	 *  - the second one at the same indentation level
+	 */
+	IndentOutdent = 2,
+	/**
+	 * Insert new line and outdent once (relative to the previous line's indentation).
+	 */
+	Outdent = 3
+}
+
+/**
+ * Describes what to do when pressing Enter.
+ */
+export interface EnterAction {
+	/**
+	 * Describe what to do with the indentation.
+	 */
+	indentAction: IndentAction;
+	/**
+	 * Describe whether to outdent current line.
+	 */
+	outdentCurrentLine?: boolean;
+	/**
+	 * Describes text to be appended after the new line and after the indentation.
+	 */
+	appendText?: string;
+	/**
+	 * Describes the number of characters to remove from the new line's indentation.
+	 */
+	removeText?: number;
+}
+
+/**
+ * @internal
+ */
+export class StandardAutoClosingPairConditional {
+	_standardAutoClosingPairConditionalBrand: void;
+
+	readonly open: string;
+	readonly close: string;
+	private readonly _standardTokenMask: number;
+
+	constructor(source: IAutoClosingPairConditional) {
+		this.open = source.open;
+		this.close = source.close;
+
+		// initially allowed in all tokens
+		this._standardTokenMask = 0;
+
+		if (Array.isArray(source.notIn)) {
+			for (let i = 0, len = source.notIn.length; i < len; i++) {
+				let notIn = source.notIn[i];
+				switch (notIn) {
+					case 'string':
+						this._standardTokenMask |= StandardTokenType.String;
+						break;
+					case 'comment':
+						this._standardTokenMask |= StandardTokenType.Comment;
+						break;
+					case 'regex':
+						this._standardTokenMask |= StandardTokenType.RegEx;
+						break;
+				}
+			}
+		}
+	}
+
+	public isOK(standardToken: StandardTokenType): boolean {
+		return (this._standardTokenMask & <number>standardToken) === 0;
+	}
+}

src/vs/editor/common/modes/languageConfigurationRegistry.ts

@@ -0,0 +1,776 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { CharacterPairSupport } from 'vs/editor/common/modes/supports/characterPair';
+import { BracketElectricCharacterSupport, IElectricAction } from 'vs/editor/common/modes/supports/electricCharacter';
+import { IOnEnterSupportOptions, OnEnterSupport } from 'vs/editor/common/modes/supports/onEnter';
+import { IndentRulesSupport, IndentConsts } from 'vs/editor/common/modes/supports/indentRules';
+import { RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
+import Event, { Emitter } from 'vs/base/common/event';
+import { ITokenizedModel } from 'vs/editor/common/editorCommon';
+import { onUnexpectedError } from 'vs/base/common/errors';
+import * as strings from 'vs/base/common/strings';
+import { IDisposable } from 'vs/base/common/lifecycle';
+import { DEFAULT_WORD_REGEXP, ensureValidWordDefinition } from 'vs/editor/common/model/wordHelper';
+import { createScopedLineTokens } from 'vs/editor/common/modes/supports';
+import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { Range } from 'vs/editor/common/core/range';
+import { IndentAction, EnterAction, IAutoClosingPair, LanguageConfiguration, IndentationRule } from 'vs/editor/common/modes/languageConfiguration';
+import { LanguageIdentifier, LanguageId } from 'vs/editor/common/modes';
+
+/**
+ * Interface used to support insertion of mode specific comments.
+ */
+export interface ICommentsConfiguration {
+	lineCommentToken?: string;
+	blockCommentStartToken?: string;
+	blockCommentEndToken?: string;
+}
+
+export interface IVirtualModel {
+	getLineTokens(lineNumber: number): LineTokens;
+	getLanguageIdentifier(): LanguageIdentifier;
+	getLanguageIdAtPosition(lineNumber: number, column: number): LanguageId;
+	getLineContent(lineNumber: number): string;
+}
+
+export interface IIndentConverter {
+	shiftIndent?(indentation: string): string;
+	unshiftIndent?(indentation: string): string;
+	normalizeIndentation?(indentation: string): string;
+}
+
+export class RichEditSupport {
+
+	private readonly _conf: LanguageConfiguration;
+
+	public readonly electricCharacter: BracketElectricCharacterSupport;
+	public readonly comments: ICommentsConfiguration;
+	public readonly characterPair: CharacterPairSupport;
+	public readonly wordDefinition: RegExp;
+	public readonly onEnter: OnEnterSupport;
+	public readonly indentRulesSupport: IndentRulesSupport;
+	public readonly brackets: RichEditBrackets;
+	public readonly indentationRules: IndentationRule;
+
+	constructor(languageIdentifier: LanguageIdentifier, previous: RichEditSupport, rawConf: LanguageConfiguration) {
+
+		let prev: LanguageConfiguration = null;
+		if (previous) {
+			prev = previous._conf;
+		}
+
+		this._conf = RichEditSupport._mergeConf(prev, rawConf);
+
+		if (this._conf.brackets) {
+			this.brackets = new RichEditBrackets(languageIdentifier, this._conf.brackets);
+		}
+
+		this.onEnter = RichEditSupport._handleOnEnter(this._conf);
+
+		this.comments = RichEditSupport._handleComments(this._conf);
+
+		this.characterPair = new CharacterPairSupport(this._conf);
+		this.electricCharacter = new BracketElectricCharacterSupport(this.brackets, this.characterPair.getAutoClosingPairs(), this._conf.__electricCharacterSupport);
+
+		this.wordDefinition = this._conf.wordPattern || DEFAULT_WORD_REGEXP;
+
+		this.indentationRules = this._conf.indentationRules;
+		if (this._conf.indentationRules) {
+			this.indentRulesSupport = new IndentRulesSupport(this._conf.indentationRules);
+		}
+	}
+
+	private static _mergeConf(prev: LanguageConfiguration, current: LanguageConfiguration): LanguageConfiguration {
+		return {
+			comments: (prev ? current.comments || prev.comments : current.comments),
+			brackets: (prev ? current.brackets || prev.brackets : current.brackets),
+			wordPattern: (prev ? current.wordPattern || prev.wordPattern : current.wordPattern),
+			indentationRules: (prev ? current.indentationRules || prev.indentationRules : current.indentationRules),
+			onEnterRules: (prev ? current.onEnterRules || prev.onEnterRules : current.onEnterRules),
+			autoClosingPairs: (prev ? current.autoClosingPairs || prev.autoClosingPairs : current.autoClosingPairs),
+			surroundingPairs: (prev ? current.surroundingPairs || prev.surroundingPairs : current.surroundingPairs),
+			__electricCharacterSupport: (prev ? current.__electricCharacterSupport || prev.__electricCharacterSupport : current.__electricCharacterSupport),
+		};
+	}
+
+	private static _handleOnEnter(conf: LanguageConfiguration): OnEnterSupport {
+		// on enter
+		let onEnter: IOnEnterSupportOptions = {};
+		let empty = true;
+
+		if (conf.brackets) {
+			empty = false;
+			onEnter.brackets = conf.brackets;
+		}
+		if (conf.indentationRules) {
+			empty = false;
+			onEnter.indentationRules = conf.indentationRules;
+		}
+		if (conf.onEnterRules) {
+			empty = false;
+			onEnter.regExpRules = conf.onEnterRules;
+		}
+
+		if (!empty) {
+			return new OnEnterSupport(onEnter);
+		}
+		return null;
+	}
+
+	private static _handleComments(conf: LanguageConfiguration): ICommentsConfiguration {
+		let commentRule = conf.comments;
+		if (!commentRule) {
+			return null;
+		}
+
+		// comment configuration
+		let comments: ICommentsConfiguration = {};
+
+		if (commentRule.lineComment) {
+			comments.lineCommentToken = commentRule.lineComment;
+		}
+		if (commentRule.blockComment) {
+			let [blockStart, blockEnd] = commentRule.blockComment;
+			comments.blockCommentStartToken = blockStart;
+			comments.blockCommentEndToken = blockEnd;
+		}
+
+		return comments;
+	}
+}
+
+export class LanguageConfigurationRegistryImpl {
+
+	private _entries: RichEditSupport[];
+
+	private _onDidChange: Emitter<void> = new Emitter<void>();
+	public onDidChange: Event<void> = this._onDidChange.event;
+
+	constructor() {
+		this._entries = [];
+	}
+
+	public register(languageIdentifier: LanguageIdentifier, configuration: LanguageConfiguration): IDisposable {
+		let previous = this._getRichEditSupport(languageIdentifier.id);
+		let current = new RichEditSupport(languageIdentifier, previous, configuration);
+		this._entries[languageIdentifier.id] = current;
+		this._onDidChange.fire(void 0);
+		return {
+			dispose: () => {
+				if (this._entries[languageIdentifier.id] === current) {
+					this._entries[languageIdentifier.id] = previous;
+					this._onDidChange.fire(void 0);
+				}
+			}
+		};
+	}
+
+	private _getRichEditSupport(languageId: LanguageId): RichEditSupport {
+		return this._entries[languageId] || null;
+	}
+
+	public getIndentationRules(languageId: LanguageId) {
+		let value = this._entries[languageId];
+
+		if (!value) {
+			return null;
+		}
+
+		return value.indentationRules || null;
+	}
+
+	// begin electricCharacter
+
+	private _getElectricCharacterSupport(languageId: LanguageId): BracketElectricCharacterSupport {
+		let value = this._getRichEditSupport(languageId);
+		if (!value) {
+			return null;
+		}
+		return value.electricCharacter || null;
+	}
+
+	public getElectricCharacters(languageId: LanguageId): string[] {
+		let electricCharacterSupport = this._getElectricCharacterSupport(languageId);
+		if (!electricCharacterSupport) {
+			return [];
+		}
+		return electricCharacterSupport.getElectricCharacters();
+	}
+
+	/**
+	 * Should return opening bracket type to match indentation with
+	 */
+	public onElectricCharacter(character: string, context: LineTokens, column: number): IElectricAction {
+		let scopedLineTokens = createScopedLineTokens(context, column - 1);
+		let electricCharacterSupport = this._getElectricCharacterSupport(scopedLineTokens.languageId);
+		if (!electricCharacterSupport) {
+			return null;
+		}
+		return electricCharacterSupport.onElectricCharacter(character, scopedLineTokens, column - scopedLineTokens.firstCharOffset);
+	}
+
+	// end electricCharacter
+
+	public getComments(languageId: LanguageId): ICommentsConfiguration {
+		let value = this._getRichEditSupport(languageId);
+		if (!value) {
+			return null;
+		}
+		return value.comments || null;
+	}
+
+	// begin characterPair
+
+	private _getCharacterPairSupport(languageId: LanguageId): CharacterPairSupport {
+		let value = this._getRichEditSupport(languageId);
+		if (!value) {
+			return null;
+		}
+		return value.characterPair || null;
+	}
+
+	public getAutoClosingPairs(languageId: LanguageId): IAutoClosingPair[] {
+		let characterPairSupport = this._getCharacterPairSupport(languageId);
+		if (!characterPairSupport) {
+			return [];
+		}
+		return characterPairSupport.getAutoClosingPairs();
+	}
+
+	public getSurroundingPairs(languageId: LanguageId): IAutoClosingPair[] {
+		let characterPairSupport = this._getCharacterPairSupport(languageId);
+		if (!characterPairSupport) {
+			return [];
+		}
+		return characterPairSupport.getSurroundingPairs();
+	}
+
+	public shouldAutoClosePair(character: string, context: LineTokens, column: number): boolean {
+		let scopedLineTokens = createScopedLineTokens(context, column - 1);
+		let characterPairSupport = this._getCharacterPairSupport(scopedLineTokens.languageId);
+		if (!characterPairSupport) {
+			return false;
+		}
+		return characterPairSupport.shouldAutoClosePair(character, scopedLineTokens, column - scopedLineTokens.firstCharOffset);
+	}
+
+	// end characterPair
+
+	public getWordDefinition(languageId: LanguageId): RegExp {
+		let value = this._getRichEditSupport(languageId);
+		if (!value) {
+			return ensureValidWordDefinition(null);
+		}
+		return ensureValidWordDefinition(value.wordDefinition || null);
+	}
+
+
+
+	// beigin Indent Rules
+
+	public getIndentRulesSupport(languageId: LanguageId): IndentRulesSupport {
+		let value = this._getRichEditSupport(languageId);
+		if (!value) {
+			return null;
+		}
+		return value.indentRulesSupport || null;
+	}
+
+	/**
+	 * Get nearest preceiding line which doesn't match unIndentPattern or contains all whitespace.
+	 * Result:
+	 * -1: run into the boundary of embedded languages
+	 * 0: every line above are invalid
+	 * else: nearest preceding line of the same language
+	 */
+	private getPrecedingValidLine(model: IVirtualModel, lineNumber: number, indentRulesSupport: IndentRulesSupport) {
+		let languageID = model.getLanguageIdAtPosition(lineNumber, 0);
+		if (lineNumber > 1) {
+			let lastLineNumber = lineNumber - 1;
+			let resultLineNumber = -1;
+
+			for (lastLineNumber = lineNumber - 1; lastLineNumber >= 1; lastLineNumber--) {
+				if (model.getLanguageIdAtPosition(lastLineNumber, 0) !== languageID) {
+					return resultLineNumber;
+				}
+				let text = model.getLineContent(lastLineNumber);
+				if (indentRulesSupport.shouldIgnore(text) || /^\s+$/.test(text) || text === '') {
+					resultLineNumber = lastLineNumber;
+					continue;
+				}
+
+				return lastLineNumber;
+			}
+		}
+
+		return -1;
+	}
+
+	/**
+	 * Get inherited indentation from above lines.
+	 * 1. Find the nearest preceding line which doesn't match unIndentedLinePattern.
+	 * 2. If this line matches indentNextLinePattern or increaseIndentPattern, it means that the indent level of `lineNumber` should be 1 greater than this line.
+	 * 3. If this line doesn't match any indent rules
+	 *   a. check whether the line above it matches indentNextLinePattern
+	 *   b. If not, the indent level of this line is the result
+	 *   c. If so, it means the indent of this line is *temporary*, go upward utill we find a line whose indent is not temporary (the same workflow a -> b -> c).
+	 * 4. Otherwise, we fail to get an inherited indent from aboves. Return null and we should not touch the indent of `lineNumber`
+	 *
+	 * This function only return the inherited indent based on above lines, it doesn't check whether current line should decrease or not.
+	 */
+	public getInheritIndentForLine(model: IVirtualModel, lineNumber: number, honorIntentialIndent: boolean = true): { indentation: string, action: IndentAction, line?: number } {
+		let indentRulesSupport = this.getIndentRulesSupport(model.getLanguageIdentifier().id);
+		if (!indentRulesSupport) {
+			return null;
+		}
+
+		if (lineNumber <= 1) {
+			return {
+				indentation: '',
+				action: null
+			};
+		}
+
+		let precedingUnIgnoredLine = this.getPrecedingValidLine(model, lineNumber, indentRulesSupport);
+		if (precedingUnIgnoredLine < 0) {
+			return null;
+		} else if (precedingUnIgnoredLine < 1) {
+			return {
+				indentation: '',
+				action: null
+			};
+		}
+
+		let precedingUnIgnoredLineContent = model.getLineContent(precedingUnIgnoredLine);
+
+		if (indentRulesSupport.shouldIncrease(precedingUnIgnoredLineContent) || indentRulesSupport.shouldIndentNextLine(precedingUnIgnoredLineContent)) {
+			return {
+				indentation: strings.getLeadingWhitespace(precedingUnIgnoredLineContent),
+				action: IndentAction.Indent,
+				line: precedingUnIgnoredLine
+			};
+		} else if (indentRulesSupport.shouldDecrease(precedingUnIgnoredLineContent)) {
+			return {
+				indentation: strings.getLeadingWhitespace(precedingUnIgnoredLineContent),
+				action: null,
+				line: precedingUnIgnoredLine
+			};
+		} else {
+			// precedingUnIgnoredLine can not be ignored.
+			// it doesn't increase indent of following lines
+			// it doesn't increase just next line
+			// so current line is not affect by precedingUnIgnoredLine
+			// and then we should get a correct inheritted indentation from above lines
+			if (precedingUnIgnoredLine === 1) {
+				return {
+					indentation: strings.getLeadingWhitespace(model.getLineContent(precedingUnIgnoredLine)),
+					action: null,
+					line: precedingUnIgnoredLine
+				};
+			}
+
+			let previousLine = precedingUnIgnoredLine - 1;
+
+			let previousLineIndentMetadata = indentRulesSupport.getIndentMetadata(model.getLineContent(previousLine));
+			if (!(previousLineIndentMetadata & (IndentConsts.INCREASE_MASK | IndentConsts.DECREASE_MASK)) &&
+				(previousLineIndentMetadata & IndentConsts.INDENT_NEXTLINE_MASK)) {
+				let stopLine = 0;
+				for (let i = previousLine - 1; i > 0; i--) {
+					if (indentRulesSupport.shouldIndentNextLine(model.getLineContent(i))) {
+						continue;
+					}
+					stopLine = i;
+					break;
+				}
+
+				return {
+					indentation: strings.getLeadingWhitespace(model.getLineContent(stopLine + 1)),
+					action: null,
+					line: stopLine + 1
+				};
+			}
+
+			if (honorIntentialIndent) {
+				return {
+					indentation: strings.getLeadingWhitespace(model.getLineContent(precedingUnIgnoredLine)),
+					action: null,
+					line: precedingUnIgnoredLine
+				};
+			} else {
+				// search from precedingUnIgnoredLine until we find one whose indent is not temporary
+				for (let i = precedingUnIgnoredLine; i > 0; i--) {
+					let lineContent = model.getLineContent(i);
+					if (indentRulesSupport.shouldIncrease(lineContent)) {
+						return {
+							indentation: strings.getLeadingWhitespace(lineContent),
+							action: IndentAction.Indent,
+							line: i
+						};
+					} else if (indentRulesSupport.shouldIndentNextLine(lineContent)) {
+						let stopLine = 0;
+						for (let j = i - 1; j > 0; j--) {
+							if (indentRulesSupport.shouldIndentNextLine(model.getLineContent(i))) {
+								continue;
+							}
+							stopLine = j;
+							break;
+						}
+
+						return {
+							indentation: strings.getLeadingWhitespace(model.getLineContent(stopLine + 1)),
+							action: null,
+							line: stopLine + 1
+						};
+					} else if (indentRulesSupport.shouldDecrease(lineContent)) {
+						return {
+							indentation: strings.getLeadingWhitespace(lineContent),
+							action: null,
+							line: i
+						};
+					}
+				}
+
+				return {
+					indentation: strings.getLeadingWhitespace(model.getLineContent(1)),
+					action: null,
+					line: 1
+				};
+			}
+		}
+	}
+
+	public getGoodIndentForLine(virtualModel: IVirtualModel, languageId: LanguageId, lineNumber: number, indentConverter: IIndentConverter): string {
+		let indentRulesSupport = this.getIndentRulesSupport(languageId);
+		if (!indentRulesSupport) {
+			return null;
+		}
+
+		let indent = this.getInheritIndentForLine(virtualModel, lineNumber);
+		let lineContent = virtualModel.getLineContent(lineNumber);
+
+		if (indent) {
+			let inheritLine = indent.line;
+			if (inheritLine !== undefined) {
+				let onEnterSupport = this._getOnEnterSupport(languageId);
+				let enterResult: EnterAction = null;
+				try {
+					enterResult = onEnterSupport.onEnter('', virtualModel.getLineContent(inheritLine), '');
+				} catch (e) {
+					onUnexpectedError(e);
+				}
+
+				if (enterResult) {
+					let indentation = strings.getLeadingWhitespace(virtualModel.getLineContent(inheritLine));
+
+					if (enterResult.removeText) {
+						indentation = indentation.substring(0, indentation.length - enterResult.removeText);
+					}
+
+					if (
+						(enterResult.indentAction === IndentAction.Indent) ||
+						(enterResult.indentAction === IndentAction.IndentOutdent)
+					) {
+						indentation = indentConverter.shiftIndent(indentation);
+					} else if (enterResult.indentAction === IndentAction.Outdent) {
+						indentation = indentConverter.unshiftIndent(indentation);
+					}
+
+					if (indentRulesSupport.shouldDecrease(lineContent)) {
+						indentation = indentConverter.unshiftIndent(indentation);
+					}
+
+					if (enterResult.appendText) {
+						indentation += enterResult.appendText;
+					}
+
+					return strings.getLeadingWhitespace(indentation);
+				}
+			}
+
+			if (indentRulesSupport.shouldDecrease(lineContent)) {
+				if (indent.action === IndentAction.Indent) {
+					return indent.indentation;
+				} else {
+					return indentConverter.unshiftIndent(indent.indentation);
+				}
+			} else {
+				if (indent.action === IndentAction.Indent) {
+					return indentConverter.shiftIndent(indent.indentation);
+				} else {
+					return indent.indentation;
+				}
+			}
+		}
+		return null;
+	}
+
+	public getIndentForEnter(model: ITokenizedModel, range: Range, indentConverter: IIndentConverter, autoIndent: boolean): { beforeEnter: string, afterEnter: string } {
+		model.forceTokenization(range.startLineNumber);
+		let lineTokens = model.getLineTokens(range.startLineNumber);
+
+		let beforeEnterText;
+		let afterEnterText;
+		let scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1);
+		let scopedLineText = scopedLineTokens.getLineContent();
+
+		let embeddedLanguage = false;
+		if (scopedLineTokens.firstCharOffset > 0 && lineTokens.getLanguageId(0) !== scopedLineTokens.languageId) {
+			// we are in the embeded language content
+			embeddedLanguage = true; // if embeddedLanguage is true, then we don't touch the indentation of current line
+			beforeEnterText = scopedLineText.substr(0, range.startColumn - 1 - scopedLineTokens.firstCharOffset);
+		} else {
+			beforeEnterText = lineTokens.getLineContent().substring(0, range.startColumn - 1);
+		}
+
+		if (range.isEmpty()) {
+			afterEnterText = scopedLineText.substr(range.startColumn - 1 - scopedLineTokens.firstCharOffset);
+		} else {
+			const endScopedLineTokens = this.getScopedLineTokens(model, range.endLineNumber, range.endColumn);
+			afterEnterText = endScopedLineTokens.getLineContent().substr(range.endColumn - 1 - scopedLineTokens.firstCharOffset);
+		}
+
+		let indentRulesSupport = this.getIndentRulesSupport(scopedLineTokens.languageId);
+
+		if (!indentRulesSupport) {
+			return null;
+		}
+
+		let beforeEnterResult = beforeEnterText;
+		let beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterText);
+
+		if (!autoIndent && !embeddedLanguage) {
+			let beforeEnterIndentAction = this.getInheritIndentForLine(model, range.startLineNumber);
+
+			if (indentRulesSupport.shouldDecrease(beforeEnterText)) {
+				if (beforeEnterIndentAction) {
+					beforeEnterIndent = beforeEnterIndentAction.indentation;
+					if (beforeEnterIndentAction.action !== IndentAction.Indent) {
+						beforeEnterIndent = indentConverter.unshiftIndent(beforeEnterIndent);
+					}
+				}
+			}
+
+			beforeEnterResult = beforeEnterIndent + strings.ltrim(strings.ltrim(beforeEnterText, ' '), '\t');
+		}
+
+		let virtualModel: IVirtualModel = {
+			getLineTokens: (lineNumber: number) => {
+				return model.getLineTokens(lineNumber);
+			},
+			getLanguageIdentifier: () => {
+				return model.getLanguageIdentifier();
+			},
+			getLanguageIdAtPosition: (lineNumber: number, column: number) => {
+				return model.getLanguageIdAtPosition(lineNumber, column);
+			},
+			getLineContent: (lineNumber: number) => {
+				if (lineNumber === range.startLineNumber) {
+					return beforeEnterResult;
+				} else {
+					return model.getLineContent(lineNumber);
+				}
+			}
+		};
+
+		let currentLineIndent = strings.getLeadingWhitespace(lineTokens.getLineContent());
+		let afterEnterAction = this.getInheritIndentForLine(virtualModel, range.startLineNumber + 1);
+		if (!afterEnterAction) {
+			let beforeEnter = embeddedLanguage ? currentLineIndent : beforeEnterIndent;
+			return {
+				beforeEnter: beforeEnter,
+				afterEnter: beforeEnter
+			};
+		}
+
+		let afterEnterIndent = embeddedLanguage ? currentLineIndent : afterEnterAction.indentation;
+
+		if (afterEnterAction.action === IndentAction.Indent) {
+			afterEnterIndent = indentConverter.shiftIndent(afterEnterIndent);
+		}
+
+		if (indentRulesSupport.shouldDecrease(afterEnterText)) {
+			afterEnterIndent = indentConverter.unshiftIndent(afterEnterIndent);
+		}
+
+		return {
+			beforeEnter: embeddedLanguage ? currentLineIndent : beforeEnterIndent,
+			afterEnter: afterEnterIndent
+		};
+	}
+
+	/**
+	 * We should always allow intentional indentation. It means, if users change the indentation of `lineNumber` and the content of
+	 * this line doesn't match decreaseIndentPattern, we should not adjust the indentation.
+	 */
+	public getIndentActionForType(model: ITokenizedModel, range: Range, ch: string, indentConverter: IIndentConverter): string {
+		let scopedLineTokens = this.getScopedLineTokens(model, range.startLineNumber, range.startColumn);
+		let indentRulesSupport = this.getIndentRulesSupport(scopedLineTokens.languageId);
+		if (!indentRulesSupport) {
+			return null;
+		}
+
+		let scopedLineText = scopedLineTokens.getLineContent();
+		let beforeTypeText = scopedLineText.substr(0, range.startColumn - 1 - scopedLineTokens.firstCharOffset);
+		let afterTypeText;
+
+		// selection support
+		if (range.isEmpty()) {
+			afterTypeText = scopedLineText.substr(range.startColumn - 1 - scopedLineTokens.firstCharOffset);
+		} else {
+			const endScopedLineTokens = this.getScopedLineTokens(model, range.endLineNumber, range.endColumn);
+			afterTypeText = endScopedLineTokens.getLineContent().substr(range.endColumn - 1 - scopedLineTokens.firstCharOffset);
+		}
+
+		// If previous content already matches decreaseIndentPattern, it means indentation of this line should already be adjusted
+		// Users might change the indentation by purpose and we should honor that instead of readjusting.
+		if (!indentRulesSupport.shouldDecrease(beforeTypeText + afterTypeText) && indentRulesSupport.shouldDecrease(beforeTypeText + ch + afterTypeText)) {
+			// after typing `ch`, the content matches decreaseIndentPattern, we should adjust the indent to a good manner.
+			// 1. Get inherited indent action
+			let r = this.getInheritIndentForLine(model, range.startLineNumber, false);
+			if (!r) {
+				return null;
+			}
+
+			let indentation = r.indentation;
+
+			if (r.action !== IndentAction.Indent) {
+				indentation = indentConverter.unshiftIndent(indentation);
+			}
+
+			return indentation;
+		}
+
+		return null;
+	}
+
+	public getIndentMetadata(model: ITokenizedModel, lineNumber: number): number {
+		let indentRulesSupport = this.getIndentRulesSupport(model.getLanguageIdentifier().id);
+		if (!indentRulesSupport) {
+			return null;
+		}
+
+		if (lineNumber < 1 || lineNumber > model.getLineCount()) {
+			return null;
+		}
+
+		return indentRulesSupport.getIndentMetadata(model.getLineContent(lineNumber));
+	}
+
+	// end Indent Rules
+
+	// begin onEnter
+
+	private _getOnEnterSupport(languageId: LanguageId): OnEnterSupport {
+		let value = this._getRichEditSupport(languageId);
+		if (!value) {
+			return null;
+		}
+		return value.onEnter || null;
+	}
+
+	public getRawEnterActionAtPosition(model: ITokenizedModel, lineNumber: number, column: number): EnterAction {
+		let r = this.getEnterAction(model, new Range(lineNumber, column, lineNumber, column));
+
+		return r ? r.enterAction : null;
+	}
+
+	public getEnterAction(model: ITokenizedModel, range: Range): { enterAction: EnterAction; indentation: string; } {
+		let indentation = this.getIndentationAtPosition(model, range.startLineNumber, range.startColumn);
+
+		let scopedLineTokens = this.getScopedLineTokens(model, range.startLineNumber, range.startColumn);
+		let onEnterSupport = this._getOnEnterSupport(scopedLineTokens.languageId);
+		if (!onEnterSupport) {
+			return null;
+		}
+
+		let scopedLineText = scopedLineTokens.getLineContent();
+		let beforeEnterText = scopedLineText.substr(0, range.startColumn - 1 - scopedLineTokens.firstCharOffset);
+		let afterEnterText;
+
+		// selection support
+		if (range.isEmpty()) {
+			afterEnterText = scopedLineText.substr(range.startColumn - 1 - scopedLineTokens.firstCharOffset);
+		} else {
+			const endScopedLineTokens = this.getScopedLineTokens(model, range.endLineNumber, range.endColumn);
+			afterEnterText = endScopedLineTokens.getLineContent().substr(range.endColumn - 1 - scopedLineTokens.firstCharOffset);
+		}
+
+		let lineNumber = range.startLineNumber;
+		let oneLineAboveText = '';
+
+		if (lineNumber > 1 && scopedLineTokens.firstCharOffset === 0) {
+			// This is not the first line and the entire line belongs to this mode
+			let oneLineAboveScopedLineTokens = this.getScopedLineTokens(model, lineNumber - 1);
+			if (oneLineAboveScopedLineTokens.languageId === scopedLineTokens.languageId) {
+				// The line above ends with text belonging to the same mode
+				oneLineAboveText = oneLineAboveScopedLineTokens.getLineContent();
+			}
+		}
+
+		let enterResult: EnterAction = null;
+		try {
+			enterResult = onEnterSupport.onEnter(oneLineAboveText, beforeEnterText, afterEnterText);
+		} catch (e) {
+			onUnexpectedError(e);
+		}
+
+		if (!enterResult) {
+			return null;
+		} else {
+			// Here we add `\t` to appendText first because enterAction is leveraging appendText and removeText to change indentation.
+			if (!enterResult.appendText) {
+				if (
+					(enterResult.indentAction === IndentAction.Indent) ||
+					(enterResult.indentAction === IndentAction.IndentOutdent)
+				) {
+					enterResult.appendText = '\t';
+				} else {
+					enterResult.appendText = '';
+				}
+			}
+		}
+
+		if (enterResult.removeText) {
+			indentation = indentation.substring(0, indentation.length - enterResult.removeText);
+		}
+
+		return {
+			enterAction: enterResult,
+			indentation: indentation,
+		};
+	}
+
+	public getIndentationAtPosition(model: ITokenizedModel, lineNumber: number, column: number): string {
+		let lineText = model.getLineContent(lineNumber);
+		let indentation = strings.getLeadingWhitespace(lineText);
+		if (indentation.length > column - 1) {
+			indentation = indentation.substring(0, column - 1);
+		}
+
+		return indentation;
+	}
+
+	private getScopedLineTokens(model: ITokenizedModel, lineNumber: number, columnNumber?: number) {
+		model.forceTokenization(lineNumber);
+		let lineTokens = model.getLineTokens(lineNumber);
+		let column = isNaN(columnNumber) ? model.getLineMaxColumn(lineNumber) - 1 : columnNumber - 1;
+		let scopedLineTokens = createScopedLineTokens(lineTokens, column);
+		return scopedLineTokens;
+	}
+
+	// end onEnter
+
+	public getBracketsSupport(languageId: LanguageId): RichEditBrackets {
+		let value = this._getRichEditSupport(languageId);
+		if (!value) {
+			return null;
+		}
+		return value.brackets || null;
+	}
+}
+
+export const LanguageConfigurationRegistry = new LanguageConfigurationRegistryImpl();

src/vs/editor/common/modes/languageFeatureRegistry.ts

@@ -0,0 +1,163 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import Event, { Emitter } from 'vs/base/common/event';
+import { IDisposable } from 'vs/base/common/lifecycle';
+import { IReadOnlyModel } from 'vs/editor/common/editorCommon';
+import { LanguageSelector, score } from 'vs/editor/common/modes/languageSelector';
+
+interface Entry<T> {
+	selector: LanguageSelector;
+	provider: T;
+	_score: number;
+	_time: number;
+}
+
+export default class LanguageFeatureRegistry<T> {
+
+	private _clock: number = 0;
+	private _entries: Entry<T>[] = [];
+	private _onDidChange: Emitter<number> = new Emitter<number>();
+
+	constructor() {
+	}
+
+	get onDidChange(): Event<number> {
+		return this._onDidChange.event;
+	}
+
+	register(selector: LanguageSelector, provider: T): IDisposable {
+
+		let entry: Entry<T> = {
+			selector,
+			provider,
+			_score: -1,
+			_time: this._clock++
+		};
+
+		this._entries.push(entry);
+		this._lastCandidate = undefined;
+		this._onDidChange.fire(this._entries.length);
+
+		return {
+			dispose: () => {
+				if (entry) {
+					let idx = this._entries.indexOf(entry);
+					if (idx >= 0) {
+						this._entries.splice(idx, 1);
+						this._lastCandidate = undefined;
+						this._onDidChange.fire(this._entries.length);
+						entry = undefined;
+					}
+				}
+			}
+		};
+	}
+
+	has(model: IReadOnlyModel): boolean {
+		return this.all(model).length > 0;
+	}
+
+	all(model: IReadOnlyModel): T[] {
+		if (!model || model.isTooLargeForHavingARichMode()) {
+			return [];
+		}
+
+		this._updateScores(model);
+		const result: T[] = [];
+
+		// from registry
+		for (let entry of this._entries) {
+			if (entry._score > 0) {
+				result.push(entry.provider);
+			}
+		}
+
+		return result;
+	}
+
+	ordered(model: IReadOnlyModel): T[] {
+		const result: T[] = [];
+		this._orderedForEach(model, entry => result.push(entry.provider));
+		return result;
+	}
+
+	orderedGroups(model: IReadOnlyModel): T[][] {
+		const result: T[][] = [];
+		let lastBucket: T[];
+		let lastBucketScore: number;
+
+		this._orderedForEach(model, entry => {
+			if (lastBucket && lastBucketScore === entry._score) {
+				lastBucket.push(entry.provider);
+			} else {
+				lastBucketScore = entry._score;
+				lastBucket = [entry.provider];
+				result.push(lastBucket);
+			}
+		});
+
+		return result;
+	}
+
+	private _orderedForEach(model: IReadOnlyModel, callback: (provider: Entry<T>) => any): void {
+
+		if (!model || model.isTooLargeForHavingARichMode()) {
+			return;
+		}
+
+		this._updateScores(model);
+
+		for (let from = 0; from < this._entries.length; from++) {
+			let entry = this._entries[from];
+			if (entry._score > 0) {
+				callback(entry);
+			}
+		}
+	}
+
+	private _lastCandidate: { uri: string; language: string; };
+
+	private _updateScores(model: IReadOnlyModel): void {
+
+		let candidate = {
+			uri: model.uri.toString(),
+			language: model.getLanguageIdentifier().language
+		};
+
+		if (this._lastCandidate
+			&& this._lastCandidate.language === candidate.language
+			&& this._lastCandidate.uri === candidate.uri) {
+
+			// nothing has changed
+			return;
+		}
+
+		this._lastCandidate = candidate;
+
+		for (let entry of this._entries) {
+			entry._score = score(entry.selector, model.uri, model.getLanguageIdentifier().language);
+		}
+
+		// needs sorting
+		this._entries.sort(LanguageFeatureRegistry._compareByScoreAndTime);
+	}
+
+	private static _compareByScoreAndTime(a: Entry<any>, b: Entry<any>): number {
+		if (a._score < b._score) {
+			return 1;
+		} else if (a._score > b._score) {
+			return -1;
+		} else if (a._time < b._time) {
+			return 1;
+		} else if (a._time > b._time) {
+			return -1;
+		} else {
+			return 0;
+		}
+	}
+}

src/vs/editor/common/modes/languageSelector.ts

@@ -0,0 +1,90 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import URI from 'vs/base/common/uri';
+import { match as matchGlobPattern } from 'vs/base/common/glob'; // TODO@Alex
+
+export interface LanguageFilter {
+	language?: string;
+	scheme?: string;
+	pattern?: string;
+}
+
+export type LanguageSelector = string | LanguageFilter | (string | LanguageFilter)[];
+
+export default function matches(selection: LanguageSelector, uri: URI, language: string): boolean {
+	return score(selection, uri, language) > 0;
+}
+
+export function score(selector: LanguageSelector, candidateUri: URI, candidateLanguage: string): number {
+
+	if (Array.isArray(selector)) {
+		// array -> take max individual value
+		let ret = 0;
+		for (const filter of selector) {
+			const value = score(filter, candidateUri, candidateLanguage);
+			if (value === 10) {
+				return value; // already at the highest
+			}
+			if (value > ret) {
+				ret = value;
+			}
+		}
+		return ret;
+
+	} else if (typeof selector === 'string') {
+		// short-hand notion, desugars to
+		// 'fooLang' -> [{ language: 'fooLang', scheme: 'file' }, { language: 'fooLang', scheme: 'untitled' }]
+		// '*' -> { language: '*', scheme: '*' }
+		if (selector === '*') {
+			return 5;
+		} else if (selector === candidateLanguage) {
+			return 10;
+		} else {
+			return 0;
+		}
+
+	} else if (selector) {
+		// filter -> select accordingly, use defaults for scheme
+		const { language, pattern, scheme } = selector;
+
+		let ret = 0;
+
+		if (scheme) {
+			if (scheme === candidateUri.scheme) {
+				ret = 10;
+			} else if (scheme === '*') {
+				ret = 5;
+			} else {
+				return 0;
+			}
+		}
+
+		if (language) {
+			if (language === candidateLanguage) {
+				ret = 10;
+			} else if (language === '*') {
+				ret = Math.max(ret, 5);
+			} else {
+				return 0;
+			}
+		}
+
+		if (pattern) {
+			if (pattern === candidateUri.fsPath || matchGlobPattern(pattern, candidateUri.fsPath)) {
+				ret = 10;
+			} else {
+				return 0;
+			}
+		}
+
+		return ret;
+
+	} else {
+		return 0;
+	}
+}

src/vs/editor/common/modes/linkComputer.ts

@@ -0,0 +1,286 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { ILink } from 'vs/editor/common/modes';
+import { CharCode } from 'vs/base/common/charCode';
+import { CharacterClassifier } from 'vs/editor/common/core/characterClassifier';
+import { Uint8Matrix } from 'vs/editor/common/core/uint';
+
+export interface ILinkComputerTarget {
+	getLineCount(): number;
+	getLineContent(lineNumber: number): string;
+}
+
+const enum State {
+	Invalid = 0,
+	Start = 1,
+	H = 2,
+	HT = 3,
+	HTT = 4,
+	HTTP = 5,
+	F = 6,
+	FI = 7,
+	FIL = 8,
+	BeforeColon = 9,
+	AfterColon = 10,
+	AlmostThere = 11,
+	End = 12,
+	Accept = 13
+}
+
+type Edge = [State, number, State];
+
+class StateMachine {
+
+	private _states: Uint8Matrix;
+	private _maxCharCode: number;
+
+	constructor(edges: Edge[]) {
+		let maxCharCode = 0;
+		let maxState = State.Invalid;
+		for (let i = 0, len = edges.length; i < len; i++) {
+			let [from, chCode, to] = edges[i];
+			if (chCode > maxCharCode) {
+				maxCharCode = chCode;
+			}
+			if (from > maxState) {
+				maxState = from;
+			}
+			if (to > maxState) {
+				maxState = to;
+			}
+		}
+
+		maxCharCode++;
+		maxState++;
+
+		let states = new Uint8Matrix(maxState, maxCharCode, State.Invalid);
+		for (let i = 0, len = edges.length; i < len; i++) {
+			let [from, chCode, to] = edges[i];
+			states.set(from, chCode, to);
+		}
+
+		this._states = states;
+		this._maxCharCode = maxCharCode;
+	}
+
+	public nextState(currentState: State, chCode: number): State {
+		if (chCode < 0 || chCode >= this._maxCharCode) {
+			return State.Invalid;
+		}
+		return this._states.get(currentState, chCode);
+	}
+}
+
+// State machine for http:// or https:// or file://
+let _stateMachine: StateMachine = null;
+function getStateMachine(): StateMachine {
+	if (_stateMachine === null) {
+		_stateMachine = new StateMachine([
+			[State.Start, CharCode.h, State.H],
+			[State.Start, CharCode.H, State.H],
+			[State.Start, CharCode.f, State.F],
+			[State.Start, CharCode.F, State.F],
+
+			[State.H, CharCode.t, State.HT],
+			[State.H, CharCode.T, State.HT],
+
+			[State.HT, CharCode.t, State.HTT],
+			[State.HT, CharCode.T, State.HTT],
+
+			[State.HTT, CharCode.p, State.HTTP],
+			[State.HTT, CharCode.P, State.HTTP],
+
+			[State.HTTP, CharCode.s, State.BeforeColon],
+			[State.HTTP, CharCode.S, State.BeforeColon],
+			[State.HTTP, CharCode.Colon, State.AfterColon],
+
+			[State.F, CharCode.i, State.FI],
+			[State.F, CharCode.I, State.FI],
+
+			[State.FI, CharCode.l, State.FIL],
+			[State.FI, CharCode.L, State.FIL],
+
+			[State.FIL, CharCode.e, State.BeforeColon],
+			[State.FIL, CharCode.E, State.BeforeColon],
+
+			[State.BeforeColon, CharCode.Colon, State.AfterColon],
+
+			[State.AfterColon, CharCode.Slash, State.AlmostThere],
+
+			[State.AlmostThere, CharCode.Slash, State.End],
+		]);
+	}
+	return _stateMachine;
+}
+
+
+const enum CharacterClass {
+	None = 0,
+	ForceTermination = 1,
+	CannotEndIn = 2
+}
+
+let _classifier: CharacterClassifier<CharacterClass> = null;
+function getClassifier(): CharacterClassifier<CharacterClass> {
+	if (_classifier === null) {
+		_classifier = new CharacterClassifier<CharacterClass>(CharacterClass.None);
+
+		const FORCE_TERMINATION_CHARACTERS = ' \t<>\'\"、。｡､，．：；？！＠＃＄％＆＊‘“〈《「『【〔（［｛｢｣｝］）〕】』」》〉”’｀～…';
+		for (let i = 0; i < FORCE_TERMINATION_CHARACTERS.length; i++) {
+			_classifier.set(FORCE_TERMINATION_CHARACTERS.charCodeAt(i), CharacterClass.ForceTermination);
+		}
+
+		const CANNOT_END_WITH_CHARACTERS = '.,;';
+		for (let i = 0; i < CANNOT_END_WITH_CHARACTERS.length; i++) {
+			_classifier.set(CANNOT_END_WITH_CHARACTERS.charCodeAt(i), CharacterClass.CannotEndIn);
+		}
+	}
+	return _classifier;
+}
+
+class LinkComputer {
+
+	private static _createLink(classifier: CharacterClassifier<CharacterClass>, line: string, lineNumber: number, linkBeginIndex: number, linkEndIndex: number): ILink {
+		// Do not allow to end link in certain characters...
+		let lastIncludedCharIndex = linkEndIndex - 1;
+		do {
+			const chCode = line.charCodeAt(lastIncludedCharIndex);
+			const chClass = classifier.get(chCode);
+			if (chClass !== CharacterClass.CannotEndIn) {
+				break;
+			}
+			lastIncludedCharIndex--;
+		} while (lastIncludedCharIndex > linkBeginIndex);
+
+		return {
+			range: {
+				startLineNumber: lineNumber,
+				startColumn: linkBeginIndex + 1,
+				endLineNumber: lineNumber,
+				endColumn: lastIncludedCharIndex + 2
+			},
+			url: line.substring(linkBeginIndex, lastIncludedCharIndex + 1)
+		};
+	}
+
+	public static computeLinks(model: ILinkComputerTarget): ILink[] {
+		const stateMachine = getStateMachine();
+		const classifier = getClassifier();
+
+		let result: ILink[] = [];
+		for (let i = 1, lineCount = model.getLineCount(); i <= lineCount; i++) {
+			const line = model.getLineContent(i);
+			const len = line.length;
+
+			let j = 0;
+			let linkBeginIndex = 0;
+			let linkBeginChCode = 0;
+			let state = State.Start;
+			let hasOpenParens = false;
+			let hasOpenSquareBracket = false;
+			let hasOpenCurlyBracket = false;
+
+			while (j < len) {
+
+				let resetStateMachine = false;
+				const chCode = line.charCodeAt(j);
+
+				if (state === State.Accept) {
+					let chClass: CharacterClass;
+					switch (chCode) {
+						case CharCode.OpenParen:
+							hasOpenParens = true;
+							chClass = CharacterClass.None;
+							break;
+						case CharCode.CloseParen:
+							chClass = (hasOpenParens ? CharacterClass.None : CharacterClass.ForceTermination);
+							break;
+						case CharCode.OpenSquareBracket:
+							hasOpenSquareBracket = true;
+							chClass = CharacterClass.None;
+							break;
+						case CharCode.CloseSquareBracket:
+							chClass = (hasOpenSquareBracket ? CharacterClass.None : CharacterClass.ForceTermination);
+							break;
+						case CharCode.OpenCurlyBrace:
+							hasOpenCurlyBracket = true;
+							chClass = CharacterClass.None;
+							break;
+						case CharCode.CloseCurlyBrace:
+							chClass = (hasOpenCurlyBracket ? CharacterClass.None : CharacterClass.ForceTermination);
+							break;
+						/* The following three rules make it that ' or " or ` are allowed inside links if the link began with a different one */
+						case CharCode.SingleQuote:
+							chClass = (linkBeginChCode === CharCode.DoubleQuote || linkBeginChCode === CharCode.BackTick) ? CharacterClass.None : CharacterClass.ForceTermination;
+							break;
+						case CharCode.DoubleQuote:
+							chClass = (linkBeginChCode === CharCode.SingleQuote || linkBeginChCode === CharCode.BackTick) ? CharacterClass.None : CharacterClass.ForceTermination;
+							break;
+						case CharCode.BackTick:
+							chClass = (linkBeginChCode === CharCode.SingleQuote || linkBeginChCode === CharCode.DoubleQuote) ? CharacterClass.None : CharacterClass.ForceTermination;
+							break;
+						default:
+							chClass = classifier.get(chCode);
+					}
+
+					// Check if character terminates link
+					if (chClass === CharacterClass.ForceTermination) {
+						result.push(LinkComputer._createLink(classifier, line, i, linkBeginIndex, j));
+						resetStateMachine = true;
+					}
+				} else if (state === State.End) {
+					const chClass = classifier.get(chCode);
+
+					// Check if character terminates link
+					if (chClass === CharacterClass.ForceTermination) {
+						resetStateMachine = true;
+					} else {
+						state = State.Accept;
+					}
+				} else {
+					state = stateMachine.nextState(state, chCode);
+					if (state === State.Invalid) {
+						resetStateMachine = true;
+					}
+				}
+
+				if (resetStateMachine) {
+					state = State.Start;
+					hasOpenParens = false;
+					hasOpenSquareBracket = false;
+					hasOpenCurlyBracket = false;
+
+					// Record where the link started
+					linkBeginIndex = j + 1;
+					linkBeginChCode = chCode;
+				}
+
+				j++;
+			}
+
+			if (state === State.Accept) {
+				result.push(LinkComputer._createLink(classifier, line, i, linkBeginIndex, len));
+			}
+
+		}
+
+		return result;
+	}
+}
+
+/**
+ * Returns an array of all links contains in the provided
+ * document. *Note* that this operation is computational
+ * expensive and should not run in the UI thread.
+ */
+export function computeLinks(model: ILinkComputerTarget): ILink[] {
+	if (!model || typeof model.getLineCount !== 'function' || typeof model.getLineContent !== 'function') {
+		// Unknown caller!
+		return [];
+	}
+	return LinkComputer.computeLinks(model);
+}

src/vs/editor/common/modes/modesRegistry.ts

@@ -0,0 +1,63 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as nls from 'vs/nls';
+import Event, { Emitter } from 'vs/base/common/event';
+import { Registry } from 'vs/platform/registry/common/platform';
+import { ILanguageExtensionPoint } from 'vs/editor/common/services/modeService';
+import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
+import { LanguageIdentifier, LanguageId } from 'vs/editor/common/modes';
+
+// Define extension point ids
+export var Extensions = {
+	ModesRegistry: 'editor.modesRegistry'
+};
+
+export class EditorModesRegistry {
+
+	private _languages: ILanguageExtensionPoint[];
+
+	private _onDidAddLanguages: Emitter<ILanguageExtensionPoint[]> = new Emitter<ILanguageExtensionPoint[]>();
+	public onDidAddLanguages: Event<ILanguageExtensionPoint[]> = this._onDidAddLanguages.event;
+
+	constructor() {
+		this._languages = [];
+	}
+
+	// --- languages
+
+	public registerLanguage(def: ILanguageExtensionPoint): void {
+		this._languages.push(def);
+		this._onDidAddLanguages.fire([def]);
+	}
+	public registerLanguages(def: ILanguageExtensionPoint[]): void {
+		this._languages = this._languages.concat(def);
+		this._onDidAddLanguages.fire(def);
+	}
+	public getLanguages(): ILanguageExtensionPoint[] {
+		return this._languages.slice(0);
+	}
+}
+
+export var ModesRegistry = new EditorModesRegistry();
+Registry.add(Extensions.ModesRegistry, ModesRegistry);
+
+export const PLAINTEXT_MODE_ID = 'plaintext';
+export const PLAINTEXT_LANGUAGE_IDENTIFIER = new LanguageIdentifier(PLAINTEXT_MODE_ID, LanguageId.PlainText);
+
+ModesRegistry.registerLanguage({
+	id: PLAINTEXT_MODE_ID,
+	extensions: ['.txt', '.gitignore'],
+	aliases: [nls.localize('plainText.alias', "Plain Text"), 'text'],
+	mimetypes: ['text/plain']
+});
+LanguageConfigurationRegistry.register(PLAINTEXT_LANGUAGE_IDENTIFIER, {
+	brackets: [
+		['(', ')'],
+		['[', ']'],
+		['{', '}'],
+	]
+});

src/vs/editor/common/modes/nullMode.ts

@@ -0,0 +1,43 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { IState, ColorId, MetadataConsts, LanguageIdentifier, FontStyle, StandardTokenType, LanguageId } from 'vs/editor/common/modes';
+import { Token, TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token';
+
+class NullStateImpl implements IState {
+
+	public clone(): IState {
+		return this;
+	}
+
+	public equals(other: IState): boolean {
+		return (this === other);
+	}
+}
+
+export const NULL_STATE: IState = new NullStateImpl();
+
+export const NULL_MODE_ID = 'vs.editor.nullMode';
+
+export const NULL_LANGUAGE_IDENTIFIER = new LanguageIdentifier(NULL_MODE_ID, LanguageId.Null);
+
+export function nullTokenize(modeId: string, buffer: string, state: IState, deltaOffset: number): TokenizationResult {
+	return new TokenizationResult([new Token(deltaOffset, '', modeId)], state);
+}
+
+export function nullTokenize2(languageId: LanguageId, buffer: string, state: IState, deltaOffset: number): TokenizationResult2 {
+	let tokens = new Uint32Array(2);
+	tokens[0] = deltaOffset;
+	tokens[1] = (
+		(languageId << MetadataConsts.LANGUAGEID_OFFSET)
+		| (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET)
+		| (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET)
+		| (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET)
+		| (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET)
+	) >>> 0;
+
+	return new TokenizationResult2(tokens, state);
+}

src/vs/editor/common/modes/supports.ts

@@ -0,0 +1,89 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as modes from 'vs/editor/common/modes';
+import { LineTokens } from 'vs/editor/common/core/lineTokens';
+
+export function createScopedLineTokens(context: LineTokens, offset: number): ScopedLineTokens {
+	let tokenCount = context.getTokenCount();
+	let tokenIndex = context.findTokenIndexAtOffset(offset);
+	let desiredLanguageId = context.getLanguageId(tokenIndex);
+
+	let lastTokenIndex = tokenIndex;
+	while (lastTokenIndex + 1 < tokenCount && context.getLanguageId(lastTokenIndex + 1) === desiredLanguageId) {
+		lastTokenIndex++;
+	}
+
+	let firstTokenIndex = tokenIndex;
+	while (firstTokenIndex > 0 && context.getLanguageId(firstTokenIndex - 1) === desiredLanguageId) {
+		firstTokenIndex--;
+	}
+
+	return new ScopedLineTokens(
+		context,
+		desiredLanguageId,
+		firstTokenIndex,
+		lastTokenIndex + 1,
+		context.getTokenStartOffset(firstTokenIndex),
+		context.getTokenEndOffset(lastTokenIndex)
+	);
+}
+
+export class ScopedLineTokens {
+	_scopedLineTokensBrand: void;
+
+	public readonly languageId: modes.LanguageId;
+	private readonly _actual: LineTokens;
+	private readonly _firstTokenIndex: number;
+	private readonly _lastTokenIndex: number;
+	public readonly firstCharOffset: number;
+	private readonly _lastCharOffset: number;
+
+	constructor(
+		actual: LineTokens,
+		languageId: modes.LanguageId,
+		firstTokenIndex: number,
+		lastTokenIndex: number,
+		firstCharOffset: number,
+		lastCharOffset: number
+	) {
+		this._actual = actual;
+		this.languageId = languageId;
+		this._firstTokenIndex = firstTokenIndex;
+		this._lastTokenIndex = lastTokenIndex;
+		this.firstCharOffset = firstCharOffset;
+		this._lastCharOffset = lastCharOffset;
+	}
+
+	public getLineContent(): string {
+		var actualLineContent = this._actual.getLineContent();
+		return actualLineContent.substring(this.firstCharOffset, this._lastCharOffset);
+	}
+
+	public getTokenCount(): number {
+		return this._lastTokenIndex - this._firstTokenIndex;
+	}
+
+	public findTokenIndexAtOffset(offset: number): number {
+		return this._actual.findTokenIndexAtOffset(offset + this.firstCharOffset) - this._firstTokenIndex;
+	}
+
+	public getTokenStartOffset(tokenIndex: number): number {
+		return this._actual.getTokenStartOffset(tokenIndex + this._firstTokenIndex) - this.firstCharOffset;
+	}
+
+	public getStandardTokenType(tokenIndex: number): modes.StandardTokenType {
+		return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex);
+	}
+}
+
+const enum IgnoreBracketsInTokens {
+	value = modes.StandardTokenType.Comment | modes.StandardTokenType.String | modes.StandardTokenType.RegEx
+}
+
+export function ignoreBracketsInToken(standardTokenType: modes.StandardTokenType): boolean {
+	return (standardTokenType & IgnoreBracketsInTokens.value) !== 0;
+}

src/vs/editor/common/modes/supports/characterPair.ts

@@ -0,0 +1,54 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { ScopedLineTokens } from 'vs/editor/common/modes/supports';
+import { CharacterPair, IAutoClosingPair, IAutoClosingPairConditional, StandardAutoClosingPairConditional } from 'vs/editor/common/modes/languageConfiguration';
+
+export class CharacterPairSupport {
+
+	private readonly _autoClosingPairs: StandardAutoClosingPairConditional[];
+	private readonly _surroundingPairs: IAutoClosingPair[];
+
+	constructor(config: { brackets?: CharacterPair[]; autoClosingPairs?: IAutoClosingPairConditional[], surroundingPairs?: IAutoClosingPair[] }) {
+		if (config.autoClosingPairs) {
+			this._autoClosingPairs = config.autoClosingPairs.map(el => new StandardAutoClosingPairConditional(el));
+		} else if (config.brackets) {
+			this._autoClosingPairs = config.brackets.map(b => new StandardAutoClosingPairConditional({ open: b[0], close: b[1] }));
+		} else {
+			this._autoClosingPairs = [];
+		}
+
+		this._surroundingPairs = config.surroundingPairs || this._autoClosingPairs;
+	}
+
+	public getAutoClosingPairs(): IAutoClosingPair[] {
+		return this._autoClosingPairs;
+	}
+
+	public shouldAutoClosePair(character: string, context: ScopedLineTokens, column: number): boolean {
+		// Always complete on empty line
+		if (context.getTokenCount() === 0) {
+			return true;
+		}
+
+		let tokenIndex = context.findTokenIndexAtOffset(column - 2);
+		let standardTokenType = context.getStandardTokenType(tokenIndex);
+
+		for (let i = 0; i < this._autoClosingPairs.length; ++i) {
+			let autoClosingPair = this._autoClosingPairs[i];
+
+			if (autoClosingPair.open === character) {
+				return autoClosingPair.isOK(standardTokenType);
+			}
+		}
+
+		return true;
+	}
+
+	public getSurroundingPairs(): IAutoClosingPair[] {
+		return this._surroundingPairs;
+	}
+}

src/vs/editor/common/modes/supports/electricCharacter.ts

@@ -0,0 +1,146 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { ScopedLineTokens, ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
+import { BracketsUtils, RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
+import { IAutoClosingPairConditional, IBracketElectricCharacterContribution, StandardAutoClosingPairConditional } from 'vs/editor/common/modes/languageConfiguration';
+
+/**
+ * Interface used to support electric characters
+ * @internal
+ */
+export interface IElectricAction {
+	// Only one of the following properties should be defined:
+
+	// The line will be indented at the same level of the line
+	// which contains the matching given bracket type.
+	matchOpenBracket?: string;
+
+	// The text will be appended after the electric character.
+	appendText?: string;
+}
+
+export class BracketElectricCharacterSupport {
+
+	private readonly _richEditBrackets: RichEditBrackets;
+	private readonly _complexAutoClosePairs: StandardAutoClosingPairConditional[];
+
+	constructor(richEditBrackets: RichEditBrackets, autoClosePairs: IAutoClosingPairConditional[], contribution: IBracketElectricCharacterContribution) {
+		contribution = contribution || {};
+		this._richEditBrackets = richEditBrackets;
+		this._complexAutoClosePairs = autoClosePairs.filter(pair => pair.open.length > 1 && !!pair.close).map(el => new StandardAutoClosingPairConditional(el));
+		if (contribution.docComment) {
+			// IDocComment is legacy, only partially supported
+			this._complexAutoClosePairs.push(new StandardAutoClosingPairConditional({ open: contribution.docComment.open, close: contribution.docComment.close }));
+		}
+	}
+
+	public getElectricCharacters(): string[] {
+		var result: string[] = [];
+
+		if (this._richEditBrackets) {
+			for (let i = 0, len = this._richEditBrackets.brackets.length; i < len; i++) {
+				let bracketPair = this._richEditBrackets.brackets[i];
+				let lastChar = bracketPair.close.charAt(bracketPair.close.length - 1);
+				result.push(lastChar);
+			}
+		}
+
+		// auto close
+		for (let pair of this._complexAutoClosePairs) {
+			result.push(pair.open.charAt(pair.open.length - 1));
+		}
+
+		// Filter duplicate entries
+		result = result.filter((item, pos, array) => {
+			return array.indexOf(item) === pos;
+		});
+
+		return result;
+	}
+
+	public onElectricCharacter(character: string, context: ScopedLineTokens, column: number): IElectricAction {
+		return (this._onElectricAutoClose(character, context, column) ||
+			this._onElectricAutoIndent(character, context, column));
+	}
+
+	private _onElectricAutoIndent(character: string, context: ScopedLineTokens, column: number): IElectricAction {
+
+		if (!this._richEditBrackets || this._richEditBrackets.brackets.length === 0) {
+			return null;
+		}
+
+		let tokenIndex = context.findTokenIndexAtOffset(column - 1);
+		if (ignoreBracketsInToken(context.getStandardTokenType(tokenIndex))) {
+			return null;
+		}
+
+		let reversedBracketRegex = this._richEditBrackets.reversedRegex;
+		let text = context.getLineContent().substring(0, column - 1) + character;
+
+		let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, 1, text, 0, text.length);
+		if (!r) {
+			return null;
+		}
+
+		let bracketText = text.substring(r.startColumn - 1, r.endColumn - 1);
+		bracketText = bracketText.toLowerCase();
+
+		let isOpen = this._richEditBrackets.textIsOpenBracket[bracketText];
+		if (isOpen) {
+			return null;
+		}
+
+		let textBeforeBracket = text.substring(0, r.startColumn - 1);
+		if (!/^\s*$/.test(textBeforeBracket)) {
+			// There is other text on the line before the bracket
+			return null;
+		}
+
+		return {
+			matchOpenBracket: bracketText
+		};
+	}
+
+	private _onElectricAutoClose(character: string, context: ScopedLineTokens, column: number): IElectricAction {
+		if (!this._complexAutoClosePairs.length) {
+			return null;
+		}
+
+		let line = context.getLineContent();
+
+		for (let i = 0, len = this._complexAutoClosePairs.length; i < len; i++) {
+			let pair = this._complexAutoClosePairs[i];
+
+			// See if the right electric character was pressed
+			if (character !== pair.open.charAt(pair.open.length - 1)) {
+				continue;
+			}
+
+			// check if the full open bracket matches
+			let actual = line.substring(line.length - pair.open.length + 1) + character;
+			if (actual !== pair.open) {
+				continue;
+			}
+
+			let lastTokenIndex = context.findTokenIndexAtOffset(column - 1);
+			let lastTokenStandardType = context.getStandardTokenType(lastTokenIndex);
+			// If we're in a scope listed in 'notIn', do nothing
+			if (!pair.isOK(lastTokenStandardType)) {
+				continue;
+			}
+
+			// If this line already contains the closing tag, do nothing.
+			if (line.indexOf(pair.close, column - 1) >= 0) {
+				continue;
+			}
+
+			return { appendText: pair.close };
+		}
+
+		return null;
+	}
+}

src/vs/editor/common/modes/supports/indentRules.ts

@@ -0,0 +1,102 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as strings from 'vs/base/common/strings';
+import { IndentationRule, IndentAction } from 'vs/editor/common/modes/languageConfiguration';
+
+export const enum IndentConsts {
+	INCREASE_MASK = 0b00000001,
+	DECREASE_MASK = 0b00000010,
+	INDENT_NEXTLINE_MASK = 0b00000100,
+	UNINDENT_MASK = 0b00001000,
+};
+
+export class IndentRulesSupport {
+
+	private readonly _indentationRules: IndentationRule;
+
+	constructor(indentationRules: IndentationRule) {
+		this._indentationRules = indentationRules;
+	}
+
+	public onType(text: string): IndentAction {
+		if (this._indentationRules) {
+			if (this._indentationRules.unIndentedLinePattern && this._indentationRules.unIndentedLinePattern.test(text)) {
+				return null;
+			}
+
+			if (this._indentationRules.decreaseIndentPattern && this._indentationRules.decreaseIndentPattern.test(text)) {
+				return IndentAction.Outdent;
+			}
+		}
+		return null;
+	}
+
+	public containNonWhitespace(text: string): boolean {
+		// the text doesn't contain any non-whitespace character.
+		let nonWhitespaceIdx = strings.lastNonWhitespaceIndex(text);
+
+		if (nonWhitespaceIdx >= 0) {
+			return true;
+		}
+
+		return false;
+	}
+
+	public shouldIncrease(text: string): boolean {
+		if (this._indentationRules) {
+			if (this._indentationRules.increaseIndentPattern && this._indentationRules.increaseIndentPattern.test(text)) {
+				return true;
+			}
+			// if (this._indentationRules.indentNextLinePattern && this._indentationRules.indentNextLinePattern.test(text)) {
+			// 	return true;
+			// }
+		}
+		return false;
+	}
+
+	public shouldDecrease(text: string): boolean {
+		if (this._indentationRules && this._indentationRules.decreaseIndentPattern && this._indentationRules.decreaseIndentPattern.test(text)) {
+			return true;
+		}
+		return false;
+	}
+
+	public shouldIndentNextLine(text: string): boolean {
+		if (this._indentationRules && this._indentationRules.indentNextLinePattern && this._indentationRules.indentNextLinePattern.test(text)) {
+			return true;
+		}
+
+		return false;
+	}
+
+	public shouldIgnore(text: string): boolean {
+		// the text matches `unIndentedLinePattern`
+		if (this._indentationRules && this._indentationRules.unIndentedLinePattern && this._indentationRules.unIndentedLinePattern.test(text)) {
+			return true;
+		}
+
+		return false;
+	}
+
+	public getIndentMetadata(text: string): number {
+		let ret = 0;
+		if (this.shouldIncrease(text)) {
+			ret += IndentConsts.INCREASE_MASK;
+		}
+		if (this.shouldDecrease(text)) {
+			ret += IndentConsts.DECREASE_MASK;
+		}
+		if (this.shouldIndentNextLine(text)) {
+			ret += IndentConsts.INDENT_NEXTLINE_MASK;
+		}
+		if (this.shouldIgnore(text)) {
+			ret += IndentConsts.UNINDENT_MASK;
+		}
+		return ret;
+	}
+}
+

src/vs/editor/common/modes/supports/inplaceReplaceSupport.ts

@@ -0,0 +1,100 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { IInplaceReplaceSupportResult } from 'vs/editor/common/modes';
+import { IRange } from 'vs/editor/common/core/range';
+
+export class BasicInplaceReplace {
+
+	public static INSTANCE = new BasicInplaceReplace();
+
+	public navigateValueSet(range1: IRange, text1: string, range2: IRange, text2: string, up: boolean): IInplaceReplaceSupportResult {
+
+		if (range1 && text1) {
+			let result = this.doNavigateValueSet(text1, up);
+			if (result) {
+				return {
+					range: range1,
+					value: result
+				};
+			}
+		}
+
+		if (range2 && text2) {
+			let result = this.doNavigateValueSet(text2, up);
+			if (result) {
+				return {
+					range: range2,
+					value: result
+				};
+			}
+		}
+
+		return null;
+	}
+
+	private doNavigateValueSet(text: string, up: boolean): string {
+		let numberResult = this.numberReplace(text, up);
+		if (numberResult !== null) {
+			return numberResult;
+		}
+		return this.textReplace(text, up);
+	}
+
+	private numberReplace(value: string, up: boolean): string {
+		var precision = Math.pow(10, value.length - (value.lastIndexOf('.') + 1)),
+			n1 = Number(value),
+			n2 = parseFloat(value);
+
+		if (!isNaN(n1) && !isNaN(n2) && n1 === n2) {
+
+			if (n1 === 0 && !up) {
+				return null; // don't do negative
+				//			} else if(n1 === 9 && up) {
+				//				return null; // don't insert 10 into a number
+			} else {
+				n1 = Math.floor(n1 * precision);
+				n1 += up ? precision : -precision;
+				return String(n1 / precision);
+			}
+		}
+
+		return null;
+	}
+
+	private _defaultValueSet: string[][] = [
+		['true', 'false'],
+		['True', 'False'],
+		['Private', 'Public', 'Friend', 'ReadOnly', 'Partial', 'Protected', 'WriteOnly'],
+		['public', 'protected', 'private'],
+	];
+
+	private textReplace(value: string, up: boolean): string {
+		return this.valueSetsReplace(this._defaultValueSet, value, up);
+	}
+
+	private valueSetsReplace(valueSets: string[][], value: string, up: boolean): string {
+		var result: string = null;
+		for (let i = 0, len = valueSets.length; result === null && i < len; i++) {
+			result = this.valueSetReplace(valueSets[i], value, up);
+		}
+		return result;
+	}
+
+	private valueSetReplace(valueSet: string[], value: string, up: boolean): string {
+		var idx = valueSet.indexOf(value);
+		if (idx >= 0) {
+			idx += up ? +1 : -1;
+			if (idx < 0) {
+				idx = valueSet.length - 1;
+			} else {
+				idx %= valueSet.length;
+			}
+			return valueSet[idx];
+		}
+		return null;
+	}
+}

src/vs/editor/common/modes/supports/onEnter.ts

@@ -0,0 +1,116 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { onUnexpectedError } from 'vs/base/common/errors';
+import * as strings from 'vs/base/common/strings';
+import { CharacterPair, IndentationRule, IndentAction, EnterAction, OnEnterRule } from 'vs/editor/common/modes/languageConfiguration';
+
+export interface IOnEnterSupportOptions {
+	brackets?: CharacterPair[];
+	indentationRules?: IndentationRule;
+	regExpRules?: OnEnterRule[];
+}
+
+interface IProcessedBracketPair {
+	open: string;
+	close: string;
+	openRegExp: RegExp;
+	closeRegExp: RegExp;
+}
+
+export class OnEnterSupport {
+
+	private readonly _brackets: IProcessedBracketPair[];
+	private readonly _indentationRules: IndentationRule;
+	private readonly _regExpRules: OnEnterRule[];
+
+	constructor(opts?: IOnEnterSupportOptions) {
+		opts = opts || {};
+		opts.brackets = opts.brackets || [
+			['(', ')'],
+			['{', '}'],
+			['[', ']']
+		];
+
+		this._brackets = opts.brackets.map((bracket) => {
+			return {
+				open: bracket[0],
+				openRegExp: OnEnterSupport._createOpenBracketRegExp(bracket[0]),
+				close: bracket[1],
+				closeRegExp: OnEnterSupport._createCloseBracketRegExp(bracket[1]),
+			};
+		});
+		this._regExpRules = opts.regExpRules || [];
+		this._indentationRules = opts.indentationRules;
+	}
+
+	public onEnter(oneLineAboveText: string, beforeEnterText: string, afterEnterText: string): EnterAction {
+		// (1): `regExpRules`
+		for (let i = 0, len = this._regExpRules.length; i < len; i++) {
+			let rule = this._regExpRules[i];
+			if (rule.beforeText.test(beforeEnterText)) {
+				if (rule.afterText) {
+					if (rule.afterText.test(afterEnterText)) {
+						return rule.action;
+					}
+				} else {
+					return rule.action;
+				}
+			}
+		}
+
+		// (2): Special indent-outdent
+		if (beforeEnterText.length > 0 && afterEnterText.length > 0) {
+			for (let i = 0, len = this._brackets.length; i < len; i++) {
+				let bracket = this._brackets[i];
+				if (bracket.openRegExp.test(beforeEnterText) && bracket.closeRegExp.test(afterEnterText)) {
+					return { indentAction: IndentAction.IndentOutdent };
+				}
+			}
+		}
+
+
+		// (4): Open bracket based logic
+		if (beforeEnterText.length > 0) {
+			for (let i = 0, len = this._brackets.length; i < len; i++) {
+				let bracket = this._brackets[i];
+				if (bracket.openRegExp.test(beforeEnterText)) {
+					return { indentAction: IndentAction.Indent };
+				}
+			}
+		}
+
+		return null;
+	}
+
+	private static _createOpenBracketRegExp(bracket: string): RegExp {
+		var str = strings.escapeRegExpCharacters(bracket);
+		if (!/\B/.test(str.charAt(0))) {
+			str = '\\b' + str;
+		}
+		str += '\\s*$';
+		return OnEnterSupport._safeRegExp(str);
+	}
+
+	private static _createCloseBracketRegExp(bracket: string): RegExp {
+		var str = strings.escapeRegExpCharacters(bracket);
+		if (!/\B/.test(str.charAt(str.length - 1))) {
+			str = str + '\\b';
+		}
+		str = '^\\s*' + str;
+		return OnEnterSupport._safeRegExp(str);
+	}
+
+	private static _safeRegExp(def: string): RegExp {
+		try {
+			return new RegExp(def);
+		} catch (err) {
+			onUnexpectedError(err);
+			return null;
+		}
+	}
+}
+

src/vs/editor/common/modes/supports/richEditBrackets.ts

@@ -0,0 +1,193 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as strings from 'vs/base/common/strings';
+import { Range } from 'vs/editor/common/core/range';
+import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration';
+import { LanguageIdentifier } from 'vs/editor/common/modes';
+
+interface ISimpleInternalBracket {
+	open: string;
+	close: string;
+}
+
+export class RichEditBracket {
+	_richEditBracketBrand: void;
+
+	readonly languageIdentifier: LanguageIdentifier;
+	readonly open: string;
+	readonly close: string;
+	readonly forwardRegex: RegExp;
+	readonly reversedRegex: RegExp;
+
+	constructor(languageIdentifier: LanguageIdentifier, open: string, close: string, forwardRegex: RegExp, reversedRegex: RegExp) {
+		this.languageIdentifier = languageIdentifier;
+		this.open = open;
+		this.close = close;
+		this.forwardRegex = forwardRegex;
+		this.reversedRegex = reversedRegex;
+	}
+}
+
+export class RichEditBrackets {
+	_richEditBracketsBrand: void;
+
+	public readonly brackets: RichEditBracket[];
+	public readonly forwardRegex: RegExp;
+	public readonly reversedRegex: RegExp;
+	public readonly maxBracketLength: number;
+	public readonly textIsBracket: { [text: string]: RichEditBracket; };
+	public readonly textIsOpenBracket: { [text: string]: boolean; };
+
+	constructor(languageIdentifier: LanguageIdentifier, brackets: CharacterPair[]) {
+		this.brackets = brackets.map((b) => {
+			return new RichEditBracket(
+				languageIdentifier,
+				b[0],
+				b[1],
+				getRegexForBracketPair({ open: b[0], close: b[1] }),
+				getReversedRegexForBracketPair({ open: b[0], close: b[1] })
+			);
+		});
+		this.forwardRegex = getRegexForBrackets(this.brackets);
+		this.reversedRegex = getReversedRegexForBrackets(this.brackets);
+
+		this.textIsBracket = {};
+		this.textIsOpenBracket = {};
+
+		let maxBracketLength = 0;
+		this.brackets.forEach((b) => {
+			this.textIsBracket[b.open.toLowerCase()] = b;
+			this.textIsBracket[b.close.toLowerCase()] = b;
+			this.textIsOpenBracket[b.open.toLowerCase()] = true;
+			this.textIsOpenBracket[b.close.toLowerCase()] = false;
+			maxBracketLength = Math.max(maxBracketLength, b.open.length);
+			maxBracketLength = Math.max(maxBracketLength, b.close.length);
+		});
+		this.maxBracketLength = maxBracketLength;
+	}
+}
+
+function once<T, R>(keyFn: (input: T) => string, computeFn: (input: T) => R): (input: T) => R {
+	let cache: { [key: string]: R; } = {};
+	return (input: T): R => {
+		let key = keyFn(input);
+		if (!cache.hasOwnProperty(key)) {
+			cache[key] = computeFn(input);
+		}
+		return cache[key];
+	};
+}
+
+var getRegexForBracketPair = once<ISimpleInternalBracket, RegExp>(
+	(input) => `${input.open};${input.close}`,
+	(input) => {
+		return createOrRegex([input.open, input.close]);
+	}
+);
+
+var getReversedRegexForBracketPair = once<ISimpleInternalBracket, RegExp>(
+	(input) => `${input.open};${input.close}`,
+	(input) => {
+		return createOrRegex([toReversedString(input.open), toReversedString(input.close)]);
+	}
+);
+
+var getRegexForBrackets = once<ISimpleInternalBracket[], RegExp>(
+	(input) => input.map(b => `${b.open};${b.close}`).join(';'),
+	(input) => {
+		let pieces: string[] = [];
+		input.forEach((b) => {
+			pieces.push(b.open);
+			pieces.push(b.close);
+		});
+		return createOrRegex(pieces);
+	}
+);
+
+var getReversedRegexForBrackets = once<ISimpleInternalBracket[], RegExp>(
+	(input) => input.map(b => `${b.open};${b.close}`).join(';'),
+	(input) => {
+		let pieces: string[] = [];
+		input.forEach((b) => {
+			pieces.push(toReversedString(b.open));
+			pieces.push(toReversedString(b.close));
+		});
+		return createOrRegex(pieces);
+	}
+);
+
+function createOrRegex(pieces: string[]): RegExp {
+	let regexStr = `(${pieces.map(strings.escapeRegExpCharacters).join(')|(')})`;
+	return strings.createRegExp(regexStr, true);
+}
+
+let toReversedString = (function () {
+
+	function reverse(str: string): string {
+		let reversedStr = '';
+		for (let i = str.length - 1; i >= 0; i--) {
+			reversedStr += str.charAt(i);
+		}
+		return reversedStr;
+	}
+
+	let lastInput: string = null;
+	let lastOutput: string = null;
+	return function toReversedString(str: string): string {
+		if (lastInput !== str) {
+			lastInput = str;
+			lastOutput = reverse(lastInput);
+		}
+		return lastOutput;
+	};
+})();
+
+export class BracketsUtils {
+
+	private static _findPrevBracketInText(reversedBracketRegex: RegExp, lineNumber: number, reversedText: string, offset: number): Range {
+		let m = reversedText.match(reversedBracketRegex);
+
+		if (!m) {
+			return null;
+		}
+
+		let matchOffset = reversedText.length - m.index;
+		let matchLength = m[0].length;
+		let absoluteMatchOffset = offset + matchOffset;
+
+		return new Range(lineNumber, absoluteMatchOffset - matchLength + 1, lineNumber, absoluteMatchOffset + 1);
+	}
+
+	public static findPrevBracketInToken(reversedBracketRegex: RegExp, lineNumber: number, lineText: string, currentTokenStart: number, currentTokenEnd: number): Range {
+		// Because JS does not support backwards regex search, we search forwards in a reversed string with a reversed regex ;)
+		let reversedLineText = toReversedString(lineText);
+		let reversedTokenText = reversedLineText.substring(lineText.length - currentTokenEnd, lineText.length - currentTokenStart);
+
+		return this._findPrevBracketInText(reversedBracketRegex, lineNumber, reversedTokenText, currentTokenStart);
+	}
+
+	public static findNextBracketInText(bracketRegex: RegExp, lineNumber: number, text: string, offset: number): Range {
+		let m = text.match(bracketRegex);
+
+		if (!m) {
+			return null;
+		}
+
+		let matchOffset = m.index;
+		let matchLength = m[0].length;
+		let absoluteMatchOffset = offset + matchOffset;
+
+		return new Range(lineNumber, absoluteMatchOffset + 1, lineNumber, absoluteMatchOffset + 1 + matchLength);
+	}
+
+	public static findNextBracketInToken(bracketRegex: RegExp, lineNumber: number, lineText: string, currentTokenStart: number, currentTokenEnd: number): Range {
+		let currentTokenText = lineText.substring(currentTokenStart, currentTokenEnd);
+
+		return this.findNextBracketInText(bracketRegex, lineNumber, currentTokenText, currentTokenStart);
+	}
+
+}

src/vs/editor/common/modes/supports/tokenization.ts

@@ -0,0 +1,404 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { ColorId, FontStyle, MetadataConsts, LanguageId, StandardTokenType } from 'vs/editor/common/modes';
+import { Color } from 'vs/base/common/color';
+
+export interface ITokenThemeRule {
+	token: string;
+	foreground?: string;
+	background?: string;
+	fontStyle?: string;
+}
+
+export class ParsedTokenThemeRule {
+	_parsedThemeRuleBrand: void;
+
+	readonly token: string;
+	readonly index: number;
+
+	/**
+	 * -1 if not set. An or mask of `FontStyle` otherwise.
+	 */
+	readonly fontStyle: FontStyle;
+	readonly foreground: string;
+	readonly background: string;
+
+	constructor(
+		token: string,
+		index: number,
+		fontStyle: number,
+		foreground: string,
+		background: string,
+	) {
+		this.token = token;
+		this.index = index;
+		this.fontStyle = fontStyle;
+		this.foreground = foreground;
+		this.background = background;
+	}
+}
+
+/**
+ * Parse a raw theme into rules.
+ */
+export function parseTokenTheme(source: ITokenThemeRule[]): ParsedTokenThemeRule[] {
+	if (!source || !Array.isArray(source)) {
+		return [];
+	}
+	let result: ParsedTokenThemeRule[] = [], resultLen = 0;
+	for (let i = 0, len = source.length; i < len; i++) {
+		let entry = source[i];
+
+		let fontStyle: number = FontStyle.NotSet;
+		if (typeof entry.fontStyle === 'string') {
+			fontStyle = FontStyle.None;
+
+			let segments = entry.fontStyle.split(' ');
+			for (let j = 0, lenJ = segments.length; j < lenJ; j++) {
+				let segment = segments[j];
+				switch (segment) {
+					case 'italic':
+						fontStyle = fontStyle | FontStyle.Italic;
+						break;
+					case 'bold':
+						fontStyle = fontStyle | FontStyle.Bold;
+						break;
+					case 'underline':
+						fontStyle = fontStyle | FontStyle.Underline;
+						break;
+				}
+			}
+		}
+
+		let foreground: string = null;
+		if (typeof entry.foreground === 'string') {
+			foreground = entry.foreground;
+		}
+
+		let background: string = null;
+		if (typeof entry.background === 'string') {
+			background = entry.background;
+		}
+
+		result[resultLen++] = new ParsedTokenThemeRule(
+			entry.token || '',
+			i,
+			fontStyle,
+			foreground,
+			background
+		);
+	}
+
+	return result;
+}
+
+/**
+ * Resolve rules (i.e. inheritance).
+ */
+function resolveParsedTokenThemeRules(parsedThemeRules: ParsedTokenThemeRule[]): TokenTheme {
+
+	// Sort rules lexicographically, and then by index if necessary
+	parsedThemeRules.sort((a, b) => {
+		let r = strcmp(a.token, b.token);
+		if (r !== 0) {
+			return r;
+		}
+		return a.index - b.index;
+	});
+
+	// Determine defaults
+	let defaultFontStyle = FontStyle.None;
+	let defaultForeground = '000000';
+	let defaultBackground = 'ffffff';
+	while (parsedThemeRules.length >= 1 && parsedThemeRules[0].token === '') {
+		let incomingDefaults = parsedThemeRules.shift();
+		if (incomingDefaults.fontStyle !== FontStyle.NotSet) {
+			defaultFontStyle = incomingDefaults.fontStyle;
+		}
+		if (incomingDefaults.foreground !== null) {
+			defaultForeground = incomingDefaults.foreground;
+		}
+		if (incomingDefaults.background !== null) {
+			defaultBackground = incomingDefaults.background;
+		}
+	}
+	let colorMap = new ColorMap();
+	// ensure default foreground gets id 1 and default background gets id 2
+	let defaults = new ThemeTrieElementRule(defaultFontStyle, colorMap.getId(defaultForeground), colorMap.getId(defaultBackground));
+
+	let root = new ThemeTrieElement(defaults);
+	for (let i = 0, len = parsedThemeRules.length; i < len; i++) {
+		let rule = parsedThemeRules[i];
+		root.insert(rule.token, rule.fontStyle, colorMap.getId(rule.foreground), colorMap.getId(rule.background));
+	}
+
+	return new TokenTheme(colorMap, root);
+}
+
+export class ColorMap {
+
+	private _lastColorId: number;
+	private _id2color: Color[];
+	private _color2id: Map<string, ColorId>;
+
+	constructor() {
+		this._lastColorId = 0;
+		this._id2color = [];
+		this._color2id = new Map<string, ColorId>();
+	}
+
+	public getId(color: string): ColorId {
+		if (color === null) {
+			return 0;
+		}
+		color = color.toUpperCase();
+		if (!/^[0-9A-F]{6}$/.test(color)) {
+			throw new Error('Illegal color name: ' + color);
+		}
+		let value = this._color2id.get(color);
+		if (value) {
+			return value;
+		}
+		value = ++this._lastColorId;
+		this._color2id.set(color, value);
+		this._id2color[value] = Color.fromHex('#' + color);
+		return value;
+	}
+
+	public getColorMap(): Color[] {
+		return this._id2color.slice(0);
+	}
+
+}
+
+export class TokenTheme {
+
+	public static createFromRawTokenTheme(source: ITokenThemeRule[]): TokenTheme {
+		return this.createFromParsedTokenTheme(parseTokenTheme(source));
+	}
+
+	public static createFromParsedTokenTheme(source: ParsedTokenThemeRule[]): TokenTheme {
+		return resolveParsedTokenThemeRules(source);
+	}
+
+	private readonly _colorMap: ColorMap;
+	private readonly _root: ThemeTrieElement;
+	private readonly _cache: Map<string, number>;
+
+	constructor(colorMap: ColorMap, root: ThemeTrieElement) {
+		this._colorMap = colorMap;
+		this._root = root;
+		this._cache = new Map<string, number>();
+	}
+
+	public getColorMap(): Color[] {
+		return this._colorMap.getColorMap();
+	}
+
+	/**
+	 * used for testing purposes
+	 */
+	public getThemeTrieElement(): ExternalThemeTrieElement {
+		return this._root.toExternalThemeTrieElement();
+	}
+
+	public _match(token: string): ThemeTrieElementRule {
+		return this._root.match(token);
+	}
+
+	public match(languageId: LanguageId, token: string): number {
+		// The cache contains the metadata without the language bits set.
+		let result = this._cache.get(token);
+		if (typeof result === 'undefined') {
+			let rule = this._match(token);
+			let standardToken = toStandardTokenType(token);
+			result = (
+				rule.metadata
+				| (standardToken << MetadataConsts.TOKEN_TYPE_OFFSET)
+			) >>> 0;
+			this._cache.set(token, result);
+		}
+
+		return (
+			result
+			| (languageId << MetadataConsts.LANGUAGEID_OFFSET)
+		) >>> 0;
+	}
+}
+
+const STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex)\b/;
+export function toStandardTokenType(tokenType: string): StandardTokenType {
+	let m = tokenType.match(STANDARD_TOKEN_TYPE_REGEXP);
+	if (!m) {
+		return StandardTokenType.Other;
+	}
+	switch (m[1]) {
+		case 'comment':
+			return StandardTokenType.Comment;
+		case 'string':
+			return StandardTokenType.String;
+		case 'regex':
+			return StandardTokenType.RegEx;
+	}
+	throw new Error('Unexpected match for standard token type!');
+}
+
+export function strcmp(a: string, b: string): number {
+	if (a < b) {
+		return -1;
+	}
+	if (a > b) {
+		return 1;
+	}
+	return 0;
+}
+
+export class ThemeTrieElementRule {
+	_themeTrieElementRuleBrand: void;
+
+	private _fontStyle: FontStyle;
+	private _foreground: ColorId;
+	private _background: ColorId;
+	public metadata: number;
+
+	constructor(fontStyle: FontStyle, foreground: ColorId, background: ColorId) {
+		this._fontStyle = fontStyle;
+		this._foreground = foreground;
+		this._background = background;
+		this.metadata = (
+			(this._fontStyle << MetadataConsts.FONT_STYLE_OFFSET)
+			| (this._foreground << MetadataConsts.FOREGROUND_OFFSET)
+			| (this._background << MetadataConsts.BACKGROUND_OFFSET)
+		) >>> 0;
+	}
+
+	public clone(): ThemeTrieElementRule {
+		return new ThemeTrieElementRule(this._fontStyle, this._foreground, this._background);
+	}
+
+	public static cloneArr(arr: ThemeTrieElementRule[]): ThemeTrieElementRule[] {
+		let r: ThemeTrieElementRule[] = [];
+		for (let i = 0, len = arr.length; i < len; i++) {
+			r[i] = arr[i].clone();
+		}
+		return r;
+	}
+
+	public acceptOverwrite(fontStyle: FontStyle, foreground: ColorId, background: ColorId): void {
+		if (fontStyle !== FontStyle.NotSet) {
+			this._fontStyle = fontStyle;
+		}
+		if (foreground !== ColorId.None) {
+			this._foreground = foreground;
+		}
+		if (background !== ColorId.None) {
+			this._background = background;
+		}
+		this.metadata = (
+			(this._fontStyle << MetadataConsts.FONT_STYLE_OFFSET)
+			| (this._foreground << MetadataConsts.FOREGROUND_OFFSET)
+			| (this._background << MetadataConsts.BACKGROUND_OFFSET)
+		) >>> 0;
+	}
+}
+
+export class ExternalThemeTrieElement {
+
+	public readonly mainRule: ThemeTrieElementRule;
+	public readonly children: { [segment: string]: ExternalThemeTrieElement };
+
+	constructor(mainRule: ThemeTrieElementRule, children?: { [segment: string]: ExternalThemeTrieElement }) {
+		this.mainRule = mainRule;
+		this.children = children || Object.create(null);
+	}
+}
+
+export class ThemeTrieElement {
+	_themeTrieElementBrand: void;
+
+	private readonly _mainRule: ThemeTrieElementRule;
+	private readonly _children: Map<string, ThemeTrieElement>;
+
+	constructor(mainRule: ThemeTrieElementRule) {
+		this._mainRule = mainRule;
+		this._children = new Map<string, ThemeTrieElement>();
+	}
+
+	/**
+	 * used for testing purposes
+	 */
+	public toExternalThemeTrieElement(): ExternalThemeTrieElement {
+		let children: { [segment: string]: ExternalThemeTrieElement } = Object.create(null);
+		this._children.forEach((element, index) => {
+			children[index] = element.toExternalThemeTrieElement();
+		});
+		return new ExternalThemeTrieElement(this._mainRule, children);
+	}
+
+	public match(token: string): ThemeTrieElementRule {
+		if (token === '') {
+			return this._mainRule;
+		}
+
+		let dotIndex = token.indexOf('.');
+		let head: string;
+		let tail: string;
+		if (dotIndex === -1) {
+			head = token;
+			tail = '';
+		} else {
+			head = token.substring(0, dotIndex);
+			tail = token.substring(dotIndex + 1);
+		}
+
+		let child = this._children.get(head);
+		if (typeof child !== 'undefined') {
+			return child.match(tail);
+		}
+
+		return this._mainRule;
+	}
+
+	public insert(token: string, fontStyle: FontStyle, foreground: ColorId, background: ColorId): void {
+		if (token === '') {
+			// Merge into the main rule
+			this._mainRule.acceptOverwrite(fontStyle, foreground, background);
+			return;
+		}
+
+		let dotIndex = token.indexOf('.');
+		let head: string;
+		let tail: string;
+		if (dotIndex === -1) {
+			head = token;
+			tail = '';
+		} else {
+			head = token.substring(0, dotIndex);
+			tail = token.substring(dotIndex + 1);
+		}
+
+		let child = this._children.get(head);
+		if (typeof child === 'undefined') {
+			child = new ThemeTrieElement(this._mainRule.clone());
+			this._children.set(head, child);
+		}
+
+		child.insert(tail, fontStyle, foreground, background);
+	}
+}
+
+export function generateTokensCSSForColorMap(colorMap: Color[]): string {
+	let rules: string[] = [];
+	for (let i = 1, len = colorMap.length; i < len; i++) {
+		let color = colorMap[i];
+		rules[i] = `.mtk${i} { color: ${color}; }`;
+	}
+	rules.push('.mtki { font-style: italic; }');
+	rules.push('.mtkb { font-weight: bold; }');
+	rules.push('.mtku { text-decoration: underline; }');
+	return rules.join('\n');
+}

src/vs/editor/common/modes/textToHtmlTokenizer.ts

@@ -0,0 +1,127 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import * as strings from 'vs/base/common/strings';
+import { IState, ITokenizationSupport, TokenizationRegistry, LanguageId } from 'vs/editor/common/modes';
+import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
+import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { CharCode } from 'vs/base/common/charCode';
+import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';
+
+export function tokenizeToString(text: string, languageId: string): string {
+	return _tokenizeToString(text, _getSafeTokenizationSupport(languageId));
+}
+
+export function tokenizeLineToHTML(text: string, viewLineTokens: ViewLineToken[], colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string {
+	let result = `<div>`;
+	let charIndex = startOffset;
+	let tabsCharDelta = 0;
+
+	for (let tokenIndex = 0, lenJ = viewLineTokens.length; tokenIndex < lenJ; tokenIndex++) {
+		const token = viewLineTokens[tokenIndex];
+		const tokenEndIndex = token.endIndex;
+
+		if (token.endIndex <= startOffset) {
+			continue;
+		}
+
+		let partContent = '';
+
+		for (; charIndex < tokenEndIndex && charIndex < endOffset; charIndex++) {
+			const charCode = text.charCodeAt(charIndex);
+
+			switch (charCode) {
+				case CharCode.Tab:
+					let insertSpacesCount = tabSize - (charIndex + tabsCharDelta) % tabSize;
+					tabsCharDelta += insertSpacesCount - 1;
+					while (insertSpacesCount > 0) {
+						partContent += '&nbsp;';
+						insertSpacesCount--;
+					}
+					break;
+
+				case CharCode.LessThan:
+					partContent += '&lt;';
+					break;
+
+				case CharCode.GreaterThan:
+					partContent += '&gt;';
+					break;
+
+				case CharCode.Ampersand:
+					partContent += '&amp;';
+					break;
+
+				case CharCode.Null:
+					partContent += '&#00;';
+					break;
+
+				case CharCode.UTF8_BOM:
+				case CharCode.LINE_SEPARATOR_2028:
+					partContent += '\ufffd';
+					break;
+
+				case CharCode.CarriageReturn:
+					// zero width space, because carriage return would introduce a line break
+					partContent += '&#8203';
+					break;
+
+				default:
+					partContent += String.fromCharCode(charCode);
+			}
+		}
+
+		result += `<span style="${token.getInlineStyle(colorMap)}">${partContent}</span>`;
+
+		if (token.endIndex > endOffset || charIndex >= endOffset) {
+			break;
+		}
+	}
+
+	result += `</div>`;
+	return result;
+}
+
+function _getSafeTokenizationSupport(languageId: string): ITokenizationSupport {
+	let tokenizationSupport = TokenizationRegistry.get(languageId);
+	if (tokenizationSupport) {
+		return tokenizationSupport;
+	}
+	return {
+		getInitialState: () => NULL_STATE,
+		tokenize: undefined,
+		tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
+	};
+}
+
+function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport): string {
+	let result = `<div class="monaco-tokenized-source">`;
+	let lines = text.split(/\r\n|\r|\n/);
+	let currentState = tokenizationSupport.getInitialState();
+	for (let i = 0, len = lines.length; i < len; i++) {
+		let line = lines[i];
+
+		if (i > 0) {
+			result += `<br/>`;
+		}
+
+		let tokenizationResult = tokenizationSupport.tokenize2(line, currentState, 0);
+		let lineTokens = new LineTokens(tokenizationResult.tokens, line);
+		let viewLineTokens = lineTokens.inflate();
+
+		let startOffset = 0;
+		for (let j = 0, lenJ = viewLineTokens.length; j < lenJ; j++) {
+			const viewLineToken = viewLineTokens[j];
+			result += `<span class="${viewLineToken.getType()}">${strings.escape(line.substring(startOffset, viewLineToken.endIndex))}</span>`;
+			startOffset = viewLineToken.endIndex;
+		}
+
+		currentState = tokenizationResult.endState;
+	}
+
+	result += `</div>`;
+	return result;
+}

src/vs/editor/common/modes/tokenizationRegistry.ts

@@ -0,0 +1,70 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import { IDisposable } from 'vs/base/common/lifecycle';
+import Event, { Emitter } from 'vs/base/common/event';
+import { ColorId, ITokenizationRegistry, ITokenizationSupport, ITokenizationSupportChangedEvent } from 'vs/editor/common/modes';
+import { Color } from 'vs/base/common/color';
+
+export class TokenizationRegistryImpl implements ITokenizationRegistry {
+
+	private _map: { [language: string]: ITokenizationSupport };
+
+	private _onDidChange: Emitter<ITokenizationSupportChangedEvent> = new Emitter<ITokenizationSupportChangedEvent>();
+	public onDidChange: Event<ITokenizationSupportChangedEvent> = this._onDidChange.event;
+
+	private _colorMap: Color[];
+
+	constructor() {
+		this._map = Object.create(null);
+		this._colorMap = null;
+	}
+
+	public fire(languages: string[]): void {
+		this._onDidChange.fire({
+			changedLanguages: languages,
+			changedColorMap: false
+		});
+	}
+
+	public register(language: string, support: ITokenizationSupport): IDisposable {
+		this._map[language] = support;
+		this.fire([language]);
+		return {
+			dispose: () => {
+				if (this._map[language] !== support) {
+					return;
+				}
+				delete this._map[language];
+				this.fire([language]);
+			}
+		};
+	}
+
+	public get(language: string): ITokenizationSupport {
+		return (this._map[language] || null);
+	}
+
+	public setColorMap(colorMap: Color[]): void {
+		this._colorMap = colorMap;
+		this._onDidChange.fire({
+			changedLanguages: Object.keys(this._map),
+			changedColorMap: true
+		});
+	}
+
+	public getColorMap(): Color[] {
+		return this._colorMap;
+	}
+
+	public getDefaultForeground(): Color {
+		return this._colorMap[ColorId.DefaultForeground];
+	}
+
+	public getDefaultBackground(): Color {
+		return this._colorMap[ColorId.DefaultBackground];
+	}
+}