Merge from master

src/vs/editor/common/modes/abstractMode.ts

@@ -2,7 +2,6 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
 import { IMode, LanguageIdentifier } from 'vs/editor/common/modes';
 

src/vs/editor/common/modes/languageConfiguration.ts

@@ -2,7 +2,6 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
 import { StandardTokenType } from 'vs/editor/common/modes';
 
@@ -13,11 +12,11 @@ export interface CommentRule {
 	/**
 	 * The line comment token, like `// this is a comment`
 	 */
-	lineComment?: string;
+	lineComment?: string | null;
 	/**
 	 * The block comment character pair, like `/* block comment */`
 	 */
-	blockComment?: CharacterPair;
+	blockComment?: CharacterPair | null;
 }
 
 /**
@@ -62,6 +61,13 @@ export interface LanguageConfiguration {
 	 */
 	surroundingPairs?: IAutoClosingPair[];
 
+	/**
+	 * Defines what characters must be after the cursor for bracket or quote autoclosing to occur when using the \'languageDefined\' autoclosing setting.
+	 *
+	 * This is typically the set of characters which can not start an expression, such as whitespace, closing brackets, non-unary operators, etc.
+	 */
+	autoCloseBefore?: string;
+
 	/**
 	 * The language's folding rules.
 	 */
@@ -80,7 +86,7 @@ export interface LanguageConfiguration {
  */
 export interface IndentationRule {
 	/**
-	 * If a line matches this pattern, then all the lines after it should be unindendented once (until another rule matches).
+	 * If a line matches this pattern, then all the lines after it should be unindented once (until another rule matches).
 	 */
 	decreaseIndentPattern: RegExp;
 	/**
@@ -90,11 +96,11 @@ export interface IndentationRule {
 	/**
 	 * If a line matches this pattern, then **only the next line** after it should be indented once.
 	 */
-	indentNextLinePattern?: RegExp;
+	indentNextLinePattern?: RegExp | null;
 	/**
 	 * If a line matches this pattern, then its indentation should not be changed and it should not be evaluated against the other rules.
 	 */
-	unIndentedLinePattern?: RegExp;
+	unIndentedLinePattern?: RegExp | null;
 
 }
 
@@ -114,7 +120,7 @@ export interface FoldingMarkers {
  */
 export interface FoldingRules {
 	/**
-	 * Used by the indentation based strategy to decide wheter empty lines belong to the previous or the next block.
+	 * Used by the indentation based strategy to decide whether empty lines belong to the previous or the next block.
 	 * A language adheres to the off-side rule if blocks in that language are expressed by their indentation.
 	 * See [wikipedia](https://en.wikipedia.org/wiki/Off-side_rule) for more information.
 	 * If not set, `false` is used and empty lines belong to the previous block.
@@ -139,6 +145,10 @@ export interface OnEnterRule {
 	 * This rule will only execute if the text after the cursor matches this regular expression.
 	 */
 	afterText?: RegExp;
+	/**
+	 * This rule will only execute if the text above the this line matches this regular expression.
+	 */
+	oneLineAboveText?: RegExp;
 	/**
 	 * The action to execute.
 	 */
@@ -210,10 +220,6 @@ export interface EnterAction {
 	 * Describe what to do with the indentation.
 	 */
 	indentAction: IndentAction;
-	/**
-	 * Describe whether to outdent current line.
-	 */
-	outdentCurrentLine?: boolean;
 	/**
 	 * Describes text to be appended after the new line and after the indentation.
 	 */

src/vs/editor/common/modes/languageConfigurationRegistry.ts

@@ -2,24 +2,23 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
-import { CharacterPairSupport } from 'vs/editor/common/modes/supports/characterPair';
-import { BracketElectricCharacterSupport, IElectricAction } from 'vs/editor/common/modes/supports/electricCharacter';
-import { IOnEnterSupportOptions, OnEnterSupport } from 'vs/editor/common/modes/supports/onEnter';
-import { IndentRulesSupport, IndentConsts } from 'vs/editor/common/modes/supports/indentRules';
-import { RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
-import { Event, Emitter } from 'vs/base/common/event';
-import { ITextModel } from 'vs/editor/common/model';
 import { onUnexpectedError } from 'vs/base/common/errors';
-import * as strings from 'vs/base/common/strings';
+import { Emitter, Event } from 'vs/base/common/event';
 import { IDisposable, toDisposable } from 'vs/base/common/lifecycle';
-import { DEFAULT_WORD_REGEXP, ensureValidWordDefinition } from 'vs/editor/common/model/wordHelper';
-import { createScopedLineTokens } from 'vs/editor/common/modes/supports';
+import * as strings from 'vs/base/common/strings';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
 import { Range } from 'vs/editor/common/core/range';
-import { IndentAction, EnterAction, IAutoClosingPair, LanguageConfiguration, IndentationRule, FoldingRules, IAutoClosingPairConditional } from 'vs/editor/common/modes/languageConfiguration';
-import { LanguageIdentifier, LanguageId } from 'vs/editor/common/modes';
+import { ITextModel } from 'vs/editor/common/model';
+import { DEFAULT_WORD_REGEXP, ensureValidWordDefinition } from 'vs/editor/common/model/wordHelper';
+import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes';
+import { EnterAction, FoldingRules, IAutoClosingPair, IAutoClosingPairConditional, IndentAction, IndentationRule, LanguageConfiguration } from 'vs/editor/common/modes/languageConfiguration';
+import { createScopedLineTokens } from 'vs/editor/common/modes/supports';
+import { CharacterPairSupport } from 'vs/editor/common/modes/supports/characterPair';
+import { BracketElectricCharacterSupport, IElectricAction } from 'vs/editor/common/modes/supports/electricCharacter';
+import { IndentConsts, IndentRulesSupport } from 'vs/editor/common/modes/supports/indentRules';
+import { IOnEnterSupportOptions, OnEnterSupport } from 'vs/editor/common/modes/supports/onEnter';
+import { RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
 
 /**
  * Interface used to support insertion of mode specific comments.
@@ -38,8 +37,8 @@ export interface IVirtualModel {
 }
 
 export interface IIndentConverter {
-	shiftIndent?(indentation: string): string;
-	unshiftIndent?(indentation: string): string;
+	shiftIndent(indentation: string): string;
+	unshiftIndent(indentation: string): string;
 	normalizeIndentation?(indentation: string): string;
 }
 
@@ -47,15 +46,15 @@ export class RichEditSupport {
 
 	private readonly _conf: LanguageConfiguration;
 	private readonly _languageIdentifier: LanguageIdentifier;
-	private _brackets: RichEditBrackets;
-	private _electricCharacter: BracketElectricCharacterSupport;
+	private _brackets: RichEditBrackets | null;
+	private _electricCharacter: BracketElectricCharacterSupport | null;
 
-	public readonly comments: ICommentsConfiguration;
+	public readonly comments: ICommentsConfiguration | null;
 	public readonly characterPair: CharacterPairSupport;
 	public readonly wordDefinition: RegExp;
-	public readonly onEnter: OnEnterSupport;
+	public readonly onEnter: OnEnterSupport | null;
 	public readonly indentRulesSupport: IndentRulesSupport;
-	public readonly indentationRules: IndentationRule;
+	public readonly indentationRules: IndentationRule | undefined;
 	public readonly foldingRules: FoldingRules;
 
 	constructor(languageIdentifier: LanguageIdentifier, previous: RichEditSupport, rawConf: LanguageConfiguration) {
@@ -64,7 +63,7 @@ export class RichEditSupport {
 		this._brackets = null;
 		this._electricCharacter = null;
 
-		let prev: LanguageConfiguration = null;
+		let prev: LanguageConfiguration | null = null;
 		if (previous) {
 			prev = previous._conf;
 		}
@@ -87,14 +86,14 @@ export class RichEditSupport {
 		this.foldingRules = this._conf.folding || {};
 	}
 
-	public get brackets(): RichEditBrackets {
+	public get brackets(): RichEditBrackets | null {
 		if (!this._brackets && this._conf.brackets) {
 			this._brackets = new RichEditBrackets(this._languageIdentifier, this._conf.brackets);
 		}
 		return this._brackets;
 	}
 
-	public get electricCharacter(): BracketElectricCharacterSupport {
+	public get electricCharacter(): BracketElectricCharacterSupport | null {
 		if (!this._electricCharacter) {
 			let autoClosingPairs: IAutoClosingPairConditional[] = [];
 			if (this._conf.autoClosingPairs) {
@@ -110,7 +109,7 @@ export class RichEditSupport {
 		return this._electricCharacter;
 	}
 
-	private static _mergeConf(prev: LanguageConfiguration, current: LanguageConfiguration): LanguageConfiguration {
+	private static _mergeConf(prev: LanguageConfiguration | null, current: LanguageConfiguration): LanguageConfiguration {
 		return {
 			comments: (prev ? current.comments || prev.comments : current.comments),
 			brackets: (prev ? current.brackets || prev.brackets : current.brackets),
@@ -119,12 +118,13 @@ export class RichEditSupport {
 			onEnterRules: (prev ? current.onEnterRules || prev.onEnterRules : current.onEnterRules),
 			autoClosingPairs: (prev ? current.autoClosingPairs || prev.autoClosingPairs : current.autoClosingPairs),
 			surroundingPairs: (prev ? current.surroundingPairs || prev.surroundingPairs : current.surroundingPairs),
+			autoCloseBefore: (prev ? current.autoCloseBefore || prev.autoCloseBefore : current.autoCloseBefore),
 			folding: (prev ? current.folding || prev.folding : current.folding),
 			__electricCharacterSupport: (prev ? current.__electricCharacterSupport || prev.__electricCharacterSupport : current.__electricCharacterSupport),
 		};
 	}
 
-	private static _handleOnEnter(conf: LanguageConfiguration): OnEnterSupport {
+	private static _handleOnEnter(conf: LanguageConfiguration): OnEnterSupport | null {
 		// on enter
 		let onEnter: IOnEnterSupportOptions = {};
 		let empty = true;
@@ -147,7 +147,7 @@ export class RichEditSupport {
 		return null;
 	}
 
-	private static _handleComments(conf: LanguageConfiguration): ICommentsConfiguration {
+	private static _handleComments(conf: LanguageConfiguration): ICommentsConfiguration | null {
 		let commentRule = conf.comments;
 		if (!commentRule) {
 			return null;
@@ -213,7 +213,7 @@ export class LanguageConfigurationRegistryImpl {
 
 	// begin electricCharacter
 
-	private _getElectricCharacterSupport(languageId: LanguageId): BracketElectricCharacterSupport {
+	private _getElectricCharacterSupport(languageId: LanguageId): BracketElectricCharacterSupport | null {
 		let value = this._getRichEditSupport(languageId);
 		if (!value) {
 			return null;
@@ -232,7 +232,7 @@ export class LanguageConfigurationRegistryImpl {
 	/**
 	 * Should return opening bracket type to match indentation with
 	 */
-	public onElectricCharacter(character: string, context: LineTokens, column: number): IElectricAction {
+	public onElectricCharacter(character: string, context: LineTokens, column: number): IElectricAction | null {
 		let scopedLineTokens = createScopedLineTokens(context, column - 1);
 		let electricCharacterSupport = this._getElectricCharacterSupport(scopedLineTokens.languageId);
 		if (!electricCharacterSupport) {
@@ -243,7 +243,7 @@ export class LanguageConfigurationRegistryImpl {
 
 	// end electricCharacter
 
-	public getComments(languageId: LanguageId): ICommentsConfiguration {
+	public getComments(languageId: LanguageId): ICommentsConfiguration | null {
 		let value = this._getRichEditSupport(languageId);
 		if (!value) {
 			return null;
@@ -253,7 +253,7 @@ export class LanguageConfigurationRegistryImpl {
 
 	// begin characterPair
 
-	private _getCharacterPairSupport(languageId: LanguageId): CharacterPairSupport {
+	private _getCharacterPairSupport(languageId: LanguageId): CharacterPairSupport | null {
 		let value = this._getRichEditSupport(languageId);
 		if (!value) {
 			return null;
@@ -269,6 +269,14 @@ export class LanguageConfigurationRegistryImpl {
 		return characterPairSupport.getAutoClosingPairs();
 	}
 
+	public getAutoCloseBeforeSet(languageId: LanguageId): string {
+		let characterPairSupport = this._getCharacterPairSupport(languageId);
+		if (!characterPairSupport) {
+			return CharacterPairSupport.DEFAULT_AUTOCLOSE_BEFORE_LANGUAGE_DEFINED;
+		}
+		return characterPairSupport.getAutoCloseBeforeSet();
+	}
+
 	public getSurroundingPairs(languageId: LanguageId): IAutoClosingPair[] {
 		let characterPairSupport = this._getCharacterPairSupport(languageId);
 		if (!characterPairSupport) {
@@ -306,7 +314,7 @@ export class LanguageConfigurationRegistryImpl {
 
 	// begin Indent Rules
 
-	public getIndentRulesSupport(languageId: LanguageId): IndentRulesSupport {
+	public getIndentRulesSupport(languageId: LanguageId): IndentRulesSupport | null {
 		let value = this._getRichEditSupport(languageId);
 		if (!value) {
 			return null;
@@ -356,7 +364,7 @@ export class LanguageConfigurationRegistryImpl {
 	 *
 	 * This function only return the inherited indent based on above lines, it doesn't check whether current line should decrease or not.
 	 */
-	public getInheritIndentForLine(model: IVirtualModel, lineNumber: number, honorIntentialIndent: boolean = true): { indentation: string, action: IndentAction, line?: number } {
+	public getInheritIndentForLine(model: IVirtualModel, lineNumber: number, honorIntentialIndent: boolean = true): { indentation: string; action: IndentAction | null; line?: number; } | null {
 		let indentRulesSupport = this.getIndentRulesSupport(model.getLanguageIdentifier().id);
 		if (!indentRulesSupport) {
 			return null;
@@ -477,7 +485,7 @@ export class LanguageConfigurationRegistryImpl {
 		}
 	}
 
-	public getGoodIndentForLine(virtualModel: IVirtualModel, languageId: LanguageId, lineNumber: number, indentConverter: IIndentConverter): string {
+	public getGoodIndentForLine(virtualModel: IVirtualModel, languageId: LanguageId, lineNumber: number, indentConverter: IIndentConverter): string | null {
 		let indentRulesSupport = this.getIndentRulesSupport(languageId);
 		if (!indentRulesSupport) {
 			return null;
@@ -490,9 +498,11 @@ export class LanguageConfigurationRegistryImpl {
 			let inheritLine = indent.line;
 			if (inheritLine !== undefined) {
 				let onEnterSupport = this._getOnEnterSupport(languageId);
-				let enterResult: EnterAction = null;
+				let enterResult: EnterAction | null = null;
 				try {
-					enterResult = onEnterSupport.onEnter('', virtualModel.getLineContent(inheritLine), '');
+					if (onEnterSupport) {
+						enterResult = onEnterSupport.onEnter('', virtualModel.getLineContent(inheritLine), '');
+					}
 				} catch (e) {
 					onUnexpectedError(e);
 				}
@@ -542,7 +552,7 @@ export class LanguageConfigurationRegistryImpl {
 		return null;
 	}
 
-	public getIndentForEnter(model: ITextModel, range: Range, indentConverter: IIndentConverter, autoIndent: boolean): { beforeEnter: string, afterEnter: string } {
+	public getIndentForEnter(model: ITextModel, range: Range, indentConverter: IIndentConverter, autoIndent: boolean): { beforeEnter: string, afterEnter: string } | null {
 		model.forceTokenization(range.startLineNumber);
 		let lineTokens = model.getLineTokens(range.startLineNumber);
 
@@ -640,7 +650,7 @@ export class LanguageConfigurationRegistryImpl {
 	 * We should always allow intentional indentation. It means, if users change the indentation of `lineNumber` and the content of
 	 * this line doesn't match decreaseIndentPattern, we should not adjust the indentation.
 	 */
-	public getIndentActionForType(model: ITextModel, range: Range, ch: string, indentConverter: IIndentConverter): string {
+	public getIndentActionForType(model: ITextModel, range: Range, ch: string, indentConverter: IIndentConverter): string | null {
 		let scopedLineTokens = this.getScopedLineTokens(model, range.startLineNumber, range.startColumn);
 		let indentRulesSupport = this.getIndentRulesSupport(scopedLineTokens.languageId);
 		if (!indentRulesSupport) {
@@ -681,7 +691,7 @@ export class LanguageConfigurationRegistryImpl {
 		return null;
 	}
 
-	public getIndentMetadata(model: ITextModel, lineNumber: number): number {
+	public getIndentMetadata(model: ITextModel, lineNumber: number): number | null {
 		let indentRulesSupport = this.getIndentRulesSupport(model.getLanguageIdentifier().id);
 		if (!indentRulesSupport) {
 			return null;
@@ -698,7 +708,7 @@ export class LanguageConfigurationRegistryImpl {
 
 	// begin onEnter
 
-	private _getOnEnterSupport(languageId: LanguageId): OnEnterSupport {
+	private _getOnEnterSupport(languageId: LanguageId): OnEnterSupport | null {
 		let value = this._getRichEditSupport(languageId);
 		if (!value) {
 			return null;
@@ -706,13 +716,13 @@ export class LanguageConfigurationRegistryImpl {
 		return value.onEnter || null;
 	}
 
-	public getRawEnterActionAtPosition(model: ITextModel, lineNumber: number, column: number): EnterAction {
+	public getRawEnterActionAtPosition(model: ITextModel, lineNumber: number, column: number): EnterAction | null {
 		let r = this.getEnterAction(model, new Range(lineNumber, column, lineNumber, column));
 
 		return r ? r.enterAction : null;
 	}
 
-	public getEnterAction(model: ITextModel, range: Range): { enterAction: EnterAction; indentation: string; } {
+	public getEnterAction(model: ITextModel, range: Range): { enterAction: EnterAction; indentation: string; } | null {
 		let indentation = this.getIndentationAtPosition(model, range.startLineNumber, range.startColumn);
 
 		let scopedLineTokens = this.getScopedLineTokens(model, range.startLineNumber, range.startColumn);
@@ -745,7 +755,7 @@ export class LanguageConfigurationRegistryImpl {
 			}
 		}
 
-		let enterResult: EnterAction = null;
+		let enterResult: EnterAction | null = null;
 		try {
 			enterResult = onEnterSupport.onEnter(oneLineAboveText, beforeEnterText, afterEnterText);
 		} catch (e) {
@@ -791,14 +801,14 @@ export class LanguageConfigurationRegistryImpl {
 	private getScopedLineTokens(model: ITextModel, lineNumber: number, columnNumber?: number) {
 		model.forceTokenization(lineNumber);
 		let lineTokens = model.getLineTokens(lineNumber);
-		let column = isNaN(columnNumber) ? model.getLineMaxColumn(lineNumber) - 1 : columnNumber - 1;
+		let column = (typeof columnNumber === 'undefined' ? model.getLineMaxColumn(lineNumber) - 1 : columnNumber - 1);
 		let scopedLineTokens = createScopedLineTokens(lineTokens, column);
 		return scopedLineTokens;
 	}
 
 	// end onEnter
 
-	public getBracketsSupport(languageId: LanguageId): RichEditBrackets {
+	public getBracketsSupport(languageId: LanguageId): RichEditBrackets | null {
 		let value = this._getRichEditSupport(languageId);
 		if (!value) {
 			return null;

src/vs/editor/common/modes/languageFeatureRegistry.ts

@@ -3,9 +3,7 @@
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-'use strict';
-
-import { Event, Emitter } from 'vs/base/common/event';
+import { Emitter, Event } from 'vs/base/common/event';
 import { IDisposable, toDisposable } from 'vs/base/common/lifecycle';
 import { ITextModel } from 'vs/editor/common/model';
 import { LanguageSelector, score } from 'vs/editor/common/modes/languageSelector';
@@ -24,11 +22,11 @@ function isExclusive(selector: LanguageSelector): boolean {
 	} else if (Array.isArray(selector)) {
 		return selector.every(isExclusive);
 	} else {
-		return selector.exclusive;
+		return !!selector.exclusive;
 	}
 }
 
-export default class LanguageFeatureRegistry<T> {
+export class LanguageFeatureRegistry<T> {
 
 	private _clock: number = 0;
 	private _entries: Entry<T>[] = [];
@@ -43,7 +41,7 @@ export default class LanguageFeatureRegistry<T> {
 
 	register(selector: LanguageSelector, provider: T): IDisposable {
 
-		let entry: Entry<T> = {
+		let entry: Entry<T> | undefined = {
 			selector,
 			provider,
 			_score: -1,
@@ -129,7 +127,7 @@ export default class LanguageFeatureRegistry<T> {
 		}
 	}
 
-	private _lastCandidate: { uri: string; language: string; };
+	private _lastCandidate: { uri: string; language: string; } | undefined;
 
 	private _updateScores(model: ITextModel): void {
 

src/vs/editor/common/modes/languageSelector.ts

@@ -3,10 +3,8 @@
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-'use strict';
-
-import URI from 'vs/base/common/uri';
-import { match as matchGlobPattern, IRelativePattern } from 'vs/base/common/glob'; // TODO@Alex
+import { IRelativePattern, match as matchGlobPattern } from 'vs/base/common/glob';
+import { URI } from 'vs/base/common/uri'; // TODO@Alex
 
 export interface LanguageFilter {
 	language?: string;

src/vs/editor/common/modes/linkComputer.ts

@@ -2,19 +2,18 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
-import { ILink } from 'vs/editor/common/modes';
 import { CharCode } from 'vs/base/common/charCode';
 import { CharacterClassifier } from 'vs/editor/common/core/characterClassifier';
 import { Uint8Matrix } from 'vs/editor/common/core/uint';
+import { ILink } from 'vs/editor/common/modes';
 
 export interface ILinkComputerTarget {
 	getLineCount(): number;
 	getLineContent(lineNumber: number): string;
 }
 
-const enum State {
+export const enum State {
 	Invalid = 0,
 	Start = 1,
 	H = 2,
@@ -28,12 +27,13 @@ const enum State {
 	AfterColon = 10,
 	AlmostThere = 11,
 	End = 12,
-	Accept = 13
+	Accept = 13,
+	LastKnownState = 14 // marker, custom states may follow
 }
 
-type Edge = [State, number, State];
+export type Edge = [State, number, State];
 
-class StateMachine {
+export class StateMachine {
 
 	private _states: Uint8Matrix;
 	private _maxCharCode: number;
@@ -76,7 +76,7 @@ class StateMachine {
 }
 
 // State machine for http:// or https:// or file://
-let _stateMachine: StateMachine = null;
+let _stateMachine: StateMachine | null = null;
 function getStateMachine(): StateMachine {
 	if (_stateMachine === null) {
 		_stateMachine = new StateMachine([
@@ -124,7 +124,7 @@ const enum CharacterClass {
 	CannotEndIn = 2
 }
 
-let _classifier: CharacterClassifier<CharacterClass> = null;
+let _classifier: CharacterClassifier<CharacterClass> | null = null;
 function getClassifier(): CharacterClassifier<CharacterClass> {
 	if (_classifier === null) {
 		_classifier = new CharacterClassifier<CharacterClass>(CharacterClass.None);
@@ -142,7 +142,7 @@ function getClassifier(): CharacterClassifier<CharacterClass> {
 	return _classifier;
 }
 
-class LinkComputer {
+export class LinkComputer {
 
 	private static _createLink(classifier: CharacterClassifier<CharacterClass>, line: string, lineNumber: number, linkBeginIndex: number, linkEndIndex: number): ILink {
 		// Do not allow to end link in certain characters...
@@ -184,8 +184,7 @@ class LinkComputer {
 		};
 	}
 
-	public static computeLinks(model: ILinkComputerTarget): ILink[] {
-		const stateMachine = getStateMachine();
+	public static computeLinks(model: ILinkComputerTarget, stateMachine: StateMachine = getStateMachine()): ILink[] {
 		const classifier = getClassifier();
 
 		let result: ILink[] = [];
@@ -250,7 +249,15 @@ class LinkComputer {
 						resetStateMachine = true;
 					}
 				} else if (state === State.End) {
-					const chClass = classifier.get(chCode);
+
+					let chClass: CharacterClass;
+					if (chCode === CharCode.OpenSquareBracket) {
+						// Allow for the authority part to contain ipv6 addresses which contain [ and ]
+						hasOpenSquareBracket = true;
+						chClass = CharacterClass.None;
+					} else {
+						chClass = classifier.get(chCode);
+					}
 
 					// Check if character terminates link
 					if (chClass === CharacterClass.ForceTermination) {

src/vs/editor/common/modes/modesRegistry.ts

@@ -2,14 +2,13 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
 import * as nls from 'vs/nls';
-import { Event, Emitter } from 'vs/base/common/event';
-import { Registry } from 'vs/platform/registry/common/platform';
-import { ILanguageExtensionPoint } from 'vs/editor/common/services/modeService';
+import { Emitter, Event } from 'vs/base/common/event';
+import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes';
 import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
-import { LanguageIdentifier, LanguageId } from 'vs/editor/common/modes';
+import { ILanguageExtensionPoint } from 'vs/editor/common/services/modeService';
+import { Registry } from 'vs/platform/registry/common/platform';
 
 // Define extension point ids
 export const Extensions = {

src/vs/editor/common/modes/nullMode.ts

@@ -2,10 +2,9 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
-import { IState, ColorId, MetadataConsts, LanguageIdentifier, FontStyle, StandardTokenType, LanguageId } from 'vs/editor/common/modes';
 import { Token, TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token';
+import { ColorId, FontStyle, IState, LanguageId, LanguageIdentifier, MetadataConsts, StandardTokenType } from 'vs/editor/common/modes';
 
 class NullStateImpl implements IState {
 
@@ -28,7 +27,7 @@ export function nullTokenize(modeId: string, buffer: string, state: IState, delt
 	return new TokenizationResult([new Token(deltaOffset, '', modeId)], state);
 }
 
-export function nullTokenize2(languageId: LanguageId, buffer: string, state: IState, deltaOffset: number): TokenizationResult2 {
+export function nullTokenize2(languageId: LanguageId, buffer: string, state: IState | null, deltaOffset: number): TokenizationResult2 {
 	let tokens = new Uint32Array(2);
 	tokens[0] = deltaOffset;
 	tokens[1] = (
@@ -39,5 +38,5 @@ export function nullTokenize2(languageId: LanguageId, buffer: string, state: ISt
 		| (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET)
 	) >>> 0;
 
-	return new TokenizationResult2(tokens, state);
+	return new TokenizationResult2(tokens, state === null ? NULL_STATE : state);
 }

src/vs/editor/common/modes/supports.ts

@@ -2,10 +2,9 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
-import * as modes from 'vs/editor/common/modes';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import * as modes from 'vs/editor/common/modes';
 
 export function createScopedLineTokens(context: LineTokens, offset: number): ScopedLineTokens {
 	let tokenCount = context.getCount();

src/vs/editor/common/modes/supports/characterPair.ts

@@ -2,17 +2,20 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
-import { ScopedLineTokens } from 'vs/editor/common/modes/supports';
 import { CharacterPair, IAutoClosingPair, IAutoClosingPairConditional, StandardAutoClosingPairConditional } from 'vs/editor/common/modes/languageConfiguration';
+import { ScopedLineTokens } from 'vs/editor/common/modes/supports';
 
 export class CharacterPairSupport {
 
+	static readonly DEFAULT_AUTOCLOSE_BEFORE_LANGUAGE_DEFINED = ';:.,=}])> \n\t';
+	static readonly DEFAULT_AUTOCLOSE_BEFORE_WHITESPACE = ' \n\t';
+
 	private readonly _autoClosingPairs: StandardAutoClosingPairConditional[];
 	private readonly _surroundingPairs: IAutoClosingPair[];
+	private readonly _autoCloseBefore: string;
 
-	constructor(config: { brackets?: CharacterPair[]; autoClosingPairs?: IAutoClosingPairConditional[], surroundingPairs?: IAutoClosingPair[] }) {
+	constructor(config: { brackets?: CharacterPair[]; autoClosingPairs?: IAutoClosingPairConditional[], surroundingPairs?: IAutoClosingPair[], autoCloseBefore?: string }) {
 		if (config.autoClosingPairs) {
 			this._autoClosingPairs = config.autoClosingPairs.map(el => new StandardAutoClosingPairConditional(el));
 		} else if (config.brackets) {
@@ -21,6 +24,8 @@ export class CharacterPairSupport {
 			this._autoClosingPairs = [];
 		}
 
+		this._autoCloseBefore = typeof config.autoCloseBefore === 'string' ? config.autoCloseBefore : CharacterPairSupport.DEFAULT_AUTOCLOSE_BEFORE_LANGUAGE_DEFINED;
+
 		this._surroundingPairs = config.surroundingPairs || this._autoClosingPairs;
 	}
 
@@ -28,6 +33,10 @@ export class CharacterPairSupport {
 		return this._autoClosingPairs;
 	}
 
+	public getAutoCloseBeforeSet(): string {
+		return this._autoCloseBefore;
+	}
+
 	public shouldAutoClosePair(character: string, context: ScopedLineTokens, column: number): boolean {
 		// Always complete on empty line
 		if (context.getTokenCount() === 0) {

src/vs/editor/common/modes/supports/electricCharacter.ts

@@ -2,11 +2,10 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
+import { IAutoClosingPairConditional, IBracketElectricCharacterContribution, StandardAutoClosingPairConditional } from 'vs/editor/common/modes/languageConfiguration';
 import { ScopedLineTokens, ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
 import { BracketsUtils, RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
-import { IAutoClosingPairConditional, IBracketElectricCharacterContribution, StandardAutoClosingPairConditional } from 'vs/editor/common/modes/languageConfiguration';
 
 /**
  * Interface used to support electric characters
@@ -25,10 +24,10 @@ export interface IElectricAction {
 
 export class BracketElectricCharacterSupport {
 
-	private readonly _richEditBrackets: RichEditBrackets;
+	private readonly _richEditBrackets: RichEditBrackets | null;
 	private readonly _complexAutoClosePairs: StandardAutoClosingPairConditional[];
 
-	constructor(richEditBrackets: RichEditBrackets, autoClosePairs: IAutoClosingPairConditional[], contribution: IBracketElectricCharacterContribution) {
+	constructor(richEditBrackets: RichEditBrackets | null, autoClosePairs: IAutoClosingPairConditional[], contribution: IBracketElectricCharacterContribution | undefined) {
 		contribution = contribution || {};
 		this._richEditBrackets = richEditBrackets;
 		this._complexAutoClosePairs = autoClosePairs.filter(pair => pair.open.length > 1 && !!pair.close).map(el => new StandardAutoClosingPairConditional(el));
@@ -62,12 +61,12 @@ export class BracketElectricCharacterSupport {
 		return result;
 	}
 
-	public onElectricCharacter(character: string, context: ScopedLineTokens, column: number): IElectricAction {
+	public onElectricCharacter(character: string, context: ScopedLineTokens, column: number): IElectricAction | null {
 		return (this._onElectricAutoClose(character, context, column) ||
 			this._onElectricAutoIndent(character, context, column));
 	}
 
-	private _onElectricAutoIndent(character: string, context: ScopedLineTokens, column: number): IElectricAction {
+	private _onElectricAutoIndent(character: string, context: ScopedLineTokens, column: number): IElectricAction | null {
 
 		if (!this._richEditBrackets || this._richEditBrackets.brackets.length === 0) {
 			return null;
@@ -105,7 +104,7 @@ export class BracketElectricCharacterSupport {
 		};
 	}
 
-	private _onElectricAutoClose(character: string, context: ScopedLineTokens, column: number): IElectricAction {
+	private _onElectricAutoClose(character: string, context: ScopedLineTokens, column: number): IElectricAction | null {
 		if (!this._complexAutoClosePairs.length) {
 			return null;
 		}
@@ -121,7 +120,8 @@ export class BracketElectricCharacterSupport {
 			}
 
 			// check if the full open bracket matches
-			let actual = line.substring(line.length - pair.open.length + 1) + character;
+			let start = column - pair.open.length + 1;
+			let actual = line.substring(start - 1, column - 1) + character;
 			if (actual !== pair.open) {
 				continue;
 			}

src/vs/editor/common/modes/supports/indentRules.ts

@@ -2,7 +2,6 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
 import { IndentationRule } from 'vs/editor/common/modes/languageConfiguration';
 

src/vs/editor/common/modes/supports/inplaceReplaceSupport.ts

@@ -2,16 +2,15 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
-import { IInplaceReplaceSupportResult } from 'vs/editor/common/modes';
 import { IRange } from 'vs/editor/common/core/range';
+import { IInplaceReplaceSupportResult } from 'vs/editor/common/modes';
 
 export class BasicInplaceReplace {
 
 	public static readonly INSTANCE = new BasicInplaceReplace();
 
-	public navigateValueSet(range1: IRange, text1: string, range2: IRange, text2: string, up: boolean): IInplaceReplaceSupportResult {
+	public navigateValueSet(range1: IRange, text1: string, range2: IRange, text2: string | null, up: boolean): IInplaceReplaceSupportResult | null {
 
 		if (range1 && text1) {
 			let result = this.doNavigateValueSet(text1, up);
@@ -36,7 +35,7 @@ export class BasicInplaceReplace {
 		return null;
 	}
 
-	private doNavigateValueSet(text: string, up: boolean): string {
+	private doNavigateValueSet(text: string, up: boolean): string | null {
 		let numberResult = this.numberReplace(text, up);
 		if (numberResult !== null) {
 			return numberResult;
@@ -44,7 +43,7 @@ export class BasicInplaceReplace {
 		return this.textReplace(text, up);
 	}
 
-	private numberReplace(value: string, up: boolean): string {
+	private numberReplace(value: string, up: boolean): string | null {
 		let precision = Math.pow(10, value.length - (value.lastIndexOf('.') + 1));
 		let n1 = Number(value);
 		let n2 = parseFloat(value);
@@ -72,19 +71,19 @@ export class BasicInplaceReplace {
 		['public', 'protected', 'private'],
 	];
 
-	private textReplace(value: string, up: boolean): string {
+	private textReplace(value: string, up: boolean): string | null {
 		return this.valueSetsReplace(this._defaultValueSet, value, up);
 	}
 
-	private valueSetsReplace(valueSets: string[][], value: string, up: boolean): string {
-		let result: string = null;
+	private valueSetsReplace(valueSets: string[][], value: string, up: boolean): string | null {
+		let result: string | null = null;
 		for (let i = 0, len = valueSets.length; result === null && i < len; i++) {
 			result = this.valueSetReplace(valueSets[i], value, up);
 		}
 		return result;
 	}
 
-	private valueSetReplace(valueSet: string[], value: string, up: boolean): string {
+	private valueSetReplace(valueSet: string[], value: string, up: boolean): string | null {
 		let idx = valueSet.indexOf(value);
 		if (idx >= 0) {
 			idx += up ? +1 : -1;

src/vs/editor/common/modes/supports/onEnter.ts

@@ -2,11 +2,10 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
 import { onUnexpectedError } from 'vs/base/common/errors';
 import * as strings from 'vs/base/common/strings';
-import { CharacterPair, IndentAction, EnterAction, OnEnterRule } from 'vs/editor/common/modes/languageConfiguration';
+import { CharacterPair, EnterAction, IndentAction, OnEnterRule } from 'vs/editor/common/modes/languageConfiguration';
 
 export interface IOnEnterSupportOptions {
 	brackets?: CharacterPair[];
@@ -33,32 +32,45 @@ export class OnEnterSupport {
 			['[', ']']
 		];
 
-		this._brackets = opts.brackets.map((bracket) => {
-			return {
-				open: bracket[0],
-				openRegExp: OnEnterSupport._createOpenBracketRegExp(bracket[0]),
-				close: bracket[1],
-				closeRegExp: OnEnterSupport._createCloseBracketRegExp(bracket[1]),
-			};
+		this._brackets = [];
+		opts.brackets.forEach((bracket) => {
+			const openRegExp = OnEnterSupport._createOpenBracketRegExp(bracket[0]);
+			const closeRegExp = OnEnterSupport._createCloseBracketRegExp(bracket[1]);
+			if (openRegExp && closeRegExp) {
+				this._brackets.push({
+					open: bracket[0],
+					openRegExp: openRegExp,
+					close: bracket[1],
+					closeRegExp: closeRegExp,
+				});
+			}
 		});
 		this._regExpRules = opts.regExpRules || [];
 	}
 
-	public onEnter(oneLineAboveText: string, beforeEnterText: string, afterEnterText: string): EnterAction {
+	public onEnter(oneLineAboveText: string, beforeEnterText: string, afterEnterText: string): EnterAction | null {
 		// (1): `regExpRules`
 		for (let i = 0, len = this._regExpRules.length; i < len; i++) {
 			let rule = this._regExpRules[i];
-			if (rule.beforeText.test(beforeEnterText)) {
-				if (rule.afterText) {
-					if (rule.afterText.test(afterEnterText)) {
-						return rule.action;
-					}
-				} else {
-					return rule.action;
-				}
+			const regResult = [{
+				reg: rule.beforeText,
+				text: beforeEnterText
+			}, {
+				reg: rule.afterText,
+				text: afterEnterText
+			}, {
+				reg: rule.oneLineAboveText,
+				text: oneLineAboveText
+			}].every((obj): boolean => {
+				return obj.reg ? obj.reg.test(obj.text) : true;
+			});
+
+			if (regResult) {
+				return rule.action;
 			}
 		}
 
+
 		// (2): Special indent-outdent
 		if (beforeEnterText.length > 0 && afterEnterText.length > 0) {
 			for (let i = 0, len = this._brackets.length; i < len; i++) {
@@ -83,7 +95,7 @@ export class OnEnterSupport {
 		return null;
 	}
 
-	private static _createOpenBracketRegExp(bracket: string): RegExp {
+	private static _createOpenBracketRegExp(bracket: string): RegExp | null {
 		let str = strings.escapeRegExpCharacters(bracket);
 		if (!/\B/.test(str.charAt(0))) {
 			str = '\\b' + str;
@@ -92,7 +104,7 @@ export class OnEnterSupport {
 		return OnEnterSupport._safeRegExp(str);
 	}
 
-	private static _createCloseBracketRegExp(bracket: string): RegExp {
+	private static _createCloseBracketRegExp(bracket: string): RegExp | null {
 		let str = strings.escapeRegExpCharacters(bracket);
 		if (!/\B/.test(str.charAt(str.length - 1))) {
 			str = str + '\\b';
@@ -101,7 +113,7 @@ export class OnEnterSupport {
 		return OnEnterSupport._safeRegExp(str);
 	}
 
-	private static _safeRegExp(def: string): RegExp {
+	private static _safeRegExp(def: string): RegExp | null {
 		try {
 			return new RegExp(def);
 		} catch (err) {

src/vs/editor/common/modes/supports/richEditBrackets.ts

@@ -2,12 +2,11 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
 import * as strings from 'vs/base/common/strings';
 import { Range } from 'vs/editor/common/core/range';
-import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration';
 import { LanguageIdentifier } from 'vs/editor/common/modes';
+import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration';
 
 interface ISimpleInternalBracket {
 	open: string;
@@ -142,34 +141,34 @@ let toReversedString = (function () {
 		return reversedStr;
 	}
 
-	let lastInput: string = null;
-	let lastOutput: string = null;
+	let lastInput: string | null = null;
+	let lastOutput: string | null = null;
 	return function toReversedString(str: string): string {
 		if (lastInput !== str) {
 			lastInput = str;
 			lastOutput = reverse(lastInput);
 		}
-		return lastOutput;
+		return lastOutput!;
 	};
 })();
 
 export class BracketsUtils {
 
-	private static _findPrevBracketInText(reversedBracketRegex: RegExp, lineNumber: number, reversedText: string, offset: number): Range {
+	private static _findPrevBracketInText(reversedBracketRegex: RegExp, lineNumber: number, reversedText: string, offset: number): Range | null {
 		let m = reversedText.match(reversedBracketRegex);
 
 		if (!m) {
 			return null;
 		}
 
-		let matchOffset = reversedText.length - m.index;
+		let matchOffset = reversedText.length - (m.index || 0);
 		let matchLength = m[0].length;
 		let absoluteMatchOffset = offset + matchOffset;
 
 		return new Range(lineNumber, absoluteMatchOffset - matchLength + 1, lineNumber, absoluteMatchOffset + 1);
 	}
 
-	public static findPrevBracketInToken(reversedBracketRegex: RegExp, lineNumber: number, lineText: string, currentTokenStart: number, currentTokenEnd: number): Range {
+	public static findPrevBracketInToken(reversedBracketRegex: RegExp, lineNumber: number, lineText: string, currentTokenStart: number, currentTokenEnd: number): Range | null {
 		// Because JS does not support backwards regex search, we search forwards in a reversed string with a reversed regex ;)
 		let reversedLineText = toReversedString(lineText);
 		let reversedTokenText = reversedLineText.substring(lineText.length - currentTokenEnd, lineText.length - currentTokenStart);
@@ -177,14 +176,14 @@ export class BracketsUtils {
 		return this._findPrevBracketInText(reversedBracketRegex, lineNumber, reversedTokenText, currentTokenStart);
 	}
 
-	public static findNextBracketInText(bracketRegex: RegExp, lineNumber: number, text: string, offset: number): Range {
+	public static findNextBracketInText(bracketRegex: RegExp, lineNumber: number, text: string, offset: number): Range | null {
 		let m = text.match(bracketRegex);
 
 		if (!m) {
 			return null;
 		}
 
-		let matchOffset = m.index;
+		let matchOffset = m.index || 0;
 		let matchLength = m[0].length;
 		if (matchLength === 0) {
 			return null;
@@ -194,7 +193,7 @@ export class BracketsUtils {
 		return new Range(lineNumber, absoluteMatchOffset + 1, lineNumber, absoluteMatchOffset + 1 + matchLength);
 	}
 
-	public static findNextBracketInToken(bracketRegex: RegExp, lineNumber: number, lineText: string, currentTokenStart: number, currentTokenEnd: number): Range {
+	public static findNextBracketInToken(bracketRegex: RegExp, lineNumber: number, lineText: string, currentTokenStart: number, currentTokenEnd: number): Range | null {
 		let currentTokenText = lineText.substring(currentTokenStart, currentTokenEnd);
 
 		return this.findNextBracketInText(bracketRegex, lineNumber, currentTokenText, currentTokenStart);

src/vs/editor/common/modes/supports/tokenization.ts

@@ -2,10 +2,9 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
-import { ColorId, FontStyle, MetadataConsts, LanguageId, StandardTokenType } from 'vs/editor/common/modes';
 import { Color } from 'vs/base/common/color';
+import { ColorId, FontStyle, LanguageId, MetadataConsts, StandardTokenType } from 'vs/editor/common/modes';
 
 export interface ITokenThemeRule {
 	token: string;
@@ -24,15 +23,15 @@ export class ParsedTokenThemeRule {
 	 * -1 if not set. An or mask of `FontStyle` otherwise.
 	 */
 	readonly fontStyle: FontStyle;
-	readonly foreground: string;
-	readonly background: string;
+	readonly foreground: string | null;
+	readonly background: string | null;
 
 	constructor(
 		token: string,
 		index: number,
 		fontStyle: number,
-		foreground: string,
-		background: string,
+		foreground: string | null,
+		background: string | null,
 	) {
 		this.token = token;
 		this.index = index;
@@ -74,12 +73,12 @@ export function parseTokenTheme(source: ITokenThemeRule[]): ParsedTokenThemeRule
 			}
 		}
 
-		let foreground: string = null;
+		let foreground: string | null = null;
 		if (typeof entry.foreground === 'string') {
 			foreground = entry.foreground;
 		}
 
-		let background: string = null;
+		let background: string | null = null;
 		if (typeof entry.background === 'string') {
 			background = entry.background;
 		}
@@ -115,7 +114,7 @@ function resolveParsedTokenThemeRules(parsedThemeRules: ParsedTokenThemeRule[],
 	let defaultForeground = '000000';
 	let defaultBackground = 'ffffff';
 	while (parsedThemeRules.length >= 1 && parsedThemeRules[0].token === '') {
-		let incomingDefaults = parsedThemeRules.shift();
+		let incomingDefaults = parsedThemeRules.shift()!;
 		if (incomingDefaults.fontStyle !== FontStyle.NotSet) {
 			defaultFontStyle = incomingDefaults.fontStyle;
 		}
@@ -161,7 +160,7 @@ export class ColorMap {
 		this._color2id = new Map<string, ColorId>();
 	}
 
-	public getId(color: string): ColorId {
+	public getId(color: string | null): ColorId {
 		if (color === null) {
 			return 0;
 		}

src/vs/editor/common/modes/textToHtmlTokenizer.ts

@@ -2,16 +2,26 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
-import * as strings from 'vs/base/common/strings';
-import { IState, ITokenizationSupport, TokenizationRegistry, LanguageId } from 'vs/editor/common/modes';
-import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
-import { LineTokens, IViewLineTokens } from 'vs/editor/common/core/lineTokens';
 import { CharCode } from 'vs/base/common/charCode';
+import * as strings from 'vs/base/common/strings';
+import { IViewLineTokens, LineTokens } from 'vs/editor/common/core/lineTokens';
+import { TokenizationResult2 } from 'vs/editor/common/core/token';
+import { IState, LanguageId } from 'vs/editor/common/modes';
+import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 
-export function tokenizeToString(text: string, languageId: string): string {
-	return _tokenizeToString(text, _getSafeTokenizationSupport(languageId));
+export interface IReducedTokenizationSupport {
+	getInitialState(): IState;
+	tokenize2(line: string, state: IState, offsetDelta: number): TokenizationResult2;
+}
+
+const fallback: IReducedTokenizationSupport = {
+	getInitialState: () => NULL_STATE,
+	tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
+};
+
+export function tokenizeToString(text: string, tokenizationSupport: IReducedTokenizationSupport = fallback): string {
+	return _tokenizeToString(text, tokenizationSupport || fallback);
 }
 
 export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens, colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string {
@@ -83,19 +93,7 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens
 	return result;
 }
 
-function _getSafeTokenizationSupport(languageId: string): ITokenizationSupport {
-	let tokenizationSupport = TokenizationRegistry.get(languageId);
-	if (tokenizationSupport) {
-		return tokenizationSupport;
-	}
-	return {
-		getInitialState: () => NULL_STATE,
-		tokenize: undefined,
-		tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
-	};
-}
-
-function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport): string {
+function _tokenizeToString(text: string, tokenizationSupport: IReducedTokenizationSupport): string {
 	let result = `<div class="monaco-tokenized-source">`;
 	let lines = text.split(/\r\n|\r|\n/);
 	let currentState = tokenizationSupport.getInitialState();

src/vs/editor/common/modes/tokenizationRegistry.ts

@@ -2,24 +2,25 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-'use strict';
 
-import { IDisposable, toDisposable } from 'vs/base/common/lifecycle';
-import { Event, Emitter } from 'vs/base/common/event';
-import { ColorId, ITokenizationRegistry, ITokenizationSupport, ITokenizationSupportChangedEvent } from 'vs/editor/common/modes';
 import { Color } from 'vs/base/common/color';
+import { Emitter, Event } from 'vs/base/common/event';
+import { IDisposable, toDisposable, Disposable } from 'vs/base/common/lifecycle';
+import { ColorId, ITokenizationRegistry, ITokenizationSupport, ITokenizationSupportChangedEvent } from 'vs/editor/common/modes';
 
 export class TokenizationRegistryImpl implements ITokenizationRegistry {
 
 	private _map: { [language: string]: ITokenizationSupport };
+	private _promises: { [language: string]: Thenable<IDisposable> };
 
 	private readonly _onDidChange: Emitter<ITokenizationSupportChangedEvent> = new Emitter<ITokenizationSupportChangedEvent>();
 	public readonly onDidChange: Event<ITokenizationSupportChangedEvent> = this._onDidChange.event;
 
-	private _colorMap: Color[];
+	private _colorMap: Color[] | null;
 
 	constructor() {
 		this._map = Object.create(null);
+		this._promises = Object.create(null);
 		this._colorMap = null;
 	}
 
@@ -30,7 +31,7 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
 		});
 	}
 
-	public register(language: string, support: ITokenizationSupport): IDisposable {
+	public register(language: string, support: ITokenizationSupport) {
 		this._map[language] = support;
 		this.fire([language]);
 		return toDisposable(() => {
@@ -42,6 +43,30 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
 		});
 	}
 
+	public registerPromise(language: string, supportPromise: Thenable<ITokenizationSupport | null>): Thenable<IDisposable> {
+		const promise = this._promises[language] = supportPromise.then(support => {
+			delete this._promises[language];
+			if (support) {
+				return this.register(language, support);
+			} else {
+				return Disposable.None;
+			}
+		});
+		return promise;
+	}
+
+	public getPromise(language: string): Thenable<ITokenizationSupport> | null {
+		const support = this.get(language);
+		if (support) {
+			return Promise.resolve(support);
+		}
+		const promise = this._promises[language];
+		if (promise) {
+			return promise.then(_ => this.get(language));
+		}
+		return null;
+	}
+
 	public get(language: string): ITokenizationSupport {
 		return (this._map[language] || null);
 	}
@@ -54,11 +79,14 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
 		});
 	}
 
-	public getColorMap(): Color[] {
+	public getColorMap(): Color[] | null {
 		return this._colorMap;
 	}
 
-	public getDefaultBackground(): Color {
-		return this._colorMap[ColorId.DefaultBackground];
+	public getDefaultBackground(): Color | null {
+		if (this._colorMap && this._colorMap.length > ColorId.DefaultBackground) {
+			return this._colorMap[ColorId.DefaultBackground];
+		}
+		return null;
 	}
 }