Merge VS Code 1.21 source code (#1067)

* Initial VS Code 1.21 file copy with patches

* A few more merges

* Post npm install

* Fix batch of build breaks

* Fix more build breaks

* Fix more build errors

* Fix more build breaks

* Runtime fixes 1

* Get connection dialog working with some todos

* Fix a few packaging issues

* Copy several node_modules to package build to fix loader issues

* Fix breaks from master

* A few more fixes

* Make tests pass

* First pass of license header updates

* Second pass of license header updates

* Fix restore dialog issues

* Remove add additional themes menu items

* fix select box issues where the list doesn't show up

* formatting

* Fix editor dispose issue

* Copy over node modules to correct location on all platforms

src/vs/editor/contrib/smartSelect/test/tokenSelectionSupport.test.ts

@@ -115,4 +115,4 @@ suite('TokenSelectionSupport', () => {
 				// new Range(3, 19, 3, 20)
 			]);
 	});
-});
+});

src/vs/editor/contrib/smartSelect/tokenSelectionSupport.ts

@@ -7,7 +7,7 @@
 import URI from 'vs/base/common/uri';
 import { TPromise } from 'vs/base/common/winjs.base';
 import { Range } from 'vs/editor/common/core/range';
-import { IModel } from 'vs/editor/common/editorCommon';
+import { ITextModel } from 'vs/editor/common/model';
 import { IModelService } from 'vs/editor/common/services/modelService';
 import { Node, build, find } from './tokenTree';
 import { Position } from 'vs/editor/common/core/position';
@@ -48,7 +48,7 @@ export class TokenSelectionSupport {
 		return entries;
 	}
 
-	private _doGetRangesToPosition(model: IModel, position: Position): Range[] {
+	private _doGetRangesToPosition(model: ITextModel, position: Position): Range[] {
 
 		var tree = build(model),
 			node: Node,

src/vs/editor/contrib/smartSelect/tokenTree.ts

@@ -6,8 +6,8 @@
 
 import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
-import { IModel } from 'vs/editor/common/editorCommon';
-import { LineToken } from 'vs/editor/common/core/lineTokens';
+import { ITextModel } from 'vs/editor/common/model';
+import { LineTokens } from 'vs/editor/common/core/lineTokens';
 import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
 import { BracketsUtils, RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
 import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
@@ -132,54 +132,64 @@ class RawToken {
 	public type: StandardTokenType;
 	public languageId: LanguageId;
 
-	constructor(source: LineToken, lineNumber: number, lineText: string) {
+	constructor(source: LineTokens, tokenIndex: number, lineNumber: number) {
 		this.lineNumber = lineNumber;
-		this.lineText = lineText;
-		this.startOffset = source.startOffset;
-		this.endOffset = source.endOffset;
-		this.type = source.tokenType;
-		this.languageId = source.languageId;
+		this.lineText = source.getLineContent();
+		this.startOffset = source.getStartOffset(tokenIndex);
+		this.endOffset = source.getEndOffset(tokenIndex);
+		this.type = source.getStandardTokenType(tokenIndex);
+		this.languageId = source.getLanguageId(tokenIndex);
 	}
 }
 
 class ModelRawTokenScanner {
 
-	private _model: IModel;
+	private _model: ITextModel;
 	private _lineCount: number;
 	private _versionId: number;
 	private _lineNumber: number;
-	private _lineText: string;
-	private _next: LineToken;
+	private _tokenIndex: number;
+	private _lineTokens: LineTokens;
 
-	constructor(model: IModel) {
+	constructor(model: ITextModel) {
 		this._model = model;
 		this._lineCount = this._model.getLineCount();
 		this._versionId = this._model.getVersionId();
 		this._lineNumber = 0;
-		this._lineText = null;
+		this._tokenIndex = 0;
+		this._lineTokens = null;
 		this._advance();
 	}
 
 	private _advance(): void {
-		this._next = (this._next ? this._next.next() : null);
-		while (!this._next && this._lineNumber < this._lineCount) {
+		if (this._lineTokens) {
+			this._tokenIndex++;
+			if (this._tokenIndex >= this._lineTokens.getCount()) {
+				this._lineTokens = null;
+			}
+		}
+
+		while (this._lineNumber < this._lineCount && !this._lineTokens) {
 			this._lineNumber++;
-			this._lineText = this._model.getLineContent(this._lineNumber);
 			this._model.forceTokenization(this._lineNumber);
-			let currentLineTokens = this._model.getLineTokens(this._lineNumber);
-			this._next = currentLineTokens.firstToken();
+			this._lineTokens = this._model.getLineTokens(this._lineNumber);
+			this._tokenIndex = 0;
+			if (this._lineTokens.getCount() === 0) {
+				// Skip empty lines
+				this._lineTokens = null;
+			}
 		}
 	}
 
 	public next(): RawToken {
-		if (!this._next) {
+		if (!this._lineTokens) {
 			return null;
 		}
 		if (this._model.getVersionId() !== this._versionId) {
 			return null;
 		}
 
-		let result = new RawToken(this._next, this._lineNumber, this._lineText);
+		let result = new RawToken(this._lineTokens, this._tokenIndex, this._lineNumber);
 		this._advance();
 		return result;
 	}
@@ -193,7 +203,7 @@ class TokenScanner {
 	private _cachedLanguageBrackets: RichEditBrackets;
 	private _cachedLanguageId: LanguageId;
 
-	constructor(model: IModel) {
+	constructor(model: ITextModel) {
 		this._rawTokenScanner = new ModelRawTokenScanner(model);
 		this._nextBuff = [];
 		this._cachedLanguageBrackets = null;
@@ -280,7 +290,7 @@ class TokenTreeBuilder {
 	private _stack: Token[] = [];
 	private _currentToken: Token;
 
-	constructor(model: IModel) {
+	constructor(model: ITextModel) {
 		this._scanner = new TokenScanner(model);
 	}
 
@@ -394,7 +404,7 @@ class TokenTreeBuilder {
  *	line = { block | "token" }
  *	block = "open_bracket" { line } "close_bracket"
  */
-export function build(model: IModel): Node {
+export function build(model: ITextModel): Node {
 	var node = new TokenTreeBuilder(model).build();
 	return node;
 }