Merge VS Code 1.21 source code (#1067)

* Initial VS Code 1.21 file copy with patches

* A few more merges

* Post npm install

* Fix batch of build breaks

* Fix more build breaks

* Fix more build errors

* Fix more build breaks

* Runtime fixes 1

* Get connection dialog working with some todos

* Fix a few packaging issues

* Copy several node_modules to package build to fix loader issues

* Fix breaks from master

* A few more fixes

* Make tests pass

* First pass of license header updates

* Second pass of license header updates

* Fix restore dialog issues

* Remove add additional themes menu items

* fix select box issues where the list doesn't show up

* formatting

* Fix editor dispose issue

* Copy over node modules to correct location on all platforms

src/vs/workbench/services/search/node/fileSearch.ts

@@ -118,7 +118,7 @@ export class FileWalker {
 		this.isCanceled = true;
 	}
 
-	public walk(folderQueries: IFolderSearch[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, done: (error: Error, isLimitHit: boolean) => void): void {
+	public walk(folderQueries: IFolderSearch[], extraFiles: string[], onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgress) => void, done: (error: Error, isLimitHit: boolean) => void): void {
 		this.fileWalkStartTime = Date.now();
 
 		// Support that the file pattern is a full path to a file that exists
@@ -180,7 +180,7 @@ export class FileWalker {
 
 			// For each root folder
 			flow.parallel<IFolderSearch, void>(folderQueries, (folderQuery: IFolderSearch, rootFolderDone: (err: Error, result: void) => void) => {
-				this.call(traverse, this, folderQuery, onResult, (err?: Error) => {
+				this.call(traverse, this, folderQuery, onResult, onMessage, (err?: Error) => {
 					if (err) {
 						const errorMessage = toErrorMessage(err);
 						console.error(errorMessage);
@@ -205,7 +205,7 @@ export class FileWalker {
 		}
 	}
 
-	private cmdTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
+	private cmdTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgress) => void, cb: (err?: Error) => void): void {
 		const rootFolder = folderQuery.folder;
 		const isMac = platform.isMacintosh;
 		let cmd: childProcess.ChildProcess;
@@ -227,6 +227,18 @@ export class FileWalker {
 			const ripgrep = spawnRipgrepCmd(this.config, folderQuery, this.config.includePattern, this.folderExcludePatterns.get(folderQuery.folder).expression);
 			cmd = ripgrep.cmd;
 			noSiblingsClauses = !Object.keys(ripgrep.siblingClauses).length;
+
+			process.nextTick(() => {
+				const escapedArgs = ripgrep.rgArgs.args
+					.map(arg => arg.match(/^-/) ? arg : `'${arg}'`)
+					.join(' ');
+
+				let rgCmd = `rg ${escapedArgs}\n - cwd: ${ripgrep.cwd}`;
+				if (ripgrep.rgArgs.siblingClauses) {
+					rgCmd += `\n - Sibling clauses: ${JSON.stringify(ripgrep.rgArgs.siblingClauses)}`;
+				}
+				onMessage({ message: rgCmd });
+			});
 		} else {
 			cmd = this.spawnFindCmd(folderQuery);
 		}
@@ -504,7 +516,7 @@ export class FileWalker {
 		matchDirectory(rootEntries);
 	}
 
-	private nodeJSTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
+	private nodeJSTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, onMessage: (message: IProgress) => void, done: (err?: Error) => void): void {
 		this.directoriesWalked++;
 		extfs.readdir(folderQuery.folder, (error: Error, files: string[]) => {
 			if (error || this.isCanceled || this.isLimitHit) {
@@ -730,7 +742,7 @@ export class Engine implements ISearchEngine<IRawFileMatch> {
 	}
 
 	public search(onResult: (result: IRawFileMatch) => void, onProgress: (progress: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
-		this.walker.walk(this.folderQueries, this.extraFiles, onResult, (err: Error, isLimitHit: boolean) => {
+		this.walker.walk(this.folderQueries, this.extraFiles, onResult, onProgress, (err: Error, isLimitHit: boolean) => {
 			done(err, {
 				limitHit: isLimitHit,
 				stats: this.walker.getStats()

src/vs/workbench/services/search/node/rawSearchService.ts

@@ -367,7 +367,9 @@ export class SearchService implements IRawSearchService {
 					}
 				}
 			}, (progress) => {
-				p(progress);
+				process.nextTick(() => {
+					p(progress);
+				});
 			}, (error, stats) => {
 				if (batch.length) {
 					p(batch);

src/vs/workbench/services/search/node/ripgrepFileSearch.ts

@@ -13,11 +13,17 @@ import { normalizeNFD, startsWith } from 'vs/base/common/strings';
 import { IFolderSearch, IRawSearch } from './search';
 import { foldersToIncludeGlobs, foldersToRgExcludeGlobs } from './ripgrepTextSearch';
 
+// If vscode-ripgrep is in an .asar file, then the binary is unpacked.
+const rgDiskPath = rgPath.replace(/\bnode_modules\.asar\b/, 'node_modules.asar.unpacked');
+
 export function spawnRipgrepCmd(config: IRawSearch, folderQuery: IFolderSearch, includePattern: glob.IExpression, excludePattern: glob.IExpression) {
 	const rgArgs = getRgArgs(config, folderQuery, includePattern, excludePattern);
+	const cwd = folderQuery.folder;
 	return {
-		cmd: cp.spawn(rgPath, rgArgs.globArgs, { cwd: folderQuery.folder }),
-		siblingClauses: rgArgs.siblingClauses
+		cmd: cp.spawn(rgDiskPath, rgArgs.args, { cwd }),
+		siblingClauses: rgArgs.siblingClauses,
+		rgArgs,
+		cwd
 	};
 }
 
@@ -39,6 +45,8 @@ function getRgArgs(config: IRawSearch, folderQuery: IFolderSearch, includePatter
 	if (folderQuery.disregardIgnoreFiles !== false) {
 		// Don't use .gitignore or .ignore
 		args.push('--no-ignore');
+	} else {
+		args.push('--no-ignore-parent');
 	}
 
 	// Follow symlinks
@@ -55,7 +63,7 @@ function getRgArgs(config: IRawSearch, folderQuery: IFolderSearch, includePatter
 
 	args.push('.');
 
-	return { globArgs: args, siblingClauses };
+	return { args, siblingClauses };
 }
 
 function anchor(glob: string) {

src/vs/workbench/services/search/node/ripgrepTextSearch.ts

@@ -18,10 +18,13 @@ import * as paths from 'vs/base/common/paths';
 import * as extfs from 'vs/base/node/extfs';
 import * as encoding from 'vs/base/node/encoding';
 import * as glob from 'vs/base/common/glob';
-import { ISearchLog } from 'vs/platform/search/common/search';
 import { TPromise } from 'vs/base/common/winjs.base';
 
 import { ISerializedFileMatch, ISerializedSearchComplete, IRawSearch, IFolderSearch, LineMatch, FileMatch } from './search';
+import { IProgress } from 'vs/platform/search/common/search';
+
+// If vscode-ripgrep is in an .asar file, then the binary is unpacked.
+const rgDiskPath = rgPath.replace(/\bnode_modules\.asar\b/, 'node_modules.asar.unpacked');
 
 export class RipgrepEngine {
 	private isDone = false;
@@ -44,7 +47,7 @@ export class RipgrepEngine {
 	}
 
 	// TODO@Rob - make promise-based once the old search is gone, and I don't need them to have matching interfaces anymore
-	search(onResult: (match: ISerializedFileMatch) => void, onMessage: (message: ISearchLog) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
+	search(onResult: (match: ISerializedFileMatch) => void, onMessage: (message: IProgress) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
 		if (!this.config.folderQueries.length && !this.config.extraFiles.length) {
 			process.removeListener('exit', this.killRgProcFn);
 			done(null, {
@@ -61,17 +64,18 @@ export class RipgrepEngine {
 
 		const cwd = platform.isWindows ? 'c:/' : '/';
 		process.nextTick(() => { // Allow caller to register progress callback
-			const escapedArgs = rgArgs.globArgs
+			const escapedArgs = rgArgs.args
 				.map(arg => arg.match(/^-/) ? arg : `'${arg}'`)
 				.join(' ');
 
-			const rgCmd = `rg ${escapedArgs}\n - cwd: ${cwd}\n`;
-			onMessage({ message: rgCmd });
+			let rgCmd = `rg ${escapedArgs}\n - cwd: ${cwd}`;
 			if (rgArgs.siblingClauses) {
-				onMessage({ message: ` - Sibling clauses: ${JSON.stringify(rgArgs.siblingClauses)}\n` });
+				rgCmd += `\n - Sibling clauses: ${JSON.stringify(rgArgs.siblingClauses)}`;
 			}
+
+			onMessage({ message: rgCmd });
 		});
-		this.rgProc = cp.spawn(rgPath, rgArgs.globArgs, { cwd });
+		this.rgProc = cp.spawn(rgDiskPath, rgArgs.args, { cwd });
 		process.once('exit', this.killRgProcFn);
 
 		this.ripgrepParser = new RipgrepParser(this.config.maxResults, cwd, this.config.extraFiles);
@@ -156,6 +160,11 @@ export function rgErrorMsgForDisplay(msg: string): string | undefined {
 		return firstLine.charAt(0).toUpperCase() + firstLine.substr(1);
 	}
 
+	if (strings.startsWith(firstLine, 'Literal ')) {
+		// e.g. "Literal \n not allowed"
+		return firstLine;
+	}
+
 	return undefined;
 }
 
@@ -375,8 +384,13 @@ function globExprsToRgGlobs(patterns: glob.IExpression, folder?: string, exclude
 			const value = patterns[key];
 			key = trimTrailingSlash(folder ? getAbsoluteGlob(folder, key) : key);
 
-			// glob.ts requires forward slashes
-			key = key.replace(/\\/g, '/');
+			// glob.ts requires forward slashes, but a UNC path still must start with \\
+			// #38165 and #38151
+			if (strings.startsWith(key, '\\\\')) {
+				key = '\\\\' + key.substr(2).replace(/\\/g, '/');
+			} else {
+				key = key.replace(/\\/g, '/');
+			}
 
 			if (typeof value === 'boolean' && value) {
 				globArgs.push(fixDriveC(key));
@@ -416,7 +430,7 @@ export function fixDriveC(path: string): string {
 		path;
 }
 
-function getRgArgs(config: IRawSearch): IRgGlobResult {
+function getRgArgs(config: IRawSearch) {
 	const args = ['--hidden', '--heading', '--line-number', '--color', 'ansi', '--colors', 'path:none', '--colors', 'line:none', '--colors', 'match:fg:red', '--colors', 'match:style:nobold'];
 	args.push(config.contentPattern.isCaseSensitive ? '--case-sensitive' : '--ignore-case');
 
@@ -446,6 +460,8 @@ function getRgArgs(config: IRawSearch): IRgGlobResult {
 	if (config.disregardIgnoreFiles) {
 		// Don't use .gitignore or .ignore
 		args.push('--no-ignore');
+	} else {
+		args.push('--no-ignore-parent');
 	}
 
 	// Follow symlinks
@@ -488,7 +504,7 @@ function getRgArgs(config: IRawSearch): IRgGlobResult {
 	args.push(...config.folderQueries.map(q => q.folder));
 	args.push(...config.extraFiles);
 
-	return { globArgs: args, siblingClauses };
+	return { args, siblingClauses };
 }
 
 function getSiblings(file: string): TPromise<string[]> {

src/vs/workbench/services/search/node/search.ts

@@ -7,7 +7,7 @@
 
 import { PPromise, TPromise } from 'vs/base/common/winjs.base';
 import { IExpression } from 'vs/base/common/glob';
-import { IProgress, ILineMatch, IPatternInfo, ISearchStats, ISearchLog } from 'vs/platform/search/common/search';
+import { IProgress, ILineMatch, IPatternInfo, ISearchStats } from 'vs/platform/search/common/search';
 import { ITelemetryData } from 'vs/platform/telemetry/common/telemetry';
 
 export interface IFolderSearch {
@@ -71,7 +71,7 @@ export interface ISerializedFileMatch {
 }
 
 // Type of the possible values for progress calls from the engine
-export type ISerializedSearchProgressItem = ISerializedFileMatch | ISerializedFileMatch[] | IProgress | ISearchLog;
+export type ISerializedSearchProgressItem = ISerializedFileMatch | ISerializedFileMatch[] | IProgress;
 export type IFileSearchProgressItem = IRawFileMatch | IRawFileMatch[] | IProgress;
 
 

src/vs/workbench/services/search/node/searchService.ts

@@ -10,7 +10,7 @@ import objects = require('vs/base/common/objects');
 import strings = require('vs/base/common/strings');
 import { getNextTickChannel } from 'vs/base/parts/ipc/common/ipc';
 import { Client, IIPCOptions } from 'vs/base/parts/ipc/node/ipc.cp';
-import { IProgress, LineMatch, FileMatch, ISearchComplete, ISearchProgressItem, QueryType, IFileMatch, ISearchQuery, ISearchConfiguration, ISearchService, pathIncludedInQuery, ISearchResultProvider } from 'vs/platform/search/common/search';
+import { IProgress, LineMatch, FileMatch, ISearchComplete, ISearchProgressItem, QueryType, IFileMatch, ISearchQuery, IFolderQuery, ISearchConfiguration, ISearchService, pathIncludedInQuery, ISearchResultProvider } from 'vs/platform/search/common/search';
 import { IUntitledEditorService } from 'vs/workbench/services/untitled/common/untitledEditorService';
 import { IModelService } from 'vs/editor/common/services/modelService';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
@@ -22,6 +22,8 @@ import { IDisposable } from 'vs/base/common/lifecycle';
 import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
 import { onUnexpectedError } from 'vs/base/common/errors';
 import { Schemas } from 'vs/base/common/network';
+import * as pfs from 'vs/base/node/pfs';
+import { ILogService } from 'vs/platform/log/common/log';
 
 export class SearchService implements ISearchService {
 	public _serviceBrand: any;
@@ -35,7 +37,8 @@ export class SearchService implements ISearchService {
 		@IUntitledEditorService private untitledEditorService: IUntitledEditorService,
 		@IEnvironmentService environmentService: IEnvironmentService,
 		@ITelemetryService private telemetryService: ITelemetryService,
-		@IConfigurationService private configurationService: IConfigurationService
+		@IConfigurationService private configurationService: IConfigurationService,
+		@ILogService private logService: ILogService
 	) {
 		this.diskSearch = new DiskSearch(!environmentService.isBuilt || environmentService.verbose, /*timeout=*/undefined, environmentService.debugSearch);
 		this.registerSearchResultProvider(this.diskSearch);
@@ -88,6 +91,7 @@ export class SearchService implements ISearchService {
 			// Allow caller to register progress callback
 			process.nextTick(() => localResults.values().filter((res) => !!res).forEach(onProgress));
 
+			this.logService.trace('SearchService#search', JSON.stringify(query));
 			const providerPromises = this.searchProvider.map(provider => TPromise.wrap(provider.search(query)).then(e => e,
 				err => {
 					// TODO@joh
@@ -104,6 +108,10 @@ export class SearchService implements ISearchService {
 						// Progress
 						onProgress(<IProgress>progress);
 					}
+
+					if (progress.message) {
+						this.logService.debug('SearchService#search', progress.message);
+					}
 				}
 			));
 
@@ -151,14 +159,17 @@ export class SearchService implements ISearchService {
 				}
 
 				// Support untitled files
-				if (resource.scheme === 'untitled') {
+				if (resource.scheme === Schemas.untitled) {
 					if (!this.untitledEditorService.exists(resource)) {
 						return;
 					}
 				}
 
 				// Don't support other resource schemes than files for now
-				else if (resource.scheme !== 'file') {
+				// todo@remote
+				// why is that? we should search for resources from other
+				// schemes
+				else if (resource.scheme !== Schemas.file) {
 					return;
 				}
 
@@ -187,7 +198,7 @@ export class SearchService implements ISearchService {
 	private matches(resource: uri, query: ISearchQuery): boolean {
 		// file pattern
 		if (query.filePattern) {
-			if (resource.scheme !== 'file') {
+			if (resource.scheme !== Schemas.file) {
 				return false; // if we match on file pattern, we have to ignore non file resources
 			}
 
@@ -198,7 +209,7 @@ export class SearchService implements ISearchService {
 
 		// includes
 		if (query.includePattern) {
-			if (resource.scheme !== 'file') {
+			if (resource.scheme !== Schemas.file) {
 				return false; // if we match on file patterns, we have to ignore non file resources
 			}
 		}
@@ -258,8 +269,24 @@ export class DiskSearch implements ISearchResultProvider {
 	}
 
 	public search(query: ISearchQuery): PPromise<ISearchComplete, ISearchProgressItem> {
-		let request: PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem>;
+		const folderQueries = query.folderQueries || [];
+		return TPromise.join(folderQueries.map(q => q.folder.scheme === Schemas.file && pfs.exists(q.folder.fsPath)))
+			.then(exists => {
+				const existingFolders = folderQueries.filter((q, index) => exists[index]);
+				const rawSearch = this.rawSearchQuery(query, existingFolders);
 
+				let request: PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem>;
+				if (query.type === QueryType.File) {
+					request = this.raw.fileSearch(rawSearch);
+				} else {
+					request = this.raw.textSearch(rawSearch);
+				}
+
+				return DiskSearch.collectResults(request);
+			});
+	}
+
+	private rawSearchQuery(query: ISearchQuery, existingFolders: IFolderQuery[]) {
 		let rawSearch: IRawSearch = {
 			folderQueries: [],
 			extraFiles: [],
@@ -275,18 +302,14 @@ export class DiskSearch implements ISearchResultProvider {
 			ignoreSymlinks: query.ignoreSymlinks
 		};
 
-		if (query.folderQueries) {
-			for (const q of query.folderQueries) {
-				if (q.folder.scheme === Schemas.file) {
-					rawSearch.folderQueries.push({
-						excludePattern: q.excludePattern,
-						includePattern: q.includePattern,
-						fileEncoding: q.fileEncoding,
-						disregardIgnoreFiles: q.disregardIgnoreFiles,
-						folder: q.folder.fsPath
-					});
-				}
-			}
+		for (const q of existingFolders) {
+			rawSearch.folderQueries.push({
+				excludePattern: q.excludePattern,
+				includePattern: q.includePattern,
+				fileEncoding: q.fileEncoding,
+				disregardIgnoreFiles: q.disregardIgnoreFiles,
+				folder: q.folder.fsPath
+			});
 		}
 
 		if (query.extraFileResources) {
@@ -301,13 +324,7 @@ export class DiskSearch implements ISearchResultProvider {
 			rawSearch.contentPattern = query.contentPattern;
 		}
 
-		if (query.type === QueryType.File) {
-			request = this.raw.fileSearch(rawSearch);
-		} else {
-			request = this.raw.textSearch(rawSearch);
-		}
-
-		return DiskSearch.collectResults(request);
+		return rawSearch;
 	}
 
 	public static collectResults(request: PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem>): PPromise<ISearchComplete, ISearchProgressItem> {

src/vs/workbench/services/search/node/textSearch.ts

@@ -147,20 +147,22 @@ export class Engine implements ISearchEngine<ISerializedFileMatch[]> {
 				nextBatch = [];
 				nextBatchBytes = 0;
 			}
-		}, (error, isLimitHit) => {
-			this.walkerIsDone = true;
-			this.walkerError = error;
+		},
+			onProgress,
+			(error, isLimitHit) => {
+				this.walkerIsDone = true;
+				this.walkerError = error;
 
-			// Send any remaining paths to a worker, or unwind if we're stopping
-			if (nextBatch.length) {
-				if (this.limitReached || this.isCanceled) {
-					unwind(nextBatchBytes);
+				// Send any remaining paths to a worker, or unwind if we're stopping
+				if (nextBatch.length) {
+					if (this.limitReached || this.isCanceled) {
+						unwind(nextBatchBytes);
+					} else {
+						run(nextBatch, nextBatchBytes);
+					}
 				} else {
-					run(nextBatch, nextBatchBytes);
+					unwind(0);
 				}
-			} else {
-				unwind(0);
-			}
-		});
+			});
 	}
 }

src/vs/workbench/services/search/node/worker/searchWorker.ts

@@ -169,7 +169,7 @@ export class SearchWorkerEngine {
 					return resolve(null);
 				}
 
-				const buffer = new Buffer(options.bufferLength);
+				const buffer = Buffer.allocUnsafe(options.bufferLength);
 				let line = '';
 				let lineNumber = 0;
 				let lastBufferHadTrailingCR = false;