Refresh master with initial release/0.24 snapshot (#332)

* Initial port of release/0.24 source code

* Fix additional headers

* Fix a typo in launch.json

src/vs/workbench/services/search/node/fileSearch.ts

@@ -14,7 +14,6 @@ import { isEqualOrParent } from 'vs/base/common/paths';
 import { Readable } from 'stream';
 import { TPromise } from 'vs/base/common/winjs.base';
 
-import scorer = require('vs/base/common/scorer');
 import objects = require('vs/base/common/objects');
 import arrays = require('vs/base/common/arrays');
 import platform = require('vs/base/common/platform');
@@ -26,12 +25,14 @@ import { IProgress, IUncachedSearchStats } from 'vs/platform/search/common/searc
 import extfs = require('vs/base/node/extfs');
 import flow = require('vs/base/node/flow');
 import { IRawFileMatch, ISerializedSearchComplete, IRawSearch, ISearchEngine, IFolderSearch } from './search';
+import { spawnRipgrepCmd } from './ripgrepFileSearch';
 
 enum Traversal {
 	Node = 1,
 	MacFind,
 	WindowsDir,
-	LinuxFind
+	LinuxFind,
+	Ripgrep
 }
 
 interface IDirectoryEntry {
@@ -47,10 +48,12 @@ interface IDirectoryTree {
 
 export class FileWalker {
 	private config: IRawSearch;
+	private useRipgrep: boolean;
 	private filePattern: string;
 	private normalizedFilePatternLowercase: string;
 	private includePattern: glob.ParsedExpression;
 	private maxResults: number;
+	private exists: boolean;
 	private maxFilesize: number;
 	private isLimitHit: boolean;
 	private resultCount: number;
@@ -71,9 +74,11 @@ export class FileWalker {
 
 	constructor(config: IRawSearch) {
 		this.config = config;
+		this.useRipgrep = config.useRipgrep !== false;
 		this.filePattern = config.filePattern;
 		this.includePattern = config.includePattern && glob.parse(config.includePattern);
 		this.maxResults = config.maxResults || null;
+		this.exists = config.exists;
 		this.maxFilesize = config.maxFilesize || null;
 		this.walkedPaths = Object.create(null);
 		this.resultCount = 0;
@@ -151,16 +156,19 @@ export class FileWalker {
 
 			let traverse = this.nodeJSTraversal;
 			if (!this.maxFilesize) {
-				if (platform.isMacintosh) {
+				if (this.useRipgrep) {
+					this.traversal = Traversal.Ripgrep;
+					traverse = this.cmdTraversal;
+				} else if (platform.isMacintosh) {
 					this.traversal = Traversal.MacFind;
-					traverse = this.findTraversal;
+					traverse = this.cmdTraversal;
 					// Disable 'dir' for now (#11181, #11179, #11183, #11182).
 				} /* else if (platform.isWindows) {
 					this.traversal = Traversal.WindowsDir;
 					traverse = this.windowsDirTraversal;
 				} */ else if (platform.isLinux) {
 					this.traversal = Traversal.LinuxFind;
-					traverse = this.findTraversal;
+					traverse = this.cmdTraversal;
 				}
 			}
 
@@ -173,15 +181,10 @@ export class FileWalker {
 			flow.parallel<IFolderSearch, void>(folderQueries, (folderQuery: IFolderSearch, rootFolderDone: (err: Error, result: void) => void) => {
 				this.call(traverse, this, folderQuery, onResult, (err?: Error) => {
 					if (err) {
-						if (isNodeTraversal) {
-							rootFolderDone(err, undefined);
-						} else {
-							// fallback
-							const errorMessage = toErrorMessage(err);
-							console.error(errorMessage);
-							this.errors.push(errorMessage);
-							this.nodeJSTraversal(folderQuery, onResult, err => rootFolderDone(err, undefined));
-						}
+						const errorMessage = toErrorMessage(err);
+						console.error(errorMessage);
+						this.errors.push(errorMessage);
+						rootFolderDone(err, undefined);
 					} else {
 						rootFolderDone(undefined, undefined);
 					}
@@ -200,27 +203,47 @@ export class FileWalker {
 		}
 	}
 
-	private findTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
+	private cmdTraversal(folderQuery: IFolderSearch, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
 		const rootFolder = folderQuery.folder;
 		const isMac = platform.isMacintosh;
+		let cmd: childProcess.ChildProcess;
+		const killCmd = () => cmd && cmd.kill();
+
 		let done = (err?: Error) => {
+			process.removeListener('exit', killCmd);
 			done = () => { };
 			cb(err);
 		};
 		let leftover = '';
 		let first = true;
 		const tree = this.initDirectoryTree();
-		const cmd = this.spawnFindCmd(folderQuery);
-		this.collectStdout(cmd, 'utf8', (err: Error, stdout?: string, last?: boolean) => {
+
+		const useRipgrep = this.useRipgrep;
+		let noSiblingsClauses: boolean;
+		let filePatternSeen = false;
+		if (useRipgrep) {
+			const ripgrep = spawnRipgrepCmd(this.config, folderQuery, this.config.includePattern, this.folderExcludePatterns.get(folderQuery.folder).expression);
+			cmd = ripgrep.cmd;
+			noSiblingsClauses = !Object.keys(ripgrep.siblingClauses).length;
+		} else {
+			cmd = this.spawnFindCmd(folderQuery);
+		}
+
+		process.on('exit', killCmd);
+		this.collectStdout(cmd, 'utf8', useRipgrep, (err: Error, stdout?: string, last?: boolean) => {
 			if (err) {
 				done(err);
 				return;
 			}
+			if (this.isLimitHit) {
+				done();
+				return;
+			}
 
 			// Mac: uses NFD unicode form on disk, but we want NFC
 			const normalized = leftover + (isMac ? strings.normalizeNFC(stdout) : stdout);
-			const relativeFiles = normalized.split('\n./');
-			if (first && normalized.length >= 2) {
+			const relativeFiles = normalized.split(useRipgrep ? '\n' : '\n./');
+			if (!useRipgrep && first && normalized.length >= 2) {
 				first = false;
 				relativeFiles[0] = relativeFiles[0].trim().substr(2);
 			}
@@ -240,6 +263,41 @@ export class FileWalker {
 				return;
 			}
 
+			this.cmdResultCount += relativeFiles.length;
+
+			if (useRipgrep && noSiblingsClauses) {
+				for (const relativePath of relativeFiles) {
+					if (relativePath === this.filePattern) {
+						filePatternSeen = true;
+					}
+					const basename = path.basename(relativePath);
+					this.matchFile(onResult, { base: rootFolder, relativePath, basename });
+					if (this.isLimitHit) {
+						killCmd();
+						break;
+					}
+				}
+				if (last || this.isLimitHit) {
+					if (!filePatternSeen) {
+						this.checkFilePatternRelativeMatch(folderQuery.folder, (match, size) => {
+							if (match) {
+								this.resultCount++;
+								onResult({
+									base: folderQuery.folder,
+									relativePath: this.filePattern,
+									basename: path.basename(this.filePattern),
+								});
+							}
+							done();
+						});
+					} else {
+						done();
+					}
+				}
+				return;
+			}
+
+			// TODO: Optimize siblings clauses with ripgrep here.
 			this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult);
 
 			if (last) {
@@ -304,9 +362,9 @@ export class FileWalker {
 	/**
 	 * Public for testing.
 	 */
-	public readStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string) => void): void {
+	public readStdout(cmd: childProcess.ChildProcess, encoding: string, isRipgrep: boolean, cb: (err: Error, stdout?: string) => void): void {
 		let all = '';
-		this.collectStdout(cmd, encoding, (err: Error, stdout?: string, last?: boolean) => {
+		this.collectStdout(cmd, encoding, isRipgrep, (err: Error, stdout?: string, last?: boolean) => {
 			if (err) {
 				cb(err);
 				return;
@@ -319,7 +377,7 @@ export class FileWalker {
 		});
 	}
 
-	private collectStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
+	private collectStdout(cmd: childProcess.ChildProcess, encoding: string, isRipgrep: boolean, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
 		let done = (err: Error, stdout?: string, last?: boolean) => {
 			if (err || last) {
 				done = () => { };
@@ -336,9 +394,13 @@ export class FileWalker {
 		});
 
 		cmd.on('close', (code: number) => {
-			if (code !== 0) {
-				done(new Error(`find failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
+			// ripgrep returns code=1 when no results are found
+			if (code !== 0 && (!isRipgrep || code !== 1)) {
+				done(new Error(`command failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
 			} else {
+				if (isRipgrep && this.exists && code === 0) {
+					this.isLimitHit = true;
+				}
 				done(null, '', true);
 			}
 		});
@@ -375,8 +437,6 @@ export class FileWalker {
 	}
 
 	private addDirectoryEntries({ pathToEntries }: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
-		this.cmdResultCount += relativeFiles.length;
-
 		// Support relative paths to files from a root resource (ignores excludes)
 		if (relativeFiles.indexOf(this.filePattern) !== -1) {
 			const basename = path.basename(this.filePattern);
@@ -429,6 +489,10 @@ export class FileWalker {
 
 					self.matchFile(onResult, entry);
 				}
+
+				if (self.isLimitHit) {
+					break;
+				}
 			};
 		}
 		matchDirectory(rootEntries);
@@ -599,7 +663,7 @@ export class FileWalker {
 		if (this.isFilePatternMatch(candidate.relativePath) && (!this.includePattern || this.includePattern(candidate.relativePath, candidate.basename))) {
 			this.resultCount++;
 
-			if (this.maxResults && this.resultCount > this.maxResults) {
+			if (this.exists || (this.maxResults && this.resultCount > this.maxResults)) {
 				this.isLimitHit = true;
 			}
 
@@ -617,7 +681,7 @@ export class FileWalker {
 				return true; // support the all-matching wildcard
 			}
 
-			return scorer.matches(path, this.normalizedFilePatternLowercase);
+			return strings.fuzzyContains(path, this.normalizedFilePatternLowercase);
 		}
 
 		// No patterns means we match all
@@ -682,8 +746,8 @@ class AbsoluteAndRelativeParsedExpression {
 	private absoluteParsedExpr: glob.ParsedExpression;
 	private relativeParsedExpr: glob.ParsedExpression;
 
-	constructor(expr: glob.IExpression, private root: string) {
-		this.init(expr);
+	constructor(public expression: glob.IExpression, private root: string) {
+		this.init(expression);
 	}
 
 	/**

src/vs/workbench/services/search/node/rawSearchService.ts

@@ -12,9 +12,7 @@ import gracefulFs = require('graceful-fs');
 gracefulFs.gracefulify(fs);
 
 import arrays = require('vs/base/common/arrays');
-import { compareByScore } from 'vs/base/common/comparers';
 import objects = require('vs/base/common/objects');
-import scorer = require('vs/base/common/scorer');
 import strings = require('vs/base/common/strings');
 import { PPromise, TPromise } from 'vs/base/common/winjs.base';
 import { FileWalker, Engine as FileSearchEngine } from 'vs/workbench/services/search/node/fileSearch';
@@ -22,8 +20,10 @@ import { MAX_FILE_SIZE } from 'vs/platform/files/common/files';
 import { RipgrepEngine } from 'vs/workbench/services/search/node/ripgrepTextSearch';
 import { Engine as TextSearchEngine } from 'vs/workbench/services/search/node/textSearch';
 import { TextSearchWorkerProvider } from 'vs/workbench/services/search/node/textSearchWorkerProvider';
-import { IRawSearchService, IRawSearch, IRawFileMatch, ISerializedFileMatch, ISerializedSearchProgressItem, ISerializedSearchComplete, ISearchEngine, IFileSearchProgressItem } from './search';
+import { IRawSearchService, IRawSearch, IRawFileMatch, ISerializedFileMatch, ISerializedSearchProgressItem, ISerializedSearchComplete, ISearchEngine, IFileSearchProgressItem, ITelemetryEvent } from './search';
 import { ICachedSearchStats, IProgress } from 'vs/platform/search/common/search';
+import { fuzzyContains } from 'vs/base/common/strings';
+import { compareItemsByScore, IItemAccessor, ScorerCache, prepareQuery } from 'vs/base/parts/quickopen/common/quickOpenScorer';
 
 export class SearchService implements IRawSearchService {
 
@@ -33,6 +33,8 @@ export class SearchService implements IRawSearchService {
 
 	private textSearchWorkerProvider: TextSearchWorkerProvider;
 
+	private telemetryPipe: (event: ITelemetryEvent) => void;
+
 	public fileSearch(config: IRawSearch): PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem> {
 		return this.doFileSearch(FileSearchEngine, config, SearchService.BATCH_SIZE);
 	}
@@ -81,6 +83,7 @@ export class SearchService implements IRawSearchService {
 				includePattern: config.includePattern,
 				excludePattern: config.excludePattern,
 				filePattern: config.filePattern,
+				useRipgrep: false,
 				maxFilesize: MAX_FILE_SIZE
 			}),
 			this.textSearchWorkerProvider);
@@ -140,6 +143,12 @@ export class SearchService implements IRawSearchService {
 			searchPromise = this.doSearch(engine, -1)
 				.then(result => {
 					c([result, results]);
+					if (this.telemetryPipe) {
+						this.telemetryPipe({
+							eventName: 'fileSearch',
+							data: result.stats
+						});
+					}
 				}, e, progress => {
 					if (Array.isArray(progress)) {
 						results = progress;
@@ -161,23 +170,26 @@ export class SearchService implements IRawSearchService {
 			allResultsPromise = this.preventCancellation(allResultsPromise);
 		}
 
+		let chained: TPromise<void>;
 		return new PPromise<[ISerializedSearchComplete, IRawFileMatch[]], IProgress>((c, e, p) => {
-			allResultsPromise.then(([result, results]) => {
+			chained = allResultsPromise.then(([result, results]) => {
 				const scorerCache: ScorerCache = cache ? cache.scorerCache : Object.create(null);
 				const unsortedResultTime = Date.now();
-				const sortedResults = this.sortResults(config, results, scorerCache);
-				const sortedResultTime = Date.now();
+				return this.sortResults(config, results, scorerCache)
+					.then(sortedResults => {
+						const sortedResultTime = Date.now();
 
-				c([{
-					stats: objects.assign({}, result.stats, {
-						unsortedResultTime,
-						sortedResultTime
-					}),
-					limitHit: result.limitHit || typeof config.maxResults === 'number' && results.length > config.maxResults
-				}, sortedResults]);
+						c([{
+							stats: objects.assign({}, result.stats, {
+								unsortedResultTime,
+								sortedResultTime
+							}),
+							limitHit: result.limitHit || typeof config.maxResults === 'number' && results.length > config.maxResults
+						}, sortedResults]);
+					});
 			}, e, p);
 		}, () => {
-			allResultsPromise.cancel();
+			chained.cancel();
 		});
 	}
 
@@ -198,47 +210,54 @@ export class SearchService implements IRawSearchService {
 		const cacheLookupStartTime = Date.now();
 		const cached = this.getResultsFromCache(cache, config.filePattern);
 		if (cached) {
+			let chained: TPromise<void>;
 			return new PPromise<[ISerializedSearchComplete, IRawFileMatch[]], IProgress>((c, e, p) => {
-				cached.then(([result, results, cacheStats]) => {
+				chained = cached.then(([result, results, cacheStats]) => {
 					const cacheLookupResultTime = Date.now();
-					const sortedResults = this.sortResults(config, results, cache.scorerCache);
-					const sortedResultTime = Date.now();
+					return this.sortResults(config, results, cache.scorerCache)
+						.then(sortedResults => {
+							const sortedResultTime = Date.now();
 
-					const stats: ICachedSearchStats = {
-						fromCache: true,
-						cacheLookupStartTime: cacheLookupStartTime,
-						cacheFilterStartTime: cacheStats.cacheFilterStartTime,
-						cacheLookupResultTime: cacheLookupResultTime,
-						cacheEntryCount: cacheStats.cacheFilterResultCount,
-						resultCount: results.length
-					};
-					if (config.sortByScore) {
-						stats.unsortedResultTime = cacheLookupResultTime;
-						stats.sortedResultTime = sortedResultTime;
-					}
-					if (!cacheStats.cacheWasResolved) {
-						stats.joined = result.stats;
-					}
-					c([
-						{
-							limitHit: result.limitHit || typeof config.maxResults === 'number' && results.length > config.maxResults,
-							stats: stats
-						},
-						sortedResults
-					]);
+							const stats: ICachedSearchStats = {
+								fromCache: true,
+								cacheLookupStartTime: cacheLookupStartTime,
+								cacheFilterStartTime: cacheStats.cacheFilterStartTime,
+								cacheLookupResultTime: cacheLookupResultTime,
+								cacheEntryCount: cacheStats.cacheFilterResultCount,
+								resultCount: results.length
+							};
+							if (config.sortByScore) {
+								stats.unsortedResultTime = cacheLookupResultTime;
+								stats.sortedResultTime = sortedResultTime;
+							}
+							if (!cacheStats.cacheWasResolved) {
+								stats.joined = result.stats;
+							}
+							c([
+								{
+									limitHit: result.limitHit || typeof config.maxResults === 'number' && results.length > config.maxResults,
+									stats: stats
+								},
+								sortedResults
+							]);
+						});
 				}, e, p);
 			}, () => {
-				cached.cancel();
+				chained.cancel();
 			});
 		}
 		return undefined;
 	}
 
-	private sortResults(config: IRawSearch, results: IRawFileMatch[], scorerCache: ScorerCache): IRawFileMatch[] {
-		const filePattern = config.filePattern;
-		const normalizedSearchValue = strings.stripWildcards(filePattern).toLowerCase();
-		const compare = (elementA: IRawFileMatch, elementB: IRawFileMatch) => compareByScore(elementA, elementB, FileMatchAccessor, filePattern, normalizedSearchValue, scorerCache);
-		return arrays.top(results, compare, config.maxResults);
+	private sortResults(config: IRawSearch, results: IRawFileMatch[], scorerCache: ScorerCache): TPromise<IRawFileMatch[]> {
+		// we use the same compare function that is used later when showing the results using fuzzy scoring
+		// this is very important because we are also limiting the number of results by config.maxResults
+		// and as such we want the top items to be included in this result set if the number of items
+		// exceeds config.maxResults.
+		const query = prepareQuery(config.filePattern);
+		const compare = (matchA: IRawFileMatch, matchB: IRawFileMatch) => compareItemsByScore(matchA, matchB, query, true, FileMatchItemAccessor, scorerCache);
+
+		return arrays.topAsync(results, compare, config.maxResults, 10000);
 	}
 
 	private sendProgress(results: ISerializedFileMatch[], progressCb: (batch: ISerializedFileMatch[]) => void, batchSize: number) {
@@ -291,7 +310,7 @@ export class SearchService implements IRawSearchService {
 					let entry = cachedEntries[i];
 
 					// Check if this entry is a match for the search value
-					if (!scorer.matches(entry.relativePath, normalizedSearchValueLowercase)) {
+					if (!fuzzyContains(entry.relativePath, normalizedSearchValueLowercase)) {
 						continue;
 					}
 
@@ -369,6 +388,14 @@ export class SearchService implements IRawSearchService {
 		return TPromise.as(undefined);
 	}
 
+	public fetchTelemetry(): PPromise<void, ITelemetryEvent> {
+		return new PPromise((c, e, p) => {
+			this.telemetryPipe = p;
+		}, () => {
+			this.telemetryPipe = null;
+		});
+	}
+
 	private preventCancellation<C, P>(promise: PPromise<C, P>): PPromise<C, P> {
 		return new PPromise<C, P>((c, e, p) => {
 			// Allow for piled up cancellations to come through first.
@@ -388,20 +415,20 @@ class Cache {
 	public scorerCache: ScorerCache = Object.create(null);
 }
 
-interface ScorerCache {
-	[key: string]: number;
-}
+const FileMatchItemAccessor = new class implements IItemAccessor<IRawFileMatch> {
 
-class FileMatchAccessor {
-
-	public static getLabel(match: IRawFileMatch): string {
-		return match.basename;
+	public getItemLabel(match: IRawFileMatch): string {
+		return match.basename; // e.g. myFile.txt
 	}
 
-	public static getResourcePath(match: IRawFileMatch): string {
-		return match.relativePath;
+	public getItemDescription(match: IRawFileMatch): string {
+		return match.relativePath.substr(0, match.relativePath.length - match.basename.length - 1); // e.g. some/path/to/file
 	}
-}
+
+	public getItemPath(match: IRawFileMatch): string {
+		return match.relativePath; // e.g. some/path/to/file/myFile.txt
+	}
+};
 
 interface CacheStats {
 	cacheWasResolved: boolean;

src/vs/workbench/services/search/node/ripgrepFileSearch.ts

@@ -0,0 +1,63 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as cp from 'child_process';
+import { rgPath } from 'vscode-ripgrep';
+
+import { isMacintosh as isMac } from 'vs/base/common/platform';
+import * as glob from 'vs/base/common/glob';
+import { normalizeNFD, startsWith } from 'vs/base/common/strings';
+
+import { IFolderSearch, IRawSearch } from './search';
+import { foldersToIncludeGlobs, foldersToRgExcludeGlobs } from './ripgrepTextSearch';
+
+export function spawnRipgrepCmd(config: IRawSearch, folderQuery: IFolderSearch, includePattern: glob.IExpression, excludePattern: glob.IExpression) {
+	const rgArgs = getRgArgs(config, folderQuery, includePattern, excludePattern);
+	return {
+		cmd: cp.spawn(rgPath, rgArgs.globArgs, { cwd: folderQuery.folder }),
+		siblingClauses: rgArgs.siblingClauses
+	};
+}
+
+function getRgArgs(config: IRawSearch, folderQuery: IFolderSearch, includePattern: glob.IExpression, excludePattern: glob.IExpression) {
+	const args = ['--files', '--hidden', '--case-sensitive'];
+
+	// includePattern can't have siblingClauses
+	foldersToIncludeGlobs([folderQuery], includePattern, false).forEach(globArg => {
+		args.push('-g', anchor(isMac ? normalizeNFD(globArg) : globArg));
+	});
+
+	let siblingClauses: glob.IExpression;
+
+	const rgGlobs = foldersToRgExcludeGlobs([folderQuery], excludePattern, undefined, false);
+	rgGlobs.globArgs
+		.forEach(rgGlob => args.push('-g', `!${anchor(isMac ? normalizeNFD(rgGlob) : rgGlob)}`));
+	siblingClauses = rgGlobs.siblingClauses;
+
+	if (folderQuery.disregardIgnoreFiles !== false) {
+		// Don't use .gitignore or .ignore
+		args.push('--no-ignore');
+	}
+
+	// Follow symlinks
+	if (!config.ignoreSymlinks) {
+		args.push('--follow');
+	}
+
+	if (config.exists) {
+		args.push('--quiet');
+	}
+
+	// Folder to search
+	args.push('--');
+
+	args.push('.');
+
+	return { globArgs: args, siblingClauses };
+}
+
+function anchor(glob: string) {
+	return startsWith(glob, '**') || startsWith(glob, '/') ? glob : `/${glob}`;
+}

src/vs/workbench/services/search/node/ripgrepTextSearch.ts

@@ -26,6 +26,7 @@ import { ISerializedFileMatch, ISerializedSearchComplete, IRawSearch, IFolderSea
 export class RipgrepEngine {
 	private isDone = false;
 	private rgProc: cp.ChildProcess;
+	private killRgProcFn: Function;
 	private postProcessExclusions: glob.ParsedExpression;
 
 	private ripgrepParser: RipgrepParser;
@@ -33,6 +34,7 @@ export class RipgrepEngine {
 	private resultsHandledP: TPromise<any> = TPromise.wrap(null);
 
 	constructor(private config: IRawSearch) {
+		this.killRgProcFn = () => this.rgProc && this.rgProc.kill();
 	}
 
 	cancel(): void {
@@ -44,6 +46,7 @@ export class RipgrepEngine {
 	// TODO@Rob - make promise-based once the old search is gone, and I don't need them to have matching interfaces anymore
 	search(onResult: (match: ISerializedFileMatch) => void, onMessage: (message: ISearchLog) => void, done: (error: Error, complete: ISerializedSearchComplete) => void): void {
 		if (!this.config.folderQueries.length && !this.config.extraFiles.length) {
+			process.removeListener('exit', this.killRgProcFn);
 			done(null, {
 				limitHit: false,
 				stats: null
@@ -69,8 +72,9 @@ export class RipgrepEngine {
 			}
 		});
 		this.rgProc = cp.spawn(rgPath, rgArgs.globArgs, { cwd });
+		process.once('exit', this.killRgProcFn);
 
-		this.ripgrepParser = new RipgrepParser(this.config.maxResults, cwd);
+		this.ripgrepParser = new RipgrepParser(this.config.maxResults, cwd, this.config.extraFiles);
 		this.ripgrepParser.on('result', (match: ISerializedFileMatch) => {
 			if (this.postProcessExclusions) {
 				const handleResultP = (<TPromise<string>>this.postProcessExclusions(match.path, undefined, () => getSiblings(match.path)))
@@ -87,6 +91,7 @@ export class RipgrepEngine {
 		});
 		this.ripgrepParser.on('hitLimit', () => {
 			this.cancel();
+			process.removeListener('exit', this.killRgProcFn);
 			done(null, {
 				limitHit: true,
 				stats: null
@@ -115,6 +120,7 @@ export class RipgrepEngine {
 				if (!this.isDone) {
 					this.isDone = true;
 					let displayMsg: string;
+					process.removeListener('exit', this.killRgProcFn);
 					if (stderr && !gotData && (displayMsg = this.rgErrorMsgForDisplay(stderr))) {
 						done(new Error(displayMsg), {
 							limitHit: false,
@@ -143,8 +149,10 @@ export class RipgrepEngine {
 			return firstLine;
 		}
 
-		if (strings.startsWith(firstLine, 'error parsing glob')) {
-			return firstLine;
+		if (strings.startsWith(firstLine, 'error parsing glob') ||
+			strings.startsWith(firstLine, 'unsupported encoding')) {
+			// Uppercase first letter
+			return firstLine.charAt(0).toUpperCase() + firstLine.substr(1);
 		}
 
 		return undefined;
@@ -162,12 +170,15 @@ export class RipgrepParser extends EventEmitter {
 	private remainder: string;
 	private isDone: boolean;
 	private stringDecoder: NodeStringDecoder;
+	private extraSearchFiles: string[];
 
 	private numResults = 0;
 
-	constructor(private maxResults: number, private rootFolder: string) {
+	constructor(private maxResults: number, private rootFolder: string, extraFiles?: string[]) {
 		super();
 		this.stringDecoder = new StringDecoder();
+
+		this.extraSearchFiles = extraFiles || [];
 	}
 
 	public cancel(): void {
@@ -221,15 +232,37 @@ export class RipgrepParser extends EventEmitter {
 					this.onResult();
 				}
 
-				this.fileMatch = new FileMatch(path.isAbsolute(r[1]) ? r[1] : path.join(this.rootFolder, r[1]));
+				this.fileMatch = this.getFileMatch(r[1]);
 			} else {
 				// Line is empty (or malformed)
 			}
 		}
 	}
 
+	private getFileMatch(relativeOrAbsolutePath: string): FileMatch {
+		const absPath = path.isAbsolute(relativeOrAbsolutePath) ?
+			relativeOrAbsolutePath :
+			path.join(this.rootFolder, relativeOrAbsolutePath);
+
+		return new FileMatch(absPath);
+	}
+
 	private handleMatchLine(outputLine: string, lineNum: number, text: string): void {
+		if (lineNum === 0) {
+			text = strings.stripUTF8BOM(text);
+		}
+
 		const lineMatch = new LineMatch(text, lineNum);
+		if (!this.fileMatch) {
+			// When searching a single file and no folderQueries, rg does not print the file line, so create it here
+			const singleFile = this.extraSearchFiles[0];
+			if (!singleFile) {
+				throw new Error('Got match line for unknown file');
+			}
+
+			this.fileMatch = this.getFileMatch(singleFile);
+		}
+
 		this.fileMatch.addMatch(lineMatch);
 
 		let lastMatchEndPos = 0;
@@ -363,17 +396,17 @@ export class LineMatch implements ILineMatch {
 	}
 }
 
-interface IRgGlobResult {
+export interface IRgGlobResult {
 	globArgs: string[];
 	siblingClauses: glob.IExpression;
 }
 
-function foldersToRgExcludeGlobs(folderQueries: IFolderSearch[], globalExclude: glob.IExpression, excludesToSkip: Set<string>): IRgGlobResult {
+export function foldersToRgExcludeGlobs(folderQueries: IFolderSearch[], globalExclude: glob.IExpression, excludesToSkip?: Set<string>, absoluteGlobs = true): IRgGlobResult {
 	const globArgs: string[] = [];
 	let siblingClauses: glob.IExpression = {};
 	folderQueries.forEach(folderQuery => {
 		const totalExcludePattern = objects.assign({}, folderQuery.excludePattern || {}, globalExclude || {});
-		const result = globExprsToRgGlobs(totalExcludePattern, folderQuery.folder, excludesToSkip);
+		const result = globExprsToRgGlobs(totalExcludePattern, absoluteGlobs && folderQuery.folder, excludesToSkip);
 		globArgs.push(...result.globArgs);
 		if (result.siblingClauses) {
 			siblingClauses = objects.assign(siblingClauses, result.siblingClauses);
@@ -383,18 +416,18 @@ function foldersToRgExcludeGlobs(folderQueries: IFolderSearch[], globalExclude:
 	return { globArgs, siblingClauses };
 }
 
-function foldersToIncludeGlobs(folderQueries: IFolderSearch[], globalInclude: glob.IExpression): string[] {
-	const globArgs = [];
+export function foldersToIncludeGlobs(folderQueries: IFolderSearch[], globalInclude: glob.IExpression, absoluteGlobs = true): string[] {
+	const globArgs: string[] = [];
 	folderQueries.forEach(folderQuery => {
 		const totalIncludePattern = objects.assign({}, globalInclude || {}, folderQuery.includePattern || {});
-		const result = globExprsToRgGlobs(totalIncludePattern, folderQuery.folder);
+		const result = globExprsToRgGlobs(totalIncludePattern, absoluteGlobs && folderQuery.folder);
 		globArgs.push(...result.globArgs);
 	});
 
 	return globArgs;
 }
 
-function globExprsToRgGlobs(patterns: glob.IExpression, folder: string, excludesToSkip?: Set<string>): IRgGlobResult {
+function globExprsToRgGlobs(patterns: glob.IExpression, folder?: string, excludesToSkip?: Set<string>): IRgGlobResult {
 	const globArgs: string[] = [];
 	let siblingClauses: glob.IExpression = null;
 	Object.keys(patterns)
@@ -403,8 +436,15 @@ function globExprsToRgGlobs(patterns: glob.IExpression, folder: string, excludes
 				return;
 			}
 
+			if (!key) {
+				return;
+			}
+
 			const value = patterns[key];
-			key = getAbsoluteGlob(folder, key);
+			key = trimTrailingSlash(folder ? getAbsoluteGlob(folder, key) : key);
+
+			// glob.ts requires forward slashes
+			key = key.replace(/\\/g, '/');
 
 			if (typeof value === 'boolean' && value) {
 				globArgs.push(fixDriveC(key));
@@ -427,11 +467,9 @@ function globExprsToRgGlobs(patterns: glob.IExpression, folder: string, excludes
  * Exported for testing
  */
 export function getAbsoluteGlob(folder: string, key: string): string {
-	const absolute = paths.isAbsolute(key) ?
+	return paths.isAbsolute(key) ?
 		key :
 		path.join(folder, key);
-
-	return trimTrailingSlash(absolute);
 }
 
 function trimTrailingSlash(str: string): string {
@@ -479,7 +517,9 @@ function getRgArgs(config: IRawSearch): IRgGlobResult {
 	}
 
 	// Follow symlinks
-	args.push('--follow');
+	if (!config.ignoreSymlinks) {
+		args.push('--follow');
+	}
 
 	// Set default encoding if only one folder is opened
 	if (config.folderQueries.length === 1 && config.folderQueries[0].fileEncoding && config.folderQueries[0].fileEncoding !== 'utf8') {

src/vs/workbench/services/search/node/search.ts

@@ -8,22 +8,26 @@
 import { PPromise, TPromise } from 'vs/base/common/winjs.base';
 import { IExpression } from 'vs/base/common/glob';
 import { IProgress, ILineMatch, IPatternInfo, ISearchStats, ISearchLog } from 'vs/platform/search/common/search';
+import { ITelemetryData } from 'vs/platform/telemetry/common/telemetry';
 
 export interface IFolderSearch {
 	folder: string;
 	excludePattern?: IExpression;
 	includePattern?: IExpression;
 	fileEncoding?: string;
+	disregardIgnoreFiles?: boolean;
 }
 
 export interface IRawSearch {
 	folderQueries: IFolderSearch[];
+	ignoreSymlinks?: boolean;
 	extraFiles?: string[];
 	filePattern?: string;
 	excludePattern?: IExpression;
 	includePattern?: IExpression;
 	contentPattern?: IPatternInfo;
 	maxResults?: number;
+	exists?: boolean;
 	sortByScore?: boolean;
 	cacheKey?: string;
 	maxFilesize?: number;
@@ -31,10 +35,16 @@ export interface IRawSearch {
 	disregardIgnoreFiles?: boolean;
 }
 
+export interface ITelemetryEvent {
+	eventName: string;
+	data: ITelemetryData;
+}
+
 export interface IRawSearchService {
 	fileSearch(search: IRawSearch): PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem>;
 	textSearch(search: IRawSearch): PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem>;
 	clearCache(cacheKey: string): TPromise<void>;
+	fetchTelemetry(): PPromise<void, ITelemetryEvent>;
 }
 
 export interface IRawFileMatch {

src/vs/workbench/services/search/node/searchIpc.ts

@@ -7,12 +7,13 @@
 
 import { PPromise, TPromise } from 'vs/base/common/winjs.base';
 import { IChannel } from 'vs/base/parts/ipc/common/ipc';
-import { IRawSearchService, IRawSearch, ISerializedSearchComplete, ISerializedSearchProgressItem } from './search';
+import { IRawSearchService, IRawSearch, ISerializedSearchComplete, ISerializedSearchProgressItem, ITelemetryEvent } from './search';
 
 export interface ISearchChannel extends IChannel {
 	call(command: 'fileSearch', search: IRawSearch): PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem>;
 	call(command: 'textSearch', search: IRawSearch): PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem>;
 	call(command: 'clearCache', cacheKey: string): TPromise<void>;
+	call(command: 'fetchTelemetry'): PPromise<void, ITelemetryEvent>;
 	call(command: string, arg: any): TPromise<any>;
 }
 
@@ -20,11 +21,12 @@ export class SearchChannel implements ISearchChannel {
 
 	constructor(private service: IRawSearchService) { }
 
-	call(command: string, arg: any): TPromise<any> {
+	call(command: string, arg?: any): TPromise<any> {
 		switch (command) {
 			case 'fileSearch': return this.service.fileSearch(arg);
 			case 'textSearch': return this.service.textSearch(arg);
 			case 'clearCache': return this.service.clearCache(arg);
+			case 'fetchTelemetry': return this.service.fetchTelemetry();
 		}
 		return undefined;
 	}
@@ -42,7 +44,11 @@ export class SearchChannelClient implements IRawSearchService {
 		return this.channel.call('textSearch', search);
 	}
 
-	public clearCache(cacheKey: string): TPromise<void> {
+	clearCache(cacheKey: string): TPromise<void> {
 		return this.channel.call('clearCache', cacheKey);
 	}
+
+	fetchTelemetry(): PPromise<void, ITelemetryEvent> {
+		return this.channel.call('fetchTelemetry');
+	}
 }

src/vs/workbench/services/search/node/searchService.ts

@@ -7,20 +7,22 @@
 import { PPromise, TPromise } from 'vs/base/common/winjs.base';
 import uri from 'vs/base/common/uri';
 import objects = require('vs/base/common/objects');
-import scorer = require('vs/base/common/scorer');
 import strings = require('vs/base/common/strings');
 import { getNextTickChannel } from 'vs/base/parts/ipc/common/ipc';
-import { Client } from 'vs/base/parts/ipc/node/ipc.cp';
+import { Client, IIPCOptions } from 'vs/base/parts/ipc/node/ipc.cp';
 import { IProgress, LineMatch, FileMatch, ISearchComplete, ISearchProgressItem, QueryType, IFileMatch, ISearchQuery, ISearchConfiguration, ISearchService, pathIncludedInQuery, ISearchResultProvider } from 'vs/platform/search/common/search';
 import { IUntitledEditorService } from 'vs/workbench/services/untitled/common/untitledEditorService';
 import { IModelService } from 'vs/editor/common/services/modelService';
 import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
-import { IRawSearch, IFolderSearch, ISerializedSearchComplete, ISerializedSearchProgressItem, ISerializedFileMatch, IRawSearchService } from './search';
+import { IRawSearch, ISerializedSearchComplete, ISerializedSearchProgressItem, ISerializedFileMatch, IRawSearchService, ITelemetryEvent } from './search';
 import { ISearchChannel, SearchChannelClient } from './searchIpc';
-import { IEnvironmentService } from 'vs/platform/environment/common/environment';
+import { IEnvironmentService, IDebugParams } from 'vs/platform/environment/common/environment';
 import { ResourceMap } from 'vs/base/common/map';
 import { IDisposable } from 'vs/base/common/lifecycle';
+import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
+import { onUnexpectedError } from 'vs/base/common/errors';
+import { Schemas } from 'vs/base/common/network';
 
 export class SearchService implements ISearchService {
 	public _serviceBrand: any;
@@ -33,10 +35,12 @@ export class SearchService implements ISearchService {
 		@IUntitledEditorService private untitledEditorService: IUntitledEditorService,
 		@IEnvironmentService environmentService: IEnvironmentService,
 		@IWorkspaceContextService private contextService: IWorkspaceContextService,
+		@ITelemetryService private telemetryService: ITelemetryService,
 		@IConfigurationService private configurationService: IConfigurationService
 	) {
-		this.diskSearch = new DiskSearch(!environmentService.isBuilt || environmentService.verbose);
+		this.diskSearch = new DiskSearch(!environmentService.isBuilt || environmentService.verbose, /*timeout=*/undefined, environmentService.debugSearch);
 		this.registerSearchResultProvider(this.diskSearch);
+		this.forwardTelemetry();
 	}
 
 	public registerSearchResultProvider(provider: ISearchResultProvider): IDisposable {
@@ -188,7 +192,7 @@ export class SearchService implements ISearchService {
 				return false; // if we match on file pattern, we have to ignore non file resources
 			}
 
-			if (!scorer.matches(resource.fsPath, strings.stripWildcards(query.filePattern).toLowerCase())) {
+			if (!strings.fuzzyContains(resource.fsPath, strings.stripWildcards(query.filePattern).toLowerCase())) {
 				return false;
 			}
 		}
@@ -206,31 +210,47 @@ export class SearchService implements ISearchService {
 	public clearCache(cacheKey: string): TPromise<void> {
 		return this.diskSearch.clearCache(cacheKey);
 	}
+
+	private forwardTelemetry() {
+		this.diskSearch.fetchTelemetry()
+			.then(null, onUnexpectedError, event => {
+				this.telemetryService.publicLog(event.eventName, event.data);
+			});
+	}
 }
 
 export class DiskSearch implements ISearchResultProvider {
 
 	private raw: IRawSearchService;
 
-	constructor(verboseLogging: boolean, timeout: number = 60 * 60 * 1000) {
+	constructor(verboseLogging: boolean, timeout: number = 60 * 60 * 1000, searchDebug?: IDebugParams) {
+		const opts: IIPCOptions = {
+			serverName: 'Search',
+			timeout: timeout,
+			args: ['--type=searchService'],
+			// See https://github.com/Microsoft/vscode/issues/27665
+			// Pass in fresh execArgv to the forked process such that it doesn't inherit them from `process.execArgv`.
+			// e.g. Launching the extension host process with `--inspect-brk=xxx` and then forking a process from the extension host
+			// results in the forked process inheriting `--inspect-brk=xxx`.
+			freshExecArgv: true,
+			env: {
+				AMD_ENTRYPOINT: 'vs/workbench/services/search/node/searchApp',
+				PIPE_LOGGING: 'true',
+				VERBOSE_LOGGING: verboseLogging
+			}
+		};
+
+		if (searchDebug) {
+			if (searchDebug.break && searchDebug.port) {
+				opts.debugBrk = searchDebug.port;
+			} else if (!searchDebug.break && searchDebug.port) {
+				opts.debug = searchDebug.port;
+			}
+		}
+
 		const client = new Client(
 			uri.parse(require.toUrl('bootstrap')).fsPath,
-			{
-				serverName: 'Search',
-				timeout: timeout,
-				args: ['--type=searchService'],
-				// See https://github.com/Microsoft/vscode/issues/27665
-				// Pass in fresh execArgv to the forked process such that it doesn't inherit them from `process.execArgv`.
-				// e.g. Launching the extension host process with `--inspect-brk=xxx` and then forking a process from the extension host
-				// results in the forked process inheriting `--inspect-brk=xxx`.
-				freshExecArgv: true,
-				env: {
-					AMD_ENTRYPOINT: 'vs/workbench/services/search/node/searchApp',
-					PIPE_LOGGING: 'true',
-					VERBOSE_LOGGING: verboseLogging
-				}
-			}
-		);
+			opts);
 
 		const channel = getNextTickChannel(client.getChannel<ISearchChannel>('search'));
 		this.raw = new SearchChannelClient(channel);
@@ -240,25 +260,42 @@ export class DiskSearch implements ISearchResultProvider {
 		let request: PPromise<ISerializedSearchComplete, ISerializedSearchProgressItem>;
 
 		let rawSearch: IRawSearch = {
-			folderQueries: query.folderQueries ? query.folderQueries.map(q => {
-				return <IFolderSearch>{
-					excludePattern: q.excludePattern,
-					includePattern: q.includePattern,
-					fileEncoding: q.fileEncoding,
-					folder: q.folder.fsPath
-				};
-			}) : [],
-			extraFiles: query.extraFileResources ? query.extraFileResources.map(r => r.fsPath) : [],
+			folderQueries: [],
+			extraFiles: [],
 			filePattern: query.filePattern,
 			excludePattern: query.excludePattern,
 			includePattern: query.includePattern,
 			maxResults: query.maxResults,
+			exists: query.exists,
 			sortByScore: query.sortByScore,
 			cacheKey: query.cacheKey,
 			useRipgrep: query.useRipgrep,
-			disregardIgnoreFiles: query.disregardIgnoreFiles
+			disregardIgnoreFiles: query.disregardIgnoreFiles,
+			ignoreSymlinks: query.ignoreSymlinks
 		};
 
+		if (query.folderQueries) {
+			for (const q of query.folderQueries) {
+				if (q.folder.scheme === Schemas.file) {
+					rawSearch.folderQueries.push({
+						excludePattern: q.excludePattern,
+						includePattern: q.includePattern,
+						fileEncoding: q.fileEncoding,
+						disregardIgnoreFiles: q.disregardIgnoreFiles,
+						folder: q.folder.fsPath
+					});
+				}
+			}
+		}
+
+		if (query.extraFileResources) {
+			for (const r of query.extraFileResources) {
+				if (r.scheme === Schemas.file) {
+					rawSearch.extraFiles.push(r.fsPath);
+				}
+			}
+		}
+
 		if (query.type === QueryType.Text) {
 			rawSearch.contentPattern = query.contentPattern;
 		}
@@ -318,4 +355,8 @@ export class DiskSearch implements ISearchResultProvider {
 	public clearCache(cacheKey: string): TPromise<void> {
 		return this.raw.clearCache(cacheKey);
 	}
+
+	public fetchTelemetry(): PPromise<void, ITelemetryEvent> {
+		return this.raw.fetchTelemetry();
+	}
 }