Merge VS Code 1.26.1 (#2394)

* Squash merge commits for 1.26 (#1) (#2323)

* Polish tag search as per feedback (#55269)

* Polish tag search as per feedback

* Updated regex

* Allow users to opt-out of features that send online requests in the background (#55097)

* settings sweep #54690

* Minor css tweaks to enable eoverflow elipsis in more places (#55277)

* fix an issue with titlebarheight when not scaling with zoom

* Settings descriptions update #54690

* fixes #55209

* Settings editor - many padding fixes

* More space above level 2 label

* Fixing Cannot debug npm script using Yarn #55103

* Settings editor - show ellipsis when description overflows

* Settings editor - ... fix measuring around links, relayout

* Setting descriptions

* Settings editor - fix ... for some short lines, fix select container width

* Settings editor - overlay trees so scrollable shadow is full width

* Fix #54133 - missing extension settings after reload

* Settings color token description tweak

* Settings editor - disable overflow indicator temporarily, needs to be faster

* Added command to Run the selected npm script

* fixes #54452

* fixes #54929

* fixes #55248

* prefix command with extension name

* Contribute run selected to the context menu

* node-debug@1.26.6

* Allow terminal rendererType to be swapped out at runtime

Part of #53274
Fixes #55344

* Settings editor - fix not focusing search when restoring editor
setInput must be actually async. Will be fixed naturally when we aren't using winJS promises...

* Settings editor - TOC should only expand the section with a selected item

* Bump node-debug2

* Settings editor - Tree focus outlines

* Settings editor - don't blink the scrollbar when toc selection changes
And hide TOC correctly when the editor is narrow

* Settings editor - header rows should not be selectable

* fixes #54877

* change debug assignee to isi

* Settings sweep (#54690)

* workaround for #55051

* Settings sweep (#54690)

* settings sweep

#54690

* Don't try closing tags when you type > after another >

* Describe what implementation code lens does

Fixes #55370

* fix javadoc formatter setting description

* fixes #55325

* update to officical TS version

* Settings editor - Even more padding, use semibold instead of bold

* Fix #55357 - fix TOC twistie

* fixes #55288

* explorer: refresh on di change file system provider registration

fixes #53256

* Disable push to Linux repo to test standalone publisher

* New env var to notify log level to extensions #54001

* Disable snippets in extension search (when not in suggest dropdown) (#55281)

* Disable snippits in extension search (when not in suggest dropdown)

* Add monaco input contributions

* Fix bug preventing snippetSuggestions from taking effect in sub-editors

* Latest emmet helper to fix #52366

* Fix comment updates for threads within same file

* Allow extensions to log telemetry to log files #54001

* Pull latest css grammar

* files.exclude control - use same style for "add" vs "edit"

* files.exclude control - focus/keyboard behavior

* don't show menubar too early

* files.exclude - better styling

* Place cursor at end of extensions search box on autofill (#55254)

* Place cursor at end of extensions search box on autofill

* Use position instead of selection

* fix linux build issue (empty if block)

* Settings editor - fix extension category prefixes

* Settings editor - add simple ellipsis for first line that overflows, doesn't cover case when first line does not overflow but there is more text, TODO

* File/Text search provider docs

* Fixes #52655

* Include epoch (#55008)

* Fixes #53385

* Fixes #49480

*  VS Code Insiders (Users) not opening Fixes #55353

* Better handling of the case when the extension host fails to start

* Fixes #53966

*  Remove confusing Start from wordPartLeft commands ID

* vscode-xterm@3.6.0-beta12

Fixes #55488

* Initial size is set to infinity!! Fixes #55461

* Polish embeddedEditorBackground

* configuration service misses event

* Fix #55224 - fix duplicate results in multiroot workspace from splitting the diskseach query

* Select all not working in issue reporter on mac, fixes #55424

* Disable fuzzy matching for extensions autosuggest (#55498)

* Fix clipping of extensions search border in some third party themes (#55504)

* fixes #55538

* Fix bug causing an aria alert to not be shown the third time
 (and odd numbers thereafter)

* Settings editor - work around rendering glitch with webkit-line-clamp

* Settings editor - revert earlier '...' changes

* Settings editor - move enumDescription to its own div, because it disturbs -webkit-line-clamp for some reason

* Settings editor - better overflow indicator

* Don't show existing filters in autocomplete (#55495)

* Dont show existing filters in autocomplete

* Simplify

* Settings Editor: Add aria labels for input elements Fixes: #54836 (#55543)

* fixes #55223

* Update vscode-css-languageservice to 3.0.10-next.1

* Fix #55509 - settings navigation

* Fix #55519

* Fix #55520

* FIx #55524

* Fix #55556 - include wordSeparators in all search queries, so findTextInFiles can respect isWordMatch correctly

* oss updates for endgame

* Fix unit tests

* fixes #55522

* Avoid missing manifest error from bubbling up #54757

* Settings format crawl

* Search provider - Fix FileSearchProvider to return array, not progress

* Fix #55598

* Settings editor - fix NPE rendering settings with no description

* dont render inden guides in search box (#55600)

* fixes #55454

* More settings crawl

* Another change for #55598 - maxResults applies to FileSearch and TextSearch but not FileIndex

* Fix FileSearchProvider unit tests for progress change

* fixes #55561

* Settings description update for #54690

* Update setting descriptions for online services

* Minor edits

* fixes #55513

* fixes #55451

* Fix #55612 - fix findTextInFiles cancellation

* fixes #55539

* More setting description tweaks

* Setting to disable online experiments #54354

* fixes #55507

* fixes #55515

* Show online services action only in Insiders for now

* Settings editor - change toc behavior default to 'filter'

* Settings editor - nicer filter count style during search

* Fix #55617 - search viewlet icons

* Settings editor - better styling for element count indicator

* SearchProvider - fix NPE when searching extraFileResources

* Allow extends to work without json suffix

Fixes #16905

* Remove accessability options logic entirely

Follow up on #55451

* use latest version of DAP

* fixes #55490

* fixes #55122

* fixes #52332

* Avoid assumptions about git: URIs (fixes #36236)

* relative path for descriptions

* resourece: get rid of isFile context key

fixes #48275

* Register previous ids for compatibility (#53497)

* more tuning for #48275

* no need to always re-read "files explorer"

fixes #52003

* read out active composites properly

fixes #51967

* Update link colors for hc theme to meet color contrast ratio, fixes #55651

Also updated link color for `textLinkActiveForeground` to be the same as `textLinkForeground` as it wasn't properly updated

* detect 'winpty-agent.exe'; fixes #55672

* node-debug@1.26.7

* reset counter on new label

* Settings editor - fix multiple setting links in one description

* Settings editor - color code blocks in setting descriptions, fix #55532

* Settings editor - hover color in TOC

* Settings editor - fix navigation NPE

* Settings editor - fix text control width

* Settings editor - maybe fix #55684

* Fix bug causing cursor to not move on paste

* fixes #53582

* Use ctrlCmd instead of ctrl for go down from search box

* fixes #55264

* fixes #55456

* filter for spcaes before triggering search (#55611)

* Fix #55698 - don't lose filtered TOC counts when refreshing TOC

* fixes #55421

* fixes #28979

* fixes #55576

* only add check for updates to windows/linux help

* readonly files: append decoration to label

fixes #53022

* debug: do not show toolbar while initialising

fixes #55026

* Opening launch.json should not activate debug extensions

fixes #55029

* fixes #55435

* fixes #55434

* fixes #55439

* trigger menu only on altkey up

* Fix #50555 - fix settings editor memory leak

* Fix #55712 - no need to focus 'a' anymore when restoring control focus after tree render

* fixes #55335

* proper fix for readonly model

fixes #53022

* improve FoldingRangeKind spec (for #55686)

* Use class with static fields (fixes #55494)

* Fixes #53671

* fixes #54630

* [html] should disable ionic suggestions by default. Currently forces deprecated Ionic v1 suggestions in .html files while typing. Fixes #53324

* cleanup deps

* debug issues back to andre

* update electron for smoketest

* Fix #55757 - prevent settings tabs from overflowing

* Fix #53897 - revert setting menu defaults to old editor

* Add enum descriptions to `typescript.preferences.importModuleSpecifier`

* Fix #55767 - leaking style elements from settings editor

* Fix #55521 - prevent flashing when clicking in exclude control

* Update Git modified color for contrast ratio, fixes #53140

* Revert "Merge branch 'master' of github.com:Microsoft/vscode"

This reverts commit bf46b6bfbae0cab99c2863e1244a916181fa9fbc, reversing
changes made to e275a424483dfb4ed33b428c97d5e2c441d6b917.

* Revert "Revert "Merge branch 'master' of github.com:Microsoft/vscode""

This reverts commit 53949d963f39e40757557c6526332354a31d9154.

* don't ask to install an incomplete menu

* Fix NPE in terminal AccessibilityManager

Fixes #55744

* don't display fallback menu unless we've closed the last window

* fixes #55547

* Fix smoke tests for extension search box

* Update OSSREADME.json for Electron 2.0.5

* Update distro

Includes Chromium license changes

* fix #55455

* fix #55865

* fixes #55893

* Fix bug causing workspace recommendations to go away upon ignoring a recommendation (#55805)

* Fix bug causing workspace recommendations to go away upon ignoring a recommendation

* ONly show on @recommended or @recommended:workspace

* Make more consistant

* Fix #55911

* Understand json activity (#55926)

* Understand json file activity

* Refactoring

* adding composer.json

* Distro update for experiments

* use terminal.processId for auto-attach; fixes #55918

* Reject invalid URI with vscode.openFolder (for #55891)

* improve win32 setup system vs user detection

fixes #55840

fixes #55840

delay winreg import

related to #55840

show notification earlier

related to #55840

fix #55840

update inno setup message

related to #55840

* Fix #55593 - this code only operates on local paths, so use fsPath and Uri.file instead

* Bring back the old menu due to electron 2.0 issues (#55913)

* add the old menu back for native menus

* make menu labels match

* `vscode.openFolder`: treat missing URI schema gracefully (for #55891)

* delay EH reattach; fixes #55955

* Mark all json files under appSettingsHome as settings

* Use localized strings for telemetry opt-out

* Exception when saving file editor opened from remote file provider (fixes #55051)

* Remove terminal menu from stable

Fixes 56003

* VSCode Insiders crashes on open with TypeError: Cannot read property 'lastIndexOf' of undefined. Fixes #54933

* improve fix for #55891

* fix #55916

* Improve #55891

* increase EH debugging restart delay; fixes #55955

* Revert "Don't include non-resource entries in history quick pick"

This reverts commit 37209a838e9f7e9abe6dc53ed73cdf1e03b72060.

* Diff editor: horizontal scrollbar height is smaller (fixes #56062)

* improve openFolder uri fix (correctly treat backslashes)

* fixes #56116
repair ipc for native menubar keybindings

* Fix #56240 - Open the JSON settings editor instead of the UI editor

* Fix #55536

* uriDisplay: if no formatter is registered fall back to getPathlabel

fixes #56104

* VSCode hangs when opening python file. Fixes #56377

* VS Code Hangs When Opening Specific PowerShell File. Fixes #56430

* Fix #56433 - search extraFileResources even when no folders open

* Workaround #55649

* Fix in master #56371

* Fix tests #56371

* Fix in master #56317

* increase version to 1.26.1

* Fixes #56387: Handle SIGPIPE in extension host

* fixes #56185

* Fix merge issues (part 1)

* Fix build breaks (part 1)

* Build breaks (part 2)

* Build breaks (part 3)

* More build breaks (part 4)

* Fix build breaks (part 5)

* WIP

* Fix menus

* Render query result and message panels (#2363)

* Put back query editor hot exit changes

* Fix grid changes that broke profiler (#2365)

* Update APIs for saving query editor state

* Fix restore view state for profiler and edit data

* Updating custom default themes to support 4.5:1 contrast ratio

* Test updates

* Fix Extension Manager and Windows Setup

* Update license headers

* Add appveyor and travis files back

* Fix hidden modal dropdown issue

extensions/search-rg/.vscodeignore

@@ -0,0 +1,4 @@
+build/**
+src/**
+test/**
+tsconfig.json

extensions/search-rg/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "search-rg",
+  "description": "%description%",
+  "displayName": "%displayName%",
+  "version": "1.0.0",
+  "author": "vscode",
+  "publisher": "vscode",
+  "license": "MIT",
+  "aiKey": "AIF-d9b70cd4-b9f9-4d70-929b-a071c400b217",
+  "enableProposedApi": true,
+  "engines": {
+    "vscode": "*"
+  },
+  "categories": [],
+  "dependencies": {
+    "vscode-extension-telemetry": "0.0.18",
+    "vscode-nls": "^3.2.4",
+    "vscode-ripgrep": "^1.0.1"
+  },
+  "devDependencies": {
+    "@types/node": "8.0.33",
+    "@types/semver": "5.4.0",
+    "vscode": "^1.1.17"
+  },
+  "scripts": {},
+  "activationEvents": [
+    "onSearch:file",
+    "*"
+  ],
+  "main": "./out/extension",
+  "contributes": {
+    "configuration": {
+      "title": "Search (ripgrep)",
+      "properties": {
+        "searchRipgrep.enable": {
+          "type": "boolean",
+          "default": false,
+          "scope": "window",
+          "description": "%searchRipgrep.enable.description%"
+        }
+      }
+    }
+  }
+}

extensions/search-rg/package.nls.json

@@ -0,0 +1,5 @@
+{
+	"displayName": "Search (ripgrep)",
+	"description": "Provides search using Ripgrep.",
+	"searchRipgrep.enable.description": "(Experimental) Enable this experimental search provider extension. When enabled, it takes precedence over vscode's built-in search."
+}

extensions/search-rg/src/extension.ts

@@ -0,0 +1,57 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import { RipgrepFileSearchEngine } from './ripgrepFileSearch';
+import { RipgrepTextSearchEngine } from './ripgrepTextSearch';
+import { joinPath } from './utils';
+
+export function activate(): void {
+	if (vscode.workspace.getConfiguration('searchRipgrep').get('enable')) {
+		const outputChannel = vscode.window.createOutputChannel('search-rg');
+
+		const provider = new RipgrepSearchProvider(outputChannel);
+		vscode.workspace.registerFileIndexProvider('file', provider);
+		vscode.workspace.registerTextSearchProvider('file', provider);
+	}
+}
+
+type SearchEngine = RipgrepFileSearchEngine | RipgrepTextSearchEngine;
+
+class RipgrepSearchProvider implements vscode.FileIndexProvider, vscode.TextSearchProvider {
+	private inProgress: Set<SearchEngine> = new Set();
+
+	constructor(private outputChannel: vscode.OutputChannel) {
+		process.once('exit', () => this.dispose());
+	}
+
+	provideTextSearchResults(query: vscode.TextSearchQuery, options: vscode.TextSearchOptions, progress: vscode.Progress<vscode.TextSearchResult>, token: vscode.CancellationToken): Thenable<void> {
+		const engine = new RipgrepTextSearchEngine(this.outputChannel);
+		return this.withEngine(engine, () => engine.provideTextSearchResults(query, options, progress, token));
+	}
+
+	provideFileIndex(options: vscode.FileSearchOptions, token: vscode.CancellationToken): Thenable<vscode.Uri[]> {
+		const engine = new RipgrepFileSearchEngine(this.outputChannel);
+
+		const results: vscode.Uri[] = [];
+		const onResult = relativePathMatch => {
+			results.push(joinPath(options.folder, relativePathMatch));
+		};
+
+		return this.withEngine(engine, () => engine.provideFileSearchResults(options, { report: onResult }, token))
+			.then(() => results);
+	}
+
+	private withEngine(engine: SearchEngine, fn: () => Thenable<void>): Thenable<void> {
+		this.inProgress.add(engine);
+		return fn().then(() => {
+			this.inProgress.delete(engine);
+		});
+	}
+
+	private dispose() {
+		this.inProgress.forEach(engine => engine.cancel());
+	}
+}

extensions/search-rg/src/normalization.ts

@@ -0,0 +1,36 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/**
+ * The normalize() method returns the Unicode Normalization Form of a given string. The form will be
+ * the Normalization Form Canonical Composition.
+ *
+ * @see {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/normalize}
+ */
+export const canNormalize = typeof ((<any>'').normalize) === 'function';
+
+export function normalizeNFC(str: string): string {
+	return normalize(str, 'NFC');
+}
+
+export function normalizeNFD(str: string): string {
+	return normalize(str, 'NFD');
+}
+
+const nonAsciiCharactersPattern = /[^\u0000-\u0080]/;
+function normalize(str: string, form: string): string {
+	if (!canNormalize || !str) {
+		return str;
+	}
+
+	let res: string;
+	if (nonAsciiCharactersPattern.test(str)) {
+		res = (<any>str).normalize(form);
+	} else {
+		res = str;
+	}
+
+	return res;
+}

extensions/search-rg/src/ripgrep.ts

@@ -0,0 +1,10 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { rgPath } from 'vscode-ripgrep';
+
+export { rgPath };

extensions/search-rg/src/ripgrepFileSearch.ts

@@ -0,0 +1,213 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as cp from 'child_process';
+import { Readable } from 'stream';
+import { NodeStringDecoder, StringDecoder } from 'string_decoder';
+import * as vscode from 'vscode';
+import { normalizeNFC, normalizeNFD } from './normalization';
+import { rgPath } from './ripgrep';
+import { rgErrorMsgForDisplay } from './ripgrepTextSearch';
+import { anchorGlob } from './utils';
+
+const isMac = process.platform === 'darwin';
+
+// If vscode-ripgrep is in an .asar file, then the binary is unpacked.
+const rgDiskPath = rgPath.replace(/\bnode_modules\.asar\b/, 'node_modules.asar.unpacked');
+
+export class RipgrepFileSearchEngine {
+	private rgProc: cp.ChildProcess;
+	private isDone: boolean;
+
+	constructor(private outputChannel: vscode.OutputChannel) { }
+
+	cancel() {
+		this.isDone = true;
+		if (this.rgProc) {
+			this.rgProc.kill();
+		}
+	}
+
+	provideFileSearchResults(options: vscode.FileSearchOptions, progress: vscode.Progress<string>, token: vscode.CancellationToken): Thenable<void> {
+		this.outputChannel.appendLine(`provideFileSearchResults ${JSON.stringify({
+			...options,
+			...{
+				folder: options.folder.toString()
+			}
+		})}`);
+
+		return new Promise((resolve, reject) => {
+			token.onCancellationRequested(() => this.cancel());
+
+			const rgArgs = getRgArgs(options);
+
+			const cwd = options.folder.fsPath;
+
+			const escapedArgs = rgArgs
+				.map(arg => arg.match(/^-/) ? arg : `'${arg}'`)
+				.join(' ');
+			this.outputChannel.appendLine(`rg ${escapedArgs}\n - cwd: ${cwd}\n`);
+
+			this.rgProc = cp.spawn(rgDiskPath, rgArgs, { cwd });
+
+			this.rgProc.on('error', e => {
+				console.log(e);
+				reject(e);
+			});
+
+			let leftover = '';
+			this.collectStdout(this.rgProc, (err, stdout, last) => {
+				if (err) {
+					reject(err);
+					return;
+				}
+
+				// Mac: uses NFD unicode form on disk, but we want NFC
+				const normalized = leftover + (isMac ? normalizeNFC(stdout) : stdout);
+				const relativeFiles = normalized.split('\n');
+
+				if (last) {
+					const n = relativeFiles.length;
+					relativeFiles[n - 1] = relativeFiles[n - 1].trim();
+					if (!relativeFiles[n - 1]) {
+						relativeFiles.pop();
+					}
+				} else {
+					leftover = relativeFiles.pop();
+				}
+
+				if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
+					reject(new Error('Splitting up files failed'));
+					return;
+				}
+
+				relativeFiles.forEach(relativeFile => {
+					progress.report(relativeFile);
+				});
+
+				if (last) {
+					if (this.isDone) {
+						resolve();
+					} else {
+						// Trigger last result
+						this.rgProc = null;
+						if (err) {
+							reject(err);
+						} else {
+							resolve();
+						}
+					}
+				}
+			});
+		});
+	}
+
+	private collectStdout(cmd: cp.ChildProcess, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
+		let onData = (err: Error, stdout?: string, last?: boolean) => {
+			if (err || last) {
+				onData = () => { };
+			}
+
+			cb(err, stdout, last);
+		};
+
+		let gotData = false;
+		if (cmd.stdout) {
+			// Should be non-null, but #38195
+			this.forwardData(cmd.stdout, onData);
+			cmd.stdout.once('data', () => gotData = true);
+		} else {
+			this.outputChannel.appendLine('stdout is null');
+		}
+
+		let stderr: Buffer[];
+		if (cmd.stderr) {
+			// Should be non-null, but #38195
+			stderr = this.collectData(cmd.stderr);
+		} else {
+			this.outputChannel.appendLine('stderr is null');
+		}
+
+		cmd.on('error', (err: Error) => {
+			onData(err);
+		});
+
+		cmd.on('close', (code: number) => {
+			// ripgrep returns code=1 when no results are found
+			let stderrText, displayMsg: string;
+			if (!gotData && (stderrText = this.decodeData(stderr)) && (displayMsg = rgErrorMsgForDisplay(stderrText))) {
+				onData(new Error(`command failed with error code ${code}: ${displayMsg}`));
+			} else {
+				onData(null, '', true);
+			}
+		});
+	}
+
+	private forwardData(stream: Readable, cb: (err: Error, stdout?: string) => void): NodeStringDecoder {
+		const decoder = new StringDecoder();
+		stream.on('data', (data: Buffer) => {
+			cb(null, decoder.write(data));
+		});
+		return decoder;
+	}
+
+	private collectData(stream: Readable): Buffer[] {
+		const buffers: Buffer[] = [];
+		stream.on('data', (data: Buffer) => {
+			buffers.push(data);
+		});
+		return buffers;
+	}
+
+	private decodeData(buffers: Buffer[]): string {
+		const decoder = new StringDecoder();
+		return buffers.map(buffer => decoder.write(buffer)).join('');
+	}
+}
+
+function getRgArgs(options: vscode.FileSearchOptions): string[] {
+	const args = ['--files', '--hidden', '--case-sensitive'];
+
+	options.includes.forEach(globArg => {
+		const inclusion = anchorGlob(globArg);
+		args.push('-g', inclusion);
+		if (isMac) {
+			const normalized = normalizeNFD(inclusion);
+			if (normalized !== inclusion) {
+				args.push('-g', normalized);
+			}
+		}
+	});
+
+	options.excludes.forEach(globArg => {
+		const exclusion = `!${anchorGlob(globArg)}`;
+		args.push('-g', exclusion);
+		if (isMac) {
+			const normalized = normalizeNFD(exclusion);
+			if (normalized !== exclusion) {
+				args.push('-g', normalized);
+			}
+		}
+	});
+
+	if (options.useIgnoreFiles) {
+		args.push('--no-ignore-parent');
+	} else {
+		// Don't use .gitignore or .ignore
+		args.push('--no-ignore');
+	}
+
+	// Follow symlinks
+	if (options.followSymlinks) {
+		args.push('--follow');
+	}
+
+	// Folder to search
+	args.push('--');
+
+	args.push('.');
+
+	return args;
+}

extensions/search-rg/src/ripgrepTextSearch.ts

@@ -0,0 +1,448 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as cp from 'child_process';
+import { EventEmitter } from 'events';
+import * as path from 'path';
+import { NodeStringDecoder, StringDecoder } from 'string_decoder';
+import * as vscode from 'vscode';
+import { rgPath } from './ripgrep';
+import { anchorGlob } from './utils';
+
+// If vscode-ripgrep is in an .asar file, then the binary is unpacked.
+const rgDiskPath = rgPath.replace(/\bnode_modules\.asar\b/, 'node_modules.asar.unpacked');
+
+// TODO@roblou move to SearchService
+const MAX_TEXT_RESULTS = 10000;
+
+export class RipgrepTextSearchEngine {
+	private isDone = false;
+	private rgProc: cp.ChildProcess;
+
+	private ripgrepParser: RipgrepParser;
+
+	constructor(private outputChannel: vscode.OutputChannel) { }
+
+	cancel() {
+		this.isDone = true;
+
+		if (this.rgProc) {
+			this.rgProc.kill();
+		}
+
+		if (this.ripgrepParser) {
+			this.ripgrepParser.cancel();
+		}
+	}
+
+	provideTextSearchResults(query: vscode.TextSearchQuery, options: vscode.TextSearchOptions, progress: vscode.Progress<vscode.TextSearchResult>, token: vscode.CancellationToken): Thenable<void> {
+		this.outputChannel.appendLine(`provideTextSearchResults ${query.pattern}, ${JSON.stringify({
+			...options,
+			...{
+				folder: options.folder.toString()
+			}
+		})}`);
+
+		return new Promise((resolve, reject) => {
+			token.onCancellationRequested(() => this.cancel());
+
+			const rgArgs = getRgArgs(query, options);
+
+			const cwd = options.folder.fsPath;
+
+			const escapedArgs = rgArgs
+				.map(arg => arg.match(/^-/) ? arg : `'${arg}'`)
+				.join(' ');
+			this.outputChannel.appendLine(`rg ${escapedArgs}\n - cwd: ${cwd}`);
+
+			this.rgProc = cp.spawn(rgDiskPath, rgArgs, { cwd });
+			this.rgProc.on('error', e => {
+				console.error(e);
+				this.outputChannel.append('Error: ' + (e && e.message));
+				reject(e);
+			});
+
+			let gotResult = false;
+			this.ripgrepParser = new RipgrepParser(MAX_TEXT_RESULTS, cwd);
+			this.ripgrepParser.on('result', (match: vscode.TextSearchResult) => {
+				gotResult = true;
+				progress.report(match);
+			});
+
+			this.ripgrepParser.on('hitLimit', () => {
+				this.cancel();
+			});
+
+			this.rgProc.stdout.on('data', data => {
+				this.ripgrepParser.handleData(data);
+			});
+
+			let gotData = false;
+			this.rgProc.stdout.once('data', () => gotData = true);
+
+			let stderr = '';
+			this.rgProc.stderr.on('data', data => {
+				const message = data.toString();
+				this.outputChannel.append(message);
+				stderr += message;
+			});
+
+			this.rgProc.on('close', code => {
+				this.outputChannel.appendLine(gotData ? 'Got data from stdout' : 'No data from stdout');
+				this.outputChannel.appendLine(gotResult ? 'Got result from parser' : 'No result from parser');
+				this.outputChannel.appendLine('');
+				if (this.isDone) {
+					resolve();
+				} else {
+					// Trigger last result
+					this.ripgrepParser.flush();
+					this.rgProc = null;
+					let displayMsg: string;
+					if (stderr && !gotData && (displayMsg = rgErrorMsgForDisplay(stderr))) {
+						reject(new Error(displayMsg));
+					} else {
+						resolve();
+					}
+				}
+			});
+		});
+	}
+}
+
+/**
+ * Read the first line of stderr and return an error for display or undefined, based on a whitelist.
+ * Ripgrep produces stderr output which is not from a fatal error, and we only want the search to be
+ * "failed" when a fatal error was produced.
+ */
+export function rgErrorMsgForDisplay(msg: string): string | undefined {
+	const firstLine = msg.split('\n')[0].trim();
+
+	if (firstLine.startsWith('Error parsing regex')) {
+		return firstLine;
+	}
+
+	if (firstLine.startsWith('error parsing glob') ||
+		firstLine.startsWith('unsupported encoding')) {
+		// Uppercase first letter
+		return firstLine.charAt(0).toUpperCase() + firstLine.substr(1);
+	}
+
+	if (firstLine === `Literal '\\n' not allowed.`) {
+		// I won't localize this because none of the Ripgrep error messages are localized
+		return `Literal '\\n' currently not supported`;
+	}
+
+	if (firstLine.startsWith('Literal ')) {
+		// Other unsupported chars
+		return firstLine;
+	}
+
+	return undefined;
+}
+
+export class RipgrepParser extends EventEmitter {
+	private static readonly RESULT_REGEX = /^\u001b\[0m(\d+)\u001b\[0m:(.*)(\r?)/;
+	private static readonly FILE_REGEX = /^\u001b\[0m(.+)\u001b\[0m$/;
+	private static readonly ESC_CODE = '\u001b'.charCodeAt(0);
+
+	// public for test
+	public static readonly MATCH_START_MARKER = '\u001b[0m\u001b[31m';
+	public static readonly MATCH_END_MARKER = '\u001b[0m';
+
+	private currentFile: string;
+	private remainder: string;
+	private isDone: boolean;
+	private stringDecoder: NodeStringDecoder;
+
+	private numResults = 0;
+
+	constructor(private maxResults: number, private rootFolder: string) {
+		super();
+		this.stringDecoder = new StringDecoder();
+	}
+
+	public cancel(): void {
+		this.isDone = true;
+	}
+
+	public flush(): void {
+		this.handleDecodedData(this.stringDecoder.end());
+	}
+
+	public handleData(data: Buffer | string): void {
+		const dataStr = typeof data === 'string' ? data : this.stringDecoder.write(data);
+		this.handleDecodedData(dataStr);
+	}
+
+	private handleDecodedData(decodedData: string): void {
+		// If the previous data chunk didn't end in a newline, prepend it to this chunk
+		const dataStr = this.remainder ?
+			this.remainder + decodedData :
+			decodedData;
+
+		const dataLines: string[] = dataStr.split(/\r\n|\n/);
+		this.remainder = dataLines[dataLines.length - 1] ? dataLines.pop() : null;
+
+		for (let l = 0; l < dataLines.length; l++) {
+			const outputLine = dataLines[l].trim();
+			if (this.isDone) {
+				break;
+			}
+
+			let r: RegExpMatchArray;
+			if (r = outputLine.match(RipgrepParser.RESULT_REGEX)) {
+				const lineNum = parseInt(r[1]) - 1;
+				let matchText = r[2];
+
+				// workaround https://github.com/BurntSushi/ripgrep/issues/416
+				// If the match line ended with \r, append a match end marker so the match isn't lost
+				if (r[3]) {
+					matchText += RipgrepParser.MATCH_END_MARKER;
+				}
+
+				// Line is a result - add to collected results for the current file path
+				this.handleMatchLine(outputLine, lineNum, matchText);
+			} else if (r = outputLine.match(RipgrepParser.FILE_REGEX)) {
+				this.currentFile = r[1];
+			} else {
+				// Line is empty (or malformed)
+			}
+		}
+	}
+
+	private handleMatchLine(outputLine: string, lineNum: number, lineText: string): void {
+		if (lineNum === 0) {
+			lineText = stripUTF8BOM(lineText);
+		}
+
+		// if (!this.currentFile) {
+		// 	// When searching a single file and no folderQueries, rg does not print the file line, so create it here
+		// 	const singleFile = this.extraSearchFiles[0];
+		// 	if (!singleFile) {
+		// 		throw new Error('Got match line for unknown file');
+		// 	}
+
+		// 	this.currentFile = this.getFileUri(singleFile);
+		// }
+
+		let lastMatchEndPos = 0;
+		let matchTextStartPos = -1;
+
+		// Track positions with color codes subtracted - offsets in the final text preview result
+		let matchTextStartRealIdx = -1;
+		let textRealIdx = 0;
+		let hitLimit = false;
+
+		const realTextParts: string[] = [];
+
+		const lineMatches: vscode.Range[] = [];
+
+		for (let i = 0; i < lineText.length - (RipgrepParser.MATCH_END_MARKER.length - 1);) {
+			if (lineText.charCodeAt(i) === RipgrepParser.ESC_CODE) {
+				if (lineText.substr(i, RipgrepParser.MATCH_START_MARKER.length) === RipgrepParser.MATCH_START_MARKER) {
+					// Match start
+					const chunk = lineText.slice(lastMatchEndPos, i);
+					realTextParts.push(chunk);
+					i += RipgrepParser.MATCH_START_MARKER.length;
+					matchTextStartPos = i;
+					matchTextStartRealIdx = textRealIdx;
+				} else if (lineText.substr(i, RipgrepParser.MATCH_END_MARKER.length) === RipgrepParser.MATCH_END_MARKER) {
+					// Match end
+					const chunk = lineText.slice(matchTextStartPos, i);
+					realTextParts.push(chunk);
+					if (!hitLimit) {
+						const startCol = matchTextStartRealIdx;
+						const endCol = textRealIdx;
+
+						// actually have to finish parsing the line, and use the real ones
+						lineMatches.push(new vscode.Range(lineNum, startCol, lineNum, endCol));
+					}
+
+					matchTextStartPos = -1;
+					matchTextStartRealIdx = -1;
+					i += RipgrepParser.MATCH_END_MARKER.length;
+					lastMatchEndPos = i;
+					this.numResults++;
+
+					// Check hit maxResults limit
+					if (this.numResults >= this.maxResults) {
+						// Finish the line, then report the result below
+						hitLimit = true;
+					}
+				} else {
+					// ESC char in file
+					i++;
+					textRealIdx++;
+				}
+			} else {
+				// Some other char
+				i++;
+				textRealIdx++;
+			}
+		}
+
+		const chunk = lineText.slice(lastMatchEndPos);
+		realTextParts.push(chunk);
+
+		// Get full real text line without color codes
+		const preview = realTextParts.join('');
+
+		lineMatches
+			.map(range => {
+				return <vscode.TextSearchResult>{
+					uri: vscode.Uri.file(path.join(this.rootFolder, this.currentFile)),
+					range,
+					preview: {
+						text: preview,
+						match: new vscode.Range(0, range.start.character, 0, range.end.character)
+					}
+				};
+			})
+			.forEach(match => this.onResult(match));
+
+		if (hitLimit) {
+			this.cancel();
+			this.emit('hitLimit');
+		}
+	}
+
+	private onResult(match: vscode.TextSearchResult): void {
+		this.emit('result', match);
+	}
+}
+
+function getRgArgs(query: vscode.TextSearchQuery, options: vscode.TextSearchOptions): string[] {
+	const args = ['--hidden', '--heading', '--line-number', '--color', 'ansi', '--colors', 'path:none', '--colors', 'line:none', '--colors', 'match:fg:red', '--colors', 'match:style:nobold'];
+	args.push(query.isCaseSensitive ? '--case-sensitive' : '--ignore-case');
+
+	options.includes
+		.map(anchorGlob)
+		.forEach(globArg => args.push('-g', globArg));
+
+	options.excludes
+		.map(anchorGlob)
+		.forEach(rgGlob => args.push('-g', `!${rgGlob}`));
+
+	if (options.maxFileSize) {
+		args.push('--max-filesize', options.maxFileSize + '');
+	}
+
+	if (options.useIgnoreFiles) {
+		args.push('--no-ignore-parent');
+	} else {
+		// Don't use .gitignore or .ignore
+		args.push('--no-ignore');
+	}
+
+	if (options.followSymlinks) {
+		args.push('--follow');
+	}
+
+	if (options.encoding) {
+		args.push('--encoding', options.encoding);
+	}
+
+	// Ripgrep handles -- as a -- arg separator. Only --.
+	// - is ok, --- is ok, --some-flag is handled as query text. Need to special case.
+	if (query.pattern === '--') {
+		query.isRegExp = true;
+		query.pattern = '\\-\\-';
+	}
+
+	let searchPatternAfterDoubleDashes: string;
+	if (query.isWordMatch) {
+		const regexp = createRegExp(query.pattern, query.isRegExp, { wholeWord: query.isWordMatch });
+		const regexpStr = regexp.source.replace(/\\\//g, '/'); // RegExp.source arbitrarily returns escaped slashes. Search and destroy.
+		args.push('--regexp', regexpStr);
+	} else if (query.isRegExp) {
+		args.push('--regexp', fixRegexEndingPattern(query.pattern));
+	} else {
+		searchPatternAfterDoubleDashes = query.pattern;
+		args.push('--fixed-strings');
+	}
+
+	args.push('--no-config');
+
+	// Folder to search
+	args.push('--');
+
+	if (searchPatternAfterDoubleDashes) {
+		// Put the query after --, in case the query starts with a dash
+		args.push(searchPatternAfterDoubleDashes);
+	}
+
+	args.push('.');
+
+	return args;
+}
+
+// TODO@roblou organize away
+
+interface RegExpOptions {
+	matchCase?: boolean;
+	wholeWord?: boolean;
+	multiline?: boolean;
+	global?: boolean;
+}
+
+function createRegExp(searchString: string, isRegex: boolean, options: RegExpOptions = {}): RegExp {
+	if (!searchString) {
+		throw new Error('Cannot create regex from empty string');
+	}
+	if (!isRegex) {
+		searchString = escapeRegExpCharacters(searchString);
+	}
+	if (options.wholeWord) {
+		if (!/\B/.test(searchString.charAt(0))) {
+			searchString = '\\b' + searchString;
+		}
+		if (!/\B/.test(searchString.charAt(searchString.length - 1))) {
+			searchString = searchString + '\\b';
+		}
+	}
+	let modifiers = '';
+	if (options.global) {
+		modifiers += 'g';
+	}
+	if (!options.matchCase) {
+		modifiers += 'i';
+	}
+	if (options.multiline) {
+		modifiers += 'm';
+	}
+
+	return new RegExp(searchString, modifiers);
+}
+
+/**
+ * Escapes regular expression characters in a given string
+ */
+function escapeRegExpCharacters(value: string): string {
+	return value.replace(/[\-\\\{\}\*\+\?\|\^\$\.\[\]\(\)\#]/g, '\\$&');
+}
+
+// -- UTF-8 BOM
+
+const UTF8_BOM = 65279;
+
+const UTF8_BOM_CHARACTER = String.fromCharCode(UTF8_BOM);
+
+function startsWithUTF8BOM(str: string): boolean {
+	return (str && str.length > 0 && str.charCodeAt(0) === UTF8_BOM);
+}
+
+function stripUTF8BOM(str: string): string {
+	return startsWithUTF8BOM(str) ? str.substr(1) : str;
+}
+
+function fixRegexEndingPattern(pattern: string): string {
+	// Replace an unescaped $ at the end of the pattern with \r?$
+	// Match $ preceeded by none or even number of literal \
+	return pattern.match(/([^\\]|^)(\\\\)*\$$/) ?
+		pattern.replace(/\$$/, '\\r?$') :
+		pattern;
+}

extensions/search-rg/src/test/index.ts

@@ -0,0 +1,28 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+//
+// PLEASE DO NOT MODIFY / DELETE UNLESS YOU KNOW WHAT YOU ARE DOING
+//
+// This file is providing the test runner to use when running extension tests.
+// By default the test runner in use is Mocha based.
+//
+// You can provide your own test runner if you want to override it by exporting
+// a function run(testRoot: string, clb: (error:Error) => void) that the extension
+// host can call to run the tests. The test runner is expected to use console.log
+// to report the results back to the caller. When the tests are finished, return
+// a possible error to the callback or null if none.
+
+const testRunner = require('vscode/lib/testrunner');
+
+// You can directly control Mocha options by uncommenting the following lines
+// See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for more info
+testRunner.configure({
+	ui: 'tdd', 		// the TDD UI is being used in extension.test.ts (suite, test, etc.)
+	useColors: process.platform !== 'win32', // colored output from test results (only windows cannot handle)
+	timeout: 60000
+});
+
+export = testRunner;

extensions/search-rg/src/typings/ref.d.ts

@@ -0,0 +1,8 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/// <reference path='../../../../src/vs/vscode.d.ts'/>
+/// <reference path='../../../../src/vs/vscode.proposed.d.ts'/>
+/// <reference types='@types/node'/>

extensions/search-rg/src/utils.ts

@@ -0,0 +1,25 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as path from 'path';
+import * as vscode from 'vscode';
+
+export function fixDriveC(_path: string): string {
+	const root = path.parse(_path).root;
+	return root.toLowerCase() === 'c:/' ?
+		_path.replace(/^c:[/\\]/i, '/') :
+		_path;
+}
+
+export function anchorGlob(glob: string): string {
+	return glob.startsWith('**') || glob.startsWith('/') ? glob : `/${glob}`;
+}
+
+export function joinPath(resource: vscode.Uri, pathFragment: string): vscode.Uri {
+	const joinedPath = path.join(resource.fsPath || '/', pathFragment);
+	return vscode.Uri.file(joinedPath);
+}

extensions/search-rg/tsconfig.json

@@ -0,0 +1,15 @@
+{
+	"compilerOptions": {
+		"target": "es5",
+		"module": "commonjs",
+		"lib": [
+			"es6",
+			"es2015.promise"
+		],
+		"outDir": "./out",
+		"experimentalDecorators": true
+	},
+	"include": [
+		"src/**/*"
+	]
+}