Vscode merge (#4582)

* Merge from vscode 37cb23d3dd4f9433d56d4ba5ea3203580719a0bd

* fix issues with merges

* bump node version in azpipe

* replace license headers

* remove duplicate launch task

* fix build errors

* fix build errors

* fix tslint issues

* working through package and linux build issues

* more work

* wip

* fix packaged builds

* working through linux build errors

* wip

* wip

* wip

* fix mac and linux file limits

* iterate linux pipeline

* disable editor typing

* revert series to parallel

* remove optimize vscode from linux

* fix linting issues

* revert testing change

* add work round for new node

* readd packaging for extensions

* fix issue with angular not resolving decorator dependencies

src/vs/workbench/services/extensions/node/proxyResolver.ts

@@ -5,10 +5,15 @@
 
 import * as http from 'http';
 import * as https from 'https';
+import * as tls from 'tls';
 import * as nodeurl from 'url';
+import * as os from 'os';
+import * as fs from 'fs';
+import * as cp from 'child_process';
 
 import { assign } from 'vs/base/common/objects';
-import { ExtHostWorkspace } from 'vs/workbench/api/node/extHostWorkspace';
+import { endsWith } from 'vs/base/common/strings';
+import { IExtHostWorkspaceProvider } from 'vs/workbench/api/node/extHostWorkspace';
 import { ExtHostConfigProvider } from 'vs/workbench/api/node/extHostConfiguration';
 import { ProxyAgent } from 'vscode-proxy-agent';
 import { MainThreadTelemetryShape } from 'vs/workbench/api/node/extHost.protocol';
@@ -16,6 +21,7 @@ import { ExtHostLogService } from 'vs/workbench/api/node/extHostLogService';
 import { toErrorMessage } from 'vs/base/common/errorMessage';
 import { ExtHostExtensionService } from 'vs/workbench/api/node/extHostExtensionService';
 import { URI } from 'vs/base/common/uri';
+import { promisify } from 'util';
 
 interface ConnectionResult {
 	proxy: string;
@@ -25,34 +31,37 @@ interface ConnectionResult {
 }
 
 export function connectProxyResolver(
-	extHostWorkspace: ExtHostWorkspace,
+	extHostWorkspace: IExtHostWorkspaceProvider,
 	configProvider: ExtHostConfigProvider,
 	extensionService: ExtHostExtensionService,
 	extHostLogService: ExtHostLogService,
 	mainThreadTelemetry: MainThreadTelemetryShape
 ) {
-	const agents = createProxyAgents(extHostWorkspace, configProvider, extHostLogService, mainThreadTelemetry);
-	const lookup = createPatchedModules(configProvider, agents);
+	const resolveProxy = setupProxyResolution(extHostWorkspace, configProvider, extHostLogService, mainThreadTelemetry);
+	const lookup = createPatchedModules(configProvider, resolveProxy);
 	return configureModuleLoading(extensionService, lookup);
 }
 
 const maxCacheEntries = 5000; // Cache can grow twice that much due to 'oldCache'.
 
-function createProxyAgents(
-	extHostWorkspace: ExtHostWorkspace,
+function setupProxyResolution(
+	extHostWorkspace: IExtHostWorkspaceProvider,
 	configProvider: ExtHostConfigProvider,
 	extHostLogService: ExtHostLogService,
 	mainThreadTelemetry: MainThreadTelemetryShape
 ) {
+	const env = process.env;
+
 	let settingsProxy = proxyFromConfigURL(configProvider.getConfiguration('http')
 		.get<string>('proxy'));
 	configProvider.onDidChangeConfiguration(e => {
 		settingsProxy = proxyFromConfigURL(configProvider.getConfiguration('http')
 			.get<string>('proxy'));
 	});
-	const env = process.env;
 	let envProxy = proxyFromConfigURL(env.https_proxy || env.HTTPS_PROXY || env.http_proxy || env.HTTP_PROXY); // Not standardized.
 
+	let envNoProxy = noProxyFromEnv(env.no_proxy || env.NO_PROXY); // Not standardized.
+
 	let cacheRolls = 0;
 	let oldCache = new Map<string, string>();
 	let cache = new Map<string, string>();
@@ -90,6 +99,7 @@ function createProxyAgents(
 	let envCount = 0;
 	let settingsCount = 0;
 	let localhostCount = 0;
+	let envNoProxyCount = 0;
 	let results: ConnectionResult[] = [];
 	function logEvent() {
 		timeout = undefined;
@@ -104,19 +114,32 @@ function createProxyAgents(
 				"envCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true },
 				"settingsCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true },
 				"localhostCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true },
+				"envNoProxyCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true },
 				"results": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth" }
 			}
 		*/
-		mainThreadTelemetry.$publicLog('resolveProxy', { count, duration, errorCount, cacheCount, cacheSize: cache.size, cacheRolls, envCount, settingsCount, localhostCount, results });
-		count = duration = errorCount = cacheCount = envCount = settingsCount = localhostCount = 0;
+		mainThreadTelemetry.$publicLog('resolveProxy', { count, duration, errorCount, cacheCount, cacheSize: cache.size, cacheRolls, envCount, settingsCount, localhostCount, envNoProxyCount, results });
+		count = duration = errorCount = cacheCount = envCount = settingsCount = localhostCount = envNoProxyCount = 0;
 		results = [];
 	}
 
-	function resolveProxy(req: http.ClientRequest, opts: http.RequestOptions, url: string, callback: (proxy?: string) => void) {
+	function resolveProxy(flags: { useProxySettings: boolean, useSystemCertificates: boolean }, req: http.ClientRequest, opts: http.RequestOptions, url: string, callback: (proxy?: string) => void) {
 		if (!timeout) {
 			timeout = setTimeout(logEvent, 10 * 60 * 1000);
 		}
 
+		useSystemCertificates(extHostLogService, flags.useSystemCertificates, opts, () => {
+			useProxySettings(flags.useProxySettings, req, opts, url, callback);
+		});
+	}
+
+	function useProxySettings(useProxySettings: boolean, req: http.ClientRequest, opts: http.RequestOptions, url: string, callback: (proxy?: string) => void) {
+
+		if (!useProxySettings) {
+			callback('DIRECT');
+			return;
+		}
+
 		const parsedUrl = nodeurl.parse(url); // Coming from Node's URL, sticking with that.
 
 		const hostname = parsedUrl.hostname;
@@ -127,6 +150,13 @@ function createProxyAgents(
 			return;
 		}
 
+		if (envNoProxy(hostname, String(parsedUrl.port || (<any>opts.agent).defaultPort))) {
+			envNoProxyCount++;
+			callback('DIRECT');
+			extHostLogService.trace('ProxyResolver#resolveProxy envNoProxy', url, 'DIRECT');
+			return;
+		}
+
 		if (settingsProxy) {
 			settingsCount++;
 			callback(settingsProxy);
@@ -168,11 +198,7 @@ function createProxyAgents(
 			});
 	}
 
-	const httpAgent: http.Agent = new ProxyAgent({ resolveProxy });
-	(<any>httpAgent).defaultPort = 80;
-	const httpsAgent: http.Agent = new ProxyAgent({ resolveProxy });
-	(<any>httpsAgent).defaultPort = 443;
-	return { http: httpAgent, https: httpsAgent };
+	return resolveProxy;
 }
 
 function collectResult(results: ConnectionResult[], resolveProxy: string, connection: string, req: http.ClientRequest) {
@@ -189,7 +215,7 @@ function collectResult(results: ConnectionResult[], resolveProxy: string, connec
 	});
 }
 
-function findOrCreateResult(results: ConnectionResult[], proxy: string, connection: string, code: string): ConnectionResult | undefined {
+function findOrCreateResult(results: ConnectionResult[], proxy: string, connection: string, code: string): ConnectionResult {
 	for (const result of results) {
 		if (result.proxy === proxy && result.connection === connection && result.code === code) {
 			return result;
@@ -200,7 +226,7 @@ function findOrCreateResult(results: ConnectionResult[], proxy: string, connecti
 	return result;
 }
 
-function proxyFromConfigURL(configURL: string) {
+function proxyFromConfigURL(configURL: string | undefined) {
 	const url = (configURL || '').trim();
 	const i = url.indexOf('://');
 	if (i === -1) {
@@ -218,35 +244,70 @@ function proxyFromConfigURL(configURL: string) {
 	return undefined;
 }
 
-function createPatchedModules(configProvider: ExtHostConfigProvider, agents: { http: http.Agent; https: http.Agent; }) {
-	const setting = {
+function noProxyFromEnv(envValue?: string) {
+	const value = (envValue || '')
+		.trim()
+		.toLowerCase();
+
+	if (value === '*') {
+		return () => true;
+	}
+
+	const filters = value
+		.split(',')
+		.map(s => s.trim().split(':', 2))
+		.map(([name, port]) => ({ name, port }))
+		.filter(filter => !!filter.name)
+		.map(({ name, port }) => {
+			const domain = name[0] === '.' ? name : `.${name}`;
+			return { domain, port };
+		});
+	if (!filters.length) {
+		return () => false;
+	}
+	return (hostname: string, port: string) => filters.some(({ domain, port: filterPort }) => {
+		return endsWith(`.${hostname.toLowerCase()}`, domain) && (!filterPort || port === filterPort);
+	});
+}
+
+function createPatchedModules(configProvider: ExtHostConfigProvider, resolveProxy: ReturnType<typeof setupProxyResolution>) {
+	const proxySetting = {
 		config: configProvider.getConfiguration('http')
 			.get<string>('proxySupport') || 'off'
 	};
 	configProvider.onDidChangeConfiguration(e => {
-		setting.config = configProvider.getConfiguration('http')
+		proxySetting.config = configProvider.getConfiguration('http')
 			.get<string>('proxySupport') || 'off';
 	});
+	const certSetting = {
+		config: !!configProvider.getConfiguration('http')
+			.get<boolean>('systemCertificates')
+	};
+	configProvider.onDidChangeConfiguration(e => {
+		certSetting.config = !!configProvider.getConfiguration('http')
+			.get<string>('systemCertificates');
+	});
 
 	return {
 		http: {
-			off: assign({}, http, patches(http, agents.http, agents.https, { config: 'off' }, true)),
-			on: assign({}, http, patches(http, agents.http, agents.https, { config: 'on' }, true)),
-			override: assign({}, http, patches(http, agents.http, agents.https, { config: 'override' }, true)),
-			onRequest: assign({}, http, patches(http, agents.http, agents.https, setting, true)),
-			default: assign(http, patches(http, agents.http, agents.https, setting, false)) // run last
+			off: assign({}, http, patches(http, resolveProxy, { config: 'off' }, certSetting, true)),
+			on: assign({}, http, patches(http, resolveProxy, { config: 'on' }, certSetting, true)),
+			override: assign({}, http, patches(http, resolveProxy, { config: 'override' }, certSetting, true)),
+			onRequest: assign({}, http, patches(http, resolveProxy, proxySetting, certSetting, true)),
+			default: assign(http, patches(http, resolveProxy, proxySetting, certSetting, false)) // run last
 		},
 		https: {
-			off: assign({}, https, patches(https, agents.https, agents.http, { config: 'off' }, true)),
-			on: assign({}, https, patches(https, agents.https, agents.http, { config: 'on' }, true)),
-			override: assign({}, https, patches(https, agents.https, agents.http, { config: 'override' }, true)),
-			onRequest: assign({}, https, patches(https, agents.https, agents.http, setting, true)),
-			default: assign(https, patches(https, agents.https, agents.http, setting, false)) // run last
-		}
+			off: assign({}, https, patches(https, resolveProxy, { config: 'off' }, certSetting, true)),
+			on: assign({}, https, patches(https, resolveProxy, { config: 'on' }, certSetting, true)),
+			override: assign({}, https, patches(https, resolveProxy, { config: 'override' }, certSetting, true)),
+			onRequest: assign({}, https, patches(https, resolveProxy, proxySetting, certSetting, true)),
+			default: assign(https, patches(https, resolveProxy, proxySetting, certSetting, false)) // run last
+		},
+		tls: assign(tls, tlsPatches(tls))
 	};
 }
 
-function patches(originals: typeof http | typeof https, agent: http.Agent, otherAgent: http.Agent, setting: { config: string; }, onRequest: boolean) {
+function patches(originals: typeof http | typeof https, resolveProxy: ReturnType<typeof setupProxyResolution>, proxySetting: { config: string }, certSetting: { config: boolean }, onRequest: boolean) {
 	return {
 		get: patch(originals.get),
 		request: patch(originals.request)
@@ -265,12 +326,15 @@ function patches(originals: typeof http | typeof https, agent: http.Agent, other
 			}
 			options = options || {};
 
-			const config = onRequest && ((<any>options)._vscodeProxySupport || /* LS */ (<any>options)._vscodeSystemProxy) || setting.config;
-			if (config === 'off') {
+			if (options.socketPath) {
 				return original.apply(null, arguments as unknown as any[]);
 			}
 
-			if (!options.socketPath && (config === 'override' || config === 'on' && !options.agent) && options.agent !== agent && options.agent !== otherAgent) {
+			const config = onRequest && ((<any>options)._vscodeProxySupport || /* LS */ (<any>options)._vscodeSystemProxy) || proxySetting.config;
+			const useProxySettings = (config === 'override' || config === 'on' && !options.agent) && !(options.agent instanceof ProxyAgent);
+			const useSystemCertificates = certSetting.config && originals === https && !(options as https.RequestOptions).ca;
+
+			if (useProxySettings || useSystemCertificates) {
 				if (url) {
 					const parsed = typeof url === 'string' ? new nodeurl.URL(url) : url;
 					const urlOptions = {
@@ -286,7 +350,11 @@ function patches(originals: typeof http | typeof https, agent: http.Agent, other
 				} else {
 					options = { ...options };
 				}
-				options.agent = agent;
+				options.agent = new ProxyAgent({
+					resolveProxy: resolveProxy.bind(undefined, { useProxySettings, useSystemCertificates }),
+					defaultPort: originals === https ? 443 : 80,
+					originalAgent: options.agent
+				});
 				return original(options, callback);
 			}
 
@@ -296,12 +364,35 @@ function patches(originals: typeof http | typeof https, agent: http.Agent, other
 	}
 }
 
+function tlsPatches(originals: typeof tls) {
+	return {
+		createSecureContext: patch(originals.createSecureContext)
+	};
+
+	function patch(original: typeof tls.createSecureContext): typeof tls.createSecureContext {
+		return function (details: tls.SecureContextOptions): ReturnType<typeof tls.createSecureContext> {
+			const context = original.apply(null, arguments as unknown as any[]);
+			const certs = (details as any)._vscodeAdditionalCaCerts;
+			if (certs) {
+				for (const cert of certs) {
+					context.context.addCACert(cert);
+				}
+			}
+			return context;
+		};
+	}
+}
+
 function configureModuleLoading(extensionService: ExtHostExtensionService, lookup: ReturnType<typeof createPatchedModules>): Promise<void> {
 	return extensionService.getExtensionPathIndex()
 		.then(extensionPaths => {
 			const node_module = <any>require.__$__nodeRequire('module');
 			const original = node_module._load;
 			node_module._load = function load(request: string, parent: any, isMain: any) {
+				if (request === 'tls') {
+					return lookup.tls;
+				}
+
 				if (request !== 'http' && request !== 'https') {
 					return original.apply(this, arguments);
 				}
@@ -314,4 +405,120 @@ function configureModuleLoading(extensionService: ExtHostExtensionService, looku
 				return modules.default;
 			};
 		});
-}
+}
+
+function useSystemCertificates(extHostLogService: ExtHostLogService, useSystemCertificates: boolean, opts: http.RequestOptions, callback: () => void) {
+	if (useSystemCertificates) {
+		getCaCertificates(extHostLogService)
+			.then(caCertificates => {
+				if (caCertificates) {
+					if (caCertificates.append) {
+						(opts as any)._vscodeAdditionalCaCerts = caCertificates.certs;
+					} else {
+						(opts as https.RequestOptions).ca = caCertificates.certs;
+					}
+				}
+				callback();
+			})
+			.catch(err => {
+				extHostLogService.error('ProxyResolver#useSystemCertificates', toErrorMessage(err));
+			});
+	} else {
+		callback();
+	}
+}
+
+let _caCertificates: ReturnType<typeof readCaCertificates> | Promise<undefined>;
+async function getCaCertificates(extHostLogService: ExtHostLogService) {
+	if (!_caCertificates) {
+		_caCertificates = readCaCertificates()
+			.then(res => res && res.certs.length ? res : undefined)
+			.catch(err => {
+				extHostLogService.error('ProxyResolver#getCertificates', toErrorMessage(err));
+				return undefined;
+			});
+	}
+	return _caCertificates;
+}
+
+async function readCaCertificates() {
+	if (process.platform === 'win32') {
+		return readWindowsCaCertificates();
+	}
+	if (process.platform === 'darwin') {
+		return readMacCaCertificates();
+	}
+	if (process.platform === 'linux') {
+		return readLinuxCaCertificates();
+	}
+	return undefined;
+}
+
+function readWindowsCaCertificates() {
+	const winCA = require.__$__nodeRequire<any>('win-ca-lib');
+
+	let ders = [];
+	const store = winCA();
+	try {
+		let der;
+		while (der = store.next()) {
+			ders.push(der);
+		}
+	} finally {
+		store.done();
+	}
+
+	const seen = {};
+	const certs = ders.map(derToPem)
+		.filter(pem => !seen[pem] && (seen[pem] = true));
+	return {
+		certs,
+		append: true
+	};
+}
+
+async function readMacCaCertificates() {
+	const stdout = (await promisify(cp.execFile)('/usr/bin/security', ['find-certificate', '-a', '-p'], { encoding: 'utf8' })).stdout;
+	const seen = {};
+	const certs = stdout.split(/(?=-----BEGIN CERTIFICATE-----)/g)
+		.filter(pem => !!pem.length && !seen[pem] && (seen[pem] = true));
+	return {
+		certs,
+		append: true
+	};
+}
+
+const linuxCaCertificatePaths = [
+	'/etc/ssl/certs/ca-certificates.crt',
+	'/etc/ssl/certs/ca-bundle.crt',
+];
+
+async function readLinuxCaCertificates() {
+	for (const certPath of linuxCaCertificatePaths) {
+		try {
+			const content = await promisify(fs.readFile)(certPath, { encoding: 'utf8' });
+			const seen = {};
+			const certs = content.split(/(?=-----BEGIN CERTIFICATE-----)/g)
+				.filter(pem => !!pem.length && !seen[pem] && (seen[pem] = true));
+			return {
+				certs,
+				append: false
+			};
+		} catch (err) {
+			if (err.code !== 'ENOENT') {
+				throw err;
+			}
+		}
+	}
+	return undefined;
+}
+
+function derToPem(blob) {
+	const lines = ['-----BEGIN CERTIFICATE-----'];
+	const der = blob.toString('base64');
+	for (let i = 0; i < der.length; i += 64) {
+		lines.push(der.substr(i, 64));
+	}
+	lines.push('-----END CERTIFICATE-----', '');
+	return lines.join(os.EOL);
+}