Initial VS Code 1.19 source merge (#571)

* Initial 1.19 xcopy

* Fix yarn build

* Fix numerous build breaks

* Next batch of build break fixes

* More build break fixes

* Runtime breaks

* Additional post merge fixes

* Fix windows setup file

* Fix test failures.

* Update license header blocks to refer to source eula

src/vs/platform/extensionManagement/node/extensionGalleryService.ts

@@ -7,21 +7,23 @@ import { localize } from 'vs/nls';
 import { tmpdir } from 'os';
 import * as path from 'path';
 import { TPromise } from 'vs/base/common/winjs.base';
-import * as uuid from 'vs/base/common/uuid';
 import { distinct } from 'vs/base/common/arrays';
 import { getErrorMessage, isPromiseCanceledError } from 'vs/base/common/errors';
-import { StatisticType, IGalleryExtension, IExtensionGalleryService, IGalleryExtensionAsset, IQueryOptions, SortBy, SortOrder, IExtensionManifest } from 'vs/platform/extensionManagement/common/extensionManagement';
+import { StatisticType, IGalleryExtension, IExtensionGalleryService, IGalleryExtensionAsset, IQueryOptions, SortBy, SortOrder, IExtensionManifest, IExtensionIdentifier, IReportedExtension } from 'vs/platform/extensionManagement/common/extensionManagement';
 import { getGalleryExtensionId, getGalleryExtensionTelemetryData, adoptToGalleryExtensionId } from 'vs/platform/extensionManagement/common/extensionManagementUtil';
 import { assign, getOrDefault } from 'vs/base/common/objects';
 import { IRequestService } from 'vs/platform/request/node/request';
 import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
 import { IPager } from 'vs/base/common/paging';
 import { IRequestOptions, IRequestContext, download, asJson, asText } from 'vs/base/node/request';
-import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import pkg from 'vs/platform/node/package';
 import product from 'vs/platform/node/product';
 import { isVersionValid } from 'vs/platform/extensions/node/extensionValidator';
 import { IEnvironmentService } from 'vs/platform/environment/common/environment';
+import { readFile } from 'vs/base/node/pfs';
+import { writeFileAndFlushSync } from 'vs/base/node/extfs';
+import { generateUuid, isUUID } from 'vs/base/common/uuid';
+import { values } from 'vs/base/common/map';
 
 interface IRawGalleryExtensionFile {
 	assetType: string;
@@ -55,6 +57,7 @@ interface IRawGalleryExtension {
 	publisher: { displayName: string, publisherId: string, publisherName: string; };
 	versions: IRawGalleryExtensionVersion[];
 	statistics: IRawGalleryExtensionStatistics[];
+	flags: string;
 }
 
 interface IRawGalleryQueryResult {
@@ -107,6 +110,7 @@ const AssetType = {
 	Manifest: 'Microsoft.VisualStudio.Code.Manifest',
 	VSIX: 'Microsoft.VisualStudio.Services.VSIXPackage',
 	License: 'Microsoft.VisualStudio.Services.Content.License',
+	Repository: 'Microsoft.VisualStudio.Services.Links.Source'
 };
 
 const PropertyType = {
@@ -200,6 +204,26 @@ function getStatistic(statistics: IRawGalleryExtensionStatistics[], name: string
 function getVersionAsset(version: IRawGalleryExtensionVersion, type: string): IGalleryExtensionAsset {
 	const result = version.files.filter(f => f.assetType === type)[0];
 
+	if (type === AssetType.Repository) {
+		if (version.properties) {
+			const results = version.properties.filter(p => p.key === type);
+			const gitRegExp = new RegExp('((git|ssh|http(s)?)|(git@[\w\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)(/)?');
+
+			const uri = results.filter(r => gitRegExp.test(r.value))[0];
+			if (!uri) {
+				return {
+					uri: null,
+					fallbackUri: null
+				};
+			}
+
+			return {
+				uri: uri.value,
+				fallbackUri: uri.value,
+			};
+		}
+	}
+
 	if (!result) {
 		if (type === AssetType.Icon) {
 			const uri = require.toUrl('./media/defaultIcon.png');
@@ -233,6 +257,10 @@ function getEngine(version: IRawGalleryExtensionVersion): string {
 	return (values.length > 0 && values[0].value) || '';
 }
 
+function getIsPreview(flags: string): boolean {
+	return flags.indexOf('preview') !== -1;
+}
+
 function toExtension(galleryExtension: IRawGalleryExtension, extensionsGalleryUrl: string, index: number, query: Query, querySource?: string): IGalleryExtension {
 	const [version] = galleryExtension.versions;
 	const assets = {
@@ -241,7 +269,8 @@ function toExtension(galleryExtension: IRawGalleryExtension, extensionsGalleryUr
 		changelog: getVersionAsset(version, AssetType.Changelog),
 		download: getVersionAsset(version, AssetType.VSIX),
 		icon: getVersionAsset(version, AssetType.Icon),
-		license: getVersionAsset(version, AssetType.License)
+		license: getVersionAsset(version, AssetType.License),
+		repository: getVersionAsset(version, AssetType.Repository),
 	};
 
 	return {
@@ -276,31 +305,34 @@ function toExtension(galleryExtension: IRawGalleryExtension, extensionsGalleryUr
 			index: ((query.pageNumber - 1) * query.pageSize) + index,
 			searchText: query.searchText,
 			querySource
-		}
+		},
+		preview: getIsPreview(galleryExtension.flags)
 	};
 }
 
+interface IRawExtensionsReport {
+	malicious: string[];
+	slow: string[];
+}
+
 export class ExtensionGalleryService implements IExtensionGalleryService {
 
 	_serviceBrand: any;
 
 	private extensionsGalleryUrl: string;
+	private extensionsControlUrl: string;
 
-	private readonly commonHTTPHeaders: { [key: string]: string; };
+	private readonly commonHeadersPromise: TPromise<{ [key: string]: string; }>;
 
 	constructor(
 		@IRequestService private requestService: IRequestService,
 		@IEnvironmentService private environmentService: IEnvironmentService,
-		@ITelemetryService private telemetryService: ITelemetryService,
-		@IConfigurationService private configurationService: IConfigurationService
+		@ITelemetryService private telemetryService: ITelemetryService
 	) {
 		const config = product.extensionsGallery;
 		this.extensionsGalleryUrl = config && config.serviceUrl;
-		this.commonHTTPHeaders = {
-			'X-Market-Client-Id': `VSCode ${pkg.version}`,
-			'User-Agent': `VSCode ${pkg.version}`,
-			'X-Market-User-Id': this.environmentService.machineUUID
-		};
+		this.extensionsControlUrl = config && config.controlUrl;
+		this.commonHeadersPromise = resolveMarketplaceHeaders(this.environmentService);
 	}
 
 	private api(path = ''): string {
@@ -386,33 +418,34 @@ export class ExtensionGalleryService implements IExtensionGalleryService {
 	}
 
 	private queryGallery(query: Query): TPromise<{ galleryExtensions: IRawGalleryExtension[], total: number; }> {
-		const commonHeaders = this.commonHTTPHeaders;
-		const data = JSON.stringify(query.raw);
-		const headers = assign({}, commonHeaders, {
-			'Content-Type': 'application/json',
-			'Accept': 'application/json;api-version=3.0-preview.1',
-			'Accept-Encoding': 'gzip',
-			'Content-Length': data.length
-		});
+		return this.commonHeadersPromise.then(commonHeaders => {
+			const data = JSON.stringify(query.raw);
+			const headers = assign({}, commonHeaders, {
+				'Content-Type': 'application/json',
+				'Accept': 'application/json;api-version=3.0-preview.1',
+				'Accept-Encoding': 'gzip',
+				'Content-Length': data.length
+			});
 
-		return this.requestService.request({
-			type: 'POST',
-			url: this.api('/extensionquery'),
-			data,
-			headers
-		}).then(context => {
+			return this.requestService.request({
+				type: 'POST',
+				url: this.api('/extensionquery'),
+				data,
+				headers
+			}).then(context => {
 
-			if (context.res.statusCode >= 400 && context.res.statusCode < 500) {
-				return { galleryExtensions: [], total: 0 };
-			}
+				if (context.res.statusCode >= 400 && context.res.statusCode < 500) {
+					return { galleryExtensions: [], total: 0 };
+				}
 
-			return asJson<IRawGalleryQueryResult>(context).then(result => {
-				const r = result.results[0];
-				const galleryExtensions = r.extensions;
-				const resultCount = r.resultMetadata && r.resultMetadata.filter(m => m.metadataType === 'ResultCount')[0];
-				const total = resultCount && resultCount.metadataItems.filter(i => i.name === 'TotalCount')[0].count || 0;
+				return asJson<IRawGalleryQueryResult>(context).then(result => {
+					const r = result.results[0];
+					const galleryExtensions = r.extensions;
+					const resultCount = r.resultMetadata && r.resultMetadata.filter(m => m.metadataType === 'ResultCount')[0];
+					const total = resultCount && resultCount.metadataItems.filter(i => i.name === 'TotalCount')[0].count || 0;
 
-				return { galleryExtensions, total };
+					return { galleryExtensions, total };
+				});
 			});
 		});
 	}
@@ -422,35 +455,41 @@ export class ExtensionGalleryService implements IExtensionGalleryService {
 			return TPromise.as(null);
 		}
 
-		const headers = { ...this.commonHTTPHeaders, Accept: '*/*;api-version=4.0-preview.1' };
+		return this.commonHeadersPromise.then(commonHeaders => {
+			const headers = { ...commonHeaders, Accept: '*/*;api-version=4.0-preview.1' };
 
-		return this.requestService.request({
-			type: 'POST',
-			url: this.api(`/publishers/${publisher}/extensions/${name}/${version}/stats?statType=${type}`),
-			headers
-		}).then(null, () => null);
+			return this.requestService.request({
+				type: 'POST',
+				url: this.api(`/publishers/${publisher}/extensions/${name}/${version}/stats?statType=${type}`),
+				headers
+			}).then(null, () => null);
+		});
 	}
 
 	download(extension: IGalleryExtension): TPromise<string> {
-		return this.loadCompatibleVersion(extension).then(extension => {
-			const zipPath = path.join(tmpdir(), uuid.generateUuid());
-			const data = getGalleryExtensionTelemetryData(extension);
-			const startTime = new Date().getTime();
-			/* __GDPR__
-				"galleryService:downloadVSIX" : {
-					"duration": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth" },
-					"${include}": [
-						"${GalleryExtensionTelemetryData}"
-					]
+		return this.loadCompatibleVersion(extension)
+			.then(extension => {
+				if (!extension) {
+					return TPromise.wrapError(new Error(localize('notCompatibleDownload', "Unable to download because the extension compatible with current version '{0}' of VS Code is not found.", pkg.version)));
 				}
-			*/
-			const log = (duration: number) => this.telemetryService.publicLog('galleryService:downloadVSIX', assign(data, { duration }));
+				const zipPath = path.join(tmpdir(), generateUuid());
+				const data = getGalleryExtensionTelemetryData(extension);
+				const startTime = new Date().getTime();
+				/* __GDPR__
+					"galleryService:downloadVSIX" : {
+						"duration": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth" },
+						"${include}": [
+							"${GalleryExtensionTelemetryData}"
+						]
+					}
+				*/
+				const log = (duration: number) => this.telemetryService.publicLog('galleryService:downloadVSIX', assign(data, { duration }));
 
-			return this.getAsset(extension.assets.download)
-				.then(context => download(zipPath, context))
-				.then(() => log(new Date().getTime() - startTime))
-				.then(() => zipPath);
-		});
+				return this.getAsset(extension.assets.download)
+					.then(context => download(zipPath, context))
+					.then(() => log(new Date().getTime() - startTime))
+					.then(() => zipPath);
+			});
 	}
 
 	getReadme(extension: IGalleryExtension): TPromise<string> {
@@ -469,9 +508,8 @@ export class ExtensionGalleryService implements IExtensionGalleryService {
 			.then(asText);
 	}
 
-	getAllDependencies(extension: IGalleryExtension): TPromise<IGalleryExtension[]> {
-		return this.loadCompatibleVersion(<IGalleryExtension>extension)
-			.then(compatible => this.getDependenciesReccursively(compatible.properties.dependencies, [], extension));
+	loadAllDependencies(extensions: IExtensionIdentifier[]): TPromise<IGalleryExtension[]> {
+		return this.getDependenciesReccursively(extensions.map(e => e.id), []);
 	}
 
 	loadCompatibleVersion(extension: IGalleryExtension): TPromise<IGalleryExtension> {
@@ -487,22 +525,26 @@ export class ExtensionGalleryService implements IExtensionGalleryService {
 			.withAssetTypes(AssetType.Manifest, AssetType.VSIX)
 			.withFilter(FilterType.ExtensionId, extension.identifier.uuid);
 
-		return this.queryGallery(query).then(({ galleryExtensions }) => {
-			const [rawExtension] = galleryExtensions;
+		return this.queryGallery(query)
+			.then(({ galleryExtensions }) => {
+				const [rawExtension] = galleryExtensions;
 
-			if (!rawExtension) {
-				return TPromise.wrapError<IGalleryExtension>(new Error(localize('notFound', "Extension not found")));
-			}
+				if (!rawExtension) {
+					return null;
+				}
 
-			return this.getLastValidExtensionVersion(rawExtension, rawExtension.versions)
-				.then(rawVersion => {
-					extension.properties.dependencies = getDependencies(rawVersion);
-					extension.properties.engine = getEngine(rawVersion);
-					extension.assets.download = getVersionAsset(rawVersion, AssetType.VSIX);
-					extension.version = rawVersion.version;
-					return extension;
-				});
-		});
+				return this.getLastValidExtensionVersion(rawExtension, rawExtension.versions)
+					.then(rawVersion => {
+						if (rawVersion) {
+							extension.properties.dependencies = getDependencies(rawVersion);
+							extension.properties.engine = getEngine(rawVersion);
+							extension.assets.download = getVersionAsset(rawVersion, AssetType.VSIX);
+							extension.version = rawVersion.version;
+							return extension;
+						}
+						return null;
+					});
+			});
 	}
 
 	private loadDependencies(extensionNames: string[]): TPromise<IGalleryExtension[]> {
@@ -533,13 +575,10 @@ export class ExtensionGalleryService implements IExtensionGalleryService {
 		});
 	}
 
-	private getDependenciesReccursively(toGet: string[], result: IGalleryExtension[], root: IGalleryExtension): TPromise<IGalleryExtension[]> {
+	private getDependenciesReccursively(toGet: string[], result: IGalleryExtension[]): TPromise<IGalleryExtension[]> {
 		if (!toGet || !toGet.length) {
 			return TPromise.wrap(result);
 		}
-		if (toGet.indexOf(`${root.publisher}.${root.name}`) !== -1 && result.indexOf(root) === -1) {
-			result.push(root);
-		}
 		toGet = result.length ? toGet.filter(e => !ExtensionGalleryService.hasExtensionByName(result, e)) : toGet;
 		if (!toGet.length) {
 			return TPromise.wrap(result);
@@ -556,52 +595,30 @@ export class ExtensionGalleryService implements IExtensionGalleryService {
 				result = distinct(result.concat(loadedDependencies), d => d.identifier.uuid);
 				const dependencies: string[] = [];
 				dependenciesSet.forEach(d => !ExtensionGalleryService.hasExtensionByName(result, d) && dependencies.push(d));
-				return this.getDependenciesReccursively(dependencies, result, root);
+				return this.getDependenciesReccursively(dependencies, result);
 			});
 	}
 
 	private getAsset(asset: IGalleryExtensionAsset, options: IRequestOptions = {}): TPromise<IRequestContext> {
-		const baseOptions = { type: 'GET' };
-		const headers = assign({}, this.commonHTTPHeaders, options.headers || {});
-		options = assign({}, options, baseOptions, { headers });
+		return this.commonHeadersPromise.then(commonHeaders => {
+			const baseOptions = { type: 'GET' };
+			const headers = assign({}, commonHeaders, options.headers || {});
+			options = assign({}, options, baseOptions, { headers });
 
-		const url = asset.uri;
-		const fallbackUrl = asset.fallbackUri;
-		const firstOptions = assign({}, options, { url });
+			const url = asset.uri;
+			const fallbackUrl = asset.fallbackUri;
+			const firstOptions = assign({}, options, { url });
 
-		return this.requestService.request(firstOptions)
-			.then(context => {
-				if (context.res.statusCode === 200) {
-					return TPromise.as(context);
-				}
-
-				return asText(context)
-					.then(message => TPromise.wrapError<IRequestContext>(new Error(`Expected 200, got back ${context.res.statusCode} instead.\n\n${message}`)));
-			})
-			.then(null, err => {
-				if (isPromiseCanceledError(err)) {
-					return TPromise.wrapError<IRequestContext>(err);
-				}
-
-				const message = getErrorMessage(err);
-				/* __GDPR__
-					"galleryService:requestError" : {
-						"url" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
-						"cdn": { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
-						"message": { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
+			return this.requestService.request(firstOptions)
+				.then(context => {
+					if (context.res.statusCode === 200) {
+						return TPromise.as(context);
 					}
-				*/
-				this.telemetryService.publicLog('galleryService:requestError', { url, cdn: true, message });
-				/* __GDPR__
-					"galleryService:cdnFallback" : {
-						"url" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
-						"message": { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
-					}
-				*/
-				this.telemetryService.publicLog('galleryService:cdnFallback', { url, message });
 
-				const fallbackOptions = assign({}, options, { url: fallbackUrl });
-				return this.requestService.request(fallbackOptions).then(null, err => {
+					return asText(context)
+						.then(message => TPromise.wrapError<IRequestContext>(new Error(`Expected 200, got back ${context.res.statusCode} instead.\n\n${message}`)));
+				})
+				.then(null, err => {
 					if (isPromiseCanceledError(err)) {
 						return TPromise.wrapError<IRequestContext>(err);
 					}
@@ -614,10 +631,34 @@ export class ExtensionGalleryService implements IExtensionGalleryService {
 							"message": { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
 						}
 					*/
-					this.telemetryService.publicLog('galleryService:requestError', { url: fallbackUrl, cdn: false, message });
-					return TPromise.wrapError<IRequestContext>(err);
+					this.telemetryService.publicLog('galleryService:requestError', { url, cdn: true, message });
+					/* __GDPR__
+						"galleryService:cdnFallback" : {
+							"url" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
+							"message": { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
+						}
+					*/
+					this.telemetryService.publicLog('galleryService:cdnFallback', { url, message });
+
+					const fallbackOptions = assign({}, options, { url: fallbackUrl });
+					return this.requestService.request(fallbackOptions).then(null, err => {
+						if (isPromiseCanceledError(err)) {
+							return TPromise.wrapError<IRequestContext>(err);
+						}
+
+						const message = getErrorMessage(err);
+						/* __GDPR__
+							"galleryService:requestError" : {
+								"url" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
+								"cdn": { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
+								"message": { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
+							}
+						*/
+						this.telemetryService.publicLog('galleryService:requestError', { url: fallbackUrl, cdn: false, message });
+						return TPromise.wrapError<IRequestContext>(err);
+					});
 				});
-			});
+		});
 	}
 
 	private getLastValidExtensionVersion(extension: IRawGalleryExtension, versions: IRawGalleryExtensionVersion[]): TPromise<IRawGalleryExtensionVersion> {
@@ -643,7 +684,7 @@ export class ExtensionGalleryService implements IExtensionGalleryService {
 
 	private getLastValidExtensionVersionReccursively(extension: IRawGalleryExtension, versions: IRawGalleryExtensionVersion[]): TPromise<IRawGalleryExtensionVersion> {
 		if (!versions.length) {
-			return TPromise.wrapError<IRawGalleryExtensionVersion>(new Error(localize('noCompatible', "Couldn't find a compatible version of {0} with this version of Code.", extension.displayName || extension.extensionName)));
+			return null;
 		}
 
 		const version = versions[0];
@@ -678,4 +719,62 @@ export class ExtensionGalleryService implements IExtensionGalleryService {
 		}
 		return false;
 	}
+
+	getExtensionsReport(): TPromise<IReportedExtension[]> {
+		if (!this.isEnabled()) {
+			return TPromise.wrapError(new Error('No extension gallery service configured.'));
+		}
+
+		if (!this.extensionsControlUrl) {
+			return TPromise.as([]);
+		}
+
+		return this.requestService.request({ type: 'GET', url: this.extensionsControlUrl }).then(context => {
+			if (context.res.statusCode !== 200) {
+				return TPromise.wrapError(new Error('Could not get extensions report.'));
+			}
+
+			return asJson<IRawExtensionsReport>(context).then(result => {
+				const map = new Map<string, IReportedExtension>();
+
+				for (const id of result.malicious) {
+					const ext = map.get(id) || { id: { id }, malicious: true, slow: false };
+					ext.malicious = true;
+					map.set(id, ext);
+				}
+
+				return TPromise.as(values(map));
+			});
+		});
+	}
 }
+
+export function resolveMarketplaceHeaders(environmentService: IEnvironmentService): TPromise<{ [key: string]: string; }> {
+	const marketplaceMachineIdFile = path.join(environmentService.userDataPath, 'machineid');
+
+	return readFile(marketplaceMachineIdFile, 'utf8').then(contents => {
+		if (isUUID(contents)) {
+			return contents;
+		}
+
+		return TPromise.wrap(null); // invalid marketplace UUID
+	}, error => {
+		return TPromise.wrap(null); // error reading ID file
+	}).then(uuid => {
+		if (!uuid) {
+			uuid = generateUuid();
+
+			try {
+				writeFileAndFlushSync(marketplaceMachineIdFile, uuid);
+			} catch (error) {
+				//noop
+			}
+		}
+
+		return {
+			'X-Market-Client-Id': `VSCode ${pkg.version}`,
+			'User-Agent': `VSCode ${pkg.version}`,
+			'X-Market-User-Id': uuid
+		};
+	});
+}