Merge from master

build/azure-pipelines/common/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+*.js

build/azure-pipelines/common/installDistro.ts

@@ -0,0 +1,18 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as  cp from 'child_process';
+
+function yarnInstall(packageName: string): void {
+	cp.execSync(`yarn add --no-lockfile ${packageName}`);
+}
+
+const product = require('../../../product.json');
+const dependencies = product.dependencies || {} as { [name: string]: string; };
+
+Object.keys(dependencies).forEach(name => {
+	const url = dependencies[name];
+	yarnInstall(url);
+});

build/azure-pipelines/common/publish.ts

@@ -0,0 +1,286 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as fs from 'fs';
+import { execSync } from 'child_process';
+import { Readable } from 'stream';
+import * as crypto from 'crypto';
+import * as azure from 'azure-storage';
+import * as mime from 'mime';
+import * as minimist from 'minimist';
+import { DocumentClient, NewDocument } from 'documentdb';
+
+if (process.argv.length < 6) {
+	console.error('Usage: node publish.js <product> <platform> <type> <name> <version> <commit> <is_update> <file>');
+	process.exit(-1);
+}
+
+function hashStream(hashName: string, stream: Readable): Promise<string> {
+	return new Promise<string>((c, e) => {
+		const shasum = crypto.createHash(hashName);
+
+		stream
+			.on('data', shasum.update.bind(shasum))
+			.on('error', e)
+			.on('close', () => c(shasum.digest('hex')));
+	});
+}
+
+interface Config {
+	id: string;
+	frozen: boolean;
+}
+
+function createDefaultConfig(quality: string): Config {
+	return {
+		id: quality,
+		frozen: false
+	};
+}
+
+function getConfig(quality: string): Promise<Config> {
+	const client = new DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT']!, { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const collection = 'dbs/builds/colls/config';
+	const query = {
+		query: `SELECT TOP 1 * FROM c WHERE c.id = @quality`,
+		parameters: [
+			{ name: '@quality', value: quality }
+		]
+	};
+
+	return new Promise<Config>((c, e) => {
+		client.queryDocuments(collection, query).toArray((err, results) => {
+			if (err && err.code !== 409) { return e(err); }
+
+			c(!results || results.length === 0 ? createDefaultConfig(quality) : results[0] as any as Config);
+		});
+	});
+}
+
+interface Asset {
+	platform: string;
+	type: string;
+	url: string;
+	mooncakeUrl: string;
+	hash: string;
+	sha256hash: string;
+	size: number;
+	supportsFastUpdate?: boolean;
+}
+
+function createOrUpdate(commit: string, quality: string, platform: string, type: string, release: NewDocument, asset: Asset, isUpdate: boolean): Promise<void> {
+	const client = new DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT']!, { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const collection = 'dbs/builds/colls/' + quality;
+	const updateQuery = {
+		query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
+		parameters: [{ name: '@id', value: commit }]
+	};
+
+	let updateTries = 0;
+
+	function update(): Promise<void> {
+		updateTries++;
+
+		return new Promise<void>((c, e) => {
+			client.queryDocuments(collection, updateQuery).toArray((err, results) => {
+				if (err) { return e(err); }
+				if (results.length !== 1) { return e(new Error('No documents')); }
+
+				const release = results[0];
+
+				release.assets = [
+					...release.assets.filter((a: any) => !(a.platform === platform && a.type === type)),
+					asset
+				];
+
+				if (isUpdate) {
+					release.updates[platform] = type;
+				}
+
+				client.replaceDocument(release._self, release, err => {
+					if (err && err.code === 409 && updateTries < 5) { return c(update()); }
+					if (err) { return e(err); }
+
+					console.log('Build successfully updated.');
+					c();
+				});
+			});
+		});
+	}
+
+	return new Promise<void>((c, e) => {
+		client.createDocument(collection, release, err => {
+			if (err && err.code === 409) { return c(update()); }
+			if (err) { return e(err); }
+
+			console.log('Build successfully published.');
+			c();
+		});
+	});
+}
+
+async function assertContainer(blobService: azure.BlobService, quality: string): Promise<void> {
+	await new Promise((c, e) => blobService.createContainerIfNotExists(quality, { publicAccessLevel: 'blob' }, err => err ? e(err) : c()));
+}
+
+async function doesAssetExist(blobService: azure.BlobService, quality: string, blobName: string): Promise<boolean | undefined> {
+	const existsResult = await new Promise<azure.BlobService.BlobResult>((c, e) => blobService.doesBlobExist(quality, blobName, (err, r) => err ? e(err) : c(r)));
+	return existsResult.exists;
+}
+
+async function uploadBlob(blobService: azure.BlobService, quality: string, blobName: string, file: string): Promise<void> {
+	const blobOptions: azure.BlobService.CreateBlockBlobRequestOptions = {
+		contentSettings: {
+			contentType: mime.lookup(file),
+			cacheControl: 'max-age=31536000, public'
+		}
+	};
+
+	await new Promise((c, e) => blobService.createBlockBlobFromLocalFile(quality, blobName, file, blobOptions, err => err ? e(err) : c()));
+}
+
+interface PublishOptions {
+	'upload-only': boolean;
+}
+
+async function publish(commit: string, quality: string, platform: string, type: string, name: string, version: string, _isUpdate: string, file: string, opts: PublishOptions): Promise<void> {
+	const isUpdate = _isUpdate === 'true';
+
+	const queuedBy = process.env['BUILD_QUEUEDBY']!;
+	const sourceBranch = process.env['BUILD_SOURCEBRANCH']!;
+	const isReleased = quality === 'insider'
+		&& /^master$|^refs\/heads\/master$/.test(sourceBranch)
+		&& /Project Collection Service Accounts|Microsoft.VisualStudio.Services.TFS/.test(queuedBy);
+
+	console.log('Publishing...');
+	console.log('Quality:', quality);
+	console.log('Platform:', platform);
+	console.log('Type:', type);
+	console.log('Name:', name);
+	console.log('Version:', version);
+	console.log('Commit:', commit);
+	console.log('Is Update:', isUpdate);
+	console.log('Is Released:', isReleased);
+	console.log('File:', file);
+
+	const stat = await new Promise<fs.Stats>((c, e) => fs.stat(file, (err, stat) => err ? e(err) : c(stat)));
+	const size = stat.size;
+
+	console.log('Size:', size);
+
+	const stream = fs.createReadStream(file);
+	const [sha1hash, sha256hash] = await Promise.all([hashStream('sha1', stream), hashStream('sha256', stream)]);
+
+	console.log('SHA1:', sha1hash);
+	console.log('SHA256:', sha256hash);
+
+	const blobName = commit + '/' + name;
+	const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2']!;
+
+	const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY']!, `${storageAccount}.blob.core.chinacloudapi.cn`)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	// mooncake is fussy and far away, this is needed!
+	mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
+
+	await Promise.all([
+		assertContainer(blobService, quality),
+		assertContainer(mooncakeBlobService, quality)
+	]);
+
+	const [blobExists, moooncakeBlobExists] = await Promise.all([
+		doesAssetExist(blobService, quality, blobName),
+		doesAssetExist(mooncakeBlobService, quality, blobName)
+	]);
+
+	const promises: Array<Promise<void>> = [];
+
+	if (!blobExists) {
+		promises.push(uploadBlob(blobService, quality, blobName, file));
+	}
+
+	if (!moooncakeBlobExists) {
+		promises.push(uploadBlob(mooncakeBlobService, quality, blobName, file));
+	}
+
+	if (promises.length === 0) {
+		console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
+		return;
+	}
+
+	console.log('Uploading blobs to Azure storage...');
+
+	await Promise.all(promises);
+
+	console.log('Blobs successfully uploaded.');
+
+	const config = await getConfig(quality);
+
+	console.log('Quality config:', config);
+
+	const asset: Asset = {
+		platform: platform,
+		type: type,
+		url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
+		mooncakeUrl: `${process.env['MOONCAKE_CDN_URL']}/${quality}/${blobName}`,
+		hash: sha1hash,
+		sha256hash,
+		size
+	};
+
+	// Remove this if we ever need to rollback fast updates for windows
+	if (/win32/.test(platform)) {
+		asset.supportsFastUpdate = true;
+	}
+
+	console.log('Asset:', JSON.stringify(asset, null, '  '));
+
+	const release = {
+		id: commit,
+		timestamp: (new Date()).getTime(),
+		version,
+		isReleased: config.frozen ? false : isReleased,
+		sourceBranch,
+		queuedBy,
+		assets: [] as Array<Asset>,
+		updates: {} as any
+	};
+
+	if (!opts['upload-only']) {
+		release.assets.push(asset);
+
+		if (isUpdate) {
+			release.updates[platform] = type;
+		}
+	}
+
+	await createOrUpdate(commit, quality, platform, type, release, asset, isUpdate);
+}
+
+function main(): void {
+	if (process.env['VSCODE_BUILD_SKIP_PUBLISH']) {
+		console.warn('Skipping publish due to VSCODE_BUILD_SKIP_PUBLISH');
+		return;
+	}
+
+	const opts = minimist<PublishOptions>(process.argv.slice(2), {
+		boolean: ['upload-only']
+	});
+
+	const [quality, platform, type, name, version, _isUpdate, file] = opts._;
+	const commit = execSync('git rev-parse HEAD', { encoding: 'utf8' }).trim();
+
+	publish(commit, quality, platform, type, name, version, _isUpdate, file, opts).catch(err => {
+		console.error(err);
+		process.exit(1);
+	});
+}
+
+main();

build/azure-pipelines/common/symbols.ts

@@ -0,0 +1,219 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as request from 'request';
+import { createReadStream, createWriteStream, unlink, mkdir } from 'fs';
+import * as github from 'github-releases';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { promisify } from 'util';
+
+const BASE_URL = 'https://rink.hockeyapp.net/api/2/';
+const HOCKEY_APP_TOKEN_HEADER = 'X-HockeyAppToken';
+
+export interface IVersions {
+	app_versions: IVersion[];
+}
+
+export interface IVersion {
+	id: number;
+	version: string;
+}
+
+export interface IApplicationAccessor {
+	accessToken: string;
+	appId: string;
+}
+
+export interface IVersionAccessor extends IApplicationAccessor {
+	id: string;
+}
+
+enum Platform {
+	WIN_32 = 'win32-ia32',
+	WIN_64 = 'win32-x64',
+	LINUX_32 = 'linux-ia32',
+	LINUX_64 = 'linux-x64',
+	MAC_OS = 'darwin-x64'
+}
+
+function symbolsZipName(platform: Platform, electronVersion: string, insiders: boolean): string {
+	return `${insiders ? 'insiders' : 'stable'}-symbols-v${electronVersion}-${platform}.zip`;
+}
+
+const SEED = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+async function tmpFile(name: string): Promise<string> {
+	let res = '';
+	for (let i = 0; i < 8; i++) {
+		res += SEED.charAt(Math.floor(Math.random() * SEED.length));
+	}
+
+	const tmpParent = join(tmpdir(), res);
+
+	await promisify(mkdir)(tmpParent);
+
+	return join(tmpParent, name);
+}
+
+function getVersions(accessor: IApplicationAccessor): Promise<IVersions> {
+	return asyncRequest<IVersions>({
+		url: `${BASE_URL}/apps/${accessor.appId}/app_versions`,
+		method: 'GET',
+		headers: {
+			[HOCKEY_APP_TOKEN_HEADER]: accessor.accessToken
+		}
+	});
+}
+
+function createVersion(accessor: IApplicationAccessor, version: string): Promise<IVersion> {
+	return asyncRequest<IVersion>({
+		url: `${BASE_URL}/apps/${accessor.appId}/app_versions/new`,
+		method: 'POST',
+		headers: {
+			[HOCKEY_APP_TOKEN_HEADER]: accessor.accessToken
+		},
+		formData: {
+			bundle_version: version
+		}
+	});
+}
+
+function updateVersion(accessor: IVersionAccessor, symbolsPath: string) {
+	return asyncRequest<IVersions>({
+		url: `${BASE_URL}/apps/${accessor.appId}/app_versions/${accessor.id}`,
+		method: 'PUT',
+		headers: {
+			[HOCKEY_APP_TOKEN_HEADER]: accessor.accessToken
+		},
+		formData: {
+			dsym: createReadStream(symbolsPath)
+		}
+	});
+}
+
+function asyncRequest<T>(options: request.UrlOptions & request.CoreOptions): Promise<T> {
+	return new Promise<T>((resolve, reject) => {
+		request(options, (error, _response, body) => {
+			if (error) {
+				reject(error);
+			} else {
+				resolve(JSON.parse(body));
+			}
+		});
+	});
+}
+
+function downloadAsset(repository: any, assetName: string, targetPath: string, electronVersion: string) {
+	return new Promise((resolve, reject) => {
+		repository.getReleases({ tag_name: `v${electronVersion}` }, (err: any, releases: any) => {
+			if (err) {
+				reject(err);
+			} else {
+				const asset = releases[0].assets.filter((asset: any) => asset.name === assetName)[0];
+				if (!asset) {
+					reject(new Error(`Asset with name ${assetName} not found`));
+				} else {
+					repository.downloadAsset(asset, (err: any, reader: any) => {
+						if (err) {
+							reject(err);
+						} else {
+							const writer = createWriteStream(targetPath);
+							writer.on('error', reject);
+							writer.on('close', resolve);
+							reader.on('error', reject);
+
+							reader.pipe(writer);
+						}
+					});
+				}
+			}
+		});
+	});
+}
+
+interface IOptions {
+	repository: string;
+	platform: Platform;
+	versions: { code: string; insiders: boolean; electron: string; };
+	access: { hockeyAppToken: string; hockeyAppId: string; githubToken: string };
+}
+
+async function ensureVersionAndSymbols(options: IOptions) {
+
+	// Check version does not exist
+	console.log(`HockeyApp: checking for existing version ${options.versions.code} (${options.platform})`);
+	const versions = await getVersions({ accessToken: options.access.hockeyAppToken, appId: options.access.hockeyAppId });
+	if (versions.app_versions.some(v => v.version === options.versions.code)) {
+		console.log(`HockeyApp: Returning without uploading symbols because version ${options.versions.code} (${options.platform}) was already found`);
+		return;
+	}
+
+	// Download symbols for platform and electron version
+	const symbolsName = symbolsZipName(options.platform, options.versions.electron, options.versions.insiders);
+	const symbolsPath = await tmpFile('symbols.zip');
+	console.log(`HockeyApp: downloading symbols ${symbolsName} for electron ${options.versions.electron} (${options.platform}) into ${symbolsPath}`);
+	await downloadAsset(new (github as any)({ repo: options.repository, token: options.access.githubToken }), symbolsName, symbolsPath, options.versions.electron);
+
+	// Create version
+	console.log(`HockeyApp: creating new version ${options.versions.code} (${options.platform})`);
+	const version = await createVersion({ accessToken: options.access.hockeyAppToken, appId: options.access.hockeyAppId }, options.versions.code);
+
+	// Upload symbols
+	console.log(`HockeyApp: uploading symbols for version ${options.versions.code} (${options.platform})`);
+	await updateVersion({ id: String(version.id), accessToken: options.access.hockeyAppToken, appId: options.access.hockeyAppId }, symbolsPath);
+
+	// Cleanup
+	await promisify(unlink)(symbolsPath);
+}
+
+// Environment
+const pakage = require('../../../package.json');
+const product = require('../../../product.json');
+const repository = product.electronRepository;
+const electronVersion = require('../../lib/electron').getElectronVersion();
+const insiders = product.quality !== 'stable';
+let codeVersion = pakage.version;
+if (insiders) {
+	codeVersion = `${codeVersion}-insider`;
+}
+const githubToken = process.argv[2];
+const hockeyAppToken = process.argv[3];
+const is64 = process.argv[4] === 'x64';
+const hockeyAppId = process.argv[5];
+
+let platform: Platform;
+if (process.platform === 'darwin') {
+	platform = Platform.MAC_OS;
+} else if (process.platform === 'win32') {
+	platform = is64 ? Platform.WIN_64 : Platform.WIN_32;
+} else {
+	platform = is64 ? Platform.LINUX_64 : Platform.LINUX_32;
+}
+
+// Create version and upload symbols in HockeyApp
+if (repository && codeVersion && electronVersion && (product.quality === 'stable' || product.quality === 'insider')) {
+	ensureVersionAndSymbols({
+		repository,
+		platform,
+		versions: {
+			code: codeVersion,
+			insiders,
+			electron: electronVersion
+		},
+		access: {
+			githubToken,
+			hockeyAppToken,
+			hockeyAppId
+		}
+	}).then(() => {
+		console.log('HockeyApp: done');
+	}).catch(error => {
+		console.error(`HockeyApp: error (${error})`);
+	});
+} else {
+	console.log(`HockeyApp: skipping due to unexpected context (repository: ${repository}, codeVersion: ${codeVersion}, electronVersion: ${electronVersion}, quality: ${product.quality})`);
+}