Merge from vscode 31e03b8ffbb218a87e3941f2b63a249f061fe0e4 (#4986)

build/azure-pipelines/common/sync-mooncake.ts

@@ -0,0 +1,176 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as url from 'url';
+import * as azure from 'azure-storage';
+import * as mime from 'mime';
+import { DocumentClient, RetrievedDocument } from 'documentdb';
+
+function log(...args: any[]) {
+	console.log(...[`[${new Date().toISOString()}]`, ...args]);
+}
+
+function error(...args: any[]) {
+	console.error(...[`[${new Date().toISOString()}]`, ...args]);
+}
+
+if (process.argv.length < 3) {
+	error('Usage: node sync-mooncake.js <quality>');
+	process.exit(-1);
+}
+
+interface Build extends RetrievedDocument {
+	assets: Asset[];
+}
+
+interface Asset {
+	platform: string;
+	type: string;
+	url: string;
+	mooncakeUrl: string;
+	hash: string;
+	sha256hash: string;
+	size: number;
+	supportsFastUpdate?: boolean;
+}
+
+function updateBuild(commit: string, quality: string, platform: string, type: string, asset: Asset): Promise<void> {
+	const client = new DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT']!, { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const collection = 'dbs/builds/colls/' + quality;
+	const updateQuery = {
+		query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
+		parameters: [{ name: '@id', value: commit }]
+	};
+
+	let updateTries = 0;
+
+	function _update(): Promise<void> {
+		updateTries++;
+
+		return new Promise<void>((c, e) => {
+			client.queryDocuments(collection, updateQuery).toArray((err, results) => {
+				if (err) { return e(err); }
+				if (results.length !== 1) { return e(new Error('No documents')); }
+
+				const release = results[0];
+
+				release.assets = [
+					...release.assets.filter((a: any) => !(a.platform === platform && a.type === type)),
+					asset
+				];
+
+				client.replaceDocument(release._self, release, err => {
+					if (err && err.code === 409 && updateTries < 5) { return c(_update()); }
+					if (err) { return e(err); }
+
+					log('Build successfully updated.');
+					c();
+				});
+			});
+		});
+	}
+
+	return _update();
+}
+
+async function sync(commit: string, quality: string): Promise<void> {
+	log(`Synchronizing Mooncake assets for ${quality}, ${commit}...`);
+
+	const cosmosdb = new DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT']!, { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const collection = `dbs/builds/colls/${quality}`;
+	const query = {
+		query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
+		parameters: [{ name: '@id', value: commit }]
+	};
+
+	const build = await new Promise<Build>((c, e) => {
+		cosmosdb.queryDocuments(collection, query).toArray((err, results) => {
+			if (err) { return e(err); }
+			if (results.length !== 1) { return e(new Error('No documents')); }
+			c(results[0] as Build);
+		});
+	});
+
+	log(`Found build for ${commit}, with ${build.assets.length} assets`);
+
+	const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2']!;
+
+	const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY']!, `${storageAccount}.blob.core.chinacloudapi.cn`)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	// mooncake is fussy and far away, this is needed!
+	blobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
+	mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
+
+	for (const asset of build.assets) {
+		try {
+			const blobPath = url.parse(asset.url).path;
+
+			if (!blobPath) {
+				throw new Error(`Failed to parse URL: ${asset.url}`);
+			}
+
+			const blobName = blobPath.replace(/^\/\w+\//, '');
+
+			log(`Found ${blobName}`);
+
+			if (asset.mooncakeUrl) {
+				log(`  Already in Mooncake ✔️`);
+				continue;
+			}
+
+			const readStream = blobService.createReadStream(quality, blobName, undefined!);
+			const blobOptions: azure.BlobService.CreateBlockBlobRequestOptions = {
+				contentSettings: {
+					contentType: mime.lookup(blobPath),
+					cacheControl: 'max-age=31536000, public'
+				}
+			};
+
+			const writeStream = mooncakeBlobService.createWriteStreamToBlockBlob(quality, blobName, blobOptions, undefined);
+
+			log(`  Uploading to Mooncake...`);
+			await new Promise((c, e) => readStream.pipe(writeStream).on('finish', c).on('error', e));
+
+			log(`  Updating build in DB...`);
+			asset.mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
+			await updateBuild(commit, quality, asset.platform, asset.type, asset);
+
+			log(`  Done ✔️`);
+		} catch (err) {
+			error(err);
+		}
+	}
+
+	log(`All done ✔️`);
+}
+
+function main(): void {
+	if (process.env['VSCODE_BUILD_SKIP_PUBLISH']) {
+		error('Skipping publish due to VSCODE_BUILD_SKIP_PUBLISH');
+		return;
+	}
+
+	const commit = process.env['BUILD_SOURCEVERSION'];
+
+	if (!commit) {
+		error('Skipping publish due to missing BUILD_SOURCEVERSION');
+		return;
+	}
+
+	const quality = process.argv[2];
+
+	sync(commit, quality).catch(err => {
+		error(err);
+		process.exit(1);
+	});
+}
+
+main();