Merge from vscode a5cf1da01d5db3d2557132be8d30f89c38019f6c (#8525)

* Merge from vscode a5cf1da01d5db3d2557132be8d30f89c38019f6c

* remove files we don't want

* fix hygiene

* update distro

* update distro

* fix hygiene

* fix strict nulls

* distro

* distro

* fix tests

* fix tests

* add another edit

* fix viewlet icon

* fix azure dialog

* fix some padding

* fix more padding issues

build/azure-pipelines/common/createAsset.ts

@@ -0,0 +1,132 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as fs from 'fs';
+import { Readable } from 'stream';
+import * as crypto from 'crypto';
+import * as azure from 'azure-storage';
+import * as mime from 'mime';
+import { CosmosClient } from '@azure/cosmos';
+
+interface Asset {
+	platform: string;
+	type: string;
+	url: string;
+	mooncakeUrl?: string;
+	hash: string;
+	sha256hash: string;
+	size: number;
+	supportsFastUpdate?: boolean;
+}
+
+if (process.argv.length !== 6) {
+	console.error('Usage: node createAsset.js PLATFORM TYPE NAME FILE');
+	process.exit(-1);
+}
+
+function hashStream(hashName: string, stream: Readable): Promise<string> {
+	return new Promise<string>((c, e) => {
+		const shasum = crypto.createHash(hashName);
+
+		stream
+			.on('data', shasum.update.bind(shasum))
+			.on('error', e)
+			.on('close', () => c(shasum.digest('hex')));
+	});
+}
+
+async function doesAssetExist(blobService: azure.BlobService, quality: string, blobName: string): Promise<boolean | undefined> {
+	const existsResult = await new Promise<azure.BlobService.BlobResult>((c, e) => blobService.doesBlobExist(quality, blobName, (err, r) => err ? e(err) : c(r)));
+	return existsResult.exists;
+}
+
+async function uploadBlob(blobService: azure.BlobService, quality: string, blobName: string, file: string): Promise<void> {
+	const blobOptions: azure.BlobService.CreateBlockBlobRequestOptions = {
+		contentSettings: {
+			contentType: mime.lookup(file),
+			cacheControl: 'max-age=31536000, public'
+		}
+	};
+
+	await new Promise((c, e) => blobService.createBlockBlobFromLocalFile(quality, blobName, file, blobOptions, err => err ? e(err) : c()));
+}
+
+function getEnv(name: string): string {
+	const result = process.env[name];
+
+	if (typeof result === 'undefined') {
+		throw new Error('Missing env: ' + name);
+	}
+
+	return result;
+}
+
+async function main(): Promise<void> {
+	const [, , platform, type, name, file] = process.argv;
+	const quality = getEnv('VSCODE_QUALITY');
+	const commit = getEnv('BUILD_SOURCEVERSION');
+
+	console.log('Creating asset...');
+
+	const stat = await new Promise<fs.Stats>((c, e) => fs.stat(file, (err, stat) => err ? e(err) : c(stat)));
+	const size = stat.size;
+
+	console.log('Size:', size);
+
+	const stream = fs.createReadStream(file);
+	const [sha1hash, sha256hash] = await Promise.all([hashStream('sha1', stream), hashStream('sha256', stream)]);
+
+	console.log('SHA1:', sha1hash);
+	console.log('SHA256:', sha256hash);
+
+	const blobName = commit + '/' + name;
+	const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2']!;
+
+	const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	const blobExists = await doesAssetExist(blobService, quality, blobName);
+
+	if (blobExists) {
+		console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
+		return;
+	}
+
+	console.log('Uploading blobs to Azure storage...');
+
+	await uploadBlob(blobService, quality, blobName, file);
+
+	console.log('Blobs successfully uploaded.');
+
+	const asset: Asset = {
+		platform,
+		type,
+		url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
+		hash: sha1hash,
+		sha256hash,
+		size
+	};
+
+	// Remove this if we ever need to rollback fast updates for windows
+	if (/win32/.test(platform)) {
+		asset.supportsFastUpdate = true;
+	}
+
+	console.log('Asset:', JSON.stringify(asset, null, '  '));
+
+	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const scripts = client.database('builds').container(quality).scripts;
+	await scripts.storedProcedure('createAsset').execute('', [commit, asset, true]);
+}
+
+main().then(() => {
+	console.log('Asset successfully created');
+	process.exit(0);
+}, err => {
+	console.error(err);
+	process.exit(1);
+});

build/azure-pipelines/common/createBuild.ts

@@ -0,0 +1,60 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { CosmosClient } from '@azure/cosmos';
+
+if (process.argv.length !== 3) {
+	console.error('Usage: node createBuild.js VERSION');
+	process.exit(-1);
+}
+
+function getEnv(name: string): string {
+	const result = process.env[name];
+
+	if (typeof result === 'undefined') {
+		throw new Error('Missing env: ' + name);
+	}
+
+	return result;
+}
+
+async function main(): Promise<void> {
+	const [, , _version] = process.argv;
+	const quality = getEnv('VSCODE_QUALITY');
+	const commit = getEnv('BUILD_SOURCEVERSION');
+	const queuedBy = getEnv('BUILD_QUEUEDBY');
+	const sourceBranch = getEnv('BUILD_SOURCEBRANCH');
+	const version = _version + (quality === 'stable' ? '' : `-${quality}`);
+
+	console.log('Creating build...');
+	console.log('Quality:', quality);
+	console.log('Version:', version);
+	console.log('Commit:', commit);
+
+	const build = {
+		id: commit,
+		timestamp: (new Date()).getTime(),
+		version,
+		isReleased: false,
+		sourceBranch,
+		queuedBy,
+		assets: [],
+		updates: {}
+	};
+
+	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const scripts = client.database('builds').container(quality).scripts;
+	await scripts.storedProcedure('createBuild').execute('', [{ ...build, _partitionKey: '' }]);
+}
+
+main().then(() => {
+	console.log('Build successfully created');
+	process.exit(0);
+}, err => {
+	console.error(err);
+	process.exit(1);
+});

build/azure-pipelines/common/releaseBuild.ts

@@ -0,0 +1,70 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { CosmosClient } from '@azure/cosmos';
+
+function getEnv(name: string): string {
+	const result = process.env[name];
+
+	if (typeof result === 'undefined') {
+		throw new Error('Missing env: ' + name);
+	}
+
+	return result;
+}
+
+interface Config {
+	id: string;
+	frozen: boolean;
+}
+
+function createDefaultConfig(quality: string): Config {
+	return {
+		id: quality,
+		frozen: false
+	};
+}
+
+async function getConfig(client: CosmosClient, quality: string): Promise<Config> {
+	const query = `SELECT TOP 1 * FROM c WHERE c.id = "${quality}"`;
+
+	const res = await client.database('builds').container('config').items.query(query).fetchAll();
+
+	if (res.resources.length === 0) {
+		return createDefaultConfig(quality);
+	}
+
+	return res.resources[0] as Config;
+}
+
+async function main(): Promise<void> {
+	const commit = getEnv('BUILD_SOURCEVERSION');
+	const quality = getEnv('VSCODE_QUALITY');
+
+	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const config = await getConfig(client, quality);
+
+	console.log('Quality config:', config);
+
+	if (config.frozen) {
+		console.log(`Skipping release because quality ${quality} is frozen.`);
+		return;
+	}
+
+	console.log(`Releasing build ${commit}...`);
+
+	const scripts = client.database('builds').container(quality).scripts;
+	await scripts.storedProcedure('releaseBuild').execute('', [commit]);
+}
+
+main().then(() => {
+	console.log('Build successfully released');
+	process.exit(0);
+}, err => {
+	console.error(err);
+	process.exit(1);
+});

build/azure-pipelines/common/sync-mooncake.ts

@@ -8,7 +8,7 @@
 import * as url from 'url';
 import * as azure from 'azure-storage';
 import * as mime from 'mime';
-import { DocumentClient, RetrievedDocument } from 'documentdb';
+import { CosmosClient } from '@azure/cosmos';
 
 function log(...args: any[]) {
 	console.log(...[`[${new Date().toISOString()}]`, ...args]);
@@ -23,7 +23,7 @@ if (process.argv.length < 3) {
 	process.exit(-1);
 }
 
-interface Build extends RetrievedDocument {
+interface Build {
 	assets: Asset[];
 }
 
@@ -38,62 +38,20 @@ interface Asset {
 	supportsFastUpdate?: boolean;
 }
 
-function updateBuild(commit: string, quality: string, platform: string, type: string, asset: Asset): Promise<void> {
-	const client = new DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT']!, { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
-	const collection = 'dbs/builds/colls/' + quality;
-	const updateQuery = {
-		query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
-		parameters: [{ name: '@id', value: commit }]
-	};
-
-	let updateTries = 0;
-
-	function _update(): Promise<void> {
-		updateTries++;
-
-		return new Promise<void>((c, e) => {
-			client.queryDocuments(collection, updateQuery).toArray((err, results) => {
-				if (err) { return e(err); }
-				if (results.length !== 1) { return e(new Error('No documents')); }
-
-				const release = results[0];
-
-				release.assets = [
-					...release.assets.filter((a: any) => !(a.platform === platform && a.type === type)),
-					asset
-				];
-
-				client.replaceDocument(release._self, release, err => {
-					if (err && err.code === 409 && updateTries < 5) { return c(_update()); }
-					if (err) { return e(err); }
-
-					log('Build successfully updated.');
-					c();
-				});
-			});
-		});
-	}
-
-	return _update();
-}
-
 async function sync(commit: string, quality: string): Promise<void> {
 	log(`Synchronizing Mooncake assets for ${quality}, ${commit}...`);
 
-	const cosmosdb = new DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT']!, { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
-	const collection = `dbs/builds/colls/${quality}`;
-	const query = {
-		query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
-		parameters: [{ name: '@id', value: commit }]
-	};
+	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const container = client.database('builds').container(quality);
 
-	const build = await new Promise<Build>((c, e) => {
-		cosmosdb.queryDocuments(collection, query).toArray((err, results) => {
-			if (err) { return e(err); }
-			if (results.length !== 1) { return e(new Error('No documents')); }
-			c(results[0] as Build);
-		});
-	});
+	const query = `SELECT TOP 1 * FROM c WHERE c.id = "${commit}"`;
+	const res = await container.items.query<Build>(query, {}).fetchAll();
+
+	if (res.resources.length !== 1) {
+		throw new Error(`No builds found for ${commit}`);
+	}
+
+	const build = res.resources[0];
 
 	log(`Found build for ${commit}, with ${build.assets.length} assets`);
 
@@ -140,8 +98,9 @@ async function sync(commit: string, quality: string): Promise<void> {
 			await new Promise((c, e) => readStream.pipe(writeStream).on('finish', c).on('error', e));
 
 			log(`  Updating build in DB...`);
-			asset.mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
-			await updateBuild(commit, quality, asset.platform, asset.type, asset);
+			const mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
+			await container.scripts.storedProcedure('setAssetMooncakeUrl')
+				.execute('', [commit, asset.platform, asset.type, mooncakeUrl]);
 
 			log(`  Done ✔️`);
 		} catch (err) {