Merge from vscode bead496a613e475819f89f08e9e882b841bc1fe8 (#14883)

* Merge from vscode bead496a613e475819f89f08e9e882b841bc1fe8

* Bump distro

* Upgrade GCC to 4.9 due to yarn install errors

* Update build image

* Fix bootstrap base url

* Bump distro

* Fix build errors

* Update source map file

* Disable checkbox for blocking migration issues (#15131)

* disable checkbox for blocking issues

* wip

* disable checkbox fixes

* fix strings

* Remove duplicate tsec command

* Default to off for tab color if settings not present

* re-skip failing tests

* Fix mocha error

* Bump sqlite version & fix notebooks search view

* Turn off esbuild warnings

* Update esbuild log level

* Fix overflowactionbar tests

* Fix ts-ignore in dropdown tests

* cleanup/fixes

* Fix hygiene

* Bundle in entire zone.js module

* Remove extra constructor param

* bump distro for web compile break

* bump distro for web compile break v2

* Undo log level change

* New distro

* Fix integration test scripts

* remove the "no yarn.lock changes" workflow

* fix scripts v2

* Update unit test scripts

* Ensure ads-kerberos2 updates in .vscodeignore

* Try fix unit tests

* Upload crash reports

* remove nogpu

* always upload crashes

* Use bash script

* Consolidate data/ext dir names

* Create in tmp directory

Co-authored-by: chlafreniere <hichise@gmail.com>
Co-authored-by: Christopher Suh <chsuh@microsoft.com>
Co-authored-by: chgagnon <chgagnon@microsoft.com>

build/azure-pipelines/common/.gitignore

@@ -1,2 +0,0 @@
-node_modules/
-*.js

build/azure-pipelines/common/computeNodeModulesCacheKey.js

@@ -0,0 +1,25 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+const { dirs } = require('../../npm/dirs');
+const ROOT = path.join(__dirname, '../../../');
+const shasum = crypto.createHash('sha1');
+shasum.update(fs.readFileSync(path.join(ROOT, 'build/.cachesalt')));
+shasum.update(fs.readFileSync(path.join(ROOT, '.yarnrc')));
+shasum.update(fs.readFileSync(path.join(ROOT, 'remote/.yarnrc')));
+// Add `yarn.lock` files
+for (let dir of dirs) {
+    const yarnLockPath = path.join(ROOT, dir, 'yarn.lock');
+    shasum.update(fs.readFileSync(yarnLockPath));
+}
+// Add any other command line arguments
+for (let i = 2; i < process.argv.length; i++) {
+    shasum.update(process.argv[i]);
+}
+process.stdout.write(shasum.digest('hex'));

build/azure-pipelines/common/computeNodeModulesCacheKey.ts

@@ -0,0 +1,32 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+const { dirs } = require('../../npm/dirs');
+
+const ROOT = path.join(__dirname, '../../../');
+
+const shasum = crypto.createHash('sha1');
+
+shasum.update(fs.readFileSync(path.join(ROOT, 'build/.cachesalt')));
+shasum.update(fs.readFileSync(path.join(ROOT, '.yarnrc')));
+shasum.update(fs.readFileSync(path.join(ROOT, 'remote/.yarnrc')));
+
+// Add `yarn.lock` files
+for (let dir of dirs) {
+	const yarnLockPath = path.join(ROOT, dir, 'yarn.lock');
+	shasum.update(fs.readFileSync(yarnLockPath));
+}
+
+// Add any other command line arguments
+for (let i = 2; i < process.argv.length; i++) {
+	shasum.update(process.argv[i]);
+}
+
+process.stdout.write(shasum.digest('hex'));

build/azure-pipelines/common/copyArtifacts.js

@@ -0,0 +1,40 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+const vfs = require("vinyl-fs");
+const path = require("path");
+const es = require("event-stream");
+const fs = require("fs");
+const files = [
+    '.build/extensions/**/*.vsix',
+    '.build/win32-x64/**/*.{exe,zip}',
+    '.build/linux/sha256hashes.txt',
+    '.build/linux/deb/amd64/deb/*.deb',
+    '.build/linux/rpm/x86_64/*.rpm',
+    '.build/linux/server/*',
+    '.build/linux/archive/*',
+    '.build/docker/*',
+    '.build/darwin/*',
+    '.build/version.json' // version information
+];
+async function main() {
+    return new Promise((resolve, reject) => {
+        const stream = vfs.src(files, { base: '.build', allowEmpty: true })
+            .pipe(es.through(file => {
+            const filePath = path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY, 
+            //Preserve intermediate directories after .build folder
+            file.path.substr(path.resolve('.build').length + 1));
+            fs.mkdirSync(path.dirname(filePath), { recursive: true });
+            fs.renameSync(file.path, filePath);
+        }));
+        stream.on('end', () => resolve());
+        stream.on('error', e => reject(e));
+    });
+}
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});

build/azure-pipelines/common/createAsset.js

@@ -0,0 +1,94 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = require("fs");
+const crypto = require("crypto");
+const azure = require("azure-storage");
+const mime = require("mime");
+const cosmos_1 = require("@azure/cosmos");
+const retry_1 = require("./retry");
+if (process.argv.length !== 6) {
+    console.error('Usage: node createAsset.js PLATFORM TYPE NAME FILE');
+    process.exit(-1);
+}
+function hashStream(hashName, stream) {
+    return new Promise((c, e) => {
+        const shasum = crypto.createHash(hashName);
+        stream
+            .on('data', shasum.update.bind(shasum))
+            .on('error', e)
+            .on('close', () => c(shasum.digest('hex')));
+    });
+}
+async function doesAssetExist(blobService, quality, blobName) {
+    const existsResult = await new Promise((c, e) => blobService.doesBlobExist(quality, blobName, (err, r) => err ? e(err) : c(r)));
+    return existsResult.exists;
+}
+async function uploadBlob(blobService, quality, blobName, filePath, fileName) {
+    const blobOptions = {
+        contentSettings: {
+            contentType: mime.lookup(filePath),
+            contentDisposition: `attachment; filename="${fileName}"`,
+            cacheControl: 'max-age=31536000, public'
+        }
+    };
+    await new Promise((c, e) => blobService.createBlockBlobFromLocalFile(quality, blobName, filePath, blobOptions, err => err ? e(err) : c()));
+}
+function getEnv(name) {
+    const result = process.env[name];
+    if (typeof result === 'undefined') {
+        throw new Error('Missing env: ' + name);
+    }
+    return result;
+}
+async function main() {
+    const [, , platform, type, fileName, filePath] = process.argv;
+    const quality = getEnv('VSCODE_QUALITY');
+    const commit = getEnv('BUILD_SOURCEVERSION');
+    console.log('Creating asset...');
+    const stat = await new Promise((c, e) => fs.stat(filePath, (err, stat) => err ? e(err) : c(stat)));
+    const size = stat.size;
+    console.log('Size:', size);
+    const stream = fs.createReadStream(filePath);
+    const [sha1hash, sha256hash] = await Promise.all([hashStream('sha1', stream), hashStream('sha256', stream)]);
+    console.log('SHA1:', sha1hash);
+    console.log('SHA256:', sha256hash);
+    const blobName = commit + '/' + fileName;
+    const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2'];
+    const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2'])
+        .withFilter(new azure.ExponentialRetryPolicyFilter(20));
+    const blobExists = await doesAssetExist(blobService, quality, blobName);
+    if (blobExists) {
+        console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
+        return;
+    }
+    console.log('Uploading blobs to Azure storage...');
+    await uploadBlob(blobService, quality, blobName, filePath, fileName);
+    console.log('Blobs successfully uploaded.');
+    const asset = {
+        platform,
+        type,
+        url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
+        hash: sha1hash,
+        sha256hash,
+        size
+    };
+    // Remove this if we ever need to rollback fast updates for windows
+    if (/win32/.test(platform)) {
+        asset.supportsFastUpdate = true;
+    }
+    console.log('Asset:', JSON.stringify(asset, null, '  '));
+    const client = new cosmos_1.CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT'], key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+    const scripts = client.database('builds').container(quality).scripts;
+    await retry_1.retry(() => scripts.storedProcedure('createAsset').execute('', [commit, asset, true]));
+}
+main().then(() => {
+    console.log('Asset successfully created');
+    process.exit(0);
+}, err => {
+    console.error(err);
+    process.exit(1);
+});

build/azure-pipelines/common/createAsset.ts

@@ -11,6 +11,7 @@ import * as crypto from 'crypto';
 import * as azure from 'azure-storage';
 import * as mime from 'mime';
 import { CosmosClient } from '@azure/cosmos';
+import { retry } from './retry';
 
 interface Asset {
 	platform: string;
@@ -121,7 +122,7 @@ async function main(): Promise<void> {
 
 	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
 	const scripts = client.database('builds').container(quality).scripts;
-	await scripts.storedProcedure('createAsset').execute('', [commit, asset, true]);
+	await retry(() => scripts.storedProcedure('createAsset').execute('', [commit, asset, true]));
 }
 
 main().then(() => {

build/azure-pipelines/common/createBuild.js

@@ -0,0 +1,51 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+const cosmos_1 = require("@azure/cosmos");
+const retry_1 = require("./retry");
+if (process.argv.length !== 3) {
+    console.error('Usage: node createBuild.js VERSION');
+    process.exit(-1);
+}
+function getEnv(name) {
+    const result = process.env[name];
+    if (typeof result === 'undefined') {
+        throw new Error('Missing env: ' + name);
+    }
+    return result;
+}
+async function main() {
+    const [, , _version] = process.argv;
+    const quality = getEnv('VSCODE_QUALITY');
+    const commit = getEnv('BUILD_SOURCEVERSION');
+    const queuedBy = getEnv('BUILD_QUEUEDBY');
+    const sourceBranch = getEnv('BUILD_SOURCEBRANCH');
+    const version = _version + (quality === 'stable' ? '' : `-${quality}`);
+    console.log('Creating build...');
+    console.log('Quality:', quality);
+    console.log('Version:', version);
+    console.log('Commit:', commit);
+    const build = {
+        id: commit,
+        timestamp: (new Date()).getTime(),
+        version,
+        isReleased: false,
+        sourceBranch,
+        queuedBy,
+        assets: [],
+        updates: {}
+    };
+    const client = new cosmos_1.CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT'], key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+    const scripts = client.database('builds').container(quality).scripts;
+    await retry_1.retry(() => scripts.storedProcedure('createBuild').execute('', [Object.assign(Object.assign({}, build), { _partitionKey: '' })]));
+}
+main().then(() => {
+    console.log('Build successfully created');
+    process.exit(0);
+}, err => {
+    console.error(err);
+    process.exit(1);
+});

build/azure-pipelines/common/createBuild.ts

@@ -6,6 +6,7 @@
 'use strict';
 
 import { CosmosClient } from '@azure/cosmos';
+import { retry } from './retry';
 
 if (process.argv.length !== 3) {
 	console.error('Usage: node createBuild.js VERSION');
@@ -48,7 +49,7 @@ async function main(): Promise<void> {
 
 	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
 	const scripts = client.database('builds').container(quality).scripts;
-	await scripts.storedProcedure('createBuild').execute('', [{ ...build, _partitionKey: '' }]);
+	await retry(() => scripts.storedProcedure('createBuild').execute('', [{ ...build, _partitionKey: '' }]));
 }
 
 main().then(() => {

build/azure-pipelines/common/extract-telemetry.sh

@@ -10,8 +10,8 @@ git clone --depth 1 https://github.com/Microsoft/vscode-node-debug2.git
 git clone --depth 1 https://github.com/Microsoft/vscode-node-debug.git
 git clone --depth 1 https://github.com/Microsoft/vscode-html-languageservice.git
 git clone --depth 1 https://github.com/Microsoft/vscode-json-languageservice.git
-node $BUILD_SOURCESDIRECTORY/build/node_modules/.bin/vscode-telemetry-extractor --sourceDir $BUILD_SOURCESDIRECTORY --excludedDir $BUILD_SOURCESDIRECTORY/extensions --outputDir . --applyEndpoints
-node $BUILD_SOURCESDIRECTORY/build/node_modules/.bin/vscode-telemetry-extractor --config $BUILD_SOURCESDIRECTORY/build/azure-pipelines/common/telemetry-config.json -o .
+node $BUILD_SOURCESDIRECTORY/node_modules/.bin/vscode-telemetry-extractor --sourceDir $BUILD_SOURCESDIRECTORY --excludedDir $BUILD_SOURCESDIRECTORY/extensions --outputDir . --applyEndpoints
+node $BUILD_SOURCESDIRECTORY/node_modules/.bin/vscode-telemetry-extractor --config $BUILD_SOURCESDIRECTORY/build/azure-pipelines/common/telemetry-config.json -o .
 mkdir -p $BUILD_SOURCESDIRECTORY/.build/telemetry
 mv declarations-resolved.json $BUILD_SOURCESDIRECTORY/.build/telemetry/telemetry-core.json
 mv config-resolved.json $BUILD_SOURCESDIRECTORY/.build/telemetry/telemetry-extensions.json

build/azure-pipelines/common/installPlaywright.js

@@ -0,0 +1,14 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+const path = require("path");
+const retry_1 = require("./retry");
+const { installBrowsersWithProgressBar } = require('playwright/lib/install/installer');
+const playwrightPath = path.dirname(require.resolve('playwright'));
+async function install() {
+    await retry_1.retry(() => installBrowsersWithProgressBar(playwrightPath));
+}
+install();

build/azure-pipelines/common/listNodeModules.js

@@ -0,0 +1,40 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = require("fs");
+const path = require("path");
+if (process.argv.length !== 3) {
+    console.error('Usage: node listNodeModules.js OUTPUT_FILE');
+    process.exit(-1);
+}
+const ROOT = path.join(__dirname, '../../../');
+function findNodeModulesFiles(location, inNodeModules, result) {
+    const entries = fs.readdirSync(path.join(ROOT, location));
+    for (const entry of entries) {
+        const entryPath = `${location}/${entry}`;
+        if (/(^\/out)|(^\/src$)|(^\/.git$)|(^\/.build$)/.test(entryPath)) {
+            continue;
+        }
+        let stat;
+        try {
+            stat = fs.statSync(path.join(ROOT, entryPath));
+        }
+        catch (err) {
+            continue;
+        }
+        if (stat.isDirectory()) {
+            findNodeModulesFiles(entryPath, inNodeModules || (entry === 'node_modules'), result);
+        }
+        else {
+            if (inNodeModules) {
+                result.push(entryPath.substr(1));
+            }
+        }
+    }
+}
+const result = [];
+findNodeModulesFiles('', false, result);
+fs.writeFileSync(process.argv[2], result.join('\n') + '\n');

build/azure-pipelines/common/publish-webview.js

@@ -0,0 +1,71 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+const azure = require("azure-storage");
+const mime = require("mime");
+const minimist = require("minimist");
+const path_1 = require("path");
+const fileNames = [
+    'fake.html',
+    'host.js',
+    'index.html',
+    'main.js',
+    'service-worker.js'
+];
+async function assertContainer(blobService, container) {
+    await new Promise((c, e) => blobService.createContainerIfNotExists(container, { publicAccessLevel: 'blob' }, err => err ? e(err) : c()));
+}
+async function doesBlobExist(blobService, container, blobName) {
+    const existsResult = await new Promise((c, e) => blobService.doesBlobExist(container, blobName, (err, r) => err ? e(err) : c(r)));
+    return existsResult.exists;
+}
+async function uploadBlob(blobService, container, blobName, file) {
+    const blobOptions = {
+        contentSettings: {
+            contentType: mime.lookup(file),
+            cacheControl: 'max-age=31536000, public'
+        }
+    };
+    await new Promise((c, e) => blobService.createBlockBlobFromLocalFile(container, blobName, file, blobOptions, err => err ? e(err) : c()));
+}
+async function publish(commit, files) {
+    console.log('Publishing...');
+    console.log('Commit:', commit);
+    const storageAccount = process.env['AZURE_WEBVIEW_STORAGE_ACCOUNT'];
+    const blobService = azure.createBlobService(storageAccount, process.env['AZURE_WEBVIEW_STORAGE_ACCESS_KEY'])
+        .withFilter(new azure.ExponentialRetryPolicyFilter(20));
+    await assertContainer(blobService, commit);
+    for (const file of files) {
+        const blobName = path_1.basename(file);
+        const blobExists = await doesBlobExist(blobService, commit, blobName);
+        if (blobExists) {
+            console.log(`Blob ${commit}, ${blobName} already exists, not publishing again.`);
+            continue;
+        }
+        console.log('Uploading blob to Azure storage...');
+        await uploadBlob(blobService, commit, blobName, file);
+    }
+    console.log('Blobs successfully uploaded.');
+}
+function main() {
+    const commit = process.env['BUILD_SOURCEVERSION'];
+    if (!commit) {
+        console.warn('Skipping publish due to missing BUILD_SOURCEVERSION');
+        return;
+    }
+    const opts = minimist(process.argv.slice(2));
+    const [directory] = opts._;
+    const files = fileNames.map(fileName => path_1.join(directory, fileName));
+    publish(commit, files).catch(err => {
+        console.error(err);
+        process.exit(1);
+    });
+}
+if (process.argv.length < 3) {
+    console.error('Usage: node publish.js <directory>');
+    process.exit(-1);
+}
+main();

build/azure-pipelines/common/publish.js

@@ -0,0 +1,224 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = require("fs");
+const crypto = require("crypto");
+const azure = require("azure-storage");
+const mime = require("mime");
+const minimist = require("minimist");
+const documentdb_1 = require("documentdb");
+// {{SQL CARBON EDIT}}
+if (process.argv.length < 9) {
+    console.error('Usage: node publish.js <product_quality> <platform> <file_type> <file_name> <version> <is_update> <file> [commit_id]');
+    process.exit(-1);
+}
+function hashStream(hashName, stream) {
+    return new Promise((c, e) => {
+        const shasum = crypto.createHash(hashName);
+        stream
+            .on('data', shasum.update.bind(shasum))
+            .on('error', e)
+            .on('close', () => c(shasum.digest('hex')));
+    });
+}
+function createDefaultConfig(quality) {
+    return {
+        id: quality,
+        frozen: false
+    };
+}
+function getConfig(quality) {
+    console.log(`Getting config for quality ${quality}`);
+    const client = new documentdb_1.DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT'], { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+    const collection = 'dbs/builds/colls/config';
+    const query = {
+        query: `SELECT TOP 1 * FROM c WHERE c.id = @quality`,
+        parameters: [
+            { name: '@quality', value: quality }
+        ]
+    };
+    return retry(() => new Promise((c, e) => {
+        client.queryDocuments(collection, query, { enableCrossPartitionQuery: true }).toArray((err, results) => {
+            if (err && err.code !== 409) {
+                return e(err);
+            }
+            c(!results || results.length === 0 ? createDefaultConfig(quality) : results[0]);
+        });
+    }));
+}
+function createOrUpdate(commit, quality, platform, type, release, asset, isUpdate) {
+    const client = new documentdb_1.DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT'], { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+    const collection = 'dbs/builds/colls/' + quality;
+    const updateQuery = {
+        query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
+        parameters: [{ name: '@id', value: commit }]
+    };
+    let updateTries = 0;
+    function update() {
+        updateTries++;
+        return new Promise((c, e) => {
+            console.log(`Querying existing documents to update...`);
+            client.queryDocuments(collection, updateQuery, { enableCrossPartitionQuery: true }).toArray((err, results) => {
+                if (err) {
+                    return e(err);
+                }
+                if (results.length !== 1) {
+                    return e(new Error('No documents'));
+                }
+                const release = results[0];
+                release.assets = [
+                    ...release.assets.filter((a) => !(a.platform === platform && a.type === type)),
+                    asset
+                ];
+                if (isUpdate) {
+                    release.updates[platform] = type;
+                }
+                console.log(`Replacing existing document with updated version`);
+                client.replaceDocument(release._self, release, err => {
+                    if (err && err.code === 409 && updateTries < 5) {
+                        return c(update());
+                    }
+                    if (err) {
+                        return e(err);
+                    }
+                    console.log('Build successfully updated.');
+                    c();
+                });
+            });
+        });
+    }
+    return retry(() => new Promise((c, e) => {
+        console.log(`Attempting to create document`);
+        client.createDocument(collection, release, err => {
+            if (err && err.code === 409) {
+                return c(update());
+            }
+            if (err) {
+                return e(err);
+            }
+            console.log('Build successfully published.');
+            c();
+        });
+    }));
+}
+async function assertContainer(blobService, quality) {
+    await new Promise((c, e) => blobService.createContainerIfNotExists(quality, { publicAccessLevel: 'blob' }, err => err ? e(err) : c()));
+}
+async function doesAssetExist(blobService, quality, blobName) {
+    const existsResult = await new Promise((c, e) => blobService.doesBlobExist(quality, blobName, (err, r) => err ? e(err) : c(r)));
+    return existsResult.exists;
+}
+async function uploadBlob(blobService, quality, blobName, file) {
+    const blobOptions = {
+        contentSettings: {
+            contentType: mime.lookup(file),
+            cacheControl: 'max-age=31536000, public'
+        }
+    };
+    await new Promise((c, e) => blobService.createBlockBlobFromLocalFile(quality, blobName, file, blobOptions, err => err ? e(err) : c()));
+}
+async function publish(commit, quality, platform, type, name, version, _isUpdate, file, opts) {
+    const isUpdate = _isUpdate === 'true';
+    const queuedBy = process.env['BUILD_QUEUEDBY'];
+    const sourceBranch = process.env['BUILD_SOURCEBRANCH'];
+    console.log('Publishing...');
+    console.log('Quality:', quality);
+    console.log('Platform:', platform);
+    console.log('Type:', type);
+    console.log('Name:', name);
+    console.log('Version:', version);
+    console.log('Commit:', commit);
+    console.log('Is Update:', isUpdate);
+    console.log('File:', file);
+    const stat = await new Promise((c, e) => fs.stat(file, (err, stat) => err ? e(err) : c(stat)));
+    const size = stat.size;
+    console.log('Size:', size);
+    const stream = fs.createReadStream(file);
+    const [sha1hash, sha256hash] = await Promise.all([hashStream('sha1', stream), hashStream('sha256', stream)]);
+    console.log('SHA1:', sha1hash);
+    console.log('SHA256:', sha256hash);
+    const blobName = commit + '/' + name;
+    const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2'];
+    const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2'])
+        .withFilter(new azure.ExponentialRetryPolicyFilter(20));
+    await assertContainer(blobService, quality);
+    const blobExists = await doesAssetExist(blobService, quality, blobName);
+    if (blobExists) {
+        console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
+        return;
+    }
+    console.log('Uploading blobs to Azure storage...');
+    await uploadBlob(blobService, quality, blobName, file);
+    console.log('Blobs successfully uploaded.');
+    const config = await getConfig(quality);
+    console.log('Quality config:', config);
+    const asset = {
+        platform: platform,
+        type: type,
+        url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
+        hash: sha1hash,
+        sha256hash,
+        size
+    };
+    // Remove this if we ever need to rollback fast updates for windows
+    if (/win32/.test(platform)) {
+        asset.supportsFastUpdate = true;
+    }
+    console.log('Asset:', JSON.stringify(asset, null, '  '));
+    // {{SQL CARBON EDIT}}
+    // Insiders: nightly build from main
+    const isReleased = (((quality === 'insider' && /^main$|^refs\/heads\/main$/.test(sourceBranch)) ||
+        (quality === 'rc1' && /^release\/|^refs\/heads\/release\//.test(sourceBranch))) &&
+        /Project Collection Service Accounts|Microsoft.VisualStudio.Services.TFS/.test(queuedBy));
+    const release = {
+        id: commit,
+        timestamp: (new Date()).getTime(),
+        version,
+        isReleased: isReleased,
+        sourceBranch,
+        queuedBy,
+        assets: [],
+        updates: {}
+    };
+    if (!opts['upload-only']) {
+        release.assets.push(asset);
+        if (isUpdate) {
+            release.updates[platform] = type;
+        }
+    }
+    await createOrUpdate(commit, quality, platform, type, release, asset, isUpdate);
+}
+const RETRY_TIMES = 10;
+async function retry(fn) {
+    for (let run = 1; run <= RETRY_TIMES; run++) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            if (!/ECONNRESET/.test(err.message)) {
+                throw err;
+            }
+            console.log(`Caught error ${err} - ${run}/${RETRY_TIMES}`);
+        }
+    }
+    throw new Error('Retried too many times');
+}
+function main() {
+    const commit = process.env['BUILD_SOURCEVERSION'];
+    if (!commit) {
+        console.warn('Skipping publish due to missing BUILD_SOURCEVERSION');
+        return;
+    }
+    const opts = minimist(process.argv.slice(2), {
+        boolean: ['upload-only']
+    });
+    const [quality, platform, type, name, version, _isUpdate, file] = opts._;
+    publish(commit, quality, platform, type, name, version, _isUpdate, file, opts).catch(err => {
+        console.error(err);
+        process.exit(1);
+    });
+}
+main();

build/azure-pipelines/common/release.js

@@ -0,0 +1,91 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+const documentdb_1 = require("documentdb");
+function createDefaultConfig(quality) {
+    return {
+        id: quality,
+        frozen: false
+    };
+}
+function getConfig(quality) {
+    const client = new documentdb_1.DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT'], { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+    const collection = 'dbs/builds/colls/config';
+    const query = {
+        query: `SELECT TOP 1 * FROM c WHERE c.id = @quality`,
+        parameters: [
+            { name: '@quality', value: quality }
+        ]
+    };
+    return new Promise((c, e) => {
+        client.queryDocuments(collection, query).toArray((err, results) => {
+            if (err && err.code !== 409) {
+                return e(err);
+            }
+            c(!results || results.length === 0 ? createDefaultConfig(quality) : results[0]);
+        });
+    });
+}
+function doRelease(commit, quality) {
+    const client = new documentdb_1.DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT'], { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+    const collection = 'dbs/builds/colls/' + quality;
+    const query = {
+        query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
+        parameters: [{ name: '@id', value: commit }]
+    };
+    let updateTries = 0;
+    function update() {
+        updateTries++;
+        return new Promise((c, e) => {
+            client.queryDocuments(collection, query).toArray((err, results) => {
+                if (err) {
+                    return e(err);
+                }
+                if (results.length !== 1) {
+                    return e(new Error('No documents'));
+                }
+                const release = results[0];
+                release.isReleased = true;
+                client.replaceDocument(release._self, release, err => {
+                    if (err && err.code === 409 && updateTries < 5) {
+                        return c(update());
+                    }
+                    if (err) {
+                        return e(err);
+                    }
+                    console.log('Build successfully updated.');
+                    c();
+                });
+            });
+        });
+    }
+    return update();
+}
+async function release(commit, quality) {
+    const config = await getConfig(quality);
+    console.log('Quality config:', config);
+    if (config.frozen) {
+        console.log(`Skipping release because quality ${quality} is frozen.`);
+        return;
+    }
+    await doRelease(commit, quality);
+}
+function env(name) {
+    const result = process.env[name];
+    if (!result) {
+        throw new Error(`Skipping release due to missing env: ${name}`);
+    }
+    return result;
+}
+async function main() {
+    const commit = env('BUILD_SOURCEVERSION');
+    const quality = env('VSCODE_QUALITY');
+    await release(commit, quality);
+}
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});

build/azure-pipelines/common/releaseBuild.js

@@ -0,0 +1,50 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+const cosmos_1 = require("@azure/cosmos");
+const retry_1 = require("./retry");
+function getEnv(name) {
+    const result = process.env[name];
+    if (typeof result === 'undefined') {
+        throw new Error('Missing env: ' + name);
+    }
+    return result;
+}
+function createDefaultConfig(quality) {
+    return {
+        id: quality,
+        frozen: false
+    };
+}
+async function getConfig(client, quality) {
+    const query = `SELECT TOP 1 * FROM c WHERE c.id = "${quality}"`;
+    const res = await client.database('builds').container('config').items.query(query).fetchAll();
+    if (res.resources.length === 0) {
+        return createDefaultConfig(quality);
+    }
+    return res.resources[0];
+}
+async function main() {
+    const commit = getEnv('BUILD_SOURCEVERSION');
+    const quality = getEnv('VSCODE_QUALITY');
+    const client = new cosmos_1.CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT'], key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+    const config = await getConfig(client, quality);
+    console.log('Quality config:', config);
+    if (config.frozen) {
+        console.log(`Skipping release because quality ${quality} is frozen.`);
+        return;
+    }
+    console.log(`Releasing build ${commit}...`);
+    const scripts = client.database('builds').container(quality).scripts;
+    await retry_1.retry(() => scripts.storedProcedure('releaseBuild').execute('', [commit]));
+}
+main().then(() => {
+    console.log('Build successfully released');
+    process.exit(0);
+}, err => {
+    console.error(err);
+    process.exit(1);
+});

build/azure-pipelines/common/releaseBuild.ts

@@ -6,6 +6,7 @@
 'use strict';
 
 import { CosmosClient } from '@azure/cosmos';
+import { retry } from './retry';
 
 function getEnv(name: string): string {
 	const result = process.env[name];
@@ -58,7 +59,7 @@ async function main(): Promise<void> {
 	console.log(`Releasing build ${commit}...`);
 
 	const scripts = client.database('builds').container(quality).scripts;
-	await scripts.storedProcedure('releaseBuild').execute('', [commit]);
+	await retry(() => scripts.storedProcedure('releaseBuild').execute('', [commit]));
 }
 
 main().then(() => {

build/azure-pipelines/common/retry.js

@@ -0,0 +1,25 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.retry = void 0;
+async function retry(fn) {
+    for (let run = 1; run <= 10; run++) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            if (!/ECONNRESET/.test(err.message)) {
+                throw err;
+            }
+            const millis = (Math.random() * 200) + (50 * Math.pow(1.5, run));
+            console.log(`Failed with ECONNRESET, retrying in ${millis}ms...`);
+            // maximum delay is 10th retry: ~3 seconds
+            await new Promise(c => setTimeout(c, millis));
+        }
+    }
+    throw new Error('Retried too many times');
+}
+exports.retry = retry;

build/azure-pipelines/common/retry.ts

@@ -0,0 +1,26 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+export async function retry<T>(fn: () => Promise<T>): Promise<T> {
+	for (let run = 1; run <= 10; run++) {
+		try {
+			return await fn();
+		} catch (err) {
+			if (!/ECONNRESET/.test(err.message)) {
+				throw err;
+			}
+
+			const millis = (Math.random() * 200) + (50 * Math.pow(1.5, run));
+			console.log(`Failed with ECONNRESET, retrying in ${millis}ms...`);
+
+			// maximum delay is 10th retry: ~3 seconds
+			await new Promise(c => setTimeout(c, millis));
+		}
+	}
+
+	throw new Error('Retried too many times');
+}

build/azure-pipelines/common/sync-mooncake.js

@@ -0,0 +1,87 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+Object.defineProperty(exports, "__esModule", { value: true });
+const url = require("url");
+const azure = require("azure-storage");
+const mime = require("mime");
+const cosmos_1 = require("@azure/cosmos");
+const retry_1 = require("./retry");
+function log(...args) {
+    console.log(...[`[${new Date().toISOString()}]`, ...args]);
+}
+function error(...args) {
+    console.error(...[`[${new Date().toISOString()}]`, ...args]);
+}
+if (process.argv.length < 3) {
+    error('Usage: node sync-mooncake.js <quality>');
+    process.exit(-1);
+}
+async function sync(commit, quality) {
+    log(`Synchronizing Mooncake assets for ${quality}, ${commit}...`);
+    const client = new cosmos_1.CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT'], key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+    const container = client.database('builds').container(quality);
+    const query = `SELECT TOP 1 * FROM c WHERE c.id = "${commit}"`;
+    const res = await container.items.query(query, {}).fetchAll();
+    if (res.resources.length !== 1) {
+        throw new Error(`No builds found for ${commit}`);
+    }
+    const build = res.resources[0];
+    log(`Found build for ${commit}, with ${build.assets.length} assets`);
+    const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2'];
+    const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2'])
+        .withFilter(new azure.ExponentialRetryPolicyFilter(20));
+    const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY'], `${storageAccount}.blob.core.chinacloudapi.cn`)
+        .withFilter(new azure.ExponentialRetryPolicyFilter(20));
+    // mooncake is fussy and far away, this is needed!
+    blobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
+    mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
+    for (const asset of build.assets) {
+        try {
+            const blobPath = url.parse(asset.url).path;
+            if (!blobPath) {
+                throw new Error(`Failed to parse URL: ${asset.url}`);
+            }
+            const blobName = blobPath.replace(/^\/\w+\//, '');
+            log(`Found ${blobName}`);
+            if (asset.mooncakeUrl) {
+                log(`  Already in Mooncake ✔️`);
+                continue;
+            }
+            const readStream = blobService.createReadStream(quality, blobName, undefined);
+            const blobOptions = {
+                contentSettings: {
+                    contentType: mime.lookup(blobPath),
+                    cacheControl: 'max-age=31536000, public'
+                }
+            };
+            const writeStream = mooncakeBlobService.createWriteStreamToBlockBlob(quality, blobName, blobOptions, undefined);
+            log(`  Uploading to Mooncake...`);
+            await new Promise((c, e) => readStream.pipe(writeStream).on('finish', c).on('error', e));
+            log(`  Updating build in DB...`);
+            const mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
+            await retry_1.retry(() => container.scripts.storedProcedure('setAssetMooncakeUrl')
+                .execute('', [commit, asset.platform, asset.type, mooncakeUrl]));
+            log(`  Done ✔️`);
+        }
+        catch (err) {
+            error(err);
+        }
+    }
+    log(`All done ✔️`);
+}
+function main() {
+    const commit = process.env['BUILD_SOURCEVERSION'];
+    if (!commit) {
+        error('Skipping publish due to missing BUILD_SOURCEVERSION');
+        return;
+    }
+    const quality = process.argv[2];
+    sync(commit, quality).catch(err => {
+        error(err);
+        process.exit(1);
+    });
+}
+main();

build/azure-pipelines/common/sync-mooncake.ts

@@ -9,6 +9,7 @@ import * as url from 'url';
 import * as azure from 'azure-storage';
 import * as mime from 'mime';
 import { CosmosClient } from '@azure/cosmos';
+import { retry } from './retry';
 
 function log(...args: any[]) {
 	console.log(...[`[${new Date().toISOString()}]`, ...args]);
@@ -99,8 +100,8 @@ async function sync(commit: string, quality: string): Promise<void> {
 
 			log(`  Updating build in DB...`);
 			const mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
-			await container.scripts.storedProcedure('setAssetMooncakeUrl')
-				.execute('', [commit, asset.platform, asset.type, mooncakeUrl]);
+			await retry(() => container.scripts.storedProcedure('setAssetMooncakeUrl')
+				.execute('', [commit, asset.platform, asset.type, mooncakeUrl]));
 
 			log(`  Done ✔️`);
 		} catch (err) {