ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-16 10:58:30 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge from vscode e3c4990c67c40213af168300d1cfeb71d680f877 (#16569)

Browse Source
This commit is contained in:
Cory Rivera
2021-08-25 16:28:29 -07:00
committed by GitHub
parent ab1112bfb3
commit cb7b7da0a4
1752 changed files with 59525 additions and 33878 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
build/azure-pipelines/common/createAsset.js
View File

@@ -5,15 +5,101 @@
'use strict';
Object.defineProperty(exports, "__esModule", { value: true });
const fs = require("fs");
const url = require("url");
const crypto = require("crypto");
const azure = require("azure-storage");
const mime = require("mime");
const cosmos_1 = require("@azure/cosmos");
const retry_1 = require("./retry");
if (process.argv.length !== 6) {
console.error('Usage: node createAsset.js PLATFORM TYPE NAME FILE');
if (process.argv.length !== 8) {
console.error('Usage: node createAsset.js PRODUCT OS ARCH TYPE NAME FILE');
process.exit(-1);
}
// Contains all of the logic for mapping details to our actual product names in CosmosDB
function getPlatform(product, os, arch, type) {
switch (os) {
case 'win32':
switch (product) {
case 'client':
const asset = arch === 'ia32' ? 'win32' : `win32-${arch}`;
switch (type) {
case 'archive':
return `${asset}-archive`;
case 'setup':
return asset;
case 'user-setup':
return `${asset}-user`;
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
case 'server':
if (arch === 'arm64') {
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
return arch === 'ia32' ? 'server-win32' : `server-win32-${arch}`;
case 'web':
if (arch === 'arm64') {
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
return arch === 'ia32' ? 'server-win32-web' : `server-win32-${arch}-web`;
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
case 'linux':
switch (type) {
case 'snap':
return `linux-snap-${arch}`;
case 'archive-unsigned':
switch (product) {
case 'client':
return `linux-${arch}`;
case 'server':
return `server-linux-${arch}`;
case 'web':
return arch === 'standalone' ? 'web-standalone' : `server-linux-${arch}-web`;
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
case 'deb-package':
return `linux-deb-${arch}`;
case 'rpm-package':
return `linux-rpm-${arch}`;
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
case 'darwin':
switch (product) {
case 'client':
if (arch === 'x64') {
return 'darwin';
}
return `darwin-${arch}`;
case 'server':
return 'server-darwin';
case 'web':
if (arch !== 'x64') {
throw `What should the platform be?: ${product} ${os} ${arch} ${type}`;
}
return 'server-darwin-web';
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
}
// Contains all of the logic for mapping types to our actual types in CosmosDB
function getRealType(type) {
switch (type) {
case 'user-setup':
return 'setup';
case 'deb-package':
case 'rpm-package':
return 'package';
default:
return type;
}
}
function hashStream(hashName, stream) {
return new Promise((c, e) => {
const shasum = crypto.createHash(hashName);
@@ -45,7 +131,10 @@ function getEnv(name) {
return result;
}
async function main() {
const [, , platform, type, fileName, filePath] = process.argv;
const [, , product, os, arch, unprocessedType, fileName, filePath] = process.argv;
// getPlatform needs the unprocessedType
const platform = getPlatform(product, os, arch, unprocessedType);
const type = getRealType(unprocessedType);
const quality = getEnv('VSCODE_QUALITY');
const commit = getEnv('BUILD_SOURCEVERSION');
console.log('Creating asset...');
@@ -65,14 +154,27 @@ async function main() {
console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
return;
}
console.log('Uploading blobs to Azure storage...');
await uploadBlob(blobService, quality, blobName, filePath, fileName);
const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY'], `${storageAccount}.blob.core.chinacloudapi.cn`)
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
// mooncake is fussy and far away, this is needed!
blobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
console.log('Uploading blobs to Azure storage and Mooncake Azure storage...');
await retry_1.retry(() => Promise.all([
uploadBlob(blobService, quality, blobName, filePath, fileName),
uploadBlob(mooncakeBlobService, quality, blobName, filePath, fileName)
]));
console.log('Blobs successfully uploaded.');
// TODO: Understand if blobName and blobPath are the same and replace blobPath with blobName if so.
const assetUrl = `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`;
const blobPath = url.parse(assetUrl).path;
const mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
const asset = {
platform,
type,
url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
url: assetUrl,
hash: sha1hash,
mooncakeUrl,
sha256hash,
size
};
@@ -83,7 +185,8 @@ async function main() {
console.log('Asset:', JSON.stringify(asset, null, ' '));
const client = new cosmos_1.CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT'], key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
const scripts = client.database('builds').container(quality).scripts;
await (0, retry_1.retry)(() => scripts.storedProcedure('createAsset').execute('', [commit, asset, true]));
await retry_1.retry(() => scripts.storedProcedure('createAsset').execute('', [commit, asset, true]));
console.log(` Done ✔️`);
}
main().then(() => {
console.log('Asset successfully created');

121
build/azure-pipelines/common/createAsset.ts
View File

@@ -6,6 +6,7 @@
'use strict';
import * as fs from 'fs';
import * as url from 'url';
import { Readable } from 'stream';
import * as crypto from 'crypto';
import * as azure from 'azure-storage';
@@ -24,11 +25,98 @@ interface Asset {
supportsFastUpdate?: boolean;
}
if (process.argv.length !== 6) {
console.error('Usage: node createAsset.js PLATFORM TYPE NAME FILE');
if (process.argv.length !== 8) {
console.error('Usage: node createAsset.js PRODUCT OS ARCH TYPE NAME FILE');
process.exit(-1);
}
// Contains all of the logic for mapping details to our actual product names in CosmosDB
function getPlatform(product: string, os: string, arch: string, type: string): string {
switch (os) {
case 'win32':
switch (product) {
case 'client':
const asset = arch === 'ia32' ? 'win32' : `win32-${arch}`;
switch (type) {
case 'archive':
return `${asset}-archive`;
case 'setup':
return asset;
case 'user-setup':
return `${asset}-user`;
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
case 'server':
if (arch === 'arm64') {
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
return arch === 'ia32' ? 'server-win32' : `server-win32-${arch}`;
case 'web':
if (arch === 'arm64') {
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
return arch === 'ia32' ? 'server-win32-web' : `server-win32-${arch}-web`;
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
case 'linux':
switch (type) {
case 'snap':
return `linux-snap-${arch}`;
case 'archive-unsigned':
switch (product) {
case 'client':
return `linux-${arch}`;
case 'server':
return `server-linux-${arch}`;
case 'web':
return arch === 'standalone' ? 'web-standalone' : `server-linux-${arch}-web`;
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
case 'deb-package':
return `linux-deb-${arch}`;
case 'rpm-package':
return `linux-rpm-${arch}`;
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
case 'darwin':
switch (product) {
case 'client':
if (arch === 'x64') {
return 'darwin';
}
return `darwin-${arch}`;
case 'server':
return 'server-darwin';
case 'web':
if (arch !== 'x64') {
throw `What should the platform be?: ${product} ${os} ${arch} ${type}`;
}
return 'server-darwin-web';
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
default:
throw `Unrecognized: ${product} ${os} ${arch} ${type}`;
}
}
// Contains all of the logic for mapping types to our actual types in CosmosDB
function getRealType(type: string) {
switch (type) {
case 'user-setup':
return 'setup';
case 'deb-package':
case 'rpm-package':
return 'package';
default:
return type;
}
}
function hashStream(hashName: string, stream: Readable): Promise<string> {
return new Promise<string>((c, e) => {
const shasum = crypto.createHash(hashName);
@@ -68,7 +156,10 @@ function getEnv(name: string): string {
}
async function main(): Promise<void> {
const [, , platform, type, fileName, filePath] = process.argv;
const [, , product, os, arch, unprocessedType, fileName, filePath] = process.argv;
// getPlatform needs the unprocessedType
const platform = getPlatform(product, os, arch, unprocessedType);
const type = getRealType(unprocessedType);
const quality = getEnv('VSCODE_QUALITY');
const commit = getEnv('BUILD_SOURCEVERSION');
@@ -98,17 +189,33 @@ async function main(): Promise<void> {
return;
}
console.log('Uploading blobs to Azure storage...');
const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY']!, `${storageAccount}.blob.core.chinacloudapi.cn`)
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
await uploadBlob(blobService, quality, blobName, filePath, fileName);
// mooncake is fussy and far away, this is needed!
blobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
console.log('Uploading blobs to Azure storage and Mooncake Azure storage...');
await retry(() => Promise.all([
uploadBlob(blobService, quality, blobName, filePath, fileName),
uploadBlob(mooncakeBlobService, quality, blobName, filePath, fileName)
]));
console.log('Blobs successfully uploaded.');
// TODO: Understand if blobName and blobPath are the same and replace blobPath with blobName if so.
const assetUrl = `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`;
const blobPath = url.parse(assetUrl).path;
const mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
const asset: Asset = {
platform,
type,
url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
url: assetUrl,
hash: sha1hash,
mooncakeUrl,
sha256hash,
size
};
@@ -123,6 +230,8 @@ async function main(): Promise<void> {
const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
const scripts = client.database('builds').container(quality).scripts;
await retry(() => scripts.storedProcedure('createAsset').execute('', [commit, asset, true]));
console.log(` Done ✔️`);
}
main().then(() => {

2
build/azure-pipelines/common/createBuild.js
View File

@@ -40,7 +40,7 @@ async function main() {
};
const client = new cosmos_1.CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT'], key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
const scripts = client.database('builds').container(quality).scripts;
await (0, retry_1.retry)(() => scripts.storedProcedure('createBuild').execute('', [Object.assign(Object.assign({}, build), { _partitionKey: '' })]));
await retry_1.retry(() => scripts.storedProcedure('createBuild').execute('', [Object.assign(Object.assign({}, build), { _partitionKey: '' })]));
}
main().then(() => {
console.log('Build successfully created');

12
build/azure-pipelines/common/extract-telemetry.sh
View File

@@ -4,12 +4,12 @@ set -e
cd $BUILD_STAGINGDIRECTORY
mkdir extraction
cd extraction
git clone --depth 1 https://github.com/Microsoft/vscode-extension-telemetry.git
git clone --depth 1 https://github.com/Microsoft/vscode-chrome-debug-core.git
git clone --depth 1 https://github.com/Microsoft/vscode-node-debug2.git
git clone --depth 1 https://github.com/Microsoft/vscode-node-debug.git
git clone --depth 1 https://github.com/Microsoft/vscode-html-languageservice.git
git clone --depth 1 https://github.com/Microsoft/vscode-json-languageservice.git
git clone --depth 1 https://github.com/microsoft/vscode-extension-telemetry.git
git clone --depth 1 https://github.com/microsoft/vscode-chrome-debug-core.git
git clone --depth 1 https://github.com/microsoft/vscode-node-debug2.git
git clone --depth 1 https://github.com/microsoft/vscode-node-debug.git
git clone --depth 1 https://github.com/microsoft/vscode-html-languageservice.git
git clone --depth 1 https://github.com/microsoft/vscode-json-languageservice.git
node $BUILD_SOURCESDIRECTORY/node_modules/.bin/vscode-telemetry-extractor --sourceDir $BUILD_SOURCESDIRECTORY --excludedDir $BUILD_SOURCESDIRECTORY/extensions --outputDir . --applyEndpoints
node $BUILD_SOURCESDIRECTORY/node_modules/.bin/vscode-telemetry-extractor --config $BUILD_SOURCESDIRECTORY/build/azure-pipelines/common/telemetry-config.json -o .
mkdir -p $BUILD_SOURCESDIRECTORY/.build/telemetry

4
build/azure-pipelines/common/publish-webview.js
View File

@@ -39,7 +39,7 @@ async function publish(commit, files) {
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
await assertContainer(blobService, commit);
for (const file of files) {
const blobName = (0, path_1.basename)(file);
const blobName = path_1.basename(file);
const blobExists = await doesBlobExist(blobService, commit, blobName);
if (blobExists) {
console.log(`Blob ${commit}, ${blobName} already exists, not publishing again.`);
@@ -58,7 +58,7 @@ function main() {
}
const opts = minimist(process.argv.slice(2));
const [directory] = opts._;
const files = fileNames.map(fileName => (0, path_1.join)(directory, fileName));
const files = fileNames.map(fileName => path_1.join(directory, fileName));
publish(commit, files).catch(err => {
console.error(err);
process.exit(1);

2
build/azure-pipelines/common/releaseBuild.js
View File

@@ -39,7 +39,7 @@ async function main() {
}
console.log(`Releasing build ${commit}...`);
const scripts = client.database('builds').container(quality).scripts;
await (0, retry_1.retry)(() => scripts.storedProcedure('releaseBuild').execute('', [commit]));
await retry_1.retry(() => scripts.storedProcedure('releaseBuild').execute('', [commit]));
}
main().then(() => {
console.log('Build successfully released');

87
build/azure-pipelines/common/sync-mooncake.js
View File

@@ -1,87 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
Object.defineProperty(exports, "__esModule", { value: true });
const url = require("url");
const azure = require("azure-storage");
const mime = require("mime");
const cosmos_1 = require("@azure/cosmos");
const retry_1 = require("./retry");
function log(...args) {
console.log(...[`[${new Date().toISOString()}]`, ...args]);
}
function error(...args) {
console.error(...[`[${new Date().toISOString()}]`, ...args]);
}
if (process.argv.length < 3) {
error('Usage: node sync-mooncake.js <quality>');
process.exit(-1);
}
async function sync(commit, quality) {
log(`Synchronizing Mooncake assets for ${quality}, ${commit}...`);
const client = new cosmos_1.CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT'], key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
const container = client.database('builds').container(quality);
const query = `SELECT TOP 1 * FROM c WHERE c.id = "${commit}"`;
const res = await container.items.query(query, {}).fetchAll();
if (res.resources.length !== 1) {
throw new Error(`No builds found for ${commit}`);
}
const build = res.resources[0];
log(`Found build for ${commit}, with ${build.assets.length} assets`);
const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2'];
const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2'])
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY'], `${storageAccount}.blob.core.chinacloudapi.cn`)
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
// mooncake is fussy and far away, this is needed!
blobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
for (const asset of build.assets) {
try {
const blobPath = url.parse(asset.url).path;
if (!blobPath) {
throw new Error(`Failed to parse URL: ${asset.url}`);
}
const blobName = blobPath.replace(/^\/\w+\//, '');
log(`Found ${blobName}`);
if (asset.mooncakeUrl) {
log(` Already in Mooncake ✔️`);
continue;
}
const readStream = blobService.createReadStream(quality, blobName, undefined);
const blobOptions = {
contentSettings: {
contentType: mime.lookup(blobPath),
cacheControl: 'max-age=31536000, public'
}
};
const writeStream = mooncakeBlobService.createWriteStreamToBlockBlob(quality, blobName, blobOptions, undefined);
log(` Uploading to Mooncake...`);
await new Promise((c, e) => readStream.pipe(writeStream).on('finish', c).on('error', e));
log(` Updating build in DB...`);
const mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
await (0, retry_1.retry)(() => container.scripts.storedProcedure('setAssetMooncakeUrl')
.execute('', [commit, asset.platform, asset.type, mooncakeUrl]));
log(` Done ✔️`);
}
catch (err) {
error(err);
}
}
log(`All done ✔️`);
}
function main() {
const commit = process.env['BUILD_SOURCEVERSION'];
if (!commit) {
error('Skipping publish due to missing BUILD_SOURCEVERSION');
return;
}
const quality = process.argv[2];
sync(commit, quality).catch(err => {
error(err);
process.exit(1);
});
}
main();

131
build/azure-pipelines/common/sync-mooncake.ts
View File

@@ -1,131 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as url from 'url';
import * as azure from 'azure-storage';
import * as mime from 'mime';
import { CosmosClient } from '@azure/cosmos';
import { retry } from './retry';
function log(...args: any[]) {
console.log(...[`[${new Date().toISOString()}]`, ...args]);
}
function error(...args: any[]) {
console.error(...[`[${new Date().toISOString()}]`, ...args]);
}
if (process.argv.length < 3) {
error('Usage: node sync-mooncake.js <quality>');
process.exit(-1);
}
interface Build {
assets: Asset[];
}
interface Asset {
platform: string;
type: string;
url: string;
mooncakeUrl: string;
hash: string;
sha256hash: string;
size: number;
supportsFastUpdate?: boolean;
}
async function sync(commit: string, quality: string): Promise<void> {
log(`Synchronizing Mooncake assets for ${quality}, ${commit}...`);
const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
const container = client.database('builds').container(quality);
const query = `SELECT TOP 1 * FROM c WHERE c.id = "${commit}"`;
const res = await container.items.query<Build>(query, {}).fetchAll();
if (res.resources.length !== 1) {
throw new Error(`No builds found for ${commit}`);
}
const build = res.resources[0];
log(`Found build for ${commit}, with ${build.assets.length} assets`);
const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2']!;
const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY']!, `${storageAccount}.blob.core.chinacloudapi.cn`)
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
// mooncake is fussy and far away, this is needed!
blobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
for (const asset of build.assets) {
try {
const blobPath = url.parse(asset.url).path;
if (!blobPath) {
throw new Error(`Failed to parse URL: ${asset.url}`);
}
const blobName = blobPath.replace(/^\/\w+\//, '');
log(`Found ${blobName}`);
if (asset.mooncakeUrl) {
log(` Already in Mooncake ✔️`);
continue;
}
const readStream = blobService.createReadStream(quality, blobName, undefined!);
const blobOptions: azure.BlobService.CreateBlockBlobRequestOptions = {
contentSettings: {
contentType: mime.lookup(blobPath),
cacheControl: 'max-age=31536000, public'
}
};
const writeStream = mooncakeBlobService.createWriteStreamToBlockBlob(quality, blobName, blobOptions, undefined);
log(` Uploading to Mooncake...`);
await new Promise((c, e) => readStream.pipe(writeStream).on('finish', c).on('error', e));
log(` Updating build in DB...`);
const mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
await retry(() => container.scripts.storedProcedure('setAssetMooncakeUrl')
.execute('', [commit, asset.platform, asset.type, mooncakeUrl]));
log(` Done ✔️`);
} catch (err) {
error(err);
}
}
log(`All done ✔️`);
}
function main(): void {
const commit = process.env['BUILD_SOURCEVERSION'];
if (!commit) {
error('Skipping publish due to missing BUILD_SOURCEVERSION');
return;
}
const quality = process.argv[2];
sync(commit, quality).catch(err => {
error(err);
process.exit(1);
});
}
main();