mirror of
https://github.com/ckaczor/azuredatastudio.git
synced 2026-02-16 18:46:40 -05:00
Upgrade xml2js to v0.5.0 + migration to @vscode/vsce + migration to @azure/storage-blob (#22664)
This commit is contained in:
Cheena Malhotra
@@ -6,10 +6,10 @@
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const fs = require("fs");
|
||||
const crypto = require("crypto");
|
||||
const azure = require("azure-storage");
|
||||
const mime = require("mime");
|
||||
const minimist = require("minimist");
|
||||
const documentdb_1 = require("documentdb");
|
||||
const storage_blob_1 = require("@azure/storage-blob");
|
||||
// {{SQL CARBON EDIT}}
|
||||
if (process.argv.length < 9) {
|
||||
console.error('Usage: node publish.js <product_quality> <platform> <file_type> <file_name> <version> <is_update> <file> [commit_id]');
|
||||
@@ -104,21 +104,31 @@ function createOrUpdate(commit, quality, platform, type, release, asset, isUpdat
|
||||
});
|
||||
}));
|
||||
}
|
||||
async function assertContainer(blobService, quality) {
|
||||
await new Promise((c, e) => blobService.createContainerIfNotExists(quality, { publicAccessLevel: 'blob' }, err => err ? e(err) : c()));
|
||||
async function assertContainer(containerClient) {
|
||||
let containerResponse = await containerClient.createIfNotExists({ access: 'blob' });
|
||||
return containerResponse && !!containerResponse.errorCode;
|
||||
}
|
||||
async function doesAssetExist(blobService, quality, blobName) {
|
||||
const existsResult = await new Promise((c, e) => blobService.doesBlobExist(quality, blobName, (err, r) => err ? e(err) : c(r)));
|
||||
return existsResult.exists;
|
||||
}
|
||||
async function uploadBlob(blobService, quality, blobName, file) {
|
||||
const blobOptions = {
|
||||
contentSettings: {
|
||||
contentType: mime.lookup(file),
|
||||
cacheControl: 'max-age=31536000, public'
|
||||
}
|
||||
};
|
||||
await new Promise((c, e) => blobService.createBlockBlobFromLocalFile(quality, blobName, file, blobOptions, err => err ? e(err) : c()));
|
||||
async function uploadBlob(blobClient, file) {
|
||||
var _a;
|
||||
blobClient.setHTTPHeaders({
|
||||
blobContentType: mime.lookup(file),
|
||||
blobCacheControl: 'max-age=31536000, public'
|
||||
});
|
||||
const copyPoller = await blobClient.beginCopyFromURL(file, {
|
||||
onProgress(state) {
|
||||
console.log(`Progress: ${state.copyProgress}`);
|
||||
},
|
||||
});
|
||||
while (!copyPoller.isDone()) {
|
||||
await copyPoller.poll();
|
||||
}
|
||||
const result = copyPoller.getResult();
|
||||
if (result && result.copyStatus === 'success') {
|
||||
console.log(`Blobs uploaded successfully.`);
|
||||
}
|
||||
else {
|
||||
console.error(`Blobs failed to upload, response status: ${(_a = result === null || result === void 0 ? void 0 : result._response) === null || _a === void 0 ? void 0 : _a.status}, copy status: ${result === null || result === void 0 ? void 0 : result.copyStatus}, errorcode: ${result === null || result === void 0 ? void 0 : result.errorCode}`);
|
||||
}
|
||||
}
|
||||
async function publish(commit, quality, platform, type, name, version, _isUpdate, file, opts) {
|
||||
const isUpdate = _isUpdate === 'true';
|
||||
@@ -142,54 +152,62 @@ async function publish(commit, quality, platform, type, name, version, _isUpdate
|
||||
console.log('SHA256:', sha256hash);
|
||||
const blobName = commit + '/' + name;
|
||||
const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2'];
|
||||
const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2'])
|
||||
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
|
||||
await assertContainer(blobService, quality);
|
||||
const blobExists = await doesAssetExist(blobService, quality, blobName);
|
||||
if (blobExists) {
|
||||
console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
|
||||
return;
|
||||
}
|
||||
console.log('Uploading blobs to Azure storage...');
|
||||
await uploadBlob(blobService, quality, blobName, file);
|
||||
console.log('Blobs successfully uploaded.');
|
||||
const config = await getConfig(quality);
|
||||
console.log('Quality config:', config);
|
||||
const asset = {
|
||||
platform: platform,
|
||||
type: type,
|
||||
url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
|
||||
hash: sha1hash,
|
||||
sha256hash,
|
||||
size
|
||||
};
|
||||
// Remove this if we ever need to rollback fast updates for windows
|
||||
if (/win32/.test(platform)) {
|
||||
asset.supportsFastUpdate = true;
|
||||
}
|
||||
console.log('Asset:', JSON.stringify(asset, null, ' '));
|
||||
// {{SQL CARBON EDIT}}
|
||||
// Insiders: nightly build from main
|
||||
const isReleased = (((quality === 'insider' && /^main$|^refs\/heads\/main$/.test(sourceBranch)) ||
|
||||
(quality === 'rc1' && /^release\/|^refs\/heads\/release\//.test(sourceBranch))) &&
|
||||
/Project Collection Service Accounts|Microsoft.VisualStudio.Services.TFS/.test(queuedBy));
|
||||
const release = {
|
||||
id: commit,
|
||||
timestamp: (new Date()).getTime(),
|
||||
version,
|
||||
isReleased: isReleased,
|
||||
sourceBranch,
|
||||
queuedBy,
|
||||
assets: [],
|
||||
updates: {}
|
||||
};
|
||||
if (!opts['upload-only']) {
|
||||
release.assets.push(asset);
|
||||
if (isUpdate) {
|
||||
release.updates[platform] = type;
|
||||
const storageKey = process.env['AZURE_STORAGE_ACCESS_KEY_2'];
|
||||
const connectionString = `DefaultEndpointsProtocol=https;AccountName=${storageAccount};AccountKey=${storageKey}`;
|
||||
let blobServiceClient = storage_blob_1.BlobServiceClient.fromConnectionString(connectionString, {
|
||||
retryOptions: {
|
||||
maxTries: 20,
|
||||
retryPolicyType: storage_blob_1.StorageRetryPolicyType.EXPONENTIAL
|
||||
}
|
||||
});
|
||||
let containerClient = blobServiceClient.getContainerClient(quality);
|
||||
if (await assertContainer(containerClient)) {
|
||||
const blobClient = containerClient.getBlobClient(blobName);
|
||||
const blobExists = await blobClient.exists();
|
||||
if (blobExists) {
|
||||
console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
|
||||
return;
|
||||
}
|
||||
console.log('Uploading blobs to Azure storage...');
|
||||
await uploadBlob(blobClient, file);
|
||||
const config = await getConfig(quality);
|
||||
console.log('Quality config:', config);
|
||||
const asset = {
|
||||
platform: platform,
|
||||
type: type,
|
||||
url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
|
||||
hash: sha1hash,
|
||||
sha256hash,
|
||||
size
|
||||
};
|
||||
// Remove this if we ever need to rollback fast updates for windows
|
||||
if (/win32/.test(platform)) {
|
||||
asset.supportsFastUpdate = true;
|
||||
}
|
||||
console.log('Asset:', JSON.stringify(asset, null, ' '));
|
||||
// {{SQL CARBON EDIT}}
|
||||
// Insiders: nightly build from main
|
||||
const isReleased = (((quality === 'insider' && /^main$|^refs\/heads\/main$/.test(sourceBranch)) ||
|
||||
(quality === 'rc1' && /^release\/|^refs\/heads\/release\//.test(sourceBranch))) &&
|
||||
/Project Collection Service Accounts|Microsoft.VisualStudio.Services.TFS/.test(queuedBy));
|
||||
const release = {
|
||||
id: commit,
|
||||
timestamp: (new Date()).getTime(),
|
||||
version,
|
||||
isReleased: isReleased,
|
||||
sourceBranch,
|
||||
queuedBy,
|
||||
assets: [],
|
||||
updates: {}
|
||||
};
|
||||
if (!opts['upload-only']) {
|
||||
release.assets.push(asset);
|
||||
if (isUpdate) {
|
||||
release.updates[platform] = type;
|
||||
}
|
||||
}
|
||||
await createOrUpdate(commit, quality, platform, type, release, asset, isUpdate);
|
||||
}
|
||||
await createOrUpdate(commit, quality, platform, type, release, asset, isUpdate);
|
||||
}
|
||||
const RETRY_TIMES = 10;
|
||||
async function retry(fn) {
|
||||
|
||||
@@ -8,10 +8,10 @@
|
||||
import * as fs from 'fs';
|
||||
import { Readable } from 'stream';
|
||||
import * as crypto from 'crypto';
|
||||
import * as azure from 'azure-storage';
|
||||
import * as mime from 'mime';
|
||||
import * as minimist from 'minimist';
|
||||
import { DocumentClient, NewDocument } from 'documentdb';
|
||||
import { BlobServiceClient, BlobClient, ContainerClient, StorageRetryPolicyType } from '@azure/storage-blob';
|
||||
|
||||
// {{SQL CARBON EDIT}}
|
||||
if (process.argv.length < 9) {
|
||||
@@ -127,24 +127,31 @@ function createOrUpdate(commit: string, quality: string, platform: string, type:
|
||||
}));
|
||||
}
|
||||
|
||||
async function assertContainer(blobService: azure.BlobService, quality: string): Promise<void> {
|
||||
await new Promise<void>((c, e) => blobService.createContainerIfNotExists(quality, { publicAccessLevel: 'blob' }, err => err ? e(err) : c()));
|
||||
async function assertContainer(containerClient: ContainerClient): Promise<boolean> {
|
||||
let containerResponse = await containerClient.createIfNotExists({ access: 'blob' });
|
||||
return containerResponse && !!containerResponse.errorCode;
|
||||
}
|
||||
|
||||
async function doesAssetExist(blobService: azure.BlobService, quality: string, blobName: string): Promise<boolean | undefined> {
|
||||
const existsResult = await new Promise<azure.BlobService.BlobResult>((c, e) => blobService.doesBlobExist(quality, blobName, (err, r) => err ? e(err) : c(r)));
|
||||
return existsResult.exists;
|
||||
}
|
||||
async function uploadBlob(blobClient: BlobClient, file: string): Promise<void> {
|
||||
blobClient.setHTTPHeaders({
|
||||
blobContentType: mime.lookup(file),
|
||||
blobCacheControl: 'max-age=31536000, public'
|
||||
});
|
||||
const copyPoller = await blobClient.beginCopyFromURL(file, {
|
||||
onProgress(state) {
|
||||
console.log(`Progress: ${state.copyProgress}`);
|
||||
},
|
||||
|
||||
async function uploadBlob(blobService: azure.BlobService, quality: string, blobName: string, file: string): Promise<void> {
|
||||
const blobOptions: azure.BlobService.CreateBlockBlobRequestOptions = {
|
||||
contentSettings: {
|
||||
contentType: mime.lookup(file),
|
||||
cacheControl: 'max-age=31536000, public'
|
||||
}
|
||||
};
|
||||
|
||||
await new Promise<void>((c, e) => blobService.createBlockBlobFromLocalFile(quality, blobName, file, blobOptions, err => err ? e(err) : c()));
|
||||
});
|
||||
while (!copyPoller.isDone()) {
|
||||
await copyPoller.poll();
|
||||
}
|
||||
const result = copyPoller.getResult();
|
||||
if (result && result.copyStatus === 'success') {
|
||||
console.log(`Blobs uploaded successfully.`);
|
||||
} else {
|
||||
console.error(`Blobs failed to upload, response status: ${result?._response?.status}, copy status: ${result?.copyStatus}, errorcode: ${result?.errorCode}`)
|
||||
}
|
||||
}
|
||||
|
||||
interface PublishOptions {
|
||||
@@ -180,74 +187,78 @@ async function publish(commit: string, quality: string, platform: string, type:
|
||||
|
||||
const blobName = commit + '/' + name;
|
||||
const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2']!;
|
||||
const storageKey = process.env['AZURE_STORAGE_ACCESS_KEY_2']!;
|
||||
const connectionString = `DefaultEndpointsProtocol=https;AccountName=${storageAccount};AccountKey=${storageKey}`;
|
||||
|
||||
const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
|
||||
.withFilter(new azure.ExponentialRetryPolicyFilter(20));
|
||||
|
||||
await assertContainer(blobService, quality);
|
||||
|
||||
const blobExists = await doesAssetExist(blobService, quality, blobName);
|
||||
|
||||
if (blobExists) {
|
||||
console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
|
||||
return;
|
||||
}
|
||||
console.log('Uploading blobs to Azure storage...');
|
||||
|
||||
await uploadBlob(blobService, quality, blobName, file);
|
||||
|
||||
console.log('Blobs successfully uploaded.');
|
||||
|
||||
const config = await getConfig(quality);
|
||||
|
||||
console.log('Quality config:', config);
|
||||
|
||||
const asset: Asset = {
|
||||
platform: platform,
|
||||
type: type,
|
||||
url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
|
||||
hash: sha1hash,
|
||||
sha256hash,
|
||||
size
|
||||
};
|
||||
|
||||
// Remove this if we ever need to rollback fast updates for windows
|
||||
if (/win32/.test(platform)) {
|
||||
asset.supportsFastUpdate = true;
|
||||
}
|
||||
|
||||
console.log('Asset:', JSON.stringify(asset, null, ' '));
|
||||
|
||||
// {{SQL CARBON EDIT}}
|
||||
// Insiders: nightly build from main
|
||||
const isReleased = (
|
||||
(
|
||||
(quality === 'insider' && /^main$|^refs\/heads\/main$/.test(sourceBranch)) ||
|
||||
(quality === 'rc1' && /^release\/|^refs\/heads\/release\//.test(sourceBranch))
|
||||
) &&
|
||||
/Project Collection Service Accounts|Microsoft.VisualStudio.Services.TFS/.test(queuedBy)
|
||||
);
|
||||
|
||||
const release = {
|
||||
id: commit,
|
||||
timestamp: (new Date()).getTime(),
|
||||
version,
|
||||
isReleased: isReleased,
|
||||
sourceBranch,
|
||||
queuedBy,
|
||||
assets: [] as Array<Asset>,
|
||||
updates: {} as any
|
||||
};
|
||||
|
||||
if (!opts['upload-only']) {
|
||||
release.assets.push(asset);
|
||||
|
||||
if (isUpdate) {
|
||||
release.updates[platform] = type;
|
||||
let blobServiceClient = BlobServiceClient.fromConnectionString(connectionString, {
|
||||
retryOptions: {
|
||||
maxTries: 20,
|
||||
retryPolicyType: StorageRetryPolicyType.EXPONENTIAL
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await createOrUpdate(commit, quality, platform, type, release, asset, isUpdate);
|
||||
let containerClient = blobServiceClient.getContainerClient(quality);
|
||||
if (await assertContainer(containerClient)) {
|
||||
const blobClient = containerClient.getBlobClient(blobName);
|
||||
const blobExists = await blobClient.exists();
|
||||
|
||||
if (blobExists) {
|
||||
console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Uploading blobs to Azure storage...');
|
||||
await uploadBlob(blobClient, file);
|
||||
const config = await getConfig(quality);
|
||||
|
||||
console.log('Quality config:', config);
|
||||
const asset: Asset = {
|
||||
platform: platform,
|
||||
type: type,
|
||||
url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
|
||||
hash: sha1hash,
|
||||
sha256hash,
|
||||
size
|
||||
};
|
||||
|
||||
// Remove this if we ever need to rollback fast updates for windows
|
||||
if (/win32/.test(platform)) {
|
||||
asset.supportsFastUpdate = true;
|
||||
}
|
||||
|
||||
console.log('Asset:', JSON.stringify(asset, null, ' '));
|
||||
|
||||
// {{SQL CARBON EDIT}}
|
||||
// Insiders: nightly build from main
|
||||
const isReleased = (
|
||||
(
|
||||
(quality === 'insider' && /^main$|^refs\/heads\/main$/.test(sourceBranch)) ||
|
||||
(quality === 'rc1' && /^release\/|^refs\/heads\/release\//.test(sourceBranch))
|
||||
) &&
|
||||
/Project Collection Service Accounts|Microsoft.VisualStudio.Services.TFS/.test(queuedBy)
|
||||
);
|
||||
|
||||
const release = {
|
||||
id: commit,
|
||||
timestamp: (new Date()).getTime(),
|
||||
version,
|
||||
isReleased: isReleased,
|
||||
sourceBranch,
|
||||
queuedBy,
|
||||
assets: [] as Array<Asset>,
|
||||
updates: {} as any
|
||||
};
|
||||
|
||||
if (!opts['upload-only']) {
|
||||
release.assets.push(asset);
|
||||
|
||||
if (isUpdate) {
|
||||
release.updates[platform] = type;
|
||||
}
|
||||
}
|
||||
|
||||
await createOrUpdate(commit, quality, platform, type, release, asset, isUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
const RETRY_TIMES = 10;
|
||||
|
||||
Reference in New Issue
Block a user