Merge from vscode 31e03b8ffbb218a87e3941f2b63a249f061fe0e4 (#4986)

build/azure-pipelines/common/publish.ts

@@ -65,8 +65,7 @@ interface Asset {
 	platform: string;
 	type: string;
 	url: string;
-	// {{SQL CARBON EDIT}}
-	mooncakeUrl: string | undefined;
+	mooncakeUrl?: string;
 	hash: string;
 	sha256hash: string;
 	size: number;
@@ -189,56 +188,18 @@ async function publish(commit: string, quality: string, platform: string, type:
 	const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
 		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
 
-	// {{SQL CARBON EDIT}}
 	await assertContainer(blobService, quality);
 
 	const blobExists = await doesAssetExist(blobService, quality, blobName);
 
-	const promises = [];
-
-	if (!blobExists) {
-		promises.push(uploadBlob(blobService, quality, blobName, file));
-	}
-
-	// {{SQL CARBON EDIT}}
-	if (process.env['MOONCAKE_STORAGE_ACCESS_KEY']) {
-		const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY']!, `${storageAccount}.blob.core.chinacloudapi.cn`)
-			.withFilter(new azure.ExponentialRetryPolicyFilter(20));
-
-		// mooncake is fussy and far away, this is needed!
-		mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
-
-		await Promise.all([
-			assertContainer(blobService, quality),
-			assertContainer(mooncakeBlobService, quality)
-		]);
-
-		const [blobExists, moooncakeBlobExists] = await Promise.all([
-			doesAssetExist(blobService, quality, blobName),
-			doesAssetExist(mooncakeBlobService, quality, blobName)
-		]);
-
-		const promises: Array<Promise<void>> = [];
-
-		if (!blobExists) {
-			promises.push(uploadBlob(blobService, quality, blobName, file));
-		}
-
-		if (!moooncakeBlobExists) {
-			promises.push(uploadBlob(mooncakeBlobService, quality, blobName, file));
-		}
-	} else {
-		console.log('Skipping Mooncake publishing.');
-	}
-
-	if (promises.length === 0) {
+	if (blobExists) {
 		console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
 		return;
 	}
 
 	console.log('Uploading blobs to Azure storage...');
 
-	await Promise.all(promises);
+	await uploadBlob(blobService, quality, blobName, file);
 
 	console.log('Blobs successfully uploaded.');
 
@@ -250,8 +211,6 @@ async function publish(commit: string, quality: string, platform: string, type:
 		platform: platform,
 		type: type,
 		url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
-		// {{SQL CARBON EDIT}}
-		mooncakeUrl: process.env['MOONCAKE_CDN_URL'] ? `${process.env['MOONCAKE_CDN_URL']}/${quality}/${blobName}` : undefined,
 		hash: sha1hash,
 		sha256hash,
 		size

build/azure-pipelines/common/sync-mooncake.ts

@@ -0,0 +1,176 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as url from 'url';
+import * as azure from 'azure-storage';
+import * as mime from 'mime';
+import { DocumentClient, RetrievedDocument } from 'documentdb';
+
+function log(...args: any[]) {
+	console.log(...[`[${new Date().toISOString()}]`, ...args]);
+}
+
+function error(...args: any[]) {
+	console.error(...[`[${new Date().toISOString()}]`, ...args]);
+}
+
+if (process.argv.length < 3) {
+	error('Usage: node sync-mooncake.js <quality>');
+	process.exit(-1);
+}
+
+interface Build extends RetrievedDocument {
+	assets: Asset[];
+}
+
+interface Asset {
+	platform: string;
+	type: string;
+	url: string;
+	mooncakeUrl: string;
+	hash: string;
+	sha256hash: string;
+	size: number;
+	supportsFastUpdate?: boolean;
+}
+
+function updateBuild(commit: string, quality: string, platform: string, type: string, asset: Asset): Promise<void> {
+	const client = new DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT']!, { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const collection = 'dbs/builds/colls/' + quality;
+	const updateQuery = {
+		query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
+		parameters: [{ name: '@id', value: commit }]
+	};
+
+	let updateTries = 0;
+
+	function _update(): Promise<void> {
+		updateTries++;
+
+		return new Promise<void>((c, e) => {
+			client.queryDocuments(collection, updateQuery).toArray((err, results) => {
+				if (err) { return e(err); }
+				if (results.length !== 1) { return e(new Error('No documents')); }
+
+				const release = results[0];
+
+				release.assets = [
+					...release.assets.filter((a: any) => !(a.platform === platform && a.type === type)),
+					asset
+				];
+
+				client.replaceDocument(release._self, release, err => {
+					if (err && err.code === 409 && updateTries < 5) { return c(_update()); }
+					if (err) { return e(err); }
+
+					log('Build successfully updated.');
+					c();
+				});
+			});
+		});
+	}
+
+	return _update();
+}
+
+async function sync(commit: string, quality: string): Promise<void> {
+	log(`Synchronizing Mooncake assets for ${quality}, ${commit}...`);
+
+	const cosmosdb = new DocumentClient(process.env['AZURE_DOCUMENTDB_ENDPOINT']!, { masterKey: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const collection = `dbs/builds/colls/${quality}`;
+	const query = {
+		query: 'SELECT TOP 1 * FROM c WHERE c.id = @id',
+		parameters: [{ name: '@id', value: commit }]
+	};
+
+	const build = await new Promise<Build>((c, e) => {
+		cosmosdb.queryDocuments(collection, query).toArray((err, results) => {
+			if (err) { return e(err); }
+			if (results.length !== 1) { return e(new Error('No documents')); }
+			c(results[0] as Build);
+		});
+	});
+
+	log(`Found build for ${commit}, with ${build.assets.length} assets`);
+
+	const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2']!;
+
+	const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY']!, `${storageAccount}.blob.core.chinacloudapi.cn`)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	// mooncake is fussy and far away, this is needed!
+	blobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
+	mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
+
+	for (const asset of build.assets) {
+		try {
+			const blobPath = url.parse(asset.url).path;
+
+			if (!blobPath) {
+				throw new Error(`Failed to parse URL: ${asset.url}`);
+			}
+
+			const blobName = blobPath.replace(/^\/\w+\//, '');
+
+			log(`Found ${blobName}`);
+
+			if (asset.mooncakeUrl) {
+				log(`  Already in Mooncake ✔️`);
+				continue;
+			}
+
+			const readStream = blobService.createReadStream(quality, blobName, undefined!);
+			const blobOptions: azure.BlobService.CreateBlockBlobRequestOptions = {
+				contentSettings: {
+					contentType: mime.lookup(blobPath),
+					cacheControl: 'max-age=31536000, public'
+				}
+			};
+
+			const writeStream = mooncakeBlobService.createWriteStreamToBlockBlob(quality, blobName, blobOptions, undefined);
+
+			log(`  Uploading to Mooncake...`);
+			await new Promise((c, e) => readStream.pipe(writeStream).on('finish', c).on('error', e));
+
+			log(`  Updating build in DB...`);
+			asset.mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
+			await updateBuild(commit, quality, asset.platform, asset.type, asset);
+
+			log(`  Done ✔️`);
+		} catch (err) {
+			error(err);
+		}
+	}
+
+	log(`All done ✔️`);
+}
+
+function main(): void {
+	if (process.env['VSCODE_BUILD_SKIP_PUBLISH']) {
+		error('Skipping publish due to VSCODE_BUILD_SKIP_PUBLISH');
+		return;
+	}
+
+	const commit = process.env['BUILD_SOURCEVERSION'];
+
+	if (!commit) {
+		error('Skipping publish due to missing BUILD_SOURCEVERSION');
+		return;
+	}
+
+	const quality = process.argv[2];
+
+	sync(commit, quality).catch(err => {
+		error(err);
+		process.exit(1);
+	});
+}
+
+main();

build/azure-pipelines/darwin/product-build-darwin.yml

@@ -78,7 +78,6 @@ steps:
     VERSION=`node -p "require(\"$PACKAGEJSON\").version"`
     AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
     AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
-    MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
     node build/azure-pipelines/common/publish.js \
       "$(VSCODE_QUALITY)" \
       darwin \

build/azure-pipelines/distro-build.yml

@@ -0,0 +1,29 @@
+trigger:
+  branches:
+    include: ['master', 'release/*']
+pr:
+  branches:
+    include: ['master', 'release/*']
+
+steps:
+- task: NodeTool@0
+  inputs:
+    versionSpec: "10.15.1"
+
+- script: |
+    set -e
+
+    cat << EOF > ~/.netrc
+    machine github.com
+    login vscode
+    password $(VSCODE_MIXIN_PASSWORD)
+    EOF
+
+    git config user.email "vscode@microsoft.com"
+    git config user.name "VSCode"
+
+    git remote add distro "https://github.com/$VSCODE_MIXIN_REPO.git"
+    git fetch distro
+    git merge $(node -p "require('./package.json').distro")
+
+  displayName: Merge Distro

build/azure-pipelines/linux/product-build-linux.yml

@@ -68,7 +68,6 @@ steps:
 
     AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
     AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
-    MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
     node build/azure-pipelines/common/publish.js "$VSCODE_QUALITY" "$PLATFORM_LINUX" archive-unsigned "$TARBALL_FILENAME" "$VERSION" true "$TARBALL_PATH"
 
     # Publish hockeyapp symbols
@@ -83,7 +82,6 @@ steps:
 
     AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
     AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
-    MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
     node build/azure-pipelines/common/publish.js "$VSCODE_QUALITY" "$PLATFORM_DEB" package "$DEB_FILENAME" "$VERSION" true "$DEB_PATH"
 
     # Publish RPM
@@ -95,7 +93,6 @@ steps:
 
     AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
     AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
-    MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
     node build/azure-pipelines/common/publish.js "$VSCODE_QUALITY" "$PLATFORM_RPM" package "$RPM_FILENAME" "$VERSION" true "$RPM_PATH"
 
     # Publish Snap

build/azure-pipelines/linux/snap-build-linux.yml

@@ -46,5 +46,4 @@ steps:
     # Publish snap package
     AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
     AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
-    MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
     node build/azure-pipelines/common/publish.js "$VSCODE_QUALITY" "linux-snap-$ARCH" package "$SNAP_FILENAME" "$VERSION" true "$SNAP_PATH"

build/azure-pipelines/product-build.yml

@@ -62,4 +62,18 @@ jobs:
   pool:
     vmImage: macOS 10.13
   steps:
-  - template: darwin/product-build-darwin.yml
+  - template: darwin/product-build-darwin.yml
+
+- job: Mooncake
+  pool:
+    vmImage: 'Ubuntu-16.04'
+  condition: true
+  dependsOn:
+  - Windows
+  - Windows32
+  - Linux
+  - LinuxSnap
+  - Linux32
+  - macOS
+  steps:
+  - template: sync-mooncake.yml

build/azure-pipelines/sync-mooncake.yml

@@ -0,0 +1,18 @@
+steps:
+- task: NodeTool@0
+  inputs:
+    versionSpec: "10.15.1"
+
+- task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@2
+  inputs:
+    versionSpec: "1.10.1"
+
+- script: |
+    set -e
+
+    (cd build ; yarn)
+
+    AZURE_DOCUMENTDB_MASTERKEY="$(AZURE_DOCUMENTDB_MASTERKEY)" \
+    AZURE_STORAGE_ACCESS_KEY_2="$(AZURE_STORAGE_ACCESS_KEY_2)" \
+    MOONCAKE_STORAGE_ACCESS_KEY="$(MOONCAKE_STORAGE_ACCESS_KEY)" \
+    node build/azure-pipelines/common/sync-mooncake.js "$VSCODE_QUALITY"

build/azure-pipelines/win32/product-build-win32.yml

@@ -133,7 +133,6 @@ steps:
     $Version = $PackageJson.version
     $Quality = "$env:VSCODE_QUALITY"
     $env:AZURE_STORAGE_ACCESS_KEY_2 = "$(AZURE_STORAGE_ACCESS_KEY_2)"
-    $env:MOONCAKE_STORAGE_ACCESS_KEY = "$(MOONCAKE_STORAGE_ACCESS_KEY)"
     $env:AZURE_DOCUMENTDB_MASTERKEY = "$(AZURE_DOCUMENTDB_MASTERKEY)"
 
     $assetPlatform = if ("$(VSCODE_ARCH)" -eq "ia32") { "win32" } else { "win32-x64" }

build/gulpfile.hygiene.js

@@ -87,6 +87,7 @@ const indentationFilter = [
 	'!build/azure-pipelines/**/*.js',
 	'!build/azure-pipelines/**/*.config',
 	'!**/Dockerfile',
+	'!**/Dockerfile.*',
 	'!**/*.Dockerfile',
 	'!**/*.dockerfile',
 	'!extensions/markdown-language-features/media/*.js'

build/gulpfile.vscode.js

@@ -89,6 +89,8 @@ const vscodeResources = [
 	'out-build/vs/workbench/contrib/welcome/walkThrough/**/*.md',
 	'out-build/vs/workbench/services/files/**/*.exe',
 	'out-build/vs/workbench/services/files/**/*.md',
+	'out-build/vs/workbench/services/files2/**/*.exe',
+	'out-build/vs/workbench/services/files2/**/*.md',
 	'out-build/vs/code/electron-browser/workbench/**',
 	'out-build/vs/code/electron-browser/sharedProcess/sharedProcess.js',
 	'out-build/vs/code/electron-browser/issue/issueReporter.js',
@@ -425,6 +427,8 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
 
 			result = es.merge(result, gulp.src('resources/win32/bin/code.sh', { base: 'resources/win32' })
 				.pipe(replace('@@NAME@@', product.nameShort))
+				.pipe(replace('@@PRODNAME@@', product.nameLong))
+				.pipe(replace('@@VERSION@@', version))
 				.pipe(replace('@@COMMIT@@', commit))
 				.pipe(replace('@@APPNAME@@', product.applicationName))
 				.pipe(rename(function (f) { f.basename = product.applicationName; f.extname = ''; })));
@@ -433,6 +437,7 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
 				.pipe(rename(product.nameShort + '.VisualElementsManifest.xml')));
 		} else if (platform === 'linux') {
 			result = es.merge(result, gulp.src('resources/linux/bin/code.sh', { base: '.' })
+				.pipe(replace('@@PRODNAME@@', product.nameLong))
 				.pipe(replace('@@NAME@@', product.applicationName))
 				.pipe(rename('bin/' + product.applicationName)));
 		}

build/lib/i18n.ts

@@ -24,7 +24,7 @@ function log(message: any, ...rest: any[]): void {
 }
 
 export interface Language {
-	id: string; // laguage id, e.g. zh-tw, de
+	id: string; // language id, e.g. zh-tw, de
 	translationId?: string; // language id used in translation tools, e.g zh-hant, de (optional, if not set, the id is used)
 	folderName?: string; // language specific folder name, e.g. cht, deu  (optional, if not set, the id is used)
 }
@@ -1378,4 +1378,4 @@ function decodeEntities(value: string): string {
 
 function pseudify(message: string) {
 	return '\uFF3B' + message.replace(/[aouei]/g, '$&$&') + '\uFF3D';
-}
+}

build/lib/task.js

@@ -32,7 +32,7 @@ async function _doExecute(task) {
     // Always invoke as if it were a callback task
     return new Promise((resolve, reject) => {
         if (task.length === 1) {
-            // this is a calback task
+            // this is a callback task
             task((err) => {
                 if (err) {
                     return reject(err);

build/lib/task.ts

@@ -54,7 +54,7 @@ async function _doExecute(task: Task): Promise<void> {
 	// Always invoke as if it were a callback task
 	return new Promise((resolve, reject) => {
 		if (task.length === 1) {
-			// this is a calback task
+			// this is a callback task
 			task((err) => {
 				if (err) {
 					return reject(err);

build/npm/update-localization-extension.js

@@ -74,7 +74,7 @@ function update(options) {
 			let translationPaths = [];
 			i18n.pullI18nPackFiles(server, userName, apiToken, { id: languageId }, translationPaths)
 				.on('error', (error) => {
-					console.log(`Error occured while importing translations:`);
+					console.log(`Error occurred while importing translations:`);
 					translationPaths = undefined;
 					if (Array.isArray(error)) {
 						error.forEach(console.log);
@@ -100,7 +100,7 @@ function update(options) {
 			gulp.src(path.join(location, languageId, '**', '*.xlf'))
 				.pipe(i18n.prepareI18nPackFiles(i18n.externalExtensionsWithTranslations, translationPaths, languageId === 'ps'))
 				.on('error', (error) => {
-					console.log(`Error occured while importing translations:`);
+					console.log(`Error occurred while importing translations:`);
 					translationPaths = undefined;
 					if (Array.isArray(error)) {
 						error.forEach(console.log);

build/tsconfig.json

@@ -8,7 +8,7 @@
 		"resolveJsonModule": true,
 		"experimentalDecorators": true,
 		// enable JavaScript type checking for the language service
-		// use the tsconfig.build.json for compiling wich disable JavaScript
+		// use the tsconfig.build.json for compiling which disable JavaScript
 		// type checking so that JavaScript file are not transpiled
 		"allowJs": true,
 		"checkJs": true,
@@ -22,4 +22,4 @@
 	"exclude": [
 		"node_modules/**"
 	]
-}
+}