Merge from vscode 81d7885dc2e9dc617e1522697a2966bc4025a45d (#5949)

* Merge from vscode 81d7885dc2e9dc617e1522697a2966bc4025a45d

* Fix vs unit tests and hygiene issue

* Fix strict null check issue

build/.nativeignore

@@ -1,5 +1,8 @@
 # cleanup rules for native node modules, .gitignore style
 
+nan/**
+*/node_modules/nan/**
+
 fsevents/binding.gyp
 fsevents/fsevents.cc
 fsevents/build/**
@@ -83,6 +86,7 @@ node-pty/binding.gyp
 node-pty/build/**
 node-pty/src/**
 node-pty/tools/**
+node-pty/deps/**
 !node-pty/build/Release/*.exe
 !node-pty/build/Release/*.dll
 !node-pty/build/Release/*.node

build/azure-pipelines/common/installDistro.ts

@@ -1,20 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import * as cp from 'child_process';
-import * as path from 'path';
-
-function yarnInstall(packageName: string): void {
-	cp.execSync(`yarn add --no-lockfile ${packageName}`);
-	cp.execSync(`yarn add --no-lockfile ${packageName}`, { cwd: path.join( process.cwd(), 'remote') });
-}
-
-const product = require('../../../product.json');
-const dependencies = product.dependencies || {} as { [name: string]: string; };
-
-Object.keys(dependencies).forEach(name => {
-	const url = dependencies[name];
-	yarnInstall(url);
-});

build/azure-pipelines/common/installDistroDependencies.ts

@@ -0,0 +1,38 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as cp from 'child_process';
+import * as path from 'path';
+import * as fs from 'fs';
+
+function yarnInstall(packageName: string, cwd: string): void {
+	console.log(`yarn add --no-lockfile ${packageName}`, cwd);
+	cp.execSync(`yarn add --no-lockfile ${packageName}`, { cwd, stdio: 'inherit' });
+}
+
+/**
+ * Install additional dependencies listed on each quality `package.json` file.
+ */
+function main() {
+	const quality = process.env['VSCODE_QUALITY'];
+
+	if (!quality) {
+		throw new Error('Missing VSCODE_QUALITY, can\'t install distro');
+	}
+
+	const rootPath = path.dirname(path.dirname(path.dirname(__dirname)));
+	const qualityPath = path.join(rootPath, 'quality', quality);
+	const packagePath = path.join(qualityPath, 'package.json');
+	const pkg = JSON.parse(fs.readFileSync(packagePath, 'utf8'));
+	const dependencies = pkg.dependencies || {} as { [name: string]: string; };
+
+	Object.keys(dependencies).forEach(name => {
+		const url = dependencies[name];
+		const cwd = process.argv.length < 3 ? process.cwd() : path.join(process.cwd(), process.argv[2]);
+		yarnInstall(url, cwd);
+	});
+}
+
+main();

build/azure-pipelines/common/symbols.ts

@@ -36,7 +36,6 @@ export interface IVersionAccessor extends IApplicationAccessor {
 enum Platform {
 	WIN_32 = 'win32-ia32',
 	WIN_64 = 'win32-x64',
-	LINUX_32 = 'linux-ia32',
 	LINUX_64 = 'linux-x64',
 	MAC_OS = 'darwin-x64'
 }
@@ -191,7 +190,7 @@ if (process.platform === 'darwin') {
 } else if (process.platform === 'win32') {
 	platform = is64 ? Platform.WIN_64 : Platform.WIN_32;
 } else {
-	platform = is64 ? Platform.LINUX_64 : Platform.LINUX_32;
+	platform = Platform.LINUX_64;
 }
 
 // Create version and upload symbols in HockeyApp

build/azure-pipelines/darwin/product-build-darwin.yml

@@ -34,7 +34,8 @@ steps:
     yarn gulp mixin
     yarn gulp hygiene
     yarn monaco-compile-check
-    node build/azure-pipelines/common/installDistro.js
+    node build/azure-pipelines/common/installDistroDependencies.js
+    node build/azure-pipelines/common/installDistroDependencies.js remote
     node build/lib/builtInExtensions.js
   displayName: Prepare build
 

build/azure-pipelines/linux/build-arm.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -e
+echo 'noop'

build/azure-pipelines/linux/prebuild-arm.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -e
+echo 'noop'

build/azure-pipelines/linux/product-build-linux-arm.yml

@@ -0,0 +1,65 @@
+steps:
+- task: NodeTool@0
+  inputs:
+    versionSpec: "10.15.1"
+
+- task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@2
+  inputs:
+    versionSpec: "1.10.1"
+
+- task: AzureKeyVault@1
+  displayName: 'Azure Key Vault: Get Secrets'
+  inputs:
+    azureSubscription: 'vscode-builds-subscription'
+    KeyVaultName: vscode
+
+- task: Docker@1
+  displayName: 'Pull image'
+  inputs:
+    azureSubscriptionEndpoint: 'vscode-builds-subscription'
+    azureContainerRegistry: vscodehub.azurecr.io
+    command: 'Run an image'
+    imageName: 'vscode-linux-build-agent:armhf'
+    containerCommand: uname
+
+- script: |
+    set -e
+
+    cat << EOF > ~/.netrc
+    machine monacotools.visualstudio.com
+    password $(devops-pat)
+    machine github.com
+    login vscode
+    password $(github-distro-mixin-password)
+    EOF
+
+    git config user.email "vscode@microsoft.com"
+    git config user.name "VSCode"
+    git remote add distro "https://github.com/$(VSCODE_MIXIN_REPO).git"
+    git fetch distro
+    git merge $(node -p "require('./package.json').distro")
+
+    CHILD_CONCURRENCY=1 yarn
+    yarn gulp mixin
+    yarn gulp hygiene
+    yarn monaco-compile-check
+    ./build/azure-pipelines/linux/prebuild-arm.sh
+  displayName: Prepare build
+
+- script: |
+    set -e
+    ./build/azure-pipelines/linux/build-arm.sh
+  displayName: Build
+
+- script: |
+    set -e
+    AZURE_DOCUMENTDB_MASTERKEY="$(builds-docdb-key-readwrite)" \
+    AZURE_STORAGE_ACCESS_KEY_2="$(vscode-storage-key)" \
+    VSCODE_MIXIN_PASSWORD="$(github-distro-mixin-password)" \
+    VSCODE_HOCKEYAPP_TOKEN="$(vscode-hockeyapp-token)" \
+    ./build/azure-pipelines/linux/publish-arm.sh
+  displayName: Publish
+
+- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+  displayName: 'Component Detection'
+  continueOnError: true

build/azure-pipelines/linux/product-build-linux.yml

@@ -16,9 +16,6 @@ steps:
 - script: |
     set -e
     export npm_config_arch="$(VSCODE_ARCH)"
-    if [[ "$(VSCODE_ARCH)" == "ia32" ]]; then
-      export PKG_CONFIG_PATH="/usr/lib/i386-linux-gnu/pkgconfig"
-    fi
 
     cat << EOF > ~/.netrc
     machine monacotools.visualstudio.com
@@ -38,7 +35,8 @@ steps:
     yarn gulp mixin
     yarn gulp hygiene
     yarn monaco-compile-check
-    node build/azure-pipelines/common/installDistro.js
+    node build/azure-pipelines/common/installDistroDependencies.js
+    node build/azure-pipelines/common/installDistroDependencies.js remote
     node build/lib/builtInExtensions.js
   displayName: Prepare build
 

build/azure-pipelines/linux/publish-arm.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -e
+echo 'noop'

build/azure-pipelines/linux/publish.sh

@@ -5,8 +5,6 @@ ROOT="$REPO/.."
 
 # Publish tarball
 PLATFORM_LINUX="linux-$VSCODE_ARCH"
-[[ "$VSCODE_ARCH" == "ia32" ]] && DEB_ARCH="i386" || DEB_ARCH="amd64"
-[[ "$VSCODE_ARCH" == "ia32" ]] && RPM_ARCH="i386" || RPM_ARCH="x86_64"
 BUILDNAME="VSCode-$PLATFORM_LINUX"
 BUILD="$ROOT/$BUILDNAME"
 BUILD_VERSION="$(date +%s)"
@@ -39,7 +37,7 @@ node build/azure-pipelines/common/symbols.js "$VSCODE_MIXIN_PASSWORD" "$VSCODE_H
 # Publish DEB
 yarn gulp "vscode-linux-$VSCODE_ARCH-build-deb"
 PLATFORM_DEB="linux-deb-$VSCODE_ARCH"
-[[ "$VSCODE_ARCH" == "ia32" ]] && DEB_ARCH="i386" || DEB_ARCH="amd64"
+DEB_ARCH="amd64"
 DEB_FILENAME="$(ls $REPO/.build/linux/deb/$DEB_ARCH/deb/)"
 DEB_PATH="$REPO/.build/linux/deb/$DEB_ARCH/deb/$DEB_FILENAME"
 
@@ -48,7 +46,7 @@ node build/azure-pipelines/common/publish.js "$VSCODE_QUALITY" "$PLATFORM_DEB" p
 # Publish RPM
 yarn gulp "vscode-linux-$VSCODE_ARCH-build-rpm"
 PLATFORM_RPM="linux-rpm-$VSCODE_ARCH"
-[[ "$VSCODE_ARCH" == "ia32" ]] && RPM_ARCH="i386" || RPM_ARCH="x86_64"
+RPM_ARCH="x86_64"
 RPM_FILENAME="$(ls $REPO/.build/linux/rpm/$RPM_ARCH/ | grep .rpm)"
 RPM_PATH="$REPO/.build/linux/rpm/$RPM_ARCH/$RPM_FILENAME"
 

build/azure-pipelines/product-build.yml

@@ -1,11 +1,8 @@
 resources:
   containers:
   - container: vscode-x64
-    endpoint: VSCodeHub
     image: vscodehub.azurecr.io/vscode-linux-build-agent:x64
-  - container: vscode-ia32
     endpoint: VSCodeHub
-    image: vscodehub.azurecr.io/vscode-linux-build-agent:ia32
   - container: snapcraft
     image: snapcore/snapcraft
 
@@ -49,15 +46,14 @@ jobs:
   steps:
   - template: linux/snap-build-linux.yml
 
-- job: Linux32
-  condition: eq(variables['VSCODE_BUILD_LINUX_32BIT'], 'true')
+- job: LinuxArmhf
+  condition: eq(variables['VSCODE_BUILD_LINUX_ARMHF'], 'true')
   pool:
     vmImage: 'Ubuntu-16.04'
   variables:
-    VSCODE_ARCH: ia32
-  container: vscode-ia32
+    VSCODE_ARCH: armhf
   steps:
-  - template: linux/product-build-linux.yml
+  - template: linux/product-build-linux-arm.yml
 
 - job: macOS
   condition: eq(variables['VSCODE_BUILD_MACOS'], 'true')
@@ -75,7 +71,7 @@ jobs:
   - Windows32
   - Linux
   - LinuxSnap
-  - Linux32
+  - LinuxArmhf
   - macOS
   steps:
   - template: sync-mooncake.yml

build/azure-pipelines/win32/product-build-win32.yml

@@ -35,7 +35,8 @@ steps:
     exec { yarn gulp mixin }
     exec { yarn gulp hygiene }
     exec { yarn monaco-compile-check }
-    exec { node build/azure-pipelines/common/installDistro.js }
+    exec { node build/azure-pipelines/common/installDistroDependencies.js }
+    exec { node build/azure-pipelines/common/installDistroDependencies.js remote }
     exec { node build/lib/builtInExtensions.js }
   displayName: Prepare build
 

build/gulpfile.reh.js

@@ -7,10 +7,111 @@
 
 const gulp = require('gulp');
 
+const path = require('path');
+const es = require('event-stream');
+const util = require('./lib/util');
+const task = require('./lib/task');
+
+const vfs = require('vinyl-fs');
+const flatmap = require('gulp-flatmap');
+const gunzip = require('gulp-gunzip');
+const untar = require('gulp-untar');
+const File = require('vinyl');
+const fs = require('fs');
+
+const REPO_ROOT = path.dirname(__dirname);
+
 const noop = () => { return Promise.resolve(); };
 
 gulp.task('vscode-reh-win32-ia32-min', noop);
 gulp.task('vscode-reh-win32-x64-min', noop);
 gulp.task('vscode-reh-darwin-min', noop);
 gulp.task('vscode-reh-linux-x64-min', noop);
-gulp.task('vscode-reh-linux-arm-min', noop);
+gulp.task('vscode-reh-linux-armhf-min', noop);
+
+
+function getNodeVersion() {
+	const yarnrc = fs.readFileSync(path.join(REPO_ROOT, 'remote', '.yarnrc'), 'utf8');
+	const target = /^target "(.*)"$/m.exec(yarnrc)[1];
+	return target;
+}
+
+function ensureDirs(dirPath) {
+	if (!fs.existsSync(dirPath)) {
+		ensureDirs(path.dirname(dirPath));
+		fs.mkdirSync(dirPath);
+	}
+}
+
+/* Downloads the node executable used for the remote server to ./build/node-remote */
+gulp.task(task.define('node-remote', () => {
+	const VERSION = getNodeVersion();
+	const nodePath = path.join('.build', 'node-remote');
+	const nodeVersionPath = path.join(nodePath, 'version');
+	if (!fs.existsSync(nodeVersionPath) || fs.readFileSync(nodeVersionPath).toString() !== VERSION) {
+		ensureDirs(nodePath);
+		util.rimraf(nodePath);
+		fs.writeFileSync(nodeVersionPath, VERSION);
+		return nodejs(process.platform, process.arch).pipe(vfs.dest(nodePath));
+	}
+	return vfs.src(nodePath);
+}));
+
+function nodejs(platform, arch) {
+	const VERSION = getNodeVersion();
+
+	if (arch === 'ia32') {
+		arch = 'x86';
+	}
+
+	if (platform === 'win32') {
+		const downloadPath = `/dist/v${VERSION}/win-${arch}/node.exe`;
+
+		return (
+			util.download({ host: 'nodejs.org', path: downloadPath })
+				.pipe(es.through(function (data) {
+					// base comes in looking like `https:\nodejs.org\dist\v10.2.1\win-x64\node.exe`
+					this.emit('data', new File({
+						path: data.path,
+						base: data.base.replace(/\\node\.exe$/, ''),
+						contents: data.contents,
+						stat: {
+							isFile: true,
+							mode: /* 100755 */ 33261
+						}
+					}));
+				}))
+		);
+	}
+
+	if (platform === 'darwin') {
+		arch = 'x64';
+	}
+
+	if (arch === 'armhf') {
+		arch = 'armv7l';
+	}
+
+	const downloadPath = `/dist/v${VERSION}/node-v${VERSION}-${platform}-${arch}.tar.gz`;
+
+	return (
+		util.download({ host: 'nodejs.org', path: downloadPath })
+			.pipe(flatmap(stream => stream.pipe(gunzip()).pipe(untar())))
+			.pipe(es.through(function (data) {
+				// base comes in looking like `https:/nodejs.org/dist/v8.9.3/node-v8.9.3-darwin-x64.tar.gz`
+				// => we must remove the `.tar.gz`
+				// Also, keep only bin/node
+				if (/\/bin\/node$/.test(data.path)) {
+					this.emit('data', new File({
+						path: data.path.replace(/bin\/node$/, 'node'),
+						base: data.base.replace(/\.tar\.gz$/, ''),
+						contents: data.contents,
+						stat: {
+							isFile: true,
+							mode: /* 100755 */ 33261
+						}
+					}));
+				}
+			}))
+	);
+}

build/gulpfile.vscode.js

@@ -402,6 +402,7 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
 				.pipe(replace('@@VERSION@@', version))
 				.pipe(replace('@@COMMIT@@', commit))
 				.pipe(replace('@@APPNAME@@', product.applicationName))
+				.pipe(replace('@@DATAFOLDER@@', product.dataFolderName))
 				.pipe(replace('@@QUALITY@@', quality))
 				.pipe(rename(function (f) { f.basename = product.applicationName; f.extname = ''; })));
 
@@ -537,14 +538,14 @@ gulp.task('vscode-translations-import', function () {
 // Sourcemaps
 
 gulp.task('upload-vscode-sourcemaps', () => {
-	const vs = gulp.src('out-vscode-min/**/*.map', { base: 'out-vscode-min' })
+	const vs = gulp.src('out-vscode-min/**/*.map', { base: 'out-vscode-min' }) // client source-maps only
 		.pipe(es.mapSync(f => {
 			f.path = `${f.base}/core/${f.relative}`;
 			return f;
 		}));
 
-	const extensionsOut = gulp.src('extensions/**/out/**/*.map', { base: '.' });
-	const extensionsDist = gulp.src('extensions/**/dist/**/*.map', { base: '.' });
+	const extensionsOut = gulp.src(['extensions/**/out/**/*.map', '!extensions/**/node_modules/**'], { base: '.' });
+	const extensionsDist = gulp.src(['extensions/**/dist/**/*.map', '!extensions/**/node_modules/**'], { base: '.' });
 
 	return es.merge(vs, extensionsOut, extensionsDist)
 		.pipe(es.through(function (data) {

build/gulpfile.vscode.linux.js

@@ -23,7 +23,7 @@ const commit = util.getVersion(root);
 const linuxPackageRevision = Math.floor(new Date().getTime() / 1000);
 
 function getDebPackageArch(arch) {
-	return { x64: 'amd64', ia32: 'i386', arm: 'armhf', arm64: "arm64" }[arch];
+	return { x64: 'amd64', arm: 'armhf', arm64: "arm64" }[arch];
 }
 
 function prepareDebPackage(arch) {
@@ -115,7 +115,7 @@ function getRpmBuildPath(rpmArch) {
 }
 
 function getRpmPackageArch(arch) {
-	return { x64: 'x86_64', ia32: 'i386', arm: 'armhf', arm64: "arm64" }[arch];
+	return { x64: 'x86_64', arm: 'armhf', arm64: "arm64" }[arch];
 }
 
 function prepareRpmPackage(arch) {
@@ -241,7 +241,6 @@ function buildSnapPackage(arch) {
 }
 
 const BUILD_TARGETS = [
-	{ arch: 'ia32' },
 	{ arch: 'x64' },
 	{ arch: 'arm' },
 	{ arch: 'arm64' },

build/lib/electron.js

@@ -24,7 +24,7 @@ module.exports.getElectronVersion = getElectronVersion;
 if (require.main === module) {
 	const version = getElectronVersion();
 	const versionFile = path.join(root, '.build', 'electron', 'version');
-	const isUpToDate = fs.existsSync(versionFile) && fs.readFileSync(versionFile, 'utf8') === `v${version}`;
+	const isUpToDate = fs.existsSync(versionFile) && fs.readFileSync(versionFile, 'utf8') === `${version}`;
 
 	process.exit(isUpToDate ? 0 : 1);
 }

build/lib/i18n.resources.json

@@ -38,10 +38,6 @@
 			"name": "vs/workbench/contrib/codeEditor",
 			"project": "vscode-workbench"
 		},
-		{
-			"name": "vs/workbench/contrib/codeinset",
-			"project": "vscode-workbench"
-		},
 		{
 			"name": "vs/workbench/contrib/callHierarchy",
 			"project": "vscode-workbench"

build/lib/i18n.ts

@@ -25,7 +25,7 @@ function log(message: any, ...rest: any[]): void {
 
 export interface Language {
 	id: string; // language id, e.g. zh-tw, de
-	translationId?: string; // language id used in translation tools, e.g zh-hant, de (optional, if not set, the id is used)
+	translationId?: string; // language id used in translation tools, e.g. zh-hant, de (optional, if not set, the id is used)
 	folderName?: string; // language specific folder name, e.g. cht, deu  (optional, if not set, the id is used)
 }
 

build/lib/watch/package.json

@@ -4,6 +4,7 @@
   "description": "",
   "author": "Microsoft ",
   "private": true,
+  "license": "MIT",
   "devDependencies": {
     "gulp-watch": "^4.3.9"
   }

build/package.json

@@ -1,6 +1,7 @@
 {
   "name": "azuredatastudio-oss-dev-build",
   "version": "1.0.0",
+  "license": "MIT",
   "devDependencies": {
     "@types/ansi-colors": "^3.2.0",
     "@types/azure": "0.9.19",
@@ -43,7 +44,7 @@
     "request": "^2.85.0",
     "tslint": "^5.9.1",
     "service-downloader": "github:anthonydresser/service-downloader#0.1.5",
-    "typescript": "3.4.5",
+    "typescript": "3.5.1",
     "vsce": "1.48.0",
     "xml2js": "^0.4.17"
   },

build/win32/i18n/messages.en.isl

@@ -2,7 +2,7 @@
 AddContextMenuFiles=Add "Open with %1" action to Windows Explorer file context menu
 AddContextMenuFolders=Add "Open with %1" action to Windows Explorer directory context menu
 AssociateWithFiles=Register %1 as an editor for supported file types
-AddToPath=Add to PATH (available after restart)
+AddToPath=Add to PATH (requires shell restart)
 RunAfter=Run %1 after installation
 Other=Other:
 SourceFile=%1 Source File

build/yarn.lock

@@ -3201,10 +3201,10 @@ typed-rest-client@^0.9.0:
     tunnel "0.0.4"
     underscore "1.8.3"
 
-typescript@3.4.5:
-  version "3.4.5"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.4.5.tgz#2d2618d10bb566572b8d7aad5180d84257d70a99"
-  integrity sha512-YycBxUb49UUhdNMU5aJ7z5Ej2XGmaIBL0x34vZ82fn3hGvD+bgrMrVDpatgz2f7YxUMJxMkbWxJZeAvDxVe7Vw==
+typescript@3.5.1:
+  version "3.5.1"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.5.1.tgz#ba72a6a600b2158139c5dd8850f700e231464202"
+  integrity sha512-64HkdiRv1yYZsSe4xC1WVgamNigVYjlssIoaH2HcZF0+ijsk5YK2g0G34w9wJkze8+5ow4STd22AynfO6ZYYLw==
 
 uc.micro@^1.0.1, uc.micro@^1.0.5:
   version "1.0.5"