Only package external extensions on Linux in the pipelines (#8464)

* don't package every time, only on linux

* fix build

* fix build

* fix floating promises

* add comment

* fix order of steps

* clean up packaging of the extensions

* minor cleanup

* make code easier to read and add more logging

* do some rename

* maybe?

* re order the code

* fix output path not existing

build/azure-pipelines/darwin/sql-product-build-darwin.yml

@@ -65,13 +65,13 @@ steps:
 
   - script: |
       set -e
-      yarn gulp vscode-darwin
+      yarn gulp install-sqltoolsservice
-    displayName: Build
+    displayName: Install sqltoolsservice
 
   - script: |
       set -e
-      yarn gulp install-sqltoolsservice
+      yarn gulp vscode-darwin
-    displayName: Install sqltoolsservice
+    displayName: Build
 
   - task: ArchiveFiles@2 # WHY ARE WE DOING THIS?
     displayName: 'Archive build scripts source'

build/azure-pipelines/linux/sql-product-build-linux.yml

@@ -72,6 +72,16 @@ steps:
       node build/azure-pipelines/mixin
     displayName: Mix in quality
 
+  - script: |
+      set -e
+      yarn gulp install-sqltoolsservice
+    displayName: Install sqltoolsservice
+
+  - script: |
+      set -e
+      yarn gulp install-ssmsmin
+    displayName: Install ssmsmin
+
   - script: |
       set -e
       yarn gulp vscode-linux-x64
@@ -79,8 +89,13 @@ steps:
 
   - script: |
       set -e
-      yarn gulp install-sqltoolsservice
+      yarn gulp compile-extensions
-    displayName: Install sqltoolsservice
+    displayName: Compile Extensions
+
+  - script: |
+      set -e
+      yarn gulp package-external-extensions
+    displayName: Package External extensions
 
   - task: ArchiveFiles@2 # WHY ARE WE DOING THIS?
     displayName: 'Archive build scripts source'
@@ -135,6 +150,12 @@ steps:
       Contents: '*.rpm'
       TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
+  - task: CopyFiles@2
+    displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)/vsix'
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)/../vsix'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)/vsix'
+
   - script: | # WHY ARE WE DOING THIS?
       set -e
       BUILD="$(Build.SourcesDirectory)/../azuredatastudio-linux-x64"

build/azure-pipelines/win32/sql-product-build-win32.yml

@@ -67,12 +67,6 @@ steps:
       exec { node build/azure-pipelines/mixin }
     displayName: Mix in quality
 
-  - powershell: |
-      . build/azure-pipelines/win32/exec.ps1
-      $ErrorActionPreference = "Stop"
-      exec { yarn gulp "vscode-win32-x64" }
-    displayName: Build
-
   - powershell: |
       . build/azure-pipelines/win32/exec.ps1
       $ErrorActionPreference = "Stop"
@@ -82,8 +76,8 @@ steps:
   - powershell: |
       . build/azure-pipelines/win32/exec.ps1
       $ErrorActionPreference = "Stop"
-      exec { yarn gulp "install-ssmsmin" }
+      exec { yarn gulp "vscode-win32-x64" }
-    displayName: Install ssmsmin
+    displayName: Build
 
   - task: ArchiveFiles@2 # WHY
     displayName: 'Archive build scripts source'
@@ -133,12 +127,6 @@ steps:
     condition: and(succeeded(), eq(variables['RUN_UNSTABLE_TESTS'], 'true'))
     displayName: Run unstable integration tests
 
-  - task: CopyFiles@2
-    displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)/vsix'
-    inputs:
-      SourceFolder: '$(Build.SourcesDirectory)/../vsix'
-      TargetFolder: '$(Build.ArtifactStagingDirectory)/vsix'
-
   - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
     displayName: 'ESRP CodeSigning - Build files - sha256 only'
     inputs:

build/gulpfile.sql.js

@@ -11,8 +11,13 @@ const es = require('event-stream');
 const filter = require('gulp-filter');
 const del = require('del');
 const serviceDownloader = require('service-downloader').ServiceDownloadProvider;
-const platformInfo = require('service-downloader/out/platform').PlatformInformation;
+const platform = require('service-downloader/out/platform').PlatformInformation;
 const path = require('path');
+const ext = require('./lib/extensions');
+const task = require('./lib/task');
+const glob = require('glob');
+const vsce = require('vsce');
+const mkdirp = require('mkdirp');
 
 gulp.task('clean-mssql-extension', util.rimraf('extensions/mssql/node_modules'));
 gulp.task('clean-credentials-extension', util.rimraf('extensions/credentials/node_modules'));
@@ -93,7 +98,7 @@ const formatStagedFiles = () => {
 
 function installService() {
 	let config = require('../extensions/mssql/config.json');
-	return platformInfo.getCurrent().then(p => {
+	return platform.getCurrent().then(p => {
 		let runtime = p.runtimeId;
 		// fix path since it won't be correct
 		config.installDirectory = path.join(__dirname, '../extensions/mssql/src', config.installDirectory);
@@ -113,25 +118,45 @@ gulp.task('install-sqltoolsservice', () => {
 	return installService();
 });
 
-function installSsmsMin() {
-	const config = require('../extensions/admin-tool-ext-win/config.json');
-	return platformInfo.getCurrent().then(p => {
-		const runtime = p.runtimeId;
-		// fix path since it won't be correct
-		config.installDirectory = path.join(__dirname, '..', 'extensions', 'admin-tool-ext-win', config.installDirectory);
-		var installer = new serviceDownloader(config);
-		const serviceInstallFolder = installer.getInstallDirectory(runtime);
-		const serviceCleanupFolder = path.join(serviceInstallFolder, '..');
-		console.log('Cleaning up the install folder: ' + serviceCleanupFolder);
-		return del(serviceCleanupFolder + '/*').then(() => {
-			console.log('Installing the service. Install folder: ' + serviceInstallFolder);
-			return installer.installService(runtime);
-		}, delError => {
-			console.log('failed to delete the install folder error: ' + delError);
-		});
-	});
-}
-
 gulp.task('install-ssmsmin', () => {
-	return installSsmsMin();
+	const config = require('../extensions/admin-tool-ext-win/config.json');
+	const runtime = 'Windows_64'; // admin-tool-ext is a windows only extension, and we only ship a 64 bit version, so locking the binaries as such
+	// fix path since it won't be correct
+	config.installDirectory = path.join(__dirname, '..', 'extensions', 'admin-tool-ext-win', config.installDirectory);
+	var installer = new serviceDownloader(config);
+	const serviceInstallFolder = installer.getInstallDirectory(runtime);
+	const serviceCleanupFolder = path.join(serviceInstallFolder, '..');
+	console.log('Cleaning up the install folder: ' + serviceCleanupFolder);
+	return del(serviceCleanupFolder + '/*').then(() => {
+		console.log('Installing the service. Install folder: ' + serviceInstallFolder);
+		return installer.installService(runtime);
+	}, delError => {
+		console.log('failed to delete the install folder error: ' + delError);
+	});
 });
+
+const root = path.dirname(__dirname);
+
+gulp.task('package-external-extensions', task.series(
+	task.define('bundle-external-extensions-build', () => ext.packageExternalExtensionsStream().pipe(gulp.dest('.build/external'))),
+	task.define('create-external-extension-vsix-build', () => {
+		const vsixes = glob.sync('.build/external/extensions/*/package.json').map(manifestPath => {
+			const extensionPath = path.dirname(path.join(root, manifestPath));
+			const extensionName = path.basename(extensionPath);
+			return { name: extensionName, path: extensionPath };
+		}).map(element => {
+			const pkgJson = require(path.join(element.path, 'package.json'));
+			const vsixDirectory = path.join(path.dirname(root), 'vsix');
+			mkdirp.sync(vsixDirectory);
+			const packagePath = path.join(vsixDirectory, `${pkgJson.name}-${pkgJson.version}.vsix`);
+			console.info('Creating vsix for ' + element.path + ' result:' + packagePath);
+			return vsce.createVSIX({
+				cwd: element.path,
+				packagePath: packagePath,
+				useYarn: true
+			});
+		});
+
+		return Promise.all(vsixes);
+	})
+));

build/gulpfile.vscode.js

@@ -29,7 +29,6 @@ const packageJson = require('../package.json');
 const product = require('../product.json');
 const crypto = require('crypto');
 const i18n = require('./lib/i18n');
-const ext = require('./lib/extensions'); // {{SQL CARBON EDIT}}
 const deps = require('./dependencies');
 const { config } = require('./lib/electron');
 const createAsar = require('./lib/asar').createAsar;
@@ -197,9 +196,6 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
 			.pipe(rename(function (path) { path.dirname = path.dirname.replace(new RegExp('^' + out), 'out'); }))
 			.pipe(util.setExecutableBit(['**/*.sh']));
 
-		// {{SQL CARBON EDIT}}
-		ext.packageSQLExtensions();
-
 		const extensions = gulp.src(['.build/extensions/**', '!.build/extensions/node_modules/**'], { base: '.build', dot: true }); // {{SQL CARBON EDIT}} - don't package the node_modules directory
 
 		const sources = es.merge(src, extensions)

build/lib/extensions.js

@@ -189,7 +189,7 @@ const excludedExtensions = [
     'integration-tests'
 ];
 // {{SQL CARBON EDIT}}
-const sqlExtensions = [
+const externalExtensions = [
     // This is the list of SQL extensions which the source code is included in this repository, but
     // they get packaged separately. Adding extension name here, will make the build to create
     // a separate vsix package for the extension and the extension will be excluded from the main package.
@@ -216,7 +216,7 @@ function packageLocalExtensionsStream() {
     })
         .filter(({ name }) => excludedExtensions.indexOf(name) === -1)
         .filter(({ name }) => builtInExtensions.every(b => b.name !== name))
-        .filter(({ name }) => sqlExtensions.indexOf(name) === -1); // {{SQL CARBON EDIT}} Remove SQL Extensions with separate package
+        .filter(({ name }) => externalExtensions.indexOf(name) === -1); // {{SQL CARBON EDIT}} Remove external Extensions with separate package
     const nodeModules = gulp.src('extensions/node_modules/**', { base: '.' });
     const localExtensions = localExtensionDescriptions.map(extension => {
         return fromLocal(extension.path)
@@ -235,68 +235,19 @@ function packageMarketplaceExtensionsStream() {
         .pipe(util2.setExecutableBit(['**/*.sh']));
 }
 exports.packageMarketplaceExtensionsStream = packageMarketplaceExtensionsStream;
-const vfs = require("vinyl-fs");
+function packageExternalExtensionsStream() {
-function packageSQLExtensions() {
+    const extenalExtensionDescriptions = glob.sync('extensions/*/package.json')
-    // Create package for local SQL extensions
-    //
-    const sqlBuiltInLocalExtensionDescriptions = glob.sync('extensions/*/package.json')
         .map(manifestPath => {
         const extensionPath = path.dirname(path.join(root, manifestPath));
         const extensionName = path.basename(extensionPath);
         return { name: extensionName, path: extensionPath };
     })
-        .filter(({ name }) => excludedExtensions.indexOf(name) === -1)
+        .filter(({ name }) => externalExtensions.indexOf(name) >= 0);
-        .filter(({ name }) => builtInExtensions.every(b => b.name !== name))
+    const builtExtensions = extenalExtensionDescriptions.map(extension => {
-        .filter(({ name }) => sqlExtensions.indexOf(name) >= 0);
+        return fromLocal(extension.path)
-    const visxDirectory = path.join(path.dirname(root), 'vsix');
+            .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
-    try {
-        if (!fs.existsSync(visxDirectory)) {
-            fs.mkdirSync(visxDirectory);
-        }
-    }
-    catch (err) {
-        // don't fail the build if the output directory already exists
-        console.warn(err);
-    }
-    sqlBuiltInLocalExtensionDescriptions.forEach(element => {
-        let pkgJson = JSON.parse(fs.readFileSync(path.join(element.path, 'package.json'), { encoding: 'utf8' }));
-        const packagePath = path.join(visxDirectory, `${pkgJson.name}-${pkgJson.version}.vsix`);
-        console.info('Creating vsix for ' + element.path + ' result:' + packagePath);
-        vsce.createVSIX({
-            cwd: element.path,
-            packagePath: packagePath,
-            useYarn: true
-        });
     });
+    return es.merge(builtExtensions);
 }
-exports.packageSQLExtensions = packageSQLExtensions;
+exports.packageExternalExtensionsStream = packageExternalExtensionsStream;
-function packageExtensionTask(extensionName, platform, arch) {
-    var destination = path.join(path.dirname(root), 'azuredatastudio') + (platform ? '-' + platform : '') + (arch ? '-' + arch : '');
-    if (platform === 'darwin') {
-        destination = path.join(destination, 'Azure Data Studio.app', 'Contents', 'Resources', 'app', 'extensions', extensionName);
-    }
-    else {
-        destination = path.join(destination, 'resources', 'app', 'extensions', extensionName);
-    }
-    platform = platform || process.platform;
-    return () => {
-        const root = path.resolve(path.join(__dirname, '../..'));
-        const localExtensionDescriptions = glob.sync('extensions/*/package.json')
-            .map(manifestPath => {
-            const extensionPath = path.dirname(path.join(root, manifestPath));
-            const extensionName = path.basename(extensionPath);
-            return { name: extensionName, path: extensionPath };
-        })
-            .filter(({ name }) => extensionName === name);
-        const localExtensions = es.merge(...localExtensionDescriptions.map(extension => {
-            return fromLocal(extension.path);
-        }));
-        let result = localExtensions
-            .pipe(util2.skipDirectories())
-            .pipe(util2.fixWin32DirectoryPermissions())
-            .pipe(filter(['**', '!LICENSE', '!LICENSES.chromium.html', '!version']));
-        return result.pipe(vfs.dest(destination));
-    };
-}
-exports.packageExtensionTask = packageExtensionTask;
 // {{SQL CARBON EDIT}} - End

build/lib/extensions.ts

@@ -225,7 +225,7 @@ const excludedExtensions = [
 ];
 
 // {{SQL CARBON EDIT}}
-const sqlExtensions = [
+const externalExtensions = [
 	// This is the list of SQL extensions which the source code is included in this repository, but
 	// they get packaged separately. Adding extension name here, will make the build to create
 	// a separate vsix package for the extension and the extension will be excluded from the main package.
@@ -263,7 +263,7 @@ export function packageLocalExtensionsStream(): NodeJS.ReadWriteStream {
 		})
 		.filter(({ name }) => excludedExtensions.indexOf(name) === -1)
 		.filter(({ name }) => builtInExtensions.every(b => b.name !== name))
-		.filter(({ name }) => sqlExtensions.indexOf(name) === -1); // {{SQL CARBON EDIT}} Remove SQL Extensions with separate package
+		.filter(({ name }) => externalExtensions.indexOf(name) === -1); // {{SQL CARBON EDIT}} Remove external Extensions with separate package
 
 	const nodeModules = gulp.src('extensions/node_modules/**', { base: '.' });
 	const localExtensions = localExtensionDescriptions.map(extension => {
@@ -285,74 +285,20 @@ export function packageMarketplaceExtensionsStream(): NodeJS.ReadWriteStream {
 		.pipe(util2.setExecutableBit(['**/*.sh']));
 }
 
-// {{SQL CARBON EDIT}}
+export function packageExternalExtensionsStream(): NodeJS.ReadWriteStream {
-import * as _ from 'underscore';
+	const extenalExtensionDescriptions = (<string[]>glob.sync('extensions/*/package.json'))
-import * as vfs from 'vinyl-fs';
-
-export function packageSQLExtensions() {
-
-	// Create package for local SQL extensions
-	//
-	const sqlBuiltInLocalExtensionDescriptions = glob.sync('extensions/*/package.json')
 		.map(manifestPath => {
 			const extensionPath = path.dirname(path.join(root, manifestPath));
 			const extensionName = path.basename(extensionPath);
 			return { name: extensionName, path: extensionPath };
 		})
-		.filter(({ name }) => excludedExtensions.indexOf(name) === -1)
+		.filter(({ name }) => externalExtensions.indexOf(name) >= 0);
-		.filter(({ name }) => builtInExtensions.every(b => b.name !== name))
+
-		.filter(({ name }) => sqlExtensions.indexOf(name) >= 0);
+	const builtExtensions = extenalExtensionDescriptions.map(extension => {
-	const visxDirectory = path.join(path.dirname(root), 'vsix');
+		return fromLocal(extension.path)
-	try {
+			.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
-		if (!fs.existsSync(visxDirectory)) {
-			fs.mkdirSync(visxDirectory);
-		}
-	} catch (err) {
-		// don't fail the build if the output directory already exists
-		console.warn(err);
-	}
-	sqlBuiltInLocalExtensionDescriptions.forEach(element => {
-		let pkgJson = JSON.parse(fs.readFileSync(path.join(element.path, 'package.json'), { encoding: 'utf8' }));
-		const packagePath = path.join(visxDirectory, `${pkgJson.name}-${pkgJson.version}.vsix`);
-		console.info('Creating vsix for ' + element.path + ' result:' + packagePath);
-		vsce.createVSIX({
-			cwd: element.path,
-			packagePath: packagePath,
-			useYarn: true
-		});
 	});
-}
 
-export function packageExtensionTask(extensionName: string, platform: string, arch: string) {
+	return es.merge(builtExtensions);
-	var destination = path.join(path.dirname(root), 'azuredatastudio') + (platform ? '-' + platform : '') + (arch ? '-' + arch : '');
-	if (platform === 'darwin') {
-		destination = path.join(destination, 'Azure Data Studio.app', 'Contents', 'Resources', 'app', 'extensions', extensionName);
-	} else {
-		destination = path.join(destination, 'resources', 'app', 'extensions', extensionName);
-	}
-
-	platform = platform || process.platform;
-
-	return () => {
-		const root = path.resolve(path.join(__dirname, '../..'));
-		const localExtensionDescriptions = glob.sync('extensions/*/package.json')
-			.map(manifestPath => {
-				const extensionPath = path.dirname(path.join(root, manifestPath));
-				const extensionName = path.basename(extensionPath);
-				return { name: extensionName, path: extensionPath };
-			})
-			.filter(({ name }) => extensionName === name);
-
-		const localExtensions = es.merge(...localExtensionDescriptions.map(extension => {
-			return fromLocal(extension.path);
-		}));
-
-		let result = localExtensions
-			.pipe(util2.skipDirectories())
-			.pipe(util2.fixWin32DirectoryPermissions())
-			.pipe(filter(['**', '!LICENSE', '!LICENSES.chromium.html', '!version']));
-
-		return result.pipe(vfs.dest(destination));
-	};
 }
 // {{SQL CARBON EDIT}} - End