Export-XLF function for ADS (#15709)

* wip commit

* added function to generate XLFs for ADS

* code cleanup

* removed unnecessary locfunc

* updated extensions and locfunc

* changed wording to be more clear

* added working single extension compile

* added export all extensions

* added more comments and closing sql carbon edit braces.

* consolidated gulpfile.extensions changes

* changed name to remove ADS part.

* changed gulpfile name use

build/gulpfile.extensions.js

@@ -274,6 +274,49 @@ exports.compileExtensionsBuildTask = compileExtensionsBuildTask;
 
 //#endregion
 
+// {{SQL CARBON EDIT}}
+//#region XLF Creation
+
+//Get every extension in 'extensions' to create XLF files.
+const exportCompilations = glob.sync('**/package.json', {
+	cwd: extensionsPath,
+	ignore: ['**/out/**', '**/node_modules/**', 'package.json']
+});
+
+//Run the localization packaging task on all extensions in ADS.
+const exportTasks = exportCompilations.map(function (packageFile) {
+	const locFunc = require('./lib/locFunc');
+	const relativeDirname = path.dirname(packageFile);
+
+	const extensionName = relativeDirname.replace(/\//g, '-');
+	const packageTask = task.define(`localization-package-extension:${extensionName}`, task.series(() => {
+		return locFunc.packageSingleExtensionStream(extensionName)
+			.pipe(gulp.dest('.build'));
+	}));
+
+	// Tasks
+	gulp.task(packageTask);
+
+	return { packageTask };
+});
+
+const packageLocalizationExtensionsTask = task.define('package-localization-extensions-task', task.series(...exportTasks.map(t => t.packageTask)));
+gulp.task(packageLocalizationExtensionsTask);
+
+//Builds all ADS extensions including external/excluded extensions (only for creating XLF files, not for compiling extensions for shipping)
+const compileLocalizationExtensionsBuildTask = task.define('compile-localization-extensions-build', task.series(
+	cleanExtensionsBuildTask,
+	compileExtensionsTask,
+	task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build'))),
+	packageLocalizationExtensionsTask,
+));
+
+gulp.task(compileLocalizationExtensionsBuildTask);
+exports.compileLocalizationExtensionsBuildTask = compileLocalizationExtensionsBuildTask;
+
+//#endregion
+// {{SQL CARBON EDIT}} end
+
 const compileWebExtensionsTask = task.define('compile-web', () => buildWebExtensions(false));
 gulp.task(compileWebExtensionsTask);
 exports.compileWebExtensionsTask = compileWebExtensionsTask;

build/gulpfile.vscode.js

@@ -30,7 +30,7 @@ const { getProductionDependencies } = require('./lib/dependencies');
 const { config } = require('./lib/electron');
 const createAsar = require('./lib/asar').createAsar;
 const { compileBuildTask } = require('./gulpfile.compile');
-const { compileExtensionsBuildTask } = require('./gulpfile.extensions');
+const { compileExtensionsBuildTask, compileLocalizationExtensionsBuildTask } = require('./gulpfile.extensions');  // {{SQL CARBON EDIT}} Must handle localization code.
 
 // Build
 const vscodeEntryPoints = _.flatten([
@@ -108,6 +108,18 @@ const optimizeVSCodeTask = task.define('optimize-vscode', task.series(
 ));
 gulp.task(optimizeVSCodeTask);
 
+// {{SQL CARBON EDIT}} Gulp task that exports any extensions found in the build folder as an XLF to an external folder.
+const exportXLFFolderTask = task.define('export-xlf-folder',
+	function () {
+		const pathToExtensions = '.build/extensions/*';
+		return es.merge(
+			gulp.src(pathToExtensions).pipe(i18n.createXlfFilesForExtensions())
+		).pipe(vfs.dest('../export-xlfs'));
+	}
+);
+gulp.task(exportXLFFolderTask);
+// {{SQL CARBON EDIT}} end
+
 const sourceMappingURLBase = `https://sqlopsbuilds.blob.core.windows.net/sourcemaps/${commit}`;
 const minifyVSCodeTask = task.define('minify-vscode', task.series(
 	optimizeVSCodeTask,
@@ -457,6 +469,16 @@ gulp.task(task.define(
 	)
 ));
 
+// {{SQL CARBON EDIT}} Localization gulp task, similar to vscode-translations-export but for all extensions.
+gulp.task(task.define(
+	'export-xlfs',
+	task.series(
+		compileLocalizationExtensionsBuildTask,
+		exportXLFFolderTask
+	)
+));
+// {{SQL CARBON EDIT}} end
+
 gulp.task('vscode-translations-pull', function () {
 	return es.merge([...i18n.defaultLanguages, ...i18n.extraLanguages].map(language => {
 		let includeDefault = !!innoSetupConfig[language.id].defaultInfo;

build/lib/extensions.js

@@ -4,7 +4,7 @@
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.translatePackageJSON = exports.packageRebuildExtensionsStream = exports.cleanRebuildExtensions = exports.packageExternalExtensionsStream = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.fromMarketplace = void 0;
+exports.translatePackageJSON = exports.packageRebuildExtensionsStream = exports.cleanRebuildExtensions = exports.packageExternalExtensionsStream = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.fromMarketplace = exports.fromLocalNormal = exports.fromLocal = void 0;
 const es = require("event-stream");
 const fs = require("fs");
 const glob = require("glob");
@@ -71,6 +71,7 @@ function fromLocal(extensionPath, forWeb) {
     }
     return input;
 }
+exports.fromLocal = fromLocal;
 function fromLocalWebpack(extensionPath, webpackConfigFileName) {
     const result = es.through();
     const packagedDependencies = [];
@@ -163,6 +164,7 @@ function fromLocalNormal(extensionPath) {
         .catch(err => result.emit('error', err));
     return result.pipe(stats_1.createStatsStream(path.basename(extensionPath)));
 }
+exports.fromLocalNormal = fromLocalNormal;
 const baseHeaders = {
     'X-Market-Client-Id': 'VSCode Build',
     'User-Agent': 'VSCode Build',

build/lib/extensions.ts

@@ -54,7 +54,7 @@ function updateExtensionPackageJSON(input: Stream, update: (data: any) => any):
 		.pipe(packageJsonFilter.restore);
 }
 
-function fromLocal(extensionPath: string, forWeb: boolean): Stream {
+export function fromLocal(extensionPath: string, forWeb: boolean): Stream { // {{SQL CARBON EDIT}} - Needed in locFunc
 	const webpackConfigFileName = forWeb ? 'extension-browser.webpack.config.js' : 'extension.webpack.config.js';
 
 	const isWebPacked = fs.existsSync(path.join(extensionPath, webpackConfigFileName));
@@ -171,7 +171,7 @@ function fromLocalWebpack(extensionPath: string, webpackConfigFileName: string):
 	return result.pipe(createStatsStream(path.basename(extensionPath)));
 }
 
-function fromLocalNormal(extensionPath: string): Stream {
+export function fromLocalNormal(extensionPath: string): Stream { // {{SQL CARBON EDIT}} - Needed in locFunc
 	const result = es.through();
 
 	const vsce = require('vsce') as typeof import('vsce');

build/lib/locFunc.js

@@ -4,14 +4,12 @@
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.packageLangpacksStream = void 0;
+exports.packageSingleExtensionStream = exports.packageLangpacksStream = void 0;
 const es = require("event-stream");
 const path = require("path");
-const fs = require("fs");
-const stats_1 = require("./stats");
-const File = require("vinyl");
 const glob = require("glob");
 const rename = require("gulp-rename");
+const ext = require("./extensions");
 const root = path.dirname(path.dirname(__dirname));
 // Modified packageLocalExtensionsStream from extensions.ts, but for langpacks.
 function packageLangpacksStream() {
@@ -22,28 +20,24 @@ function packageLangpacksStream() {
         return { name: langpackName, path: langpackPath };
     });
     const builtLangpacks = langpackDescriptions.map(langpack => {
-        return fromLocalNormal(langpack.path)
+        return ext.fromLocalNormal(langpack.path)
             .pipe(rename(p => p.dirname = `langpacks/${langpack.name}/${p.dirname}`));
     });
     return es.merge(builtLangpacks);
 }
 exports.packageLangpacksStream = packageLangpacksStream;
-//copied from extensions.
+// Modified packageLocalExtensionsStream but for any ADS extensions including excluded/external ones.
-function fromLocalNormal(extensionPath) {
+function packageSingleExtensionStream(name) {
-    const result = es.through();
+    const extenalExtensionDescriptions = glob.sync(`extensions/${name}/package.json`)
-    const vsce = require('vsce');
+        .map(manifestPath => {
-    vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn })
+        const extensionPath = path.dirname(path.join(root, manifestPath));
-        .then(fileNames => {
+        const extensionName = path.basename(extensionPath);
-        const files = fileNames
+        return { name: extensionName, path: extensionPath };
-            .map(fileName => path.join(extensionPath, fileName))
+    });
-            .map(filePath => new File({
+    const builtExtension = extenalExtensionDescriptions.map(extension => {
-            path: filePath,
+        return ext.fromLocal(extension.path, false)
-            stat: fs.statSync(filePath),
+            .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
-            base: extensionPath,
+    });
-            contents: fs.createReadStream(filePath)
+    return es.merge(builtExtension);
-        }));
-        es.readArray(files).pipe(result);
-    })
-        .catch(err => result.emit('error', err));
-    return result.pipe(stats_1.createStatsStream(path.basename(extensionPath)));
 }
+exports.packageSingleExtensionStream = packageSingleExtensionStream;

build/lib/locFunc.ts

@@ -5,12 +5,9 @@
 
 import * as es from 'event-stream';
 import * as path from 'path';
-import * as fs from 'fs';
-import { createStatsStream } from './stats';
-import * as File from 'vinyl';
-import { Stream } from 'stream';
 import * as glob from 'glob';
 import rename = require('gulp-rename');
+import ext = require('./extensions');
 
 const root = path.dirname(path.dirname(__dirname));
 
@@ -24,33 +21,26 @@ export function packageLangpacksStream(): NodeJS.ReadWriteStream {
 		})
 
 	const builtLangpacks = langpackDescriptions.map(langpack => {
-		return fromLocalNormal(langpack.path)
+		return ext.fromLocalNormal(langpack.path)
 			.pipe(rename(p => p.dirname = `langpacks/${langpack.name}/${p.dirname}`));
 	});
 
 	return es.merge(builtLangpacks);
 }
 
-//copied from extensions.
+// Modified packageLocalExtensionsStream but for any ADS extensions including excluded/external ones.
-function fromLocalNormal(extensionPath: string): Stream {
+export function packageSingleExtensionStream(name : string): NodeJS.ReadWriteStream {
-	const result = es.through();
+	const extenalExtensionDescriptions = (<string[]>glob.sync(`extensions/${name}/package.json`))
-
+		.map(manifestPath => {
-	const vsce = require('vsce') as typeof import('vsce');
+			const extensionPath = path.dirname(path.join(root, manifestPath));
-
+			const extensionName = path.basename(extensionPath);
-	vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn })
+			return { name: extensionName, path: extensionPath };
-		.then(fileNames => {
-			const files = fileNames
-				.map(fileName => path.join(extensionPath, fileName))
-				.map(filePath => new File({
-					path: filePath,
-					stat: fs.statSync(filePath),
-					base: extensionPath,
-					contents: fs.createReadStream(filePath) as any
-				}));
-
-			es.readArray(files).pipe(result);
 		})
-		.catch(err => result.emit('error', err));
 
-	return result.pipe(createStatsStream(path.basename(extensionPath)));
+	const builtExtension = extenalExtensionDescriptions.map(extension => {
+		return ext.fromLocal(extension.path, false)
+			.pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`));
+	});
+
+	return es.merge(builtExtension);
 }