From 1dc4a437eb975d9589f253337aa685e75a20b37f Mon Sep 17 00:00:00 2001 From: Alex Ma Date: Tue, 22 Jun 2021 09:57:15 -0700 Subject: [PATCH] Export-XLF function for ADS (#15709) * wip commit * added function to generate XLFs for ADS * code cleanup * removed unnecessary locfunc * updated extensions and locfunc * changed wording to be more clear * added working single extension compile * added export all extensions * added more comments and closing sql carbon edit braces. * consolidated gulpfile.extensions changes * changed name to remove ADS part. * changed gulpfile name use --- build/gulpfile.extensions.js | 43 ++++++++++++++++++++++++++++++++++++ build/gulpfile.vscode.js | 24 +++++++++++++++++++- build/lib/extensions.js | 4 +++- build/lib/extensions.ts | 4 ++-- build/lib/locFunc.js | 40 ++++++++++++++------------------- build/lib/locFunc.ts | 40 +++++++++++++-------------------- 6 files changed, 103 insertions(+), 52 deletions(-) diff --git a/build/gulpfile.extensions.js b/build/gulpfile.extensions.js index 0f9c51c456..4f38911459 100644 --- a/build/gulpfile.extensions.js +++ b/build/gulpfile.extensions.js @@ -274,6 +274,49 @@ exports.compileExtensionsBuildTask = compileExtensionsBuildTask; //#endregion +// {{SQL CARBON EDIT}} +//#region XLF Creation + +//Get every extension in 'extensions' to create XLF files. +const exportCompilations = glob.sync('**/package.json', { + cwd: extensionsPath, + ignore: ['**/out/**', '**/node_modules/**', 'package.json'] +}); + +//Run the localization packaging task on all extensions in ADS. +const exportTasks = exportCompilations.map(function (packageFile) { + const locFunc = require('./lib/locFunc'); + const relativeDirname = path.dirname(packageFile); + + const extensionName = relativeDirname.replace(/\//g, '-'); + const packageTask = task.define(`localization-package-extension:${extensionName}`, task.series(() => { + return locFunc.packageSingleExtensionStream(extensionName) + .pipe(gulp.dest('.build')); + })); + + // Tasks + gulp.task(packageTask); + + return { packageTask }; +}); + +const packageLocalizationExtensionsTask = task.define('package-localization-extensions-task', task.series(...exportTasks.map(t => t.packageTask))); +gulp.task(packageLocalizationExtensionsTask); + +//Builds all ADS extensions including external/excluded extensions (only for creating XLF files, not for compiling extensions for shipping) +const compileLocalizationExtensionsBuildTask = task.define('compile-localization-extensions-build', task.series( + cleanExtensionsBuildTask, + compileExtensionsTask, + task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build'))), + packageLocalizationExtensionsTask, +)); + +gulp.task(compileLocalizationExtensionsBuildTask); +exports.compileLocalizationExtensionsBuildTask = compileLocalizationExtensionsBuildTask; + +//#endregion +// {{SQL CARBON EDIT}} end + const compileWebExtensionsTask = task.define('compile-web', () => buildWebExtensions(false)); gulp.task(compileWebExtensionsTask); exports.compileWebExtensionsTask = compileWebExtensionsTask; diff --git a/build/gulpfile.vscode.js b/build/gulpfile.vscode.js index 6b6739286d..8c56961d59 100644 --- a/build/gulpfile.vscode.js +++ b/build/gulpfile.vscode.js @@ -30,7 +30,7 @@ const { getProductionDependencies } = require('./lib/dependencies'); const { config } = require('./lib/electron'); const createAsar = require('./lib/asar').createAsar; const { compileBuildTask } = require('./gulpfile.compile'); -const { compileExtensionsBuildTask } = require('./gulpfile.extensions'); +const { compileExtensionsBuildTask, compileLocalizationExtensionsBuildTask } = require('./gulpfile.extensions'); // {{SQL CARBON EDIT}} Must handle localization code. // Build const vscodeEntryPoints = _.flatten([ @@ -108,6 +108,18 @@ const optimizeVSCodeTask = task.define('optimize-vscode', task.series( )); gulp.task(optimizeVSCodeTask); +// {{SQL CARBON EDIT}} Gulp task that exports any extensions found in the build folder as an XLF to an external folder. +const exportXLFFolderTask = task.define('export-xlf-folder', + function () { + const pathToExtensions = '.build/extensions/*'; + return es.merge( + gulp.src(pathToExtensions).pipe(i18n.createXlfFilesForExtensions()) + ).pipe(vfs.dest('../export-xlfs')); + } +); +gulp.task(exportXLFFolderTask); +// {{SQL CARBON EDIT}} end + const sourceMappingURLBase = `https://sqlopsbuilds.blob.core.windows.net/sourcemaps/${commit}`; const minifyVSCodeTask = task.define('minify-vscode', task.series( optimizeVSCodeTask, @@ -457,6 +469,16 @@ gulp.task(task.define( ) )); +// {{SQL CARBON EDIT}} Localization gulp task, similar to vscode-translations-export but for all extensions. +gulp.task(task.define( + 'export-xlfs', + task.series( + compileLocalizationExtensionsBuildTask, + exportXLFFolderTask + ) +)); +// {{SQL CARBON EDIT}} end + gulp.task('vscode-translations-pull', function () { return es.merge([...i18n.defaultLanguages, ...i18n.extraLanguages].map(language => { let includeDefault = !!innoSetupConfig[language.id].defaultInfo; diff --git a/build/lib/extensions.js b/build/lib/extensions.js index a088357652..08529f9ae7 100644 --- a/build/lib/extensions.js +++ b/build/lib/extensions.js @@ -4,7 +4,7 @@ * Licensed under the Source EULA. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ Object.defineProperty(exports, "__esModule", { value: true }); -exports.translatePackageJSON = exports.packageRebuildExtensionsStream = exports.cleanRebuildExtensions = exports.packageExternalExtensionsStream = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.fromMarketplace = void 0; +exports.translatePackageJSON = exports.packageRebuildExtensionsStream = exports.cleanRebuildExtensions = exports.packageExternalExtensionsStream = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.fromMarketplace = exports.fromLocalNormal = exports.fromLocal = void 0; const es = require("event-stream"); const fs = require("fs"); const glob = require("glob"); @@ -71,6 +71,7 @@ function fromLocal(extensionPath, forWeb) { } return input; } +exports.fromLocal = fromLocal; function fromLocalWebpack(extensionPath, webpackConfigFileName) { const result = es.through(); const packagedDependencies = []; @@ -163,6 +164,7 @@ function fromLocalNormal(extensionPath) { .catch(err => result.emit('error', err)); return result.pipe(stats_1.createStatsStream(path.basename(extensionPath))); } +exports.fromLocalNormal = fromLocalNormal; const baseHeaders = { 'X-Market-Client-Id': 'VSCode Build', 'User-Agent': 'VSCode Build', diff --git a/build/lib/extensions.ts b/build/lib/extensions.ts index 6184437210..a8ac5f40e0 100644 --- a/build/lib/extensions.ts +++ b/build/lib/extensions.ts @@ -54,7 +54,7 @@ function updateExtensionPackageJSON(input: Stream, update: (data: any) => any): .pipe(packageJsonFilter.restore); } -function fromLocal(extensionPath: string, forWeb: boolean): Stream { +export function fromLocal(extensionPath: string, forWeb: boolean): Stream { // {{SQL CARBON EDIT}} - Needed in locFunc const webpackConfigFileName = forWeb ? 'extension-browser.webpack.config.js' : 'extension.webpack.config.js'; const isWebPacked = fs.existsSync(path.join(extensionPath, webpackConfigFileName)); @@ -171,7 +171,7 @@ function fromLocalWebpack(extensionPath: string, webpackConfigFileName: string): return result.pipe(createStatsStream(path.basename(extensionPath))); } -function fromLocalNormal(extensionPath: string): Stream { +export function fromLocalNormal(extensionPath: string): Stream { // {{SQL CARBON EDIT}} - Needed in locFunc const result = es.through(); const vsce = require('vsce') as typeof import('vsce'); diff --git a/build/lib/locFunc.js b/build/lib/locFunc.js index c4de449859..432b951bc4 100644 --- a/build/lib/locFunc.js +++ b/build/lib/locFunc.js @@ -4,14 +4,12 @@ * Licensed under the Source EULA. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ Object.defineProperty(exports, "__esModule", { value: true }); -exports.packageLangpacksStream = void 0; +exports.packageSingleExtensionStream = exports.packageLangpacksStream = void 0; const es = require("event-stream"); const path = require("path"); -const fs = require("fs"); -const stats_1 = require("./stats"); -const File = require("vinyl"); const glob = require("glob"); const rename = require("gulp-rename"); +const ext = require("./extensions"); const root = path.dirname(path.dirname(__dirname)); // Modified packageLocalExtensionsStream from extensions.ts, but for langpacks. function packageLangpacksStream() { @@ -22,28 +20,24 @@ function packageLangpacksStream() { return { name: langpackName, path: langpackPath }; }); const builtLangpacks = langpackDescriptions.map(langpack => { - return fromLocalNormal(langpack.path) + return ext.fromLocalNormal(langpack.path) .pipe(rename(p => p.dirname = `langpacks/${langpack.name}/${p.dirname}`)); }); return es.merge(builtLangpacks); } exports.packageLangpacksStream = packageLangpacksStream; -//copied from extensions. -function fromLocalNormal(extensionPath) { - const result = es.through(); - const vsce = require('vsce'); - vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn }) - .then(fileNames => { - const files = fileNames - .map(fileName => path.join(extensionPath, fileName)) - .map(filePath => new File({ - path: filePath, - stat: fs.statSync(filePath), - base: extensionPath, - contents: fs.createReadStream(filePath) - })); - es.readArray(files).pipe(result); - }) - .catch(err => result.emit('error', err)); - return result.pipe(stats_1.createStatsStream(path.basename(extensionPath))); +// Modified packageLocalExtensionsStream but for any ADS extensions including excluded/external ones. +function packageSingleExtensionStream(name) { + const extenalExtensionDescriptions = glob.sync(`extensions/${name}/package.json`) + .map(manifestPath => { + const extensionPath = path.dirname(path.join(root, manifestPath)); + const extensionName = path.basename(extensionPath); + return { name: extensionName, path: extensionPath }; + }); + const builtExtension = extenalExtensionDescriptions.map(extension => { + return ext.fromLocal(extension.path, false) + .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`)); + }); + return es.merge(builtExtension); } +exports.packageSingleExtensionStream = packageSingleExtensionStream; diff --git a/build/lib/locFunc.ts b/build/lib/locFunc.ts index e121701731..c99aa05593 100644 --- a/build/lib/locFunc.ts +++ b/build/lib/locFunc.ts @@ -5,12 +5,9 @@ import * as es from 'event-stream'; import * as path from 'path'; -import * as fs from 'fs'; -import { createStatsStream } from './stats'; -import * as File from 'vinyl'; -import { Stream } from 'stream'; import * as glob from 'glob'; import rename = require('gulp-rename'); +import ext = require('./extensions'); const root = path.dirname(path.dirname(__dirname)); @@ -24,33 +21,26 @@ export function packageLangpacksStream(): NodeJS.ReadWriteStream { }) const builtLangpacks = langpackDescriptions.map(langpack => { - return fromLocalNormal(langpack.path) + return ext.fromLocalNormal(langpack.path) .pipe(rename(p => p.dirname = `langpacks/${langpack.name}/${p.dirname}`)); }); return es.merge(builtLangpacks); } -//copied from extensions. -function fromLocalNormal(extensionPath: string): Stream { - const result = es.through(); - - const vsce = require('vsce') as typeof import('vsce'); - - vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn }) - .then(fileNames => { - const files = fileNames - .map(fileName => path.join(extensionPath, fileName)) - .map(filePath => new File({ - path: filePath, - stat: fs.statSync(filePath), - base: extensionPath, - contents: fs.createReadStream(filePath) as any - })); - - es.readArray(files).pipe(result); +// Modified packageLocalExtensionsStream but for any ADS extensions including excluded/external ones. +export function packageSingleExtensionStream(name : string): NodeJS.ReadWriteStream { + const extenalExtensionDescriptions = (glob.sync(`extensions/${name}/package.json`)) + .map(manifestPath => { + const extensionPath = path.dirname(path.join(root, manifestPath)); + const extensionName = path.basename(extensionPath); + return { name: extensionName, path: extensionPath }; }) - .catch(err => result.emit('error', err)); - return result.pipe(createStatsStream(path.basename(extensionPath))); + const builtExtension = extenalExtensionDescriptions.map(extension => { + return ext.fromLocal(extension.path, false) + .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`)); + }); + + return es.merge(builtExtension); }