From 82e0ede92163d71e0e2e98888c0a94b697f30eb2 Mon Sep 17 00:00:00 2001 From: Alex Ma Date: Thu, 20 May 2021 20:03:25 -0700 Subject: [PATCH] Vsix build task for langpacks (#15489) * added refreshlangpack command * added beginning update-localization yml file, also changed langpack location in RLE * added regex tester * moved xlf files into new folder structure * small change to update-localization * added yaml script for gulp refresh * added missing dash. * added better update-localization yml file * remove update-localization as its not necessary * added small changes to langpack-compile * remove upload sourcemaps and write version information * added more languages * added vsce packaging * added automatic langpack handling * added built locFunc * fixed refresh-langpack-extension * working langpack vsix generator made * added langpacks to files in copyArtifacts and product-build-linux * changed command to package-langpacks * removed unnecessary language flags. * invalid ADS extensions filter explained * Fix for regex * removed unnecessary fields to change, and removed langpack-compile * added doc comments. * moved xlf files back to old place. * WIP translation redirect * isolated vsix build task * fixed spaces in locFunc.ts --- build/azure-pipelines/common/copyArtifacts.js | 1 + build/azure-pipelines/common/copyArtifacts.ts | 1 + .../linux/sql-product-build-linux.yml | 9 ++- build/gulpfile.sql.js | 26 +++++++++ build/lib/locFunc.js | 49 ++++++++++++++++ build/lib/locFunc.ts | 56 +++++++++++++++++++ 6 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 build/lib/locFunc.js create mode 100644 build/lib/locFunc.ts diff --git a/build/azure-pipelines/common/copyArtifacts.js b/build/azure-pipelines/common/copyArtifacts.js index 45f5e4f481..c26923b7bf 100644 --- a/build/azure-pipelines/common/copyArtifacts.js +++ b/build/azure-pipelines/common/copyArtifacts.js @@ -9,6 +9,7 @@ const path = require("path"); const es = require("event-stream"); const fs = require("fs"); const files = [ + '.build/langpacks/**/*.vsix', '.build/extensions/**/*.vsix', '.build/win32-x64/**/*.{exe,zip}', '.build/linux/sha256hashes.txt', diff --git a/build/azure-pipelines/common/copyArtifacts.ts b/build/azure-pipelines/common/copyArtifacts.ts index ae66c0c26a..d2c0e8e542 100644 --- a/build/azure-pipelines/common/copyArtifacts.ts +++ b/build/azure-pipelines/common/copyArtifacts.ts @@ -11,6 +11,7 @@ import * as es from 'event-stream'; import * as fs from 'fs'; const files = [ + '.build/langpacks/**/*.vsix', // langpacks '.build/extensions/**/*.vsix', // external extensions '.build/win32-x64/**/*.{exe,zip}', // windows binaries '.build/linux/sha256hashes.txt', // linux hashes diff --git a/build/azure-pipelines/linux/sql-product-build-linux.yml b/build/azure-pipelines/linux/sql-product-build-linux.yml index df05728b19..9a34aa0492 100644 --- a/build/azure-pipelines/linux/sql-product-build-linux.yml +++ b/build/azure-pipelines/linux/sql-product-build-linux.yml @@ -106,6 +106,11 @@ steps: yarn gulp package-external-extensions displayName: Package External extensions + - script: | + set -e + yarn gulp package-langpacks + displayName: Package Langpacks + - script: | set -e service xvfb start @@ -190,7 +195,7 @@ steps: inputs: ConnectedServiceName: 'Code Signing' FolderPath: '$(Build.SourcesDirectory)/.build' - Pattern: 'extensions/*.vsix' + Pattern: 'extensions/*.vsix,langpacks/*.vsix' signConfigType: inlineSignParams inlineOperation: | [ @@ -215,7 +220,7 @@ steps: } ] SessionTimeout: 120 - displayName: 'Signing Extensions' + displayName: 'Signing Extensions and Langpacks' condition: and(succeeded(), eq(variables['signed'], true)) - script: | diff --git a/build/gulpfile.sql.js b/build/gulpfile.sql.js index b73d0949d3..9ddf893c75 100644 --- a/build/gulpfile.sql.js +++ b/build/gulpfile.sql.js @@ -10,6 +10,7 @@ const es = require('event-stream'); const filter = require('gulp-filter'); const path = require('path'); const ext = require('./lib/extensions'); +const loc = require('./lib/locFunc'); const task = require('./lib/task'); const glob = require('glob'); const vsce = require('vsce'); @@ -115,6 +116,31 @@ gulp.task('package-external-extensions', task.series( }) )); +gulp.task('package-langpacks', task.series( + task.define('bundle-external-langpack-build', () => loc.packageLangpacksStream().pipe(gulp.dest('.build/external'))), + task.define('create-external-langpack-vsix-build', () => { + const vsixes = glob.sync('.build/external/langpacks/*/package.json').map(manifestPath => { + const extensionPath = path.dirname(path.join(root, manifestPath)); + const extensionName = path.basename(extensionPath); + return { name: extensionName, path: extensionPath }; + }).map(element => { + const pkgJson = require(path.join(element.path, 'package.json')); + const vsixDirectory = path.join(root, '.build', 'langpacks'); + mkdirp.sync(vsixDirectory); + const packagePath = path.join(vsixDirectory, `${pkgJson.name}-${pkgJson.version}.vsix`); + console.info('Creating vsix for ' + element.path + ' result:' + packagePath); + return vsce.createVSIX({ + cwd: element.path, + packagePath: packagePath, + useYarn: true + }); + }); + + return Promise.all(vsixes); + }) +)); + + gulp.task('package-rebuild-extensions', task.series( task.define('clean-rebuild-extensions', () => ext.cleanRebuildExtensions('.build/extensions')), task.define('rebuild-extensions-build', () => ext.packageRebuildExtensionsStream().pipe(gulp.dest('.build'))), diff --git a/build/lib/locFunc.js b/build/lib/locFunc.js new file mode 100644 index 0000000000..c4de449859 --- /dev/null +++ b/build/lib/locFunc.js @@ -0,0 +1,49 @@ +"use strict"; +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the Source EULA. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.packageLangpacksStream = void 0; +const es = require("event-stream"); +const path = require("path"); +const fs = require("fs"); +const stats_1 = require("./stats"); +const File = require("vinyl"); +const glob = require("glob"); +const rename = require("gulp-rename"); +const root = path.dirname(path.dirname(__dirname)); +// Modified packageLocalExtensionsStream from extensions.ts, but for langpacks. +function packageLangpacksStream() { + const langpackDescriptions = glob.sync('i18n/*/package.json') + .map(manifestPath => { + const langpackPath = path.dirname(path.join(root, manifestPath)); + const langpackName = path.basename(langpackPath); + return { name: langpackName, path: langpackPath }; + }); + const builtLangpacks = langpackDescriptions.map(langpack => { + return fromLocalNormal(langpack.path) + .pipe(rename(p => p.dirname = `langpacks/${langpack.name}/${p.dirname}`)); + }); + return es.merge(builtLangpacks); +} +exports.packageLangpacksStream = packageLangpacksStream; +//copied from extensions. +function fromLocalNormal(extensionPath) { + const result = es.through(); + const vsce = require('vsce'); + vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn }) + .then(fileNames => { + const files = fileNames + .map(fileName => path.join(extensionPath, fileName)) + .map(filePath => new File({ + path: filePath, + stat: fs.statSync(filePath), + base: extensionPath, + contents: fs.createReadStream(filePath) + })); + es.readArray(files).pipe(result); + }) + .catch(err => result.emit('error', err)); + return result.pipe(stats_1.createStatsStream(path.basename(extensionPath))); +} diff --git a/build/lib/locFunc.ts b/build/lib/locFunc.ts new file mode 100644 index 0000000000..e121701731 --- /dev/null +++ b/build/lib/locFunc.ts @@ -0,0 +1,56 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the Source EULA. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as es from 'event-stream'; +import * as path from 'path'; +import * as fs from 'fs'; +import { createStatsStream } from './stats'; +import * as File from 'vinyl'; +import { Stream } from 'stream'; +import * as glob from 'glob'; +import rename = require('gulp-rename'); + +const root = path.dirname(path.dirname(__dirname)); + +// Modified packageLocalExtensionsStream from extensions.ts, but for langpacks. +export function packageLangpacksStream(): NodeJS.ReadWriteStream { + const langpackDescriptions = (glob.sync('i18n/*/package.json')) + .map(manifestPath => { + const langpackPath = path.dirname(path.join(root, manifestPath)); + const langpackName = path.basename(langpackPath); + return { name: langpackName, path: langpackPath }; + }) + + const builtLangpacks = langpackDescriptions.map(langpack => { + return fromLocalNormal(langpack.path) + .pipe(rename(p => p.dirname = `langpacks/${langpack.name}/${p.dirname}`)); + }); + + return es.merge(builtLangpacks); +} + +//copied from extensions. +function fromLocalNormal(extensionPath: string): Stream { + const result = es.through(); + + const vsce = require('vsce') as typeof import('vsce'); + + vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn }) + .then(fileNames => { + const files = fileNames + .map(fileName => path.join(extensionPath, fileName)) + .map(filePath => new File({ + path: filePath, + stat: fs.statSync(filePath), + base: extensionPath, + contents: fs.createReadStream(filePath) as any + })); + + es.readArray(files).pipe(result); + }) + .catch(err => result.emit('error', err)); + + return result.pipe(createStatsStream(path.basename(extensionPath))); +}