Vsix build task for langpacks (#15489)

* added refreshlangpack command

* added beginning update-localization yml file, also changed langpack location in RLE

* added regex tester

* moved xlf files into new folder structure

* small change to update-localization

* added yaml script for gulp refresh

* added missing dash.

* added better update-localization yml file

* remove update-localization as its not necessary

* added small changes to langpack-compile

* remove upload sourcemaps and write version information

* added more languages

* added vsce packaging

* added automatic langpack handling

* added built locFunc

* fixed refresh-langpack-extension

* working langpack vsix generator made

* added langpacks to files in copyArtifacts and product-build-linux

* changed command to package-langpacks

* removed unnecessary language flags.

* invalid ADS extensions filter explained

* Fix for regex

* removed unnecessary fields to change, and removed langpack-compile

* added doc comments.

* moved xlf files back to old place.

* WIP translation redirect

* isolated vsix build task

* fixed spaces in locFunc.ts

build/azure-pipelines/common/copyArtifacts.js

@@ -9,6 +9,7 @@ const path = require("path");
 const es = require("event-stream");
 const fs = require("fs");
 const files = [
+    '.build/langpacks/**/*.vsix',
     '.build/extensions/**/*.vsix',
     '.build/win32-x64/**/*.{exe,zip}',
     '.build/linux/sha256hashes.txt',

build/azure-pipelines/common/copyArtifacts.ts

@@ -11,6 +11,7 @@ import * as es from 'event-stream';
 import * as fs from 'fs';
 
 const files = [
+	'.build/langpacks/**/*.vsix', // langpacks
 	'.build/extensions/**/*.vsix', // external extensions
 	'.build/win32-x64/**/*.{exe,zip}', // windows binaries
 	'.build/linux/sha256hashes.txt', // linux hashes

build/azure-pipelines/linux/sql-product-build-linux.yml

@@ -106,6 +106,11 @@ steps:
       yarn gulp package-external-extensions
     displayName: Package External extensions
 
+  - script: |
+      set -e
+      yarn gulp package-langpacks
+    displayName: Package Langpacks
+
   - script: |
       set -e
       service xvfb start
@@ -190,7 +195,7 @@ steps:
     inputs:
       ConnectedServiceName: 'Code Signing'
       FolderPath: '$(Build.SourcesDirectory)/.build'
-      Pattern: 'extensions/*.vsix'
+      Pattern: 'extensions/*.vsix,langpacks/*.vsix'
       signConfigType: inlineSignParams
       inlineOperation: |
         [
@@ -215,7 +220,7 @@ steps:
           }
         ]
       SessionTimeout: 120
-    displayName: 'Signing Extensions'
+    displayName: 'Signing Extensions and Langpacks'
     condition: and(succeeded(), eq(variables['signed'], true))
 
   - script: |

build/gulpfile.sql.js

@@ -10,6 +10,7 @@ const es = require('event-stream');
 const filter = require('gulp-filter');
 const path = require('path');
 const ext = require('./lib/extensions');
+const loc = require('./lib/locFunc');
 const task = require('./lib/task');
 const glob = require('glob');
 const vsce = require('vsce');
@@ -115,6 +116,31 @@ gulp.task('package-external-extensions', task.series(
 	})
 ));
 
+gulp.task('package-langpacks', task.series(
+	task.define('bundle-external-langpack-build', () => loc.packageLangpacksStream().pipe(gulp.dest('.build/external'))),
+	task.define('create-external-langpack-vsix-build', () => {
+		const vsixes = glob.sync('.build/external/langpacks/*/package.json').map(manifestPath => {
+			const extensionPath = path.dirname(path.join(root, manifestPath));
+			const extensionName = path.basename(extensionPath);
+			return { name: extensionName, path: extensionPath };
+		}).map(element => {
+			const pkgJson = require(path.join(element.path, 'package.json'));
+			const vsixDirectory = path.join(root, '.build', 'langpacks');
+			mkdirp.sync(vsixDirectory);
+			const packagePath = path.join(vsixDirectory, `${pkgJson.name}-${pkgJson.version}.vsix`);
+			console.info('Creating vsix for ' + element.path + ' result:' + packagePath);
+			return vsce.createVSIX({
+				cwd: element.path,
+				packagePath: packagePath,
+				useYarn: true
+			});
+		});
+
+		return Promise.all(vsixes);
+	})
+));
+
+
 gulp.task('package-rebuild-extensions', task.series(
 	task.define('clean-rebuild-extensions', () => ext.cleanRebuildExtensions('.build/extensions')),
 	task.define('rebuild-extensions-build', () => ext.packageRebuildExtensionsStream().pipe(gulp.dest('.build'))),

build/lib/locFunc.js

@@ -0,0 +1,49 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.packageLangpacksStream = void 0;
+const es = require("event-stream");
+const path = require("path");
+const fs = require("fs");
+const stats_1 = require("./stats");
+const File = require("vinyl");
+const glob = require("glob");
+const rename = require("gulp-rename");
+const root = path.dirname(path.dirname(__dirname));
+// Modified packageLocalExtensionsStream from extensions.ts, but for langpacks.
+function packageLangpacksStream() {
+    const langpackDescriptions = glob.sync('i18n/*/package.json')
+        .map(manifestPath => {
+        const langpackPath = path.dirname(path.join(root, manifestPath));
+        const langpackName = path.basename(langpackPath);
+        return { name: langpackName, path: langpackPath };
+    });
+    const builtLangpacks = langpackDescriptions.map(langpack => {
+        return fromLocalNormal(langpack.path)
+            .pipe(rename(p => p.dirname = `langpacks/${langpack.name}/${p.dirname}`));
+    });
+    return es.merge(builtLangpacks);
+}
+exports.packageLangpacksStream = packageLangpacksStream;
+//copied from extensions.
+function fromLocalNormal(extensionPath) {
+    const result = es.through();
+    const vsce = require('vsce');
+    vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn })
+        .then(fileNames => {
+        const files = fileNames
+            .map(fileName => path.join(extensionPath, fileName))
+            .map(filePath => new File({
+            path: filePath,
+            stat: fs.statSync(filePath),
+            base: extensionPath,
+            contents: fs.createReadStream(filePath)
+        }));
+        es.readArray(files).pipe(result);
+    })
+        .catch(err => result.emit('error', err));
+    return result.pipe(stats_1.createStatsStream(path.basename(extensionPath)));
+}

build/lib/locFunc.ts

@@ -0,0 +1,56 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as es from 'event-stream';
+import * as path from 'path';
+import * as fs from 'fs';
+import { createStatsStream } from './stats';
+import * as File from 'vinyl';
+import { Stream } from 'stream';
+import * as glob from 'glob';
+import rename = require('gulp-rename');
+
+const root = path.dirname(path.dirname(__dirname));
+
+// Modified packageLocalExtensionsStream from extensions.ts, but for langpacks.
+export function packageLangpacksStream(): NodeJS.ReadWriteStream {
+	const langpackDescriptions = (<string[]>glob.sync('i18n/*/package.json'))
+		.map(manifestPath => {
+			const langpackPath = path.dirname(path.join(root, manifestPath));
+			const langpackName = path.basename(langpackPath);
+			return { name: langpackName, path: langpackPath };
+		})
+
+	const builtLangpacks = langpackDescriptions.map(langpack => {
+		return fromLocalNormal(langpack.path)
+			.pipe(rename(p => p.dirname = `langpacks/${langpack.name}/${p.dirname}`));
+	});
+
+	return es.merge(builtLangpacks);
+}
+
+//copied from extensions.
+function fromLocalNormal(extensionPath: string): Stream {
+	const result = es.through();
+
+	const vsce = require('vsce') as typeof import('vsce');
+
+	vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.Yarn })
+		.then(fileNames => {
+			const files = fileNames
+				.map(fileName => path.join(extensionPath, fileName))
+				.map(filePath => new File({
+					path: filePath,
+					stat: fs.statSync(filePath),
+					base: extensionPath,
+					contents: fs.createReadStream(filePath) as any
+				}));
+
+			es.readArray(files).pipe(result);
+		})
+		.catch(err => result.emit('error', err));
+
+	return result.pipe(createStatsStream(path.basename(extensionPath)));
+}