From aee24b202dd38d4b651c1e68bf801426d7775b5e Mon Sep 17 00:00:00 2001 From: Alex Ma Date: Tue, 29 Jun 2021 11:32:57 -0700 Subject: [PATCH] Langpack refresh gulp task (#15930) * added work in progress langpack commands * working langpack refresh function added * removed external extensions change. * reverted extensions.js * changed wording slightly * changed wording * added changelog and readme message * clarified wording again * added handling for outdated strings * fixed wording and changed promise * added another comment * added have --- build/gulpfile.sql.js | 2 + build/lib/i18n.js | 31 ++-- build/lib/i18n.ts | 10 +- build/lib/locFunc.js | 301 ++++++++++++++++++++++++++++++++++++++- build/lib/locFunc.ts | 320 +++++++++++++++++++++++++++++++++++++++++- 5 files changed, 642 insertions(+), 22 deletions(-) diff --git a/build/gulpfile.sql.js b/build/gulpfile.sql.js index 9ddf893c75..8fa94d4831 100644 --- a/build/gulpfile.sql.js +++ b/build/gulpfile.sql.js @@ -145,3 +145,5 @@ gulp.task('package-rebuild-extensions', task.series( task.define('clean-rebuild-extensions', () => ext.cleanRebuildExtensions('.build/extensions')), task.define('rebuild-extensions-build', () => ext.packageRebuildExtensionsStream().pipe(gulp.dest('.build'))), )); + +gulp.task('refresh-langpacks', () => loc.refreshLangpacks()); diff --git a/build/lib/i18n.js b/build/lib/i18n.js index a66dc7a1f9..ed591084ea 100644 --- a/build/lib/i18n.js +++ b/build/lib/i18n.js @@ -4,7 +4,7 @@ * Licensed under the Source EULA. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ Object.defineProperty(exports, "__esModule", { value: true }); -exports.prepareIslFiles = exports.prepareI18nPackFiles = exports.pullI18nPackFiles = exports.prepareI18nFiles = exports.pullSetupXlfFiles = exports.pullCoreAndExtensionsXlfFiles = exports.findObsoleteResources = exports.pushXlfFiles = exports.createXlfFilesForIsl = exports.createXlfFilesForExtensions = exports.createXlfFilesForCoreBundle = exports.getResource = exports.processNlsFiles = exports.Limiter = exports.XLF = exports.Line = exports.externalExtensionsWithTranslations = exports.extraLanguages = exports.defaultLanguages = void 0; +exports.prepareIslFiles = exports.prepareI18nPackFiles = exports.pullI18nPackFiles = exports.i18nPackVersion = exports.createI18nFile = exports.prepareI18nFiles = exports.pullSetupXlfFiles = exports.pullCoreAndExtensionsXlfFiles = exports.findObsoleteResources = exports.pushXlfFiles = exports.createXlfFilesForIsl = exports.createXlfFilesForExtensions = exports.createXlfFilesForCoreBundle = exports.getResource = exports.processNlsFiles = exports.Limiter = exports.XLF = exports.Line = exports.externalExtensionsWithTranslations = exports.extraLanguages = exports.defaultLanguages = void 0; const path = require("path"); const fs = require("fs"); const event_stream_1 = require("event-stream"); @@ -463,7 +463,7 @@ function processCoreBundleFormat(fileHeader, languages, json, emitter) { }); } function processNlsFiles(opts) { - return event_stream_1.through(function (file) { + return (0, event_stream_1.through)(function (file) { let fileName = path.basename(file.path); if (fileName === 'nls.metadata.json') { let json = null; @@ -521,7 +521,7 @@ function getResource(sourceFile) { } exports.getResource = getResource; function createXlfFilesForCoreBundle() { - return event_stream_1.through(function (file) { + return (0, event_stream_1.through)(function (file) { const basename = path.basename(file.path); if (basename === 'nls.metadata.json') { if (file.isBuffer()) { @@ -572,7 +572,7 @@ function createXlfFilesForExtensions() { let counter = 0; let folderStreamEnded = false; let folderStreamEndEmitted = false; - return event_stream_1.through(function (extensionFolder) { + return (0, event_stream_1.through)(function (extensionFolder) { const folderStream = this; const stat = fs.statSync(extensionFolder.path); if (!stat.isDirectory()) { @@ -590,7 +590,7 @@ function createXlfFilesForExtensions() { } return _xlf; } - gulp.src([`.build/extensions/${extensionName}/package.nls.json`, `.build/extensions/${extensionName}/**/nls.metadata.json`], { allowEmpty: true }).pipe(event_stream_1.through(function (file) { + gulp.src([`.build/extensions/${extensionName}/package.nls.json`, `.build/extensions/${extensionName}/**/nls.metadata.json`], { allowEmpty: true }).pipe((0, event_stream_1.through)(function (file) { if (file.isBuffer()) { const buffer = file.contents; const basename = path.basename(file.path); @@ -649,7 +649,7 @@ function createXlfFilesForExtensions() { } exports.createXlfFilesForExtensions = createXlfFilesForExtensions; function createXlfFilesForIsl() { - return event_stream_1.through(function (file) { + return (0, event_stream_1.through)(function (file) { let projectName, resourceFile; if (path.basename(file.path) === 'Default.isl') { projectName = setupProject; @@ -703,7 +703,7 @@ exports.createXlfFilesForIsl = createXlfFilesForIsl; function pushXlfFiles(apiHostname, username, password) { let tryGetPromises = []; let updateCreatePromises = []; - return event_stream_1.through(function (file) { + return (0, event_stream_1.through)(function (file) { const project = path.dirname(file.relative); const fileName = path.basename(file.path); const slug = fileName.substr(0, fileName.length - '.xlf'.length); @@ -765,7 +765,7 @@ function getAllResources(project, apiHostname, username, password) { function findObsoleteResources(apiHostname, username, password) { let resourcesByProject = Object.create(null); resourcesByProject[extensionsProject] = [].concat(exports.externalExtensionsWithTranslations); // clone - return event_stream_1.through(function (file) { + return (0, event_stream_1.through)(function (file) { const project = path.dirname(file.relative); const fileName = path.basename(file.path); const slug = fileName.substr(0, fileName.length - '.xlf'.length); @@ -942,7 +942,7 @@ function pullXlfFiles(apiHostname, username, password, language, resources) { const credentials = `${username}:${password}`; let expectedTranslationsCount = resources.length; let translationsRetrieved = 0, called = false; - return event_stream_1.readable(function (_count, callback) { + return (0, event_stream_1.readable)(function (_count, callback) { // Mark end of stream when all resources were retrieved if (translationsRetrieved === expectedTranslationsCount) { return this.emit('end'); @@ -1000,7 +1000,7 @@ function retrieveResource(language, resource, apiHostname, credentials) { } function prepareI18nFiles() { let parsePromises = []; - return event_stream_1.through(function (xlf) { + return (0, event_stream_1.through)(function (xlf) { let stream = this; let parsePromise = XLF.parse(xlf.contents.toString()); parsePromises.push(parsePromise); @@ -1038,7 +1038,8 @@ function createI18nFile(originalFilePath, messages) { contents: Buffer.from(content, 'utf8') }); } -const i18nPackVersion = '1.0.0'; +exports.createI18nFile = createI18nFile; +exports.i18nPackVersion = '1.0.0'; // {{SQL CARBON EDIT}} Needed in locfunc. function pullI18nPackFiles(apiHostname, username, password, language, resultingTranslationPaths) { return pullCoreAndExtensionsXlfFiles(apiHostname, username, password, language, exports.externalExtensionsWithTranslations) .pipe(prepareI18nPackFiles(exports.externalExtensionsWithTranslations, resultingTranslationPaths, language.id === 'ps')); @@ -1046,10 +1047,10 @@ function pullI18nPackFiles(apiHostname, username, password, language, resultingT exports.pullI18nPackFiles = pullI18nPackFiles; function prepareI18nPackFiles(externalExtensions, resultingTranslationPaths, pseudo = false) { let parsePromises = []; - let mainPack = { version: i18nPackVersion, contents: {} }; + let mainPack = { version: exports.i18nPackVersion, contents: {} }; let extensionsPacks = {}; let errors = []; - return event_stream_1.through(function (xlf) { + return (0, event_stream_1.through)(function (xlf) { let project = path.basename(path.dirname(xlf.relative)); let resource = path.basename(xlf.relative, '.xlf'); let contents = xlf.contents.toString(); @@ -1062,7 +1063,7 @@ function prepareI18nPackFiles(externalExtensions, resultingTranslationPaths, pse if (project === extensionsProject) { let extPack = extensionsPacks[resource]; if (!extPack) { - extPack = extensionsPacks[resource] = { version: i18nPackVersion, contents: {} }; + extPack = extensionsPacks[resource] = { version: exports.i18nPackVersion, contents: {} }; } const externalId = externalExtensions[resource]; if (!externalId) { // internal extension: remove 'extensions/extensionId/' segnent @@ -1110,7 +1111,7 @@ function prepareI18nPackFiles(externalExtensions, resultingTranslationPaths, pse exports.prepareI18nPackFiles = prepareI18nPackFiles; function prepareIslFiles(language, innoSetupConfig) { let parsePromises = []; - return event_stream_1.through(function (xlf) { + return (0, event_stream_1.through)(function (xlf) { let stream = this; let parsePromise = XLF.parse(xlf.contents.toString()); parsePromises.push(parsePromise); diff --git a/build/lib/i18n.ts b/build/lib/i18n.ts index 04cc474c4c..19a859dade 100644 --- a/build/lib/i18n.ts +++ b/build/lib/i18n.ts @@ -64,7 +64,7 @@ export const externalExtensionsWithTranslations = { }; -interface Map { +export interface Map { // {{SQL CARBON EDIT}} Needed in locfunc. [key: string]: V; } @@ -79,7 +79,7 @@ export interface Resource { project: string; } -interface ParsedXLF { +export interface ParsedXLF { // {{SQL CARBON EDIT}} Needed in locfunc. messages: Map; originalFilePath: string; language: string; @@ -1167,7 +1167,7 @@ export function prepareI18nFiles(): ThroughStream { }); } -function createI18nFile(originalFilePath: string, messages: any): File { +export function createI18nFile(originalFilePath: string, messages: any): File { // {{SQL CARBON EDIT}} Needed for locfunc. let result = Object.create(null); result[''] = [ '--------------------------------------------------------------------------------------------', @@ -1190,14 +1190,14 @@ function createI18nFile(originalFilePath: string, messages: any): File { }); } -interface I18nPack { +export interface I18nPack { // {{SQL CARBON EDIT}} Needed in locfunc. version: string; contents: { [path: string]: Map; }; } -const i18nPackVersion = '1.0.0'; +export const i18nPackVersion = '1.0.0'; // {{SQL CARBON EDIT}} Needed in locfunc. export interface TranslationPath { id: string; diff --git a/build/lib/locFunc.js b/build/lib/locFunc.js index 432b951bc4..4265be9c90 100644 --- a/build/lib/locFunc.js +++ b/build/lib/locFunc.js @@ -4,12 +4,20 @@ * Licensed under the Source EULA. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ Object.defineProperty(exports, "__esModule", { value: true }); -exports.packageSingleExtensionStream = exports.packageLangpacksStream = void 0; +exports.refreshLangpacks = exports.modifyI18nPackFiles = exports.packageSingleExtensionStream = exports.packageLangpacksStream = void 0; const es = require("event-stream"); const path = require("path"); const glob = require("glob"); const rename = require("gulp-rename"); const ext = require("./extensions"); +//imports for langpack refresh. +const event_stream_1 = require("event-stream"); +const i18n = require("./i18n"); +const fs = require("fs"); +const File = require("vinyl"); +const rimraf = require("rimraf"); +const gulp = require("gulp"); +const vfs = require("vinyl-fs"); const root = path.dirname(path.dirname(__dirname)); // Modified packageLocalExtensionsStream from extensions.ts, but for langpacks. function packageLangpacksStream() { @@ -41,3 +49,294 @@ function packageSingleExtensionStream(name) { return es.merge(builtExtension); } exports.packageSingleExtensionStream = packageSingleExtensionStream; +// Langpack creation functions go here. +/** + * Function combines the contents of the SQL core XLF file into the current main i18n file contianing the vs core strings. + * Based on createI18nFile in i18n.ts +*/ +function updateMainI18nFile(existingTranslationFilePath, originalFilePath, messages) { + let currFilePath = path.join(existingTranslationFilePath + '.i18n.json'); + let currentContent = fs.readFileSync(currFilePath); + let currentContentObject = JSON.parse(currentContent.toString()); + let objectContents = currentContentObject.contents; + let result = Object.create(null); + // Delete any SQL strings that are no longer part of ADS in current langpack. + for (let contentKey of Object.keys(objectContents)) { + if (contentKey.startsWith('sql') && messages.contents[contentKey] === undefined) { + delete objectContents[`${contentKey}`]; + } + } + messages.contents = Object.assign(Object.assign({}, objectContents), messages.contents); + result[''] = [ + '--------------------------------------------------------------------------------------------', + 'Copyright (c) Microsoft Corporation. All rights reserved.', + 'Licensed under the Source EULA. See License.txt in the project root for license information.', + '--------------------------------------------------------------------------------------------', + 'Do not edit this file. It is machine generated.' + ]; + for (let key of Object.keys(messages)) { + result[key] = messages[key]; + } + let content = JSON.stringify(result, null, '\t'); + if (process.platform === 'win32') { + content = content.replace(/\n/g, '\r\n'); + } + return new File({ + path: path.join(originalFilePath + '.i18n.json'), + contents: Buffer.from(content, 'utf8'), + }); +} +/** + * Function handles the processing of xlf resources and turning them into i18n.json files. + * It adds the i18n files translation paths to be added back into package.main. + * Based on prepareI18nPackFiles in i18n.ts +*/ +function modifyI18nPackFiles(existingTranslationFolder, resultingTranslationPaths, pseudo = false) { + let parsePromises = []; + let mainPack = { version: i18n.i18nPackVersion, contents: {} }; + let extensionsPacks = {}; + let errors = []; + return (0, event_stream_1.through)(function (xlf) { + let rawResource = path.basename(xlf.relative, '.xlf'); + let resource = rawResource.substring(0, rawResource.lastIndexOf('.')); + let contents = xlf.contents.toString(); + let parsePromise = pseudo ? i18n.XLF.parsePseudo(contents) : i18n.XLF.parse(contents); + parsePromises.push(parsePromise); + parsePromise.then(resolvedFiles => { + resolvedFiles.forEach(file => { + const path = file.originalFilePath; + const firstSlash = path.indexOf('/'); + //exclude core sql file from extension processing. + if (resource !== 'sql') { + let extPack = extensionsPacks[resource]; + if (!extPack) { + extPack = extensionsPacks[resource] = { version: i18n.i18nPackVersion, contents: {} }; + } + //remove extensions/extensionId section as all extensions will be webpacked. + const secondSlash = path.indexOf('/', firstSlash + 1); + extPack.contents[path.substr(secondSlash + 1)] = file.messages; + } + else { + mainPack.contents[path.substr(firstSlash + 1)] = file.messages; + } + }); + }).catch(reason => { + errors.push(reason); + }); + }, function () { + Promise.all(parsePromises) + .then(() => { + if (errors.length > 0) { + throw errors; + } + const translatedMainFile = updateMainI18nFile(existingTranslationFolder + '\\main', './main', mainPack); + this.queue(translatedMainFile); + for (let extension in extensionsPacks) { + const translatedExtFile = i18n.createI18nFile(`extensions/${extension}`, extensionsPacks[extension]); + this.queue(translatedExtFile); + //handle edge case for 'Microsoft.sqlservernotebook' where extension name is the same as extension ID. + //(Other extensions need to have publisher appended in front as their ID.) + const adsExtensionId = (extension === 'Microsoft.sqlservernotebook') ? extension : 'Microsoft.' + extension; + resultingTranslationPaths.push({ id: adsExtensionId, resourceName: `extensions/${extension}.i18n.json` }); + } + this.queue(null); + }) + .catch((reason) => { + this.emit('error', reason); + }); + }); +} +exports.modifyI18nPackFiles = modifyI18nPackFiles; +const textFields = { + "nameText": 'ads', + "displayNameText": 'Azure Data Studio', + "publisherText": 'Microsoft', + "licenseText": 'SEE SOURCE EULA LICENSE IN LICENSE.txt', + "updateText": 'cd ../vscode && npm run update-localization-extension ', + "vscodeVersion": '*', + "azdataPlaceholder": '^0.0.0', + "gitUrl": 'https://github.com/Microsoft/azuredatastudio' +}; +//list of extensions from vscode that are to be included with ADS. +const VSCODEExtensions = [ + "bat", + "configuration-editing", + "docker", + "extension-editing", + "git-ui", + "git", + "github-authentication", + "github", + "image-preview", + "json-language-features", + "json", + "markdown-basics", + "markdown-language-features", + "merge-conflict", + "microsoft-authentication", + "powershell", + "python", + "r", + "search-result", + "sql", + "theme-abyss", + "theme-defaults", + "theme-kimbie-dark", + "theme-monokai-dimmed", + "theme-monokai", + "theme-quietlight", + "theme-red", + "theme-seti", + "theme-solarized-dark", + "theme-solarized-light", + "theme-tomorrow-night-blue", + "typescript-basics", + "xml", + "yaml" +]; +/** + * A heavily modified version of update-localization-extension that runs using local xlf resources, no arguments required to pass in. + * It converts a renamed vscode langpack to an ADS one or updates the existing langpack to use current XLF resources. + * It runs this process on all langpacks currently in the ADS i18n folder. + * (Replace an individual ADS langpack folder with a corresponding vscode langpack folder renamed to "ads" instead of "vscode" + * in order to update vscode core strings and extensions for that langpack) + * + * It removes the resources of vscode that we do not support, and adds in new i18n json files created from the xlf files in the folder. + * It also merges in the sql core XLF strings with the langpack's existing core strings into a combined main i18n json file. + * + * After running this gulp task, for each language pack: + * + * 1. Remember to change the version of the langpacks to continue from the previous version of the ADS langpack. + * + * 2. Also change the azdata version to match the current ADS version number. + * + * 3. Update the changelog with the new version of the language pack. + * + * IMPORTANT: If you have run this gulp task on langpacks that originated from vscode, for each affected vscode langpack, you must + * replace the changelog and readme files with the ones from the previous ADS version of the langpack before doing the above steps. + * + * This is mainly for consistency with previous langpacks and to provide proper information to the user. +*/ +function refreshLangpacks() { + let supportedLocations = [...i18n.defaultLanguages, ...i18n.extraLanguages]; + for (let i = 0; i < supportedLocations.length; i++) { + let langId = supportedLocations[i].id; + if (langId === "zh-cn") { + langId = "zh-hans"; + } + if (langId === "zh-tw") { + langId = "zh-hant"; + } + let location = path.join('.', 'resources', 'xlf'); + let locExtFolder = path.join('.', 'i18n', `ads-language-pack-${langId}`); + try { + fs.statSync(locExtFolder); + } + catch (_a) { + console.log('Language is not included in ADS yet: ' + langId); + continue; + } + let packageJSON = JSON.parse(fs.readFileSync(path.join(locExtFolder, 'package.json')).toString()); + //processing extension fields, version and folder name must be changed manually. + packageJSON['name'] = packageJSON['name'].replace('vscode', textFields.nameText); + packageJSON['displayName'] = packageJSON['displayName'].replace('Visual Studio Code', textFields.displayNameText); + packageJSON['publisher'] = textFields.publisherText; + packageJSON['license'] = textFields.licenseText; + packageJSON['scripts']['update'] = textFields.updateText + langId; + packageJSON['engines']['vscode'] = textFields.vscodeVersion; + packageJSON['repository']['url'] = textFields.gitUrl; + packageJSON['engines']['azdata'] = textFields.azdataPlaceholder; // Remember to change this to the appropriate version at the end. + let contributes = packageJSON['contributes']; + if (!contributes) { + throw new Error('The extension must define a "localizations" contribution in the "package.json"'); + } + let localizations = contributes['localizations']; + if (!localizations) { + throw new Error('The extension must define a "localizations" contribution of type array in the "package.json"'); + } + localizations.forEach(function (localization) { + if (!localization.languageId || !localization.languageName || !localization.localizedLanguageName) { + throw new Error('Each localization contribution must define "languageId", "languageName" and "localizedLanguageName" properties.'); + } + let languageId = localization.transifexId || localization.languageId; + let translationDataFolder = path.join(locExtFolder, 'translations'); + if (languageId === "zh-cn") { + languageId = "zh-hans"; + } + if (languageId === "zh-tw") { + languageId = "zh-hant"; + } + //remove extensions not part of ADS. + if (fs.existsSync(translationDataFolder)) { + let totalExtensions = fs.readdirSync(path.join(translationDataFolder, 'extensions')); + for (let extensionTag in totalExtensions) { + let extensionFileName = totalExtensions[extensionTag]; + let xlfPath = path.join(location, `${languageId}`, extensionFileName.replace('.i18n.json', '.xlf')); + if (!(fs.existsSync(xlfPath) || VSCODEExtensions.indexOf(extensionFileName.replace('.i18n.json', '')) !== -1)) { + let filePath = path.join(translationDataFolder, 'extensions', extensionFileName); + rimraf.sync(filePath); + } + } + } + console.log(`Importing translations for ${languageId} from '${location}' to '${translationDataFolder}' ...`); + let translationPaths = []; + gulp.src(path.join(location, languageId, '**', '*.xlf')) + .pipe(modifyI18nPackFiles(translationDataFolder, translationPaths, languageId === 'ps')) + .on('error', (error) => { + console.log(`Error occurred while importing translations:`); + translationPaths = undefined; + if (Array.isArray(error)) { + error.forEach(console.log); + } + else if (error) { + console.log(error); + } + else { + console.log('Unknown error'); + } + }) + .pipe(vfs.dest(translationDataFolder)) + .on('end', function () { + if (translationPaths !== undefined) { + let nonExistantExtensions = []; + for (let curr of localization.translations) { + try { + if (curr.id === 'vscode.theme-seti') { + //handle edge case where 'theme-seti' has a different id. + curr.id = 'vscode.vscode-theme-seti'; + } + fs.statSync(path.join(translationDataFolder, curr.path.replace('./translations', ''))); + } + catch (_a) { + nonExistantExtensions.push(curr); + } + } + for (let nonExt of nonExistantExtensions) { + let index = localization.translations.indexOf(nonExt); + if (index > -1) { + localization.translations.splice(index, 1); + } + } + for (let tp of translationPaths) { + let finalPath = `./translations/${tp.resourceName}`; + let isFound = false; + for (let i = 0; i < localization.translations.length; i++) { + if (localization.translations[i].path === finalPath) { + localization.translations[i].id = tp.id; + isFound = true; + break; + } + } + if (!isFound) { + localization.translations.push({ id: tp.id, path: finalPath }); + } + } + fs.writeFileSync(path.join(locExtFolder, 'package.json'), JSON.stringify(packageJSON, null, '\t')); + } + }); + }); + } + console.log("Langpack Refresh Completed."); + return Promise.resolve(); +} +exports.refreshLangpacks = refreshLangpacks; diff --git a/build/lib/locFunc.ts b/build/lib/locFunc.ts index c99aa05593..942eed6a7a 100644 --- a/build/lib/locFunc.ts +++ b/build/lib/locFunc.ts @@ -8,6 +8,14 @@ import * as path from 'path'; import * as glob from 'glob'; import rename = require('gulp-rename'); import ext = require('./extensions'); +//imports for langpack refresh. +import { through, ThroughStream } from 'event-stream'; +import i18n = require('./i18n') +import * as fs from 'fs'; +import * as File from 'vinyl'; +import * as rimraf from 'rimraf'; +import * as gulp from 'gulp'; +import * as vfs from 'vinyl-fs'; const root = path.dirname(path.dirname(__dirname)); @@ -29,7 +37,7 @@ export function packageLangpacksStream(): NodeJS.ReadWriteStream { } // Modified packageLocalExtensionsStream but for any ADS extensions including excluded/external ones. -export function packageSingleExtensionStream(name : string): NodeJS.ReadWriteStream { +export function packageSingleExtensionStream(name: string): NodeJS.ReadWriteStream { const extenalExtensionDescriptions = (glob.sync(`extensions/${name}/package.json`)) .map(manifestPath => { const extensionPath = path.dirname(path.join(root, manifestPath)); @@ -44,3 +52,313 @@ export function packageSingleExtensionStream(name : string): NodeJS.ReadWriteStr return es.merge(builtExtension); } + +// Langpack creation functions go here. + +/** + * Function combines the contents of the SQL core XLF file into the current main i18n file contianing the vs core strings. + * Based on createI18nFile in i18n.ts +*/ +function updateMainI18nFile(existingTranslationFilePath: string, originalFilePath: string, messages: any): File { + let currFilePath = path.join(existingTranslationFilePath + '.i18n.json'); + let currentContent = fs.readFileSync(currFilePath); + let currentContentObject = JSON.parse(currentContent.toString()); + let objectContents = currentContentObject.contents; + let result = Object.create(null); + + // Delete any SQL strings that are no longer part of ADS in current langpack. + for (let contentKey of Object.keys(objectContents)) { + if(contentKey.startsWith('sql') && messages.contents[contentKey] === undefined){ + delete objectContents[`${contentKey}`] + } + } + + messages.contents = { ...objectContents, ...messages.contents }; + result[''] = [ + '--------------------------------------------------------------------------------------------', + 'Copyright (c) Microsoft Corporation. All rights reserved.', + 'Licensed under the Source EULA. See License.txt in the project root for license information.', + '--------------------------------------------------------------------------------------------', + 'Do not edit this file. It is machine generated.' + ]; + for (let key of Object.keys(messages)) { + result[key] = messages[key]; + } + let content = JSON.stringify(result, null, '\t'); + + if (process.platform === 'win32') { + content = content.replace(/\n/g, '\r\n'); + } + return new File({ + path: path.join(originalFilePath + '.i18n.json'), + + contents: Buffer.from(content, 'utf8'), + }) +} + +/** + * Function handles the processing of xlf resources and turning them into i18n.json files. + * It adds the i18n files translation paths to be added back into package.main. + * Based on prepareI18nPackFiles in i18n.ts +*/ +export function modifyI18nPackFiles(existingTranslationFolder: string, resultingTranslationPaths: i18n.TranslationPath[], pseudo = false): NodeJS.ReadWriteStream { + let parsePromises: Promise[] = []; + let mainPack: i18n.I18nPack = { version: i18n.i18nPackVersion, contents: {} }; + let extensionsPacks: i18n.Map = {}; + let errors: any[] = []; + return through(function (this: ThroughStream, xlf: File) { + let rawResource = path.basename(xlf.relative, '.xlf'); + let resource = rawResource.substring(0, rawResource.lastIndexOf('.')); + let contents = xlf.contents.toString(); + let parsePromise = pseudo ? i18n.XLF.parsePseudo(contents) : i18n.XLF.parse(contents); + parsePromises.push(parsePromise); + parsePromise.then( + resolvedFiles => { + resolvedFiles.forEach(file => { + const path = file.originalFilePath; + const firstSlash = path.indexOf('/'); + + //exclude core sql file from extension processing. + if (resource !== 'sql') { + let extPack = extensionsPacks[resource]; + if (!extPack) { + extPack = extensionsPacks[resource] = { version: i18n.i18nPackVersion, contents: {} }; + } + //remove extensions/extensionId section as all extensions will be webpacked. + const secondSlash = path.indexOf('/', firstSlash + 1); + extPack.contents[path.substr(secondSlash + 1)] = file.messages; + } else { + mainPack.contents[path.substr(firstSlash + 1)] = file.messages; + } + }); + } + ).catch(reason => { + errors.push(reason); + }); + }, function () { + Promise.all(parsePromises) + .then(() => { + if (errors.length > 0) { + throw errors; + } + const translatedMainFile = updateMainI18nFile(existingTranslationFolder + '\\main', './main', mainPack); + + this.queue(translatedMainFile); + for (let extension in extensionsPacks) { + const translatedExtFile = i18n.createI18nFile(`extensions/${extension}`, extensionsPacks[extension]); + this.queue(translatedExtFile); + + //handle edge case for 'Microsoft.sqlservernotebook' where extension name is the same as extension ID. + //(Other extensions need to have publisher appended in front as their ID.) + const adsExtensionId = (extension === 'Microsoft.sqlservernotebook') ? extension : 'Microsoft.' + extension; + resultingTranslationPaths.push({ id: adsExtensionId, resourceName: `extensions/${extension}.i18n.json` }); + } + this.queue(null); + }) + .catch((reason) => { + this.emit('error', reason); + }); + }); +} + +const textFields = { + "nameText": 'ads', + "displayNameText": 'Azure Data Studio', + "publisherText": 'Microsoft', + "licenseText": 'SEE SOURCE EULA LICENSE IN LICENSE.txt', + "updateText": 'cd ../vscode && npm run update-localization-extension ', + "vscodeVersion": '*', + "azdataPlaceholder": '^0.0.0', + "gitUrl": 'https://github.com/Microsoft/azuredatastudio' +} + +//list of extensions from vscode that are to be included with ADS. +const VSCODEExtensions = [ + "bat", + "configuration-editing", + "docker", + "extension-editing", + "git-ui", + "git", + "github-authentication", + "github", + "image-preview", + "json-language-features", + "json", + "markdown-basics", + "markdown-language-features", + "merge-conflict", + "microsoft-authentication", + "powershell", + "python", + "r", + "search-result", + "sql", + "theme-abyss", + "theme-defaults", + "theme-kimbie-dark", + "theme-monokai-dimmed", + "theme-monokai", + "theme-quietlight", + "theme-red", + "theme-seti", + "theme-solarized-dark", + "theme-solarized-light", + "theme-tomorrow-night-blue", + "typescript-basics", + "xml", + "yaml" +]; + +/** + * A heavily modified version of update-localization-extension that runs using local xlf resources, no arguments required to pass in. + * It converts a renamed vscode langpack to an ADS one or updates the existing langpack to use current XLF resources. + * It runs this process on all langpacks currently in the ADS i18n folder. + * (Replace an individual ADS langpack folder with a corresponding vscode langpack folder renamed to "ads" instead of "vscode" + * in order to update vscode core strings and extensions for that langpack) + * + * It removes the resources of vscode that we do not support, and adds in new i18n json files created from the xlf files in the folder. + * It also merges in the sql core XLF strings with the langpack's existing core strings into a combined main i18n json file. + * + * After running this gulp task, for each language pack: + * + * 1. Remember to change the version of the langpacks to continue from the previous version of the ADS langpack. + * + * 2. Also change the azdata version to match the current ADS version number. + * + * 3. Update the changelog with the new version of the language pack. + * + * IMPORTANT: If you have run this gulp task on langpacks that originated from vscode, for each affected vscode langpack, you must + * replace the changelog and readme files with the ones from the previous ADS version of the langpack before doing the above steps. + * + * This is mainly for consistency with previous langpacks and to provide proper information to the user. +*/ +export function refreshLangpacks(): Promise { + let supportedLocations = [...i18n.defaultLanguages, ...i18n.extraLanguages]; + + for (let i = 0; i < supportedLocations.length; i++) { + let langId = supportedLocations[i].id; + if (langId === "zh-cn") { + langId = "zh-hans"; + } + if (langId === "zh-tw") { + langId = "zh-hant"; + } + + let location = path.join('.', 'resources', 'xlf'); + let locExtFolder = path.join('.', 'i18n', `ads-language-pack-${langId}`); + try { + fs.statSync(locExtFolder); + } + catch { + console.log('Language is not included in ADS yet: ' + langId); + continue; + } + let packageJSON = JSON.parse(fs.readFileSync(path.join(locExtFolder, 'package.json')).toString()); + //processing extension fields, version and folder name must be changed manually. + packageJSON['name'] = packageJSON['name'].replace('vscode', textFields.nameText); + packageJSON['displayName'] = packageJSON['displayName'].replace('Visual Studio Code', textFields.displayNameText); + packageJSON['publisher'] = textFields.publisherText; + packageJSON['license'] = textFields.licenseText; + packageJSON['scripts']['update'] = textFields.updateText + langId; + packageJSON['engines']['vscode'] = textFields.vscodeVersion; + packageJSON['repository']['url'] = textFields.gitUrl + packageJSON['engines']['azdata'] = textFields.azdataPlaceholder // Remember to change this to the appropriate version at the end. + + let contributes = packageJSON['contributes']; + if (!contributes) { + throw new Error('The extension must define a "localizations" contribution in the "package.json"'); + } + let localizations = contributes['localizations']; + if (!localizations) { + throw new Error('The extension must define a "localizations" contribution of type array in the "package.json"'); + } + + localizations.forEach(function (localization: any) { + if (!localization.languageId || !localization.languageName || !localization.localizedLanguageName) { + throw new Error('Each localization contribution must define "languageId", "languageName" and "localizedLanguageName" properties.'); + } + let languageId = localization.transifexId || localization.languageId; + let translationDataFolder = path.join(locExtFolder, 'translations'); + if (languageId === "zh-cn") { + languageId = "zh-hans"; + } + if (languageId === "zh-tw") { + languageId = "zh-hant"; + } + + //remove extensions not part of ADS. + if (fs.existsSync(translationDataFolder)) { + let totalExtensions = fs.readdirSync(path.join(translationDataFolder, 'extensions')); + for (let extensionTag in totalExtensions) { + let extensionFileName = totalExtensions[extensionTag]; + let xlfPath = path.join(location, `${languageId}`, extensionFileName.replace('.i18n.json', '.xlf')) + if (!(fs.existsSync(xlfPath) || VSCODEExtensions.indexOf(extensionFileName.replace('.i18n.json', '')) !== -1)) { + let filePath = path.join(translationDataFolder, 'extensions', extensionFileName); + rimraf.sync(filePath); + } + } + } + + + console.log(`Importing translations for ${languageId} from '${location}' to '${translationDataFolder}' ...`); + let translationPaths: any = []; + gulp.src(path.join(location, languageId, '**', '*.xlf')) + .pipe(modifyI18nPackFiles(translationDataFolder, translationPaths, languageId === 'ps')) + .on('error', (error: any) => { + console.log(`Error occurred while importing translations:`); + translationPaths = undefined; + if (Array.isArray(error)) { + error.forEach(console.log); + } else if (error) { + console.log(error); + } else { + console.log('Unknown error'); + } + }) + .pipe(vfs.dest(translationDataFolder)) + .on('end', function () { + if (translationPaths !== undefined) { + let nonExistantExtensions = []; + for (let curr of localization.translations) { + try { + if (curr.id === 'vscode.theme-seti') { + //handle edge case where 'theme-seti' has a different id. + curr.id = 'vscode.vscode-theme-seti'; + } + fs.statSync(path.join(translationDataFolder, curr.path.replace('./translations', ''))); + } + catch { + nonExistantExtensions.push(curr); + } + } + for (let nonExt of nonExistantExtensions) { + let index = localization.translations.indexOf(nonExt); + if (index > -1) { + localization.translations.splice(index, 1); + } + } + for (let tp of translationPaths) { + let finalPath = `./translations/${tp.resourceName}`; + let isFound = false; + for (let i = 0; i < localization.translations.length; i++) { + if (localization.translations[i].path === finalPath) { + localization.translations[i].id = tp.id; + isFound = true; + break; + } + } + if (!isFound) { + localization.translations.push({ id: tp.id, path: finalPath }); + } + } + fs.writeFileSync(path.join(locExtFolder, 'package.json'), JSON.stringify(packageJSON, null, '\t')); + } + }); + + }); + } + console.log("Langpack Refresh Completed."); + return Promise.resolve(); +} +