Langpack refresh gulp task (#15930)

* added work in progress langpack commands

* working langpack refresh function added

* removed external extensions change.

* reverted extensions.js

* changed wording slightly

* changed wording

* added changelog and readme message

* clarified wording again

* added handling for outdated strings

* fixed wording and changed promise

* added another comment

* added have

build/gulpfile.sql.js

@@ -145,3 +145,5 @@ gulp.task('package-rebuild-extensions', task.series(
 	task.define('clean-rebuild-extensions', () => ext.cleanRebuildExtensions('.build/extensions')),
 	task.define('rebuild-extensions-build', () => ext.packageRebuildExtensionsStream().pipe(gulp.dest('.build'))),
 ));
+
+gulp.task('refresh-langpacks', () => loc.refreshLangpacks());

build/lib/i18n.js

@@ -4,7 +4,7 @@
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.prepareIslFiles = exports.prepareI18nPackFiles = exports.pullI18nPackFiles = exports.prepareI18nFiles = exports.pullSetupXlfFiles = exports.pullCoreAndExtensionsXlfFiles = exports.findObsoleteResources = exports.pushXlfFiles = exports.createXlfFilesForIsl = exports.createXlfFilesForExtensions = exports.createXlfFilesForCoreBundle = exports.getResource = exports.processNlsFiles = exports.Limiter = exports.XLF = exports.Line = exports.externalExtensionsWithTranslations = exports.extraLanguages = exports.defaultLanguages = void 0;
+exports.prepareIslFiles = exports.prepareI18nPackFiles = exports.pullI18nPackFiles = exports.i18nPackVersion = exports.createI18nFile = exports.prepareI18nFiles = exports.pullSetupXlfFiles = exports.pullCoreAndExtensionsXlfFiles = exports.findObsoleteResources = exports.pushXlfFiles = exports.createXlfFilesForIsl = exports.createXlfFilesForExtensions = exports.createXlfFilesForCoreBundle = exports.getResource = exports.processNlsFiles = exports.Limiter = exports.XLF = exports.Line = exports.externalExtensionsWithTranslations = exports.extraLanguages = exports.defaultLanguages = void 0;
 const path = require("path");
 const fs = require("fs");
 const event_stream_1 = require("event-stream");
@@ -463,7 +463,7 @@ function processCoreBundleFormat(fileHeader, languages, json, emitter) {
     });
 }
 function processNlsFiles(opts) {
-    return event_stream_1.through(function (file) {
+    return (0, event_stream_1.through)(function (file) {
         let fileName = path.basename(file.path);
         if (fileName === 'nls.metadata.json') {
             let json = null;
@@ -521,7 +521,7 @@ function getResource(sourceFile) {
 }
 exports.getResource = getResource;
 function createXlfFilesForCoreBundle() {
-    return event_stream_1.through(function (file) {
+    return (0, event_stream_1.through)(function (file) {
         const basename = path.basename(file.path);
         if (basename === 'nls.metadata.json') {
             if (file.isBuffer()) {
@@ -572,7 +572,7 @@ function createXlfFilesForExtensions() {
     let counter = 0;
     let folderStreamEnded = false;
     let folderStreamEndEmitted = false;
-    return event_stream_1.through(function (extensionFolder) {
+    return (0, event_stream_1.through)(function (extensionFolder) {
         const folderStream = this;
         const stat = fs.statSync(extensionFolder.path);
         if (!stat.isDirectory()) {
@@ -590,7 +590,7 @@ function createXlfFilesForExtensions() {
             }
             return _xlf;
         }
-        gulp.src([`.build/extensions/${extensionName}/package.nls.json`, `.build/extensions/${extensionName}/**/nls.metadata.json`], { allowEmpty: true }).pipe(event_stream_1.through(function (file) {
+        gulp.src([`.build/extensions/${extensionName}/package.nls.json`, `.build/extensions/${extensionName}/**/nls.metadata.json`], { allowEmpty: true }).pipe((0, event_stream_1.through)(function (file) {
             if (file.isBuffer()) {
                 const buffer = file.contents;
                 const basename = path.basename(file.path);
@@ -649,7 +649,7 @@ function createXlfFilesForExtensions() {
 }
 exports.createXlfFilesForExtensions = createXlfFilesForExtensions;
 function createXlfFilesForIsl() {
-    return event_stream_1.through(function (file) {
+    return (0, event_stream_1.through)(function (file) {
         let projectName, resourceFile;
         if (path.basename(file.path) === 'Default.isl') {
             projectName = setupProject;
@@ -703,7 +703,7 @@ exports.createXlfFilesForIsl = createXlfFilesForIsl;
 function pushXlfFiles(apiHostname, username, password) {
     let tryGetPromises = [];
     let updateCreatePromises = [];
-    return event_stream_1.through(function (file) {
+    return (0, event_stream_1.through)(function (file) {
         const project = path.dirname(file.relative);
         const fileName = path.basename(file.path);
         const slug = fileName.substr(0, fileName.length - '.xlf'.length);
@@ -765,7 +765,7 @@ function getAllResources(project, apiHostname, username, password) {
 function findObsoleteResources(apiHostname, username, password) {
     let resourcesByProject = Object.create(null);
     resourcesByProject[extensionsProject] = [].concat(exports.externalExtensionsWithTranslations); // clone
-    return event_stream_1.through(function (file) {
+    return (0, event_stream_1.through)(function (file) {
         const project = path.dirname(file.relative);
         const fileName = path.basename(file.path);
         const slug = fileName.substr(0, fileName.length - '.xlf'.length);
@@ -942,7 +942,7 @@ function pullXlfFiles(apiHostname, username, password, language, resources) {
     const credentials = `${username}:${password}`;
     let expectedTranslationsCount = resources.length;
     let translationsRetrieved = 0, called = false;
-    return event_stream_1.readable(function (_count, callback) {
+    return (0, event_stream_1.readable)(function (_count, callback) {
         // Mark end of stream when all resources were retrieved
         if (translationsRetrieved === expectedTranslationsCount) {
             return this.emit('end');
@@ -1000,7 +1000,7 @@ function retrieveResource(language, resource, apiHostname, credentials) {
 }
 function prepareI18nFiles() {
     let parsePromises = [];
-    return event_stream_1.through(function (xlf) {
+    return (0, event_stream_1.through)(function (xlf) {
         let stream = this;
         let parsePromise = XLF.parse(xlf.contents.toString());
         parsePromises.push(parsePromise);
@@ -1038,7 +1038,8 @@ function createI18nFile(originalFilePath, messages) {
         contents: Buffer.from(content, 'utf8')
     });
 }
-const i18nPackVersion = '1.0.0';
+exports.createI18nFile = createI18nFile;
+exports.i18nPackVersion = '1.0.0'; // {{SQL CARBON EDIT}} Needed in locfunc.
 function pullI18nPackFiles(apiHostname, username, password, language, resultingTranslationPaths) {
     return pullCoreAndExtensionsXlfFiles(apiHostname, username, password, language, exports.externalExtensionsWithTranslations)
         .pipe(prepareI18nPackFiles(exports.externalExtensionsWithTranslations, resultingTranslationPaths, language.id === 'ps'));
@@ -1046,10 +1047,10 @@ function pullI18nPackFiles(apiHostname, username, password, language, resultingT
 exports.pullI18nPackFiles = pullI18nPackFiles;
 function prepareI18nPackFiles(externalExtensions, resultingTranslationPaths, pseudo = false) {
     let parsePromises = [];
-    let mainPack = { version: i18nPackVersion, contents: {} };
+    let mainPack = { version: exports.i18nPackVersion, contents: {} };
     let extensionsPacks = {};
     let errors = [];
-    return event_stream_1.through(function (xlf) {
+    return (0, event_stream_1.through)(function (xlf) {
         let project = path.basename(path.dirname(xlf.relative));
         let resource = path.basename(xlf.relative, '.xlf');
         let contents = xlf.contents.toString();
@@ -1062,7 +1063,7 @@ function prepareI18nPackFiles(externalExtensions, resultingTranslationPaths, pse
                 if (project === extensionsProject) {
                     let extPack = extensionsPacks[resource];
                     if (!extPack) {
-                        extPack = extensionsPacks[resource] = { version: i18nPackVersion, contents: {} };
+                        extPack = extensionsPacks[resource] = { version: exports.i18nPackVersion, contents: {} };
                     }
                     const externalId = externalExtensions[resource];
                     if (!externalId) { // internal extension: remove 'extensions/extensionId/' segnent
@@ -1110,7 +1111,7 @@ function prepareI18nPackFiles(externalExtensions, resultingTranslationPaths, pse
 exports.prepareI18nPackFiles = prepareI18nPackFiles;
 function prepareIslFiles(language, innoSetupConfig) {
     let parsePromises = [];
-    return event_stream_1.through(function (xlf) {
+    return (0, event_stream_1.through)(function (xlf) {
         let stream = this;
         let parsePromise = XLF.parse(xlf.contents.toString());
         parsePromises.push(parsePromise);

build/lib/i18n.ts

@@ -64,7 +64,7 @@ export const externalExtensionsWithTranslations = {
 };
 
 
-interface Map<V> {
+export interface Map<V> {	// {{SQL CARBON EDIT}} Needed in locfunc.
 	[key: string]: V;
 }
 
@@ -79,7 +79,7 @@ export interface Resource {
 	project: string;
 }
 
-interface ParsedXLF {
+export interface ParsedXLF {	// {{SQL CARBON EDIT}} Needed in locfunc.
 	messages: Map<string>;
 	originalFilePath: string;
 	language: string;
@@ -1167,7 +1167,7 @@ export function prepareI18nFiles(): ThroughStream {
 	});
 }
 
-function createI18nFile(originalFilePath: string, messages: any): File {
+export function createI18nFile(originalFilePath: string, messages: any): File { // {{SQL CARBON EDIT}} Needed for locfunc.
 	let result = Object.create(null);
 	result[''] = [
 		'--------------------------------------------------------------------------------------------',
@@ -1190,14 +1190,14 @@ function createI18nFile(originalFilePath: string, messages: any): File {
 	});
 }
 
-interface I18nPack {
+export interface I18nPack {	// {{SQL CARBON EDIT}} Needed in locfunc.
 	version: string;
 	contents: {
 		[path: string]: Map<string>;
 	};
 }
 
-const i18nPackVersion = '1.0.0';
+export const i18nPackVersion = '1.0.0'; // {{SQL CARBON EDIT}} Needed in locfunc.
 
 export interface TranslationPath {
 	id: string;

build/lib/locFunc.js

@@ -4,12 +4,20 @@
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.packageSingleExtensionStream = exports.packageLangpacksStream = void 0;
+exports.refreshLangpacks = exports.modifyI18nPackFiles = exports.packageSingleExtensionStream = exports.packageLangpacksStream = void 0;
 const es = require("event-stream");
 const path = require("path");
 const glob = require("glob");
 const rename = require("gulp-rename");
 const ext = require("./extensions");
+//imports for langpack refresh.
+const event_stream_1 = require("event-stream");
+const i18n = require("./i18n");
+const fs = require("fs");
+const File = require("vinyl");
+const rimraf = require("rimraf");
+const gulp = require("gulp");
+const vfs = require("vinyl-fs");
 const root = path.dirname(path.dirname(__dirname));
 // Modified packageLocalExtensionsStream from extensions.ts, but for langpacks.
 function packageLangpacksStream() {
@@ -41,3 +49,294 @@ function packageSingleExtensionStream(name) {
     return es.merge(builtExtension);
 }
 exports.packageSingleExtensionStream = packageSingleExtensionStream;
+// Langpack creation functions go here.
+/**
+ * Function combines the contents of the SQL core XLF file into the current main i18n file contianing the vs core strings.
+ * Based on createI18nFile in i18n.ts
+*/
+function updateMainI18nFile(existingTranslationFilePath, originalFilePath, messages) {
+    let currFilePath = path.join(existingTranslationFilePath + '.i18n.json');
+    let currentContent = fs.readFileSync(currFilePath);
+    let currentContentObject = JSON.parse(currentContent.toString());
+    let objectContents = currentContentObject.contents;
+    let result = Object.create(null);
+    // Delete any SQL strings that are no longer part of ADS in current langpack.
+    for (let contentKey of Object.keys(objectContents)) {
+        if (contentKey.startsWith('sql') && messages.contents[contentKey] === undefined) {
+            delete objectContents[`${contentKey}`];
+        }
+    }
+    messages.contents = Object.assign(Object.assign({}, objectContents), messages.contents);
+    result[''] = [
+        '--------------------------------------------------------------------------------------------',
+        'Copyright (c) Microsoft Corporation. All rights reserved.',
+        'Licensed under the Source EULA. See License.txt in the project root for license information.',
+        '--------------------------------------------------------------------------------------------',
+        'Do not edit this file. It is machine generated.'
+    ];
+    for (let key of Object.keys(messages)) {
+        result[key] = messages[key];
+    }
+    let content = JSON.stringify(result, null, '\t');
+    if (process.platform === 'win32') {
+        content = content.replace(/\n/g, '\r\n');
+    }
+    return new File({
+        path: path.join(originalFilePath + '.i18n.json'),
+        contents: Buffer.from(content, 'utf8'),
+    });
+}
+/**
+ * Function handles the processing of xlf resources and turning them into i18n.json files.
+ * It adds the i18n files translation paths to be added back into package.main.
+ * Based on prepareI18nPackFiles in i18n.ts
+*/
+function modifyI18nPackFiles(existingTranslationFolder, resultingTranslationPaths, pseudo = false) {
+    let parsePromises = [];
+    let mainPack = { version: i18n.i18nPackVersion, contents: {} };
+    let extensionsPacks = {};
+    let errors = [];
+    return (0, event_stream_1.through)(function (xlf) {
+        let rawResource = path.basename(xlf.relative, '.xlf');
+        let resource = rawResource.substring(0, rawResource.lastIndexOf('.'));
+        let contents = xlf.contents.toString();
+        let parsePromise = pseudo ? i18n.XLF.parsePseudo(contents) : i18n.XLF.parse(contents);
+        parsePromises.push(parsePromise);
+        parsePromise.then(resolvedFiles => {
+            resolvedFiles.forEach(file => {
+                const path = file.originalFilePath;
+                const firstSlash = path.indexOf('/');
+                //exclude core sql file from extension processing.
+                if (resource !== 'sql') {
+                    let extPack = extensionsPacks[resource];
+                    if (!extPack) {
+                        extPack = extensionsPacks[resource] = { version: i18n.i18nPackVersion, contents: {} };
+                    }
+                    //remove extensions/extensionId section as all extensions will be webpacked.
+                    const secondSlash = path.indexOf('/', firstSlash + 1);
+                    extPack.contents[path.substr(secondSlash + 1)] = file.messages;
+                }
+                else {
+                    mainPack.contents[path.substr(firstSlash + 1)] = file.messages;
+                }
+            });
+        }).catch(reason => {
+            errors.push(reason);
+        });
+    }, function () {
+        Promise.all(parsePromises)
+            .then(() => {
+            if (errors.length > 0) {
+                throw errors;
+            }
+            const translatedMainFile = updateMainI18nFile(existingTranslationFolder + '\\main', './main', mainPack);
+            this.queue(translatedMainFile);
+            for (let extension in extensionsPacks) {
+                const translatedExtFile = i18n.createI18nFile(`extensions/${extension}`, extensionsPacks[extension]);
+                this.queue(translatedExtFile);
+                //handle edge case for 'Microsoft.sqlservernotebook' where extension name is the same as extension ID.
+                //(Other extensions need to have publisher appended in front as their ID.)
+                const adsExtensionId = (extension === 'Microsoft.sqlservernotebook') ? extension : 'Microsoft.' + extension;
+                resultingTranslationPaths.push({ id: adsExtensionId, resourceName: `extensions/${extension}.i18n.json` });
+            }
+            this.queue(null);
+        })
+            .catch((reason) => {
+            this.emit('error', reason);
+        });
+    });
+}
+exports.modifyI18nPackFiles = modifyI18nPackFiles;
+const textFields = {
+    "nameText": 'ads',
+    "displayNameText": 'Azure Data Studio',
+    "publisherText": 'Microsoft',
+    "licenseText": 'SEE SOURCE EULA LICENSE IN LICENSE.txt',
+    "updateText": 'cd ../vscode && npm run update-localization-extension ',
+    "vscodeVersion": '*',
+    "azdataPlaceholder": '^0.0.0',
+    "gitUrl": 'https://github.com/Microsoft/azuredatastudio'
+};
+//list of extensions from vscode that are to be included with ADS.
+const VSCODEExtensions = [
+    "bat",
+    "configuration-editing",
+    "docker",
+    "extension-editing",
+    "git-ui",
+    "git",
+    "github-authentication",
+    "github",
+    "image-preview",
+    "json-language-features",
+    "json",
+    "markdown-basics",
+    "markdown-language-features",
+    "merge-conflict",
+    "microsoft-authentication",
+    "powershell",
+    "python",
+    "r",
+    "search-result",
+    "sql",
+    "theme-abyss",
+    "theme-defaults",
+    "theme-kimbie-dark",
+    "theme-monokai-dimmed",
+    "theme-monokai",
+    "theme-quietlight",
+    "theme-red",
+    "theme-seti",
+    "theme-solarized-dark",
+    "theme-solarized-light",
+    "theme-tomorrow-night-blue",
+    "typescript-basics",
+    "xml",
+    "yaml"
+];
+/**
+ * A heavily modified version of update-localization-extension that runs using local xlf resources, no arguments required to pass in.
+ * It converts a renamed vscode langpack to an ADS one or updates the existing langpack to use current XLF resources.
+ * It runs this process on all langpacks currently in the ADS i18n folder.
+ * (Replace an individual ADS langpack folder with a corresponding vscode langpack folder renamed to "ads" instead of "vscode"
+ * in order to update vscode core strings and extensions for that langpack)
+ *
+ * It removes the resources of vscode that we do not support, and adds in new i18n json files created from the xlf files in the folder.
+ * It also merges in the sql core XLF strings with the langpack's existing core strings into a combined main i18n json file.
+ *
+ * After running this gulp task, for each language pack:
+ *
+ * 1. Remember to change the version of the langpacks to continue from the previous version of the ADS langpack.
+ *
+ * 2. Also change the azdata version to match the current ADS version number.
+ *
+ * 3. Update the changelog with the new version of the language pack.
+ *
+ * IMPORTANT: If you have run this gulp task on langpacks that originated from vscode, for each affected vscode langpack, you must
+ * replace the changelog and readme files with the ones from the previous ADS version of the langpack before doing the above steps.
+ *
+ * This is mainly for consistency with previous langpacks and to provide proper information to the user.
+*/
+function refreshLangpacks() {
+    let supportedLocations = [...i18n.defaultLanguages, ...i18n.extraLanguages];
+    for (let i = 0; i < supportedLocations.length; i++) {
+        let langId = supportedLocations[i].id;
+        if (langId === "zh-cn") {
+            langId = "zh-hans";
+        }
+        if (langId === "zh-tw") {
+            langId = "zh-hant";
+        }
+        let location = path.join('.', 'resources', 'xlf');
+        let locExtFolder = path.join('.', 'i18n', `ads-language-pack-${langId}`);
+        try {
+            fs.statSync(locExtFolder);
+        }
+        catch (_a) {
+            console.log('Language is not included in ADS yet: ' + langId);
+            continue;
+        }
+        let packageJSON = JSON.parse(fs.readFileSync(path.join(locExtFolder, 'package.json')).toString());
+        //processing extension fields, version and folder name must be changed manually.
+        packageJSON['name'] = packageJSON['name'].replace('vscode', textFields.nameText);
+        packageJSON['displayName'] = packageJSON['displayName'].replace('Visual Studio Code', textFields.displayNameText);
+        packageJSON['publisher'] = textFields.publisherText;
+        packageJSON['license'] = textFields.licenseText;
+        packageJSON['scripts']['update'] = textFields.updateText + langId;
+        packageJSON['engines']['vscode'] = textFields.vscodeVersion;
+        packageJSON['repository']['url'] = textFields.gitUrl;
+        packageJSON['engines']['azdata'] = textFields.azdataPlaceholder; // Remember to change this to the appropriate version at the end.
+        let contributes = packageJSON['contributes'];
+        if (!contributes) {
+            throw new Error('The extension must define a "localizations" contribution in the "package.json"');
+        }
+        let localizations = contributes['localizations'];
+        if (!localizations) {
+            throw new Error('The extension must define a "localizations" contribution of type array in the "package.json"');
+        }
+        localizations.forEach(function (localization) {
+            if (!localization.languageId || !localization.languageName || !localization.localizedLanguageName) {
+                throw new Error('Each localization contribution must define "languageId", "languageName" and "localizedLanguageName" properties.');
+            }
+            let languageId = localization.transifexId || localization.languageId;
+            let translationDataFolder = path.join(locExtFolder, 'translations');
+            if (languageId === "zh-cn") {
+                languageId = "zh-hans";
+            }
+            if (languageId === "zh-tw") {
+                languageId = "zh-hant";
+            }
+            //remove extensions not part of ADS.
+            if (fs.existsSync(translationDataFolder)) {
+                let totalExtensions = fs.readdirSync(path.join(translationDataFolder, 'extensions'));
+                for (let extensionTag in totalExtensions) {
+                    let extensionFileName = totalExtensions[extensionTag];
+                    let xlfPath = path.join(location, `${languageId}`, extensionFileName.replace('.i18n.json', '.xlf'));
+                    if (!(fs.existsSync(xlfPath) || VSCODEExtensions.indexOf(extensionFileName.replace('.i18n.json', '')) !== -1)) {
+                        let filePath = path.join(translationDataFolder, 'extensions', extensionFileName);
+                        rimraf.sync(filePath);
+                    }
+                }
+            }
+            console.log(`Importing translations for ${languageId} from '${location}' to '${translationDataFolder}' ...`);
+            let translationPaths = [];
+            gulp.src(path.join(location, languageId, '**', '*.xlf'))
+                .pipe(modifyI18nPackFiles(translationDataFolder, translationPaths, languageId === 'ps'))
+                .on('error', (error) => {
+                console.log(`Error occurred while importing translations:`);
+                translationPaths = undefined;
+                if (Array.isArray(error)) {
+                    error.forEach(console.log);
+                }
+                else if (error) {
+                    console.log(error);
+                }
+                else {
+                    console.log('Unknown error');
+                }
+            })
+                .pipe(vfs.dest(translationDataFolder))
+                .on('end', function () {
+                if (translationPaths !== undefined) {
+                    let nonExistantExtensions = [];
+                    for (let curr of localization.translations) {
+                        try {
+                            if (curr.id === 'vscode.theme-seti') {
+                                //handle edge case where 'theme-seti' has a different id.
+                                curr.id = 'vscode.vscode-theme-seti';
+                            }
+                            fs.statSync(path.join(translationDataFolder, curr.path.replace('./translations', '')));
+                        }
+                        catch (_a) {
+                            nonExistantExtensions.push(curr);
+                        }
+                    }
+                    for (let nonExt of nonExistantExtensions) {
+                        let index = localization.translations.indexOf(nonExt);
+                        if (index > -1) {
+                            localization.translations.splice(index, 1);
+                        }
+                    }
+                    for (let tp of translationPaths) {
+                        let finalPath = `./translations/${tp.resourceName}`;
+                        let isFound = false;
+                        for (let i = 0; i < localization.translations.length; i++) {
+                            if (localization.translations[i].path === finalPath) {
+                                localization.translations[i].id = tp.id;
+                                isFound = true;
+                                break;
+                            }
+                        }
+                        if (!isFound) {
+                            localization.translations.push({ id: tp.id, path: finalPath });
+                        }
+                    }
+                    fs.writeFileSync(path.join(locExtFolder, 'package.json'), JSON.stringify(packageJSON, null, '\t'));
+                }
+            });
+        });
+    }
+    console.log("Langpack Refresh Completed.");
+    return Promise.resolve();
+}
+exports.refreshLangpacks = refreshLangpacks;

build/lib/locFunc.ts

@@ -8,6 +8,14 @@ import * as path from 'path';
 import * as glob from 'glob';
 import rename = require('gulp-rename');
 import ext = require('./extensions');
+//imports for langpack refresh.
+import { through, ThroughStream } from 'event-stream';
+import i18n = require('./i18n')
+import * as fs from 'fs';
+import * as File from 'vinyl';
+import * as rimraf from 'rimraf';
+import * as gulp from 'gulp';
+import * as vfs from 'vinyl-fs';
 
 const root = path.dirname(path.dirname(__dirname));
 
@@ -44,3 +52,313 @@ export function packageSingleExtensionStream(name : string): NodeJS.ReadWriteStr
 
 	return es.merge(builtExtension);
 }
+
+// Langpack creation functions go here.
+
+/**
+ * Function combines the contents of the SQL core XLF file into the current main i18n file contianing the vs core strings.
+ * Based on createI18nFile in i18n.ts
+*/
+function updateMainI18nFile(existingTranslationFilePath: string, originalFilePath: string, messages: any): File {
+	let currFilePath = path.join(existingTranslationFilePath + '.i18n.json');
+	let currentContent = fs.readFileSync(currFilePath);
+	let currentContentObject = JSON.parse(currentContent.toString());
+	let objectContents = currentContentObject.contents;
+	let result = Object.create(null);
+
+	// Delete any SQL strings that are no longer part of ADS in current langpack.
+	for (let contentKey of Object.keys(objectContents)) {
+		if(contentKey.startsWith('sql') && messages.contents[contentKey] === undefined){
+			delete objectContents[`${contentKey}`]
+		}
+	}
+
+	messages.contents = { ...objectContents, ...messages.contents };
+	result[''] = [
+		'--------------------------------------------------------------------------------------------',
+		'Copyright (c) Microsoft Corporation. All rights reserved.',
+		'Licensed under the Source EULA. See License.txt in the project root for license information.',
+		'--------------------------------------------------------------------------------------------',
+		'Do not edit this file. It is machine generated.'
+	];
+	for (let key of Object.keys(messages)) {
+		result[key] = messages[key];
+	}
+	let content = JSON.stringify(result, null, '\t');
+
+	if (process.platform === 'win32') {
+		content = content.replace(/\n/g, '\r\n');
+	}
+	return new File({
+		path: path.join(originalFilePath + '.i18n.json'),
+
+		contents: Buffer.from(content, 'utf8'),
+	})
+}
+
+/**
+ * Function handles the processing of xlf resources and turning them into i18n.json files.
+ * It adds the i18n files translation paths to be added back into package.main.
+ * Based on prepareI18nPackFiles in i18n.ts
+*/
+export function modifyI18nPackFiles(existingTranslationFolder: string, resultingTranslationPaths: i18n.TranslationPath[], pseudo = false): NodeJS.ReadWriteStream {
+	let parsePromises: Promise<i18n.ParsedXLF[]>[] = [];
+	let mainPack: i18n.I18nPack = { version: i18n.i18nPackVersion, contents: {} };
+	let extensionsPacks: i18n.Map<i18n.I18nPack> = {};
+	let errors: any[] = [];
+	return through(function (this: ThroughStream, xlf: File) {
+		let rawResource = path.basename(xlf.relative, '.xlf');
+		let resource = rawResource.substring(0, rawResource.lastIndexOf('.'));
+		let contents = xlf.contents.toString();
+		let parsePromise = pseudo ? i18n.XLF.parsePseudo(contents) : i18n.XLF.parse(contents);
+		parsePromises.push(parsePromise);
+		parsePromise.then(
+			resolvedFiles => {
+				resolvedFiles.forEach(file => {
+					const path = file.originalFilePath;
+					const firstSlash = path.indexOf('/');
+
+					//exclude core sql file from extension processing.
+					if (resource !== 'sql') {
+						let extPack = extensionsPacks[resource];
+						if (!extPack) {
+							extPack = extensionsPacks[resource] = { version: i18n.i18nPackVersion, contents: {} };
+						}
+						//remove extensions/extensionId section as all extensions will be webpacked.
+						const secondSlash = path.indexOf('/', firstSlash + 1);
+						extPack.contents[path.substr(secondSlash + 1)] = file.messages;
+					} else {
+						mainPack.contents[path.substr(firstSlash + 1)] = file.messages;
+					}
+				});
+			}
+		).catch(reason => {
+			errors.push(reason);
+		});
+	}, function () {
+		Promise.all(parsePromises)
+			.then(() => {
+				if (errors.length > 0) {
+					throw errors;
+				}
+				const translatedMainFile = updateMainI18nFile(existingTranslationFolder + '\\main', './main', mainPack);
+
+				this.queue(translatedMainFile);
+				for (let extension in extensionsPacks) {
+					const translatedExtFile = i18n.createI18nFile(`extensions/${extension}`, extensionsPacks[extension]);
+					this.queue(translatedExtFile);
+
+					//handle edge case for 'Microsoft.sqlservernotebook' where extension name is the same as extension ID.
+					//(Other extensions need to have publisher appended in front as their ID.)
+					const adsExtensionId = (extension === 'Microsoft.sqlservernotebook') ? extension : 'Microsoft.' + extension;
+					resultingTranslationPaths.push({ id: adsExtensionId, resourceName: `extensions/${extension}.i18n.json` });
+				}
+				this.queue(null);
+			})
+			.catch((reason) => {
+				this.emit('error', reason);
+			});
+	});
+}
+
+const textFields = {
+	"nameText": 'ads',
+	"displayNameText": 'Azure Data Studio',
+	"publisherText": 'Microsoft',
+	"licenseText": 'SEE SOURCE EULA LICENSE IN LICENSE.txt',
+	"updateText": 'cd ../vscode && npm run update-localization-extension ',
+	"vscodeVersion": '*',
+	"azdataPlaceholder": '^0.0.0',
+	"gitUrl": 'https://github.com/Microsoft/azuredatastudio'
+}
+
+//list of extensions from vscode that are to be included with ADS.
+const VSCODEExtensions = [
+	"bat",
+	"configuration-editing",
+	"docker",
+	"extension-editing",
+	"git-ui",
+	"git",
+	"github-authentication",
+	"github",
+	"image-preview",
+	"json-language-features",
+	"json",
+	"markdown-basics",
+	"markdown-language-features",
+	"merge-conflict",
+	"microsoft-authentication",
+	"powershell",
+	"python",
+	"r",
+	"search-result",
+	"sql",
+	"theme-abyss",
+	"theme-defaults",
+	"theme-kimbie-dark",
+	"theme-monokai-dimmed",
+	"theme-monokai",
+	"theme-quietlight",
+	"theme-red",
+	"theme-seti",
+	"theme-solarized-dark",
+	"theme-solarized-light",
+	"theme-tomorrow-night-blue",
+	"typescript-basics",
+	"xml",
+	"yaml"
+];
+
+/**
+ * A heavily modified version of update-localization-extension that runs using local xlf resources, no arguments required to pass in.
+ * It converts a renamed vscode langpack to an ADS one or updates the existing langpack to use current XLF resources.
+ * It runs this process on all langpacks currently in the ADS i18n folder.
+ * (Replace an individual ADS langpack folder with a corresponding vscode langpack folder renamed to "ads" instead of "vscode"
+ * in order to update vscode core strings and extensions for that langpack)
+ *
+ * It removes the resources of vscode that we do not support, and adds in new i18n json files created from the xlf files in the folder.
+ * It also merges in the sql core XLF strings with the langpack's existing core strings into a combined main i18n json file.
+ *
+ * After running this gulp task, for each language pack:
+ *
+ * 1. Remember to change the version of the langpacks to continue from the previous version of the ADS langpack.
+ *
+ * 2. Also change the azdata version to match the current ADS version number.
+ *
+ * 3. Update the changelog with the new version of the language pack.
+ *
+ * IMPORTANT: If you have run this gulp task on langpacks that originated from vscode, for each affected vscode langpack, you must
+ * replace the changelog and readme files with the ones from the previous ADS version of the langpack before doing the above steps.
+ *
+ * This is mainly for consistency with previous langpacks and to provide proper information to the user.
+*/
+export function refreshLangpacks(): Promise<void> {
+	let supportedLocations = [...i18n.defaultLanguages, ...i18n.extraLanguages];
+
+	for (let i = 0; i < supportedLocations.length; i++) {
+		let langId = supportedLocations[i].id;
+		if (langId === "zh-cn") {
+			langId = "zh-hans";
+		}
+		if (langId === "zh-tw") {
+			langId = "zh-hant";
+		}
+
+		let location = path.join('.', 'resources', 'xlf');
+		let locExtFolder = path.join('.', 'i18n', `ads-language-pack-${langId}`);
+		try {
+			fs.statSync(locExtFolder);
+		}
+		catch {
+			console.log('Language is not included in ADS yet: ' + langId);
+			continue;
+		}
+		let packageJSON = JSON.parse(fs.readFileSync(path.join(locExtFolder, 'package.json')).toString());
+		//processing extension fields, version and folder name must be changed manually.
+		packageJSON['name'] = packageJSON['name'].replace('vscode', textFields.nameText);
+		packageJSON['displayName'] = packageJSON['displayName'].replace('Visual Studio Code', textFields.displayNameText);
+		packageJSON['publisher'] = textFields.publisherText;
+		packageJSON['license'] = textFields.licenseText;
+		packageJSON['scripts']['update'] = textFields.updateText + langId;
+		packageJSON['engines']['vscode'] = textFields.vscodeVersion;
+		packageJSON['repository']['url'] = textFields.gitUrl
+		packageJSON['engines']['azdata'] = textFields.azdataPlaceholder // Remember to change this to the appropriate version at the end.
+
+		let contributes = packageJSON['contributes'];
+		if (!contributes) {
+			throw new Error('The extension must define a "localizations" contribution in the "package.json"');
+		}
+		let localizations = contributes['localizations'];
+		if (!localizations) {
+			throw new Error('The extension must define a "localizations" contribution of type array in the "package.json"');
+		}
+
+		localizations.forEach(function (localization: any) {
+			if (!localization.languageId || !localization.languageName || !localization.localizedLanguageName) {
+				throw new Error('Each localization contribution must define "languageId", "languageName" and "localizedLanguageName" properties.');
+			}
+			let languageId = localization.transifexId || localization.languageId;
+			let translationDataFolder = path.join(locExtFolder, 'translations');
+			if (languageId === "zh-cn") {
+				languageId = "zh-hans";
+			}
+			if (languageId === "zh-tw") {
+				languageId = "zh-hant";
+			}
+
+			//remove extensions not part of ADS.
+			if (fs.existsSync(translationDataFolder)) {
+				let totalExtensions = fs.readdirSync(path.join(translationDataFolder, 'extensions'));
+				for (let extensionTag in totalExtensions) {
+					let extensionFileName = totalExtensions[extensionTag];
+					let xlfPath = path.join(location, `${languageId}`, extensionFileName.replace('.i18n.json', '.xlf'))
+					if (!(fs.existsSync(xlfPath) || VSCODEExtensions.indexOf(extensionFileName.replace('.i18n.json', '')) !== -1)) {
+						let filePath = path.join(translationDataFolder, 'extensions', extensionFileName);
+						rimraf.sync(filePath);
+					}
+				}
+			}
+
+
+			console.log(`Importing translations for ${languageId} from '${location}' to '${translationDataFolder}' ...`);
+			let translationPaths: any = [];
+			gulp.src(path.join(location, languageId, '**', '*.xlf'))
+				.pipe(modifyI18nPackFiles(translationDataFolder, translationPaths, languageId === 'ps'))
+				.on('error', (error: any) => {
+					console.log(`Error occurred while importing translations:`);
+					translationPaths = undefined;
+					if (Array.isArray(error)) {
+						error.forEach(console.log);
+					} else if (error) {
+						console.log(error);
+					} else {
+						console.log('Unknown error');
+					}
+				})
+				.pipe(vfs.dest(translationDataFolder))
+				.on('end', function () {
+					if (translationPaths !== undefined) {
+						let nonExistantExtensions = [];
+						for (let curr of localization.translations) {
+							try {
+								if (curr.id === 'vscode.theme-seti') {
+									//handle edge case where 'theme-seti' has a different id.
+									curr.id = 'vscode.vscode-theme-seti';
+								}
+								fs.statSync(path.join(translationDataFolder, curr.path.replace('./translations', '')));
+							}
+							catch {
+								nonExistantExtensions.push(curr);
+							}
+						}
+						for (let nonExt of nonExistantExtensions) {
+							let index = localization.translations.indexOf(nonExt);
+							if (index > -1) {
+								localization.translations.splice(index, 1);
+							}
+						}
+						for (let tp of translationPaths) {
+							let finalPath = `./translations/${tp.resourceName}`;
+							let isFound = false;
+							for (let i = 0; i < localization.translations.length; i++) {
+								if (localization.translations[i].path === finalPath) {
+									localization.translations[i].id = tp.id;
+									isFound = true;
+									break;
+								}
+							}
+							if (!isFound) {
+								localization.translations.push({ id: tp.id, path: finalPath });
+							}
+						}
+						fs.writeFileSync(path.join(locExtFolder, 'package.json'), JSON.stringify(packageJSON, null, '\t'));
+					}
+				});
+
+		});
+	}
+	console.log("Langpack Refresh Completed.");
+	return Promise.resolve();
+}
+