Merge VS Code 1.21 source code (#1067)

* Initial VS Code 1.21 file copy with patches

* A few more merges

* Post npm install

* Fix batch of build breaks

* Fix more build breaks

* Fix more build errors

* Fix more build breaks

* Runtime fixes 1

* Get connection dialog working with some todos

* Fix a few packaging issues

* Copy several node_modules to package build to fix loader issues

* Fix breaks from master

* A few more fixes

* Make tests pass

* First pass of license header updates

* Second pass of license header updates

* Fix restore dialog issues

* Remove add additional themes menu items

* fix select box issues where the list doesn't show up

* formatting

* Fix editor dispose issue

* Copy over node modules to correct location on all platforms

build/npm/postinstall.js

@@ -26,10 +26,11 @@ yarnInstall('extensions'); // node modules shared by all extensions
 const extensions = [
 	'vscode-colorize-tests',
 	'json',
-    'mssql',
+  'mssql',
 	'configuration-editing',
 	'extension-editing',
 	'markdown',
+	'markdown-basics',
 	'git',
 	'merge-conflict',
 	'insights-default',
@@ -42,6 +43,7 @@ extensions.forEach(extension => yarnInstall(`extensions/${extension}`));
 function yarnInstallBuildDependencies() {
 	// make sure we install the deps of build/lib/watch for the system installed
 	// node, since that is the driver of gulp
+	//@ts-ignore
 	const env = Object.assign({}, process.env);
 	const watchPath = path.join(path.dirname(__dirname), 'lib', 'watch');
 	const yarnrcPath = path.join(watchPath, '.yarnrc');
@@ -59,4 +61,5 @@ runtime "${runtime}"`;
 }
 
 yarnInstall(`build`); // node modules required for build
+yarnInstall('test/smoke'); // node modules required for smoketest
 yarnInstallBuildDependencies(); // node modules for watching, specific to host node version, not electron

build/npm/update-all-grammars.js

@@ -4,9 +4,11 @@
  *--------------------------------------------------------------------------------------------*/
 
 const cp = require('child_process');
-const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+const fs = require('fs');
+const path = require('path');
 
 function updateGrammar(location) {
+	const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm';
 	const result = cp.spawnSync(npm, ['run', 'update-grammar'], {
 		cwd: location,
 		stdio: 'inherit'
@@ -17,50 +19,17 @@ function updateGrammar(location) {
 	}
 }
 
-const extensions = [
-	'bat',
-	'clojure',
-	'coffeescript',
-	'cpp',
-	'csharp',
-	'css',
-	'diff',
-	'docker',
-	'fsharp',
-	'gitsyntax',
-	'go',
-	'groovy',
-	'handlebars',
-	'hlsl',
-	'html',
-	'ini',
-	'java',
-	// 'javascript',  updated through JavaScript
-	'json',
-	'less',
-	'lua',
-	'make',
-	'markdown',
-	'objective-c',
-	'perl',
-	'php',
-	// 'powershell', grammar not ready yet, @daviwil will ping when ready
-	'pug',
-	'python',
-	'r',
-	'razor',
-	'ruby',
-	'rust',
-	'scss',
-	'shaderlab',
-	'shellscript',
-	'sql',
-	'swift',
-	'typescript',
-	'vb',
-	'xml',
-	'yaml'
-];
+const allExtensionFolders = fs.readdirSync('extensions');
+const extensions = allExtensionFolders.filter(e => {
+	try {
+		let packageJSON = JSON.parse(fs.readFileSync(path.join('extensions', e, 'package.json')).toString());
+		return packageJSON && packageJSON.scripts && packageJSON.scripts['update-grammar'];
+	} catch (e) {
+		return false;
+	}
+});
+
+console.log(`Updating ${extensions.length} grammars...`);
 
 extensions.forEach(extension => updateGrammar(`extensions/${extension}`));
 
@@ -70,4 +39,5 @@ if (process.platform === 'win32') {
 	cp.spawn('.\scripts\test-integration.bat', [], { env: process.env, stdio: 'inherit' });
 } else {
 	cp.spawn('/bin/bash', ['./scripts/test-integration.sh'], { env: process.env, stdio: 'inherit' });
-}
+}
+

build/npm/update-grammar.js

@@ -14,14 +14,19 @@ var url = require('url');
 
 function getOptions(urlString) {
 	var _url = url.parse(urlString);
+	var headers = {
+		'User-Agent': 'VSCode'
+	};
+	var token = process.env['GITHUB_TOKEN'];
+	if (token) {
+		headers['Authorization'] = 'token ' + token
+	}
 	return {
 		protocol: _url.protocol,
 		host: _url.host,
 		port: _url.port,
 		path: _url.path,
-		headers: {
-			'User-Agent': 'NodeJS'
-		}
+		headers: headers
 	};
 }
 
@@ -32,12 +37,16 @@ function download(url, redirectCount) {
 			response.on('data', function (data) {
 				content += data.toString();
 			}).on('end', function () {
+				if (response.statusCode === 403 && response.headers['x-ratelimit-remaining'] === '0') {
+					e('GitHub API rate exceeded. Set GITHUB_TOKEN environment variable to increase rate limit.');
+					return;
+				}
 				let count = redirectCount || 0;
 				if (count < 5 && response.statusCode >= 300 && response.statusCode <= 303 || response.statusCode === 307) {
 					let location = response.headers['location'];
 					if (location) {
 						console.log("Redirected " + url + " to " + location);
-						download(location, count+1).then(c, e);
+						download(location, count + 1).then(c, e);
 						return;
 					}
 				}
@@ -59,17 +68,13 @@ function getCommitSha(repoId, repoPath) {
 				commitDate: lastCommit.commit.author.date
 			});
 		} catch (e) {
-			console.error("Failed extracting the SHA: " + content);
-			return Promise.resolve(null);
+			return Promise.reject(new Error("Failed extracting the SHA: " + content));
 		}
-	}, function () {
-		console.error('Failed loading ' + commitInfo);
-		return Promise.resolve(null);
 	});
 }
 
-exports.update = function (repoId, repoPath, dest, modifyGrammar) {
-	var contentPath = 'https://raw.githubusercontent.com/' + repoId + '/master/' + repoPath;
+exports.update = function (repoId, repoPath, dest, modifyGrammar, version = 'master') {
+	var contentPath = 'https://raw.githubusercontent.com/' + repoId + `/${version}/` + repoPath;
 	console.log('Reading from ' + contentPath);
 	return download(contentPath).then(function (content) {
 		var ext = path.extname(repoPath);
@@ -81,8 +86,7 @@ exports.update = function (repoId, repoPath, dest, modifyGrammar) {
 		} else if (ext === '.json') {
 			grammar = JSON.parse(content);
 		} else {
-			console.error('Unknown file extension: ' + ext);
-			return;
+			return Promise.reject(new Error('Unknown file extension: ' + ext));
 		}
 		if (modifyGrammar) {
 			modifyGrammar(grammar);
@@ -99,8 +103,10 @@ exports.update = function (repoId, repoPath, dest, modifyGrammar) {
 			if (info) {
 				result.version = 'https://github.com/' + repoId + '/commit/' + info.commitSha;
 			}
-			for (let key in grammar) {
-				if (!result.hasOwnProperty(key)) {
+
+			let keys = ['name', 'scopeName', 'comment', 'injections', 'patterns', 'repository'];
+			for (let key of keys) {
+				if (grammar.hasOwnProperty(key)) {
 					result[key] = grammar[key];
 				}
 			}
@@ -113,11 +119,14 @@ exports.update = function (repoId, repoPath, dest, modifyGrammar) {
 					console.log('Updated ' + path.basename(dest));
 				}
 			} catch (e) {
-				console.error(e);
+				return Promise.reject(e);
 			}
 		});
 
-	}, console.error);
+	}, console.error).catch(e => {
+		console.error(e);
+		process.exit(1);
+	});
 };
 
 if (path.basename(process.argv[1]) === 'update-grammar.js') {

build/npm/update-localization-extension.js

@@ -0,0 +1,69 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+let i18n = require("../lib/i18n");
+
+let fs = require("fs");
+let path = require("path");
+let vfs = require("vinyl-fs");
+let rimraf = require('rimraf');
+
+function update(idOrPath) {
+	if (!idOrPath) {
+		throw new Error('Argument must be the location of the localization extension.');
+	}
+	let locExtFolder = idOrPath;
+	if (/^\w{2}(-\w+)?$/.test(idOrPath)) {
+		locExtFolder = '../vscode-language-pack-' + idOrPath;
+	}
+	let locExtStat = fs.statSync(locExtFolder);
+	if (!locExtStat || !locExtStat.isDirectory) {
+		throw new Error('No directory found at ' + idOrPath);
+	}
+	let packageJSON = JSON.parse(fs.readFileSync(path.join(locExtFolder, 'package.json')).toString());
+	let contributes = packageJSON['contributes'];
+	if (!contributes) {
+		throw new Error('The extension must define a "localizations" contribution in the "package.json"');
+	}
+	let localizations = contributes['localizations'];
+	if (!localizations) {
+		throw new Error('The extension must define a "localizations" contribution of type array in the "package.json"');
+	}
+
+	localizations.forEach(function (localization) {
+		if (!localization.languageId || !localization.languageName || !localization.localizedLanguageName) {
+			throw new Error('Each localization contribution must define "languageId", "languageName" and "localizedLanguageName" properties.');
+		}
+		let server = localization.server || 'www.transifex.com';
+		let userName = localization.userName || 'api';
+		let apiToken = process.env.TRANSIFEX_API_TOKEN;
+		let languageId = localization.transifexId || localization.languageId;
+		let translationDataFolder = path.join(locExtFolder, 'translations');
+
+		if (fs.existsSync(translationDataFolder) && fs.existsSync(path.join(translationDataFolder, 'main.i18n.json'))) {
+			console.log('Clearing  \'' + translationDataFolder + '\'...');
+			rimraf.sync(translationDataFolder);
+		}
+
+		console.log('Downloading translations for \'' + languageId + '\' to \'' + translationDataFolder + '\'...');
+		const translationPaths = [];
+		i18n.pullI18nPackFiles(server, userName, apiToken, { id: languageId }, translationPaths)
+			.pipe(vfs.dest(translationDataFolder)).on('end', function () {
+				localization.translations = [];
+				for (let tp of translationPaths) {
+					localization.translations.push({ id: tp.id, path: `./translations/${tp.resourceName}`});
+				}
+				fs.writeFileSync(path.join(locExtFolder, 'package.json'), JSON.stringify(packageJSON, null, '\t'));
+			});
+
+	});
+
+
+}
+if (path.basename(process.argv[1]) === 'update-localization-extension.js') {
+	update(process.argv[2]);
+}