Merge VS Code 1.21 source code (#1067)

* Initial VS Code 1.21 file copy with patches

* A few more merges

* Post npm install

* Fix batch of build breaks

* Fix more build breaks

* Fix more build errors

* Fix more build breaks

* Runtime fixes 1

* Get connection dialog working with some todos

* Fix a few packaging issues

* Copy several node_modules to package build to fix loader issues

* Fix breaks from master

* A few more fixes

* Make tests pass

* First pass of license header updates

* Second pass of license header updates

* Fix restore dialog issues

* Remove add additional themes menu items

* fix select box issues where the list doesn't show up

* formatting

* Fix editor dispose issue

* Copy over node modules to correct location on all platforms

build/gulpfile.hygiene.js

@@ -12,7 +12,11 @@ const gulptslint = require('gulp-tslint');
 const gulpeslint = require('gulp-eslint');
 const tsfmt = require('typescript-formatter');
 const tslint = require('tslint');
+const VinylFile = require('vinyl');
 const vfs = require('vinyl-fs');
+const path = require('path');
+const fs = require('fs');
+const pall = require('p-all');
 
 /**
  * Hygiene works by creating cascading subsets of all our files and
@@ -29,55 +33,56 @@ const all = [
 	'extensions/**/*',
 	'scripts/**/*',
 	'src/**/*',
-	'test/**/*'
-];
-
-const eolFilter = [
-	'**',
-	'!ThirdPartyNotices.txt',
-	'!LICENSE.txt',
-	'!extensions/**/out/**',
-	'!test/smoke/out/**',
-	'!**/node_modules/**',
-	'!**/fixtures/**',
-	'!**/*.{svg,exe,png,bmp,scpt,bat,cmd,cur,ttf,woff,eot}',
-	'!build/{lib,tslintRules}/**/*.js',
-	'!build/monaco/**',
-	'!build/win32/**',
-	'!build/**/*.sh',
-	'!build/tfs/**/*.js',
-	'!**/Dockerfile'
+	'test/**/*',
+	'!**/node_modules/**'
 ];
 
 const indentationFilter = [
 	'**',
+
+	// except specific files
 	'!ThirdPartyNotices.txt',
-	'!**/*.md',
-	'!**/*.ps1',
-	'!**/*.template',
-	'!**/*.yaml',
-	'!**/*.yml',
-	'!**/yarn.lock',
-	'!**/lib/**',
-	'!extensions/**/*.d.ts',
-	'!src/typings/**/*.d.ts',
-	'!src/vs/*/**/*.d.ts',
-	'!**/*.d.ts.recipe',
+	'!LICENSE.txt',
+	'!src/vs/nls.js',
+	'!src/vs/css.js',
+	'!src/vs/loader.js',
+	'!src/vs/base/common/marked/raw.marked.js',
+	'!src/vs/base/common/winjs.base.raw.js',
+	'!src/vs/base/node/terminateProcess.sh',
+	'!src/vs/base/node/ps-win.ps1',
 	'!test/assert.js',
+
+	// except specific folders
+	'!test/smoke/out/**',
+	'!extensions/vscode-api-tests/testWorkspace/**',
+	'!extensions/vscode-api-tests/testWorkspace2/**',
+	'!build/monaco/**',
+	'!build/win32/**',
+
+	// except multiple specific files
 	'!**/package.json',
+	'!**/yarn.lock',
+
+	// except multiple specific folders
 	'!**/octicons/**',
-	'!**/vs/base/common/marked/raw.marked.js',
-	'!**/vs/base/common/winjs.base.raw.js',
-	'!**/vs/base/node/terminateProcess.sh',
-	'!**/vs/base/node/ps-win.ps1',
-	'!**/vs/nls.js',
-	'!**/vs/css.js',
-	'!**/vs/loader.js',
+	'!**/fixtures/**',
+	'!**/lib/**',
+	'!extensions/**/out/**',
 	'!extensions/**/snippets/**',
 	'!extensions/**/syntaxes/**',
 	'!extensions/**/themes/**',
 	'!extensions/**/colorize-fixtures/**',
-	'!extensions/vscode-api-tests/testWorkspace/**'
+
+	// except specific file types
+	'!src/vs/*/**/*.d.ts',
+	'!src/typings/**/*.d.ts',
+	'!extensions/**/*.d.ts',
+	'!**/*.{svg,exe,png,bmp,scpt,bat,cmd,cur,ttf,woff,eot,md,ps1,template,yaml,yml,d.ts.recipe}',
+	'!build/{lib,tslintRules}/**/*.js',
+	'!build/**/*.sh',
+	'!build/tfs/**/*.js',
+	'!**/Dockerfile',
+	'!extensions/markdown/media/*.js'
 ];
 
 const copyrightFilter = [
@@ -95,6 +100,7 @@ const copyrightFilter = [
 	'!**/*.xpm',
 	'!**/*.opts',
 	'!**/*.disabled',
+	'!**/*.code-workspace',
 	'!build/**/*.init',
 	'!resources/linux/snap/snapcraft.yaml',
 	'!resources/win32/bin/code.js',
@@ -124,6 +130,7 @@ const tslintFilter = [
 	'!**/node_modules/**',
 	'!extensions/typescript/test/colorize-fixtures/**',
 	'!extensions/vscode-api-tests/testWorkspace/**',
+	'!extensions/vscode-api-tests/testWorkspace2/**',
 	'!extensions/**/*.test.ts',
 	'!extensions/html/server/lib/jquery.d.ts'
 ];
@@ -144,31 +151,23 @@ gulp.task('eslint', () => {
 });
 
 gulp.task('tslint', () => {
+  // {{SQL CARBON EDIT}}
 	const options = { emitError: false };
 
 	return vfs.src(all, { base: '.', follow: true, allowEmpty: true })
 		.pipe(filter(tslintFilter))
-		.pipe(gulptslint({ rulesDirectory: 'build/lib/tslint' }))
-		.pipe(gulptslint.report(options));
+		.pipe(gulptslint.default({ rulesDirectory: 'build/lib/tslint' }))
+		.pipe(gulptslint.default.report(options));
 });
 
-const hygiene = exports.hygiene = (some, options) => {
-	options = options || {};
+function hygiene(some) {
 	let errorCount = 0;
 
-	const eol = es.through(function (file) {
-		if (/\r\n?/g.test(file.contents.toString('utf8'))) {
-			console.error(file.relative + ': Bad EOL found');
-			errorCount++;
-		}
-
-		this.emit('data', file);
-	});
-
 	const indentation = es.through(function (file) {
-		file.contents
-			.toString('utf8')
-			.split(/\r\n|\r|\n/)
+		const lines = file.contents.toString('utf8').split(/\r\n|\r|\n/);
+		file.__lines = lines;
+
+		lines
 			.forEach((line, i) => {
 				if (/^\s*$/.test(line)) {
 					// empty or whitespace lines are OK
@@ -186,9 +185,14 @@ const hygiene = exports.hygiene = (some, options) => {
 	});
 
 	const copyrights = es.through(function (file) {
-		if (file.contents.toString('utf8').indexOf(copyrightHeader) !== 0) {
-			console.error(file.relative + ': Missing or bad copyright statement');
-			errorCount++;
+		const lines = file.__lines;
+
+		for (let i = 0; i < copyrightHeaderLines.length; i++) {
+			if (lines[i] !== copyrightHeaderLines[i]) {
+				console.error(file.relative + ': Missing or bad copyright statement');
+				errorCount++;
+				break;
+			}
 		}
 
 		this.emit('data', file);
@@ -196,12 +200,20 @@ const hygiene = exports.hygiene = (some, options) => {
 
 	const formatting = es.map(function (file, cb) {
 		tsfmt.processString(file.path, file.contents.toString('utf8'), {
-			verify: true,
+			verify: false,
 			tsfmt: true,
 			// verbose: true
+			// keep checkJS happy
+			editorconfig: undefined,
+			replace: undefined,
+			tsconfig: undefined,
+			tslint: undefined
 		}).then(result => {
-			if (result.error) {
-				console.error(result.message);
+			let original = result.src.replace(/\r\n/gm, '\n');
+			let formatted = result.dest.replace(/\r\n/gm, '\n');
+
+			if (original !== formatted) {
+				console.error('File not formatted:', file.relative);
 				errorCount++;
 			}
 			cb(null, file);
@@ -211,43 +223,31 @@ const hygiene = exports.hygiene = (some, options) => {
 		});
 	});
 
-	function reportFailures(failures) {
-		failures.forEach(failure => {
-			const name = failure.name || failure.fileName;
-			const position = failure.startPosition;
-			const line = position.lineAndCharacter ? position.lineAndCharacter.line : position.line;
-			const character = position.lineAndCharacter ? position.lineAndCharacter.character : position.character;
-
-			// console.error(`${name}:${line + 1}:${character + 1}:${failure.failure}`);
-		});
-	}
+	const tslintConfiguration = tslint.Configuration.findConfiguration('tslint.json', '.');
+	const tslintOptions = { fix: false, formatter: 'json' };
+	const tsLinter = new tslint.Linter(tslintOptions);
 
 	const tsl = es.through(function (file) {
-		const configuration = tslint.Configuration.findConfiguration(null, '.');
-		const options = { formatter: 'json', rulesDirectory: 'build/lib/tslint' };
 		const contents = file.contents.toString('utf8');
-		const linter = new tslint.Linter(options);
-		linter.lint(file.relative, contents, configuration.results);
-		const result = linter.getResult();
-
-		if (result.failures.length > 0) {
-			reportFailures(result.failures);
-			errorCount += result.failures.length;
-		}
-
+		tsLinter.lint(file.relative, contents, tslintConfiguration.results);
 		this.emit('data', file);
 	});
 
-	const result = vfs.src(some || all, { base: '.', follow: true, allowEmpty: true })
+	let input;
+
+	if (Array.isArray(some) || typeof some === 'string' || !some) {
+		input = vfs.src(some || all, { base: '.', follow: true, allowEmpty: true });
+	} else {
+		input = some;
+	}
+
+	const result = input
 		.pipe(filter(f => !f.stat.isDirectory()))
-		.pipe(filter(eolFilter))
-    // {{SQL CARBON EDIT}}
-		//.pipe(options.skipEOL ? es.through() : eol)
 		.pipe(filter(indentationFilter))
 		.pipe(indentation)
-		.pipe(filter(copyrightFilter))
+		.pipe(filter(copyrightFilter));
     // {{SQL CARBON EDIT}}
-		//.pipe(copyrights);
+		// .pipe(copyrights);
 
 	const typescript = result
 		.pipe(filter(tslintFilter))
@@ -257,23 +257,52 @@ const hygiene = exports.hygiene = (some, options) => {
 	const javascript = result
 		.pipe(filter(eslintFilter))
 		.pipe(gulpeslint('src/.eslintrc'))
-		.pipe(gulpeslint.formatEach('compact'))
-		// {{SQL CARBON EDIT}}
+		.pipe(gulpeslint.formatEach('compact'));
+    // {{SQL CARBON EDIT}}
 		// .pipe(gulpeslint.failAfterError());
 
-  return es.merge(typescript, javascript)
-  		.pipe(es.through(null, function () {
-        // {{SQL CARBON EDIT}}
-  		  // if (errorCount > 0) {
-  			// 	this.emit('error', 'Hygiene failed with ' + errorCount + ' errors. Check \'build/gulpfile.hygiene.js\'.');
-  			// } else {
-  			// 	this.emit('end');
-  			// }
-  				this.emit('end');
-  		}));
-};
+	let count = 0;
+	return es.merge(typescript, javascript)
+		.pipe(es.through(function (data) {
+       // {{SQL CARBON EDIT}}
+       this.emit('end');
+		}));
+}
 
-gulp.task('hygiene', () => hygiene(''));
+function createGitIndexVinyls(paths) {
+	const cp = require('child_process');
+	const repositoryPath = process.cwd();
+
+	const fns = paths.map(relativePath => () => new Promise((c, e) => {
+		const fullPath = path.join(repositoryPath, relativePath);
+
+		fs.stat(fullPath, (err, stat) => {
+			if (err && err.code === 'ENOENT') { // ignore deletions
+				return c(null);
+			} else if (err) {
+				return e(err);
+			}
+
+			cp.exec(`git show :${relativePath}`, { maxBuffer: 2000 * 1024, encoding: 'buffer' }, (err, out) => {
+				if (err) {
+					return e(err);
+				}
+
+				c(new VinylFile({
+					path: fullPath,
+					base: repositoryPath,
+					contents: out,
+					stat
+				}));
+			});
+		});
+	}));
+
+	return pall(fns, { concurrency: 4 })
+		.then(r => r.filter(p => !!p));
+}
+
+gulp.task('hygiene', () => hygiene());
 
 // this allows us to run hygiene as a git pre-commit hook
 if (require.main === module) {
@@ -284,22 +313,19 @@ if (require.main === module) {
 		process.exit(1);
 	});
 
-	cp.exec('git config core.autocrlf', (err, out) => {
-		const skipEOL = out.trim() === 'true';
-
-		if (process.argv.length > 2) {
-			return hygiene(process.argv.slice(2), { skipEOL: skipEOL }).on('error', err => {
-				console.error();
-				console.error(err);
-				process.exit(1);
-			});
-		}
-
+	if (process.argv.length > 2) {
+		hygiene(process.argv.slice(2)).on('error', err => {
+			console.error();
+			console.error(err);
+			process.exit(1);
+		});
+	} else {
 		cp.exec('git diff --cached --name-only', { maxBuffer: 2000 * 1024 }, (err, out) => {
 			if (err) {
 				console.error();
 				console.error(err);
 				process.exit(1);
+				return;
 			}
 
 			const some = out
@@ -307,12 +333,18 @@ if (require.main === module) {
 				.filter(l => !!l);
 
 			if (some.length > 0) {
-				hygiene(some, { skipEOL: skipEOL }).on('error', err => {
-					console.error();
-					console.error(err);
-					process.exit(1);
-				});
+				console.log('Reading git index versions...');
+
+				createGitIndexVinyls(some)
+					.then(vinyls => new Promise((c, e) => hygiene(es.readArray(vinyls))
+						.on('end', () => c())
+						.on('error', e)))
+					.catch(err => {
+						console.error();
+						console.error(err);
+						process.exit(1);
+					});
 			}
 		});
-	});
+	}
 }