Merge VS Code 1.21 source code (#1067)

* Initial VS Code 1.21 file copy with patches

* A few more merges

* Post npm install

* Fix batch of build breaks

* Fix more build breaks

* Fix more build errors

* Fix more build breaks

* Runtime fixes 1

* Get connection dialog working with some todos

* Fix a few packaging issues

* Copy several node_modules to package build to fix loader issues

* Fix breaks from master

* A few more fixes

* Make tests pass

* First pass of license header updates

* Second pass of license header updates

* Fix restore dialog issues

* Remove add additional themes menu items

* fix select box issues where the list doesn't show up

* formatting

* Fix editor dispose issue

* Copy over node modules to correct location on all platforms

src/vs/base/node/extfs.ts

@@ -9,11 +9,11 @@ import * as uuid from 'vs/base/common/uuid';
 import * as strings from 'vs/base/common/strings';
 import * as platform from 'vs/base/common/platform';
 import * as flow from 'vs/base/node/flow';
-
 import * as fs from 'fs';
 import * as paths from 'path';
 import { TPromise } from 'vs/base/common/winjs.base';
 import { nfcall } from 'vs/base/common/async';
+import { encode, encodeStream } from 'vs/base/node/encoding';
 
 const loop = flow.loop;
 
@@ -43,6 +43,27 @@ export function readdir(path: string, callback: (error: Error, files: string[])
 	return fs.readdir(path, callback);
 }
 
+export interface IStatAndLink {
+	stat: fs.Stats;
+	isSymbolicLink: boolean;
+}
+
+export function statLink(path: string, callback: (error: Error, statAndIsLink: IStatAndLink) => void): void {
+	fs.lstat(path, (error, lstat) => {
+		if (error || lstat.isSymbolicLink()) {
+			fs.stat(path, (error, stat) => {
+				if (error) {
+					return callback(error, null);
+				}
+
+				callback(null, { stat, isSymbolicLink: lstat && lstat.isSymbolicLink() });
+			});
+		} else {
+			callback(null, { stat: lstat, isSymbolicLink: false });
+		}
+	});
+}
+
 export function copy(source: string, target: string, callback: (error: Error) => void, copiedSources?: { [path: string]: boolean }): void {
 	if (!copiedSources) {
 		copiedSources = Object.create(null);
@@ -54,7 +75,7 @@ export function copy(source: string, target: string, callback: (error: Error) =>
 		}
 
 		if (!stat.isDirectory()) {
-			return pipeFs(source, target, stat.mode & 511, callback);
+			return doCopyFile(source, target, stat.mode & 511, callback);
 		}
 
 		if (copiedSources[source]) {
@@ -75,6 +96,38 @@ export function copy(source: string, target: string, callback: (error: Error) =>
 	});
 }
 
+function doCopyFile(source: string, target: string, mode: number, callback: (error: Error) => void): void {
+	const reader = fs.createReadStream(source);
+	const writer = fs.createWriteStream(target, { mode });
+
+	let finished = false;
+	const finish = (error?: Error) => {
+		if (!finished) {
+			finished = true;
+
+			// in error cases, pass to callback
+			if (error) {
+				callback(error);
+			}
+
+			// we need to explicitly chmod because of https://github.com/nodejs/node/issues/1104
+			else {
+				fs.chmod(target, mode, callback);
+			}
+		}
+	};
+
+	// handle errors properly
+	reader.once('error', error => finish(error));
+	writer.once('error', error => finish(error));
+
+	// we are done (underlying fd has been closed)
+	writer.once('close', () => finish());
+
+	// start piping
+	reader.pipe(writer);
+}
+
 export function mkdirp(path: string, mode?: number): TPromise<boolean> {
 	const mkdir = () => nfcall(fs.mkdir, path, mode)
 		.then(null, (err: NodeJS.ErrnoException) => {
@@ -88,11 +141,12 @@ export function mkdirp(path: string, mode?: number): TPromise<boolean> {
 			return TPromise.wrapError<boolean>(err);
 		});
 
-	// is root?
+	// stop at root
 	if (path === paths.dirname(path)) {
 		return TPromise.as(true);
 	}
 
+	// recursively mkdir
 	return mkdir().then(null, (err: NodeJS.ErrnoException) => {
 		if (err.code === 'ENOENT') {
 			return mkdirp(paths.dirname(path), mode).then(mkdir);
@@ -102,40 +156,6 @@ export function mkdirp(path: string, mode?: number): TPromise<boolean> {
 	});
 }
 
-function pipeFs(source: string, target: string, mode: number, callback: (error: Error) => void): void {
-	let callbackHandled = false;
-
-	const readStream = fs.createReadStream(source);
-	const writeStream = fs.createWriteStream(target, { mode: mode });
-
-	const onError = (error: Error) => {
-		if (!callbackHandled) {
-			callbackHandled = true;
-			callback(error);
-		}
-	};
-
-	readStream.on('error', onError);
-	writeStream.on('error', onError);
-
-	readStream.on('end', () => {
-		(<any>writeStream).end(() => { // In this case the write stream is known to have an end signature with callback
-			if (!callbackHandled) {
-				callbackHandled = true;
-
-				fs.chmod(target, mode, callback); // we need to explicitly chmod because of https://github.com/nodejs/node/issues/1104
-			}
-		});
-	});
-
-	// In node 0.8 there is no easy way to find out when the pipe operation has finished. As such, we use the end property = false
-	// so that we are in charge of calling end() on the write stream and we will be notified when the write stream is really done.
-	// We can do this because file streams have an end() method that allows to pass in a callback.
-	// In node 0.10 there is an event 'finish' emitted from the write stream that can be used. See
-	// https://groups.google.com/forum/?fromgroups=#!topic/nodejs/YWQ1sRoXOdI
-	readStream.pipe(writeStream, { end: false });
-}
-
 // Deletes the given path by first moving it out of the workspace. This has two benefits. For one, the operation can return fast because
 // after the rename, the contents are out of the workspace although not yet deleted. The greater benefit however is that this operation
 // will fail in case any file is used by another process. fs.unlink() in node will not bail if a file unlinked is used by another process.
@@ -320,19 +340,124 @@ export function mv(source: string, target: string, callback: (error: Error) => v
 	});
 }
 
+export interface IWriteFileOptions {
+	mode?: number;
+	flag?: string;
+	encoding?: {
+		charset: string;
+		addBOM: boolean;
+	};
+}
+
+let canFlush = true;
+export function writeFileAndFlush(path: string, data: string | NodeBuffer | NodeJS.ReadableStream, options: IWriteFileOptions, callback: (error?: Error) => void): void {
+	options = ensureOptions(options);
+
+	if (typeof data === 'string' || Buffer.isBuffer(data)) {
+		doWriteFileAndFlush(path, data, options, callback);
+	} else {
+		doWriteFileStreamAndFlush(path, data, options, callback);
+	}
+}
+
+function doWriteFileStreamAndFlush(path: string, reader: NodeJS.ReadableStream, options: IWriteFileOptions, callback: (error?: Error) => void): void {
+
+	// finish only once
+	let finished = false;
+	const finish = (error?: Error) => {
+		if (!finished) {
+			finished = true;
+
+			// in error cases we need to manually close streams
+			// if the write stream was successfully opened
+			if (error) {
+				if (isOpen) {
+					writer.once('close', () => callback(error));
+					writer.close();
+				} else {
+					callback(error);
+				}
+			}
+
+			// otherwise just return without error
+			else {
+				callback();
+			}
+		}
+	};
+
+	// create writer to target. we set autoClose: false because we want to use the streams
+	// file descriptor to call fs.fdatasync to ensure the data is flushed to disk
+	const writer = fs.createWriteStream(path, { mode: options.mode, flags: options.flag, autoClose: false });
+
+	// Event: 'open'
+	// Purpose: save the fd for later use and start piping
+	// Notes: will not be called when there is an error opening the file descriptor!
+	let fd: number;
+	let isOpen: boolean;
+	writer.once('open', descriptor => {
+		fd = descriptor;
+		isOpen = true;
+
+		// if an encoding is provided, we need to pipe the stream through
+		// an encoder stream and forward the encoding related options
+		if (options.encoding) {
+			reader = reader.pipe(encodeStream(options.encoding.charset, { addBOM: options.encoding.addBOM }));
+		}
+
+		// start data piping only when we got a successful open. this ensures that we do
+		// not consume the stream when an error happens and helps to fix this issue:
+		// https://github.com/Microsoft/vscode/issues/42542
+		reader.pipe(writer);
+	});
+
+	// Event: 'error'
+	// Purpose: to return the error to the outside and to close the write stream (does not happen automatically)
+	reader.once('error', error => finish(error));
+	writer.once('error', error => finish(error));
+
+	// Event: 'finish'
+	// Purpose: use fs.fdatasync to flush the contents to disk
+	// Notes: event is called when the writer has finished writing to the underlying resource. we must call writer.close()
+	// because we have created the WriteStream with autoClose: false
+	writer.once('finish', () => {
+
+		// flush to disk
+		if (canFlush && isOpen) {
+			fs.fdatasync(fd, (syncError: Error) => {
+
+				// In some exotic setups it is well possible that node fails to sync
+				// In that case we disable flushing and warn to the console
+				if (syncError) {
+					console.warn('[node.js fs] fdatasync is now disabled for this session because it failed: ', syncError);
+					canFlush = false;
+				}
+
+				writer.close();
+			});
+		} else {
+			writer.close();
+		}
+	});
+
+	// Event: 'close'
+	// Purpose: signal we are done to the outside
+	// Notes: event is called when the writer's filedescriptor is closed
+	writer.once('close', () => finish());
+}
+
 // Calls fs.writeFile() followed by a fs.sync() call to flush the changes to disk
 // We do this in cases where we want to make sure the data is really on disk and
 // not in some cache.
 //
 // See https://github.com/nodejs/node/blob/v5.10.0/lib/fs.js#L1194
-let canFlush = true;
-export function writeFileAndFlush(path: string, data: string | NodeBuffer, options: { mode?: number; flag?: string; }, callback: (error: Error) => void): void {
-	if (!canFlush) {
-		return fs.writeFile(path, data, options, callback);
+function doWriteFileAndFlush(path: string, data: string | NodeBuffer, options: IWriteFileOptions, callback: (error?: Error) => void): void {
+	if (options.encoding) {
+		data = encode(data, options.encoding.charset, { addBOM: options.encoding.addBOM });
 	}
 
-	if (!options) {
-		options = { mode: 0o666, flag: 'w' };
+	if (!canFlush) {
+		return fs.writeFile(path, data, { mode: options.mode, flag: options.flag }, callback);
 	}
 
 	// Open the file with same flags and mode as fs.writeFile()
@@ -363,13 +488,15 @@ export function writeFileAndFlush(path: string, data: string | NodeBuffer, optio
 	});
 }
 
-export function writeFileAndFlushSync(path: string, data: string | NodeBuffer, options?: { mode?: number; flag?: string; }): void {
-	if (!canFlush) {
-		return fs.writeFileSync(path, data, options);
+export function writeFileAndFlushSync(path: string, data: string | NodeBuffer, options?: IWriteFileOptions): void {
+	options = ensureOptions(options);
+
+	if (options.encoding) {
+		data = encode(data, options.encoding.charset, { addBOM: options.encoding.addBOM });
 	}
 
-	if (!options) {
-		options = { mode: 0o666, flag: 'w' };
+	if (!canFlush) {
+		return fs.writeFileSync(path, data, { mode: options.mode, flag: options.flag });
 	}
 
 	// Open the file with same flags and mode as fs.writeFile()
@@ -392,6 +519,24 @@ export function writeFileAndFlushSync(path: string, data: string | NodeBuffer, o
 	}
 }
 
+function ensureOptions(options?: IWriteFileOptions): IWriteFileOptions {
+	if (!options) {
+		return { mode: 0o666, flag: 'w' };
+	}
+
+	const ensuredOptions: IWriteFileOptions = { mode: options.mode, flag: options.flag, encoding: options.encoding };
+
+	if (typeof ensuredOptions.mode !== 'number') {
+		ensuredOptions.mode = 0o666;
+	}
+
+	if (typeof ensuredOptions.flag !== 'string') {
+		ensuredOptions.flag = 'w';
+	}
+
+	return ensuredOptions;
+}
+
 /**
  * Copied from: https://github.com/Microsoft/vscode-node-debug/blob/master/src/node/pathUtilities.ts#L83
  *
@@ -474,21 +619,34 @@ function normalizePath(path: string): string {
 	return strings.rtrim(paths.normalize(path), paths.sep);
 }
 
-export function watch(path: string, onChange: (type: string, path: string) => void): fs.FSWatcher {
-	const watcher = fs.watch(path);
-	watcher.on('change', (type, raw) => {
-		let file: string = null;
-		if (raw) { // https://github.com/Microsoft/vscode/issues/38191
-			file = raw.toString();
-			if (platform.isMacintosh) {
-				// Mac: uses NFD unicode form on disk, but we want NFC
-				// See also https://github.com/nodejs/node/issues/2165
-				file = strings.normalizeNFC(file);
+export function watch(path: string, onChange: (type: string, path: string) => void, onError: (error: string) => void): fs.FSWatcher {
+	try {
+		const watcher = fs.watch(path);
+
+		watcher.on('change', (type, raw) => {
+			let file: string = null;
+			if (raw) { // https://github.com/Microsoft/vscode/issues/38191
+				file = raw.toString();
+				if (platform.isMacintosh) {
+					// Mac: uses NFD unicode form on disk, but we want NFC
+					// See also https://github.com/nodejs/node/issues/2165
+					file = strings.normalizeNFC(file);
+				}
 			}
-		}
 
-		onChange(type, file);
-	});
+			onChange(type, file);
+		});
 
-	return watcher;
-}
+		watcher.on('error', (code: number, signal: string) => onError(`Failed to watch ${path} for changes (${code}, ${signal})`));
+
+		return watcher;
+	} catch (error) {
+		fs.exists(path, exists => {
+			if (exists) {
+				onError(`Failed to watch ${path} for changes (${error.toString()})`);
+			}
+		});
+	}
+
+	return void 0;
+}