Merge from master

build/lib/compilation.ts

@@ -5,31 +5,39 @@
 
 'use strict';
 
-import * as gulp from 'gulp';
-import * as tsb from 'gulp-tsb';
 import * as es from 'event-stream';
-const watch = require('./watch');
-import * as nls from './nls';
-import * as util from './util';
-import { createReporter } from './reporter';
-import * as path from 'path';
+import * as fs from 'fs';
+import * as gulp from 'gulp';
 import * as bom from 'gulp-bom';
 import * as sourcemaps from 'gulp-sourcemaps';
+import * as tsb from 'gulp-tsb';
+import * as path from 'path';
 import * as _ from 'underscore';
 import * as monacodts from '../monaco/api';
-import * as fs from 'fs';
+import * as nls from './nls';
+import { createReporter } from './reporter';
+import * as util from './util';
+import * as util2 from 'gulp-util';
+const watch = require('./watch');
 
 const reporter = createReporter();
 
 function getTypeScriptCompilerOptions(src: string) {
 	const rootDir = path.join(__dirname, `../../${src}`);
-	const options = require(`../../${src}/tsconfig.json`).compilerOptions;
+	const tsconfig = require(`../../${src}/tsconfig.json`);
+	let options: { [key: string]: any };
+	if (tsconfig.extends) {
+		options = Object.assign({}, require(path.join(rootDir, tsconfig.extends)).compilerOptions, tsconfig.compilerOptions);
+	} else {
+		options = tsconfig.compilerOptions;
+	}
 	options.verbose = false;
 	options.sourceMap = true;
 	if (process.env['VSCODE_NO_SOURCEMAP']) { // To be used by developers in a hurry
 		options.sourceMap = false;
 	}
 	options.rootDir = rootDir;
+	options.baseUrl = rootDir;
 	options.sourceRoot = util.toFileUri(rootDir);
 	options.newLine = /\r\n/.test(fs.readFileSync(__filename, 'utf8')) ? 'CRLF' : 'LF';
 	return options;
@@ -40,7 +48,7 @@ function createCompile(src: string, build: boolean, emitError?: boolean): (token
 	opts.inlineSources = !!build;
 	opts.noFilesystemLookup = true;
 
-	const ts = tsb.create(opts, null, null, err => reporter(err.toString()));
+	const ts = tsb.create(opts, true, undefined, err => reporter(err.toString()));
 
 	return function (token?: util.ICancellationToken) {
 
@@ -65,12 +73,19 @@ function createCompile(src: string, build: boolean, emitError?: boolean): (token
 				sourceRoot: opts.sourceRoot
 			}))
 			.pipe(tsFilter.restore)
-			.pipe(reporter.end(emitError));
+			.pipe(reporter.end(!!emitError));
 
 		return es.duplex(input, output);
 	};
 }
 
+const typesDts = [
+	'node_modules/typescript/lib/*.d.ts',
+	'node_modules/@types/**/*.d.ts',
+	'!node_modules/@types/webpack/**/*',
+	'!node_modules/@types/uglify-js/**/*',
+];
+
 export function compileTask(src: string, out: string, build: boolean): () => NodeJS.ReadWriteStream {
 
 	return function () {
@@ -78,18 +93,18 @@ export function compileTask(src: string, out: string, build: boolean): () => Nod
 
 		const srcPipe = es.merge(
 			gulp.src(`${src}/**`, { base: `${src}` }),
-			gulp.src('node_modules/typescript/lib/lib.d.ts'),
+			gulp.src(typesDts),
 		);
 
-		// Do not write .d.ts files to disk, as they are not needed there.
-		const dtsFilter = util.filter(data => !/\.d\.ts$/.test(data.path));
+		let generator = new MonacoGenerator(false);
+		if (src === 'src') {
+			generator.execute();
+		}
 
 		return srcPipe
+			.pipe(generator.stream)
 			.pipe(compile())
-			.pipe(dtsFilter)
-			.pipe(gulp.dest(out))
-			.pipe(dtsFilter.restore)
-			.pipe(src !== 'src' ? es.through() : monacodtsTask(out, false));
+			.pipe(gulp.dest(out));
 	};
 }
 
@@ -100,80 +115,114 @@ export function watchTask(out: string, build: boolean): () => NodeJS.ReadWriteSt
 
 		const src = es.merge(
 			gulp.src('src/**', { base: 'src' }),
-			gulp.src('node_modules/typescript/lib/lib.d.ts'),
+			gulp.src(typesDts),
 		);
 		const watchSrc = watch('src/**', { base: 'src' });
 
-		// Do not write .d.ts files to disk, as they are not needed there.
-		const dtsFilter = util.filter(data => !/\.d\.ts$/.test(data.path));
+		let generator = new MonacoGenerator(true);
+		generator.execute();
 
 		return watchSrc
+			.pipe(generator.stream)
 			.pipe(util.incremental(compile, src, true))
-			.pipe(dtsFilter)
-			.pipe(gulp.dest(out))
-			.pipe(dtsFilter.restore)
-			.pipe(monacodtsTask(out, true));
+			.pipe(gulp.dest(out));
 	};
 }
 
-function monacodtsTask(out: string, isWatch: boolean): NodeJS.ReadWriteStream {
+const REPO_SRC_FOLDER = path.join(__dirname, '../../src');
 
-	const basePath = path.resolve(process.cwd(), out);
+class MonacoGenerator {
+	private readonly _isWatch: boolean;
+	public readonly stream: NodeJS.ReadWriteStream;
 
-	const neededFiles: { [file: string]: boolean; } = {};
-	monacodts.getFilesToWatch(out).forEach(function (filePath) {
-		filePath = path.normalize(filePath);
-		neededFiles[filePath] = true;
-	});
+	private readonly _watchers: fs.FSWatcher[];
+	private readonly _watchedFiles: { [filePath: string]: boolean; };
+	private readonly _fsProvider: monacodts.FSProvider;
+	private readonly _declarationResolver: monacodts.DeclarationResolver;
 
-	const inputFiles: { [file: string]: string; } = {};
-	for (let filePath in neededFiles) {
-		if (/\bsrc(\/|\\)vs\b/.test(filePath)) {
-			// This file is needed from source => simply read it now
-			inputFiles[filePath] = fs.readFileSync(filePath).toString();
+	constructor(isWatch: boolean) {
+		this._isWatch = isWatch;
+		this.stream = es.through();
+		this._watchers = [];
+		this._watchedFiles = {};
+		let onWillReadFile = (moduleId: string, filePath: string) => {
+			if (!this._isWatch) {
+				return;
+			}
+			if (this._watchedFiles[filePath]) {
+				return;
+			}
+			this._watchedFiles[filePath] = true;
+
+			const watcher = fs.watch(filePath);
+			watcher.addListener('change', () => {
+				this._declarationResolver.invalidateCache(moduleId);
+				this._executeSoon();
+			});
+			this._watchers.push(watcher);
+		};
+		this._fsProvider = new class extends monacodts.FSProvider {
+			public readFileSync(moduleId: string, filePath: string): Buffer {
+				onWillReadFile(moduleId, filePath);
+				return super.readFileSync(moduleId, filePath);
+			}
+		};
+		this._declarationResolver = new monacodts.DeclarationResolver(this._fsProvider);
+
+		if (this._isWatch) {
+			const recipeWatcher = fs.watch(monacodts.RECIPE_PATH);
+			recipeWatcher.addListener('change', () => {
+				this._executeSoon();
+			});
+			this._watchers.push(recipeWatcher);
 		}
 	}
 
-	const setInputFile = (filePath: string, contents: string) => {
-		if (inputFiles[filePath] === contents) {
-			// no change
+	private _executeSoonTimer: NodeJS.Timer | null = null;
+	private _executeSoon(): void {
+		if (this._executeSoonTimer !== null) {
+			clearTimeout(this._executeSoonTimer);
+			this._executeSoonTimer = null;
+		}
+		this._executeSoonTimer = setTimeout(() => {
+			this._executeSoonTimer = null;
+			this.execute();
+		}, 20);
+	}
+
+	public dispose(): void {
+		this._watchers.forEach(watcher => watcher.close());
+	}
+
+	private _run(): monacodts.IMonacoDeclarationResult | null {
+		let r = monacodts.run3(this._declarationResolver);
+		if (!r && !this._isWatch) {
+			// The build must always be able to generate the monaco.d.ts
+			throw new Error(`monaco.d.ts generation error - Cannot continue`);
+		}
+		return r;
+	}
+
+	private _log(message: any, ...rest: any[]): void {
+		util2.log(util2.colors.cyan('[monaco.d.ts]'), message, ...rest);
+	}
+
+	public execute(): void {
+		const startTime = Date.now();
+		const result = this._run();
+		if (!result) {
+			// nothing really changed
 			return;
 		}
-		inputFiles[filePath] = contents;
-		const neededInputFilesCount = Object.keys(neededFiles).length;
-		const availableInputFilesCount = Object.keys(inputFiles).length;
-		if (neededInputFilesCount === availableInputFilesCount) {
-			run();
+		if (result.isTheSame) {
+			return;
 		}
-	};
 
-	const run = () => {
-		const result = monacodts.run(out, inputFiles);
-		if (!result.isTheSame) {
-			if (isWatch) {
-				fs.writeFileSync(result.filePath, result.content);
-			} else {
-				fs.writeFileSync(result.filePath, result.content);
-				resultStream.emit('error', 'monaco.d.ts is no longer up to date. Please run gulp watch and commit the new file.');
-			}
+		fs.writeFileSync(result.filePath, result.content);
+		fs.writeFileSync(path.join(REPO_SRC_FOLDER, 'vs/editor/common/standalone/standaloneEnums.ts'), result.enums);
+		this._log(`monaco.d.ts is changed - total time took ${Date.now() - startTime} ms`);
+		if (!this._isWatch) {
+			this.stream.emit('error', 'monaco.d.ts is no longer up to date. Please run gulp watch and commit the new file.');
 		}
-	};
-
-	let resultStream: NodeJS.ReadWriteStream;
-
-	if (isWatch) {
-		watch('build/monaco/*').pipe(es.through(function () {
-			run();
-		}));
 	}
-
-	resultStream = es.through(function (data) {
-		const filePath = path.normalize(path.resolve(basePath, data.relative));
-		if (neededFiles[filePath]) {
-			setInputFile(filePath, data.contents.toString());
-		}
-		this.emit('data', data);
-	});
-
-	return resultStream;
 }