Merge from vscode cfc1ab4c5f816765b91fb7ead3c3427a7c8581a3

src/vs/editor/common/services/editorSimpleWorker.ts

@@ -5,7 +5,6 @@
 
 import { mergeSort } from 'vs/base/common/arrays';
 import { stringDiff } from 'vs/base/common/diff/diff';
-import { FIN, Iterator, IteratorResult } from 'vs/base/common/iterator';
 import { IDisposable } from 'vs/base/common/lifecycle';
 import { globals } from 'vs/base/common/platform';
 import { URI } from 'vs/base/common/uri';
@@ -65,7 +64,7 @@ export interface ICommonModel extends ILinkComputerTarget, IMirrorModel {
 	getLineCount(): number;
 	getLineContent(lineNumber: number): string;
 	getLineWords(lineNumber: number, wordDefinition: RegExp): IWordAtPosition[];
-	createWordIterator(wordDefinition: RegExp): Iterator<string>;
+	words(wordDefinition: RegExp): Iterable<string>;
 	getWordUntilPosition(position: IPosition, wordDefinition: RegExp): IWordAtPosition;
 	getValueInRange(range: IRange): string;
 	getWordAtPosition(position: IPosition, wordDefinition: RegExp): Range | null;
@@ -153,36 +152,37 @@ class MirrorModel extends BaseMirrorModel implements ICommonModel {
 		};
 	}
 
-	public createWordIterator(wordDefinition: RegExp): Iterator<string> {
-		let obj: { done: false; value: string; };
+
+	public words(wordDefinition: RegExp): Iterable<string> {
+
+		const lines = this._lines;
+		const wordenize = this._wordenize.bind(this);
+
 		let lineNumber = 0;
-		let lineText: string;
+		let lineText = '';
 		let wordRangesIdx = 0;
 		let wordRanges: IWordRange[] = [];
-		let next = (): IteratorResult<string> => {
 
-			if (wordRangesIdx < wordRanges.length) {
-				const value = lineText.substring(wordRanges[wordRangesIdx].start, wordRanges[wordRangesIdx].end);
-				wordRangesIdx += 1;
-				if (!obj) {
-					obj = { done: false, value: value };
-				} else {
-					obj.value = value;
+		return {
+			*[Symbol.iterator]() {
+				while (true) {
+					if (wordRangesIdx < wordRanges.length) {
+						const value = lineText.substring(wordRanges[wordRangesIdx].start, wordRanges[wordRangesIdx].end);
+						wordRangesIdx += 1;
+						yield value;
+					} else {
+						if (lineNumber < lines.length) {
+							lineText = lines[lineNumber];
+							wordRanges = wordenize(lineText, wordDefinition);
+							wordRangesIdx = 0;
+							lineNumber += 1;
+						} else {
+							break;
+						}
+					}
 				}
-				return obj;
-
-			} else if (lineNumber >= this._lines.length) {
-				return FIN;
-
-			} else {
-				lineText = this._lines[lineNumber];
-				wordRanges = this._wordenize(lineText, wordDefinition);
-				wordRangesIdx = 0;
-				lineNumber += 1;
-				return next();
 			}
 		};
-		return { next };
 	}
 
 	public getLineWords(lineNumber: number, wordDefinition: RegExp): IWordAtPosition[] {
@@ -545,12 +545,7 @@ export class EditorSimpleWorker implements IRequestHandler, IDisposable {
 			seen.add(model.getValueInRange(wordAt));
 		}
 
-		for (
-			let iter = model.createWordIterator(wordDefRegExp), e = iter.next();
-			!e.done && seen.size <= EditorSimpleWorker._suggestionsLimit;
-			e = iter.next()
-		) {
-			const word = e.value;
+		for (let word of model.words(wordDefRegExp)) {
 			if (seen.has(word)) {
 				continue;
 			}
@@ -559,6 +554,9 @@ export class EditorSimpleWorker implements IRequestHandler, IDisposable {
 				continue;
 			}
 			words.push(word);
+			if (seen.size > EditorSimpleWorker._suggestionsLimit) {
+				break;
+			}
 		}
 		return words;
 	}

src/vs/editor/common/services/modeService.ts

@@ -31,6 +31,7 @@ export interface IModeService {
 	_serviceBrand: undefined;
 
 	onDidCreateMode: Event<IMode>;
+	onLanguagesMaybeChanged: Event<void>;
 
 	// --- reading
 	isRegisteredMode(mimetypeOrModeId: string): boolean;

src/vs/editor/common/services/modeServiceImpl.ts

@@ -50,7 +50,7 @@ export class ModeServiceImpl implements IModeService {
 	public readonly onDidCreateMode: Event<IMode> = this._onDidCreateMode.event;
 
 	protected readonly _onLanguagesMaybeChanged = new Emitter<void>();
-	private readonly onLanguagesMaybeChanged: Event<void> = this._onLanguagesMaybeChanged.event;
+	public readonly onLanguagesMaybeChanged: Event<void> = this._onLanguagesMaybeChanged.event;
 
 	constructor(warnOnOverwrite = false) {
 		this._instantiatedModes = {};

src/vs/editor/common/services/modelServiceImpl.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { Emitter, Event } from 'vs/base/common/event';
-import { Disposable, IDisposable, DisposableStore } from 'vs/base/common/lifecycle';
+import { Disposable, IDisposable, DisposableStore, dispose } from 'vs/base/common/lifecycle';
 import * as platform from 'vs/base/common/platform';
 import * as errors from 'vs/base/common/errors';
 import { URI } from 'vs/base/common/uri';
@@ -207,11 +207,22 @@ export class ModelServiceImpl extends Disposable implements IModelService {
 		};
 	}
 
+	private _getEOL(resource: URI | undefined, language: string): string {
+		if (resource) {
+			return this._resourcePropertiesService.getEOL(resource, language);
+		}
+		const eol = this._configurationService.getValue<string>('files.eol', { overrideIdentifier: language });
+		if (eol && eol !== 'auto') {
+			return eol;
+		}
+		return platform.OS === platform.OperatingSystem.Linux || platform.OS === platform.OperatingSystem.Macintosh ? '\n' : '\r\n';
+	}
+
 	public getCreationOptions(language: string, resource: URI | undefined, isForSimpleWidget: boolean): ITextModelCreationOptions {
 		let creationOptions = this._modelCreationOptionsByLanguageAndResource[language + resource];
 		if (!creationOptions) {
 			const editor = this._configurationService.getValue<IRawEditorConfig>('editor', { overrideIdentifier: language, resource });
-			const eol = this._resourcePropertiesService.getEOL(resource, language);
+			const eol = this._getEOL(resource, language);
 			creationOptions = ModelServiceImpl._readModelOptions({ editor, eol }, isForSimpleWidget);
 			this._modelCreationOptionsByLanguageAndResource[language + resource] = creationOptions;
 		}
@@ -516,7 +527,7 @@ class SemanticStyling extends Disposable {
 		this._caches = new WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>();
 		if (this._themeService) {
 			// workaround for tests which use undefined... :/
-			this._register(this._themeService.onThemeChange(() => {
+			this._register(this._themeService.onDidColorThemeChange(() => {
 				this._caches = new WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>();
 			}));
 		}
@@ -651,7 +662,7 @@ class SemanticColoringProviderStyling {
 				modifierSet = modifierSet >> 1;
 			}
 
-			const tokenStyle = this._themeService.getTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
+			const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
 			if (typeof tokenStyle === 'undefined') {
 				metadata = Constants.NO_STYLING;
 			} else {
@@ -724,6 +735,7 @@ class ModelSemanticColoring extends Disposable {
 	private readonly _fetchSemanticTokens: RunOnceScheduler;
 	private _currentResponse: SemanticTokensResponse | null;
 	private _currentRequestCancellationTokenSource: CancellationTokenSource | null;
+	private _providersChangeListeners: IDisposable[];
 
 	constructor(model: ITextModel, themeService: IThemeService, stylingProvider: SemanticStyling) {
 		super();
@@ -734,16 +746,31 @@ class ModelSemanticColoring extends Disposable {
 		this._fetchSemanticTokens = this._register(new RunOnceScheduler(() => this._fetchSemanticTokensNow(), 300));
 		this._currentResponse = null;
 		this._currentRequestCancellationTokenSource = null;
+		this._providersChangeListeners = [];
 
 		this._register(this._model.onDidChangeContent(e => {
 			if (!this._fetchSemanticTokens.isScheduled()) {
 				this._fetchSemanticTokens.schedule();
 			}
 		}));
-		this._register(DocumentSemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule()));
+		const bindChangeListeners = () => {
+			dispose(this._providersChangeListeners);
+			this._providersChangeListeners = [];
+			for (const provider of DocumentSemanticTokensProviderRegistry.all(model)) {
+				if (typeof provider.onDidChange === 'function') {
+					this._providersChangeListeners.push(provider.onDidChange(() => this._fetchSemanticTokens.schedule(0)));
+				}
+			}
+		};
+		bindChangeListeners();
+		this._register(DocumentSemanticTokensProviderRegistry.onDidChange(e => {
+			bindChangeListeners();
+			this._fetchSemanticTokens.schedule();
+		}));
+
 		if (themeService) {
 			// workaround for tests which use undefined... :/
-			this._register(themeService.onThemeChange(_ => {
+			this._register(themeService.onDidColorThemeChange(_ => {
 				// clear out existing tokens
 				this._setSemanticTokens(null, null, null, []);
 				this._fetchSemanticTokens.schedule();

src/vs/editor/common/services/textResourceConfigurationService.ts

@@ -75,5 +75,5 @@ export interface ITextResourcePropertiesService {
 	/**
 	 * Returns the End of Line characters for the given resource
 	 */
-	getEOL(resource: URI | undefined, language?: string): string;
+	getEOL(resource: URI, language?: string): string;
 }