Merge from vscode a5cf1da01d5db3d2557132be8d30f89c38019f6c (#8525)

* Merge from vscode a5cf1da01d5db3d2557132be8d30f89c38019f6c

* remove files we don't want

* fix hygiene

* update distro

* update distro

* fix hygiene

* fix strict nulls

* distro

* distro

* fix tests

* fix tests

* add another edit

* fix viewlet icon

* fix azure dialog

* fix some padding

* fix more padding issues

src/vs/editor/common/services/editorSimpleWorker.ts

@@ -17,7 +17,7 @@ import * as editorCommon from 'vs/editor/common/editorCommon';
 import { EndOfLineSequence, IWordAtPosition } from 'vs/editor/common/model';
 import { IModelChangedEvent, MirrorTextModel as BaseMirrorModel } from 'vs/editor/common/model/mirrorTextModel';
 import { ensureValidWordDefinition, getWordAtText } from 'vs/editor/common/model/wordHelper';
-import { CompletionItem, CompletionItemKind, CompletionList, IInplaceReplaceSupportResult, ILink, TextEdit } from 'vs/editor/common/modes';
+import { IInplaceReplaceSupportResult, ILink, TextEdit } from 'vs/editor/common/modes';
 import { ILinkComputerTarget, computeLinks } from 'vs/editor/common/modes/linkComputer';
 import { BasicInplaceReplace } from 'vs/editor/common/modes/supports/inplaceReplaceSupport';
 import { IDiffComputationResult } from 'vs/editor/common/services/editorWorkerService';
@@ -529,44 +529,38 @@ export class EditorSimpleWorker implements IRequestHandler, IDisposable {
 
 	private static readonly _suggestionsLimit = 10000;
 
-	public async textualSuggest(modelUrl: string, position: IPosition, wordDef: string, wordDefFlags: string): Promise<CompletionList | null> {
+	public async textualSuggest(modelUrl: string, position: IPosition, wordDef: string, wordDefFlags: string): Promise<string[] | null> {
 		const model = this._getModel(modelUrl);
 		if (!model) {
 			return null;
 		}
 
-		const seen: Record<string, boolean> = Object.create(null);
-		const suggestions: CompletionItem[] = [];
+
+		const words: string[] = [];
+		const seen = new Set<string>();
 		const wordDefRegExp = new RegExp(wordDef, wordDefFlags);
-		const wordUntil = model.getWordUntilPosition(position, wordDefRegExp);
 
 		const wordAt = model.getWordAtPosition(position, wordDefRegExp);
 		if (wordAt) {
-			seen[model.getValueInRange(wordAt)] = true;
+			seen.add(model.getValueInRange(wordAt));
 		}
 
 		for (
 			let iter = model.createWordIterator(wordDefRegExp), e = iter.next();
-			!e.done && suggestions.length <= EditorSimpleWorker._suggestionsLimit;
+			!e.done && seen.size <= EditorSimpleWorker._suggestionsLimit;
 			e = iter.next()
 		) {
 			const word = e.value;
-			if (seen[word]) {
+			if (seen.has(word)) {
 				continue;
 			}
-			seen[word] = true;
+			seen.add(word);
 			if (!isNaN(Number(word))) {
 				continue;
 			}
-
-			suggestions.push({
-				kind: CompletionItemKind.Text,
-				label: word,
-				insertText: word,
-				range: { startLineNumber: position.lineNumber, startColumn: wordUntil.startColumn, endLineNumber: position.lineNumber, endColumn: wordUntil.endColumn }
-			});
+			words.push(word);
 		}
-		return { suggestions };
+		return words;
 	}
 
 

src/vs/editor/common/services/editorWorkerServiceImpl.ts

@@ -9,7 +9,7 @@ import { URI } from 'vs/base/common/uri';
 import { SimpleWorkerClient, logOnceWebWorkerWarning, IWorkerClient } from 'vs/base/common/worker/simpleWorker';
 import { DefaultWorkerFactory } from 'vs/base/worker/defaultWorkerFactory';
 import { IPosition, Position } from 'vs/editor/common/core/position';
-import { IRange } from 'vs/editor/common/core/range';
+import { IRange, Range } from 'vs/editor/common/core/range';
 import * as editorCommon from 'vs/editor/common/editorCommon';
 import { ITextModel } from 'vs/editor/common/model';
 import * as modes from 'vs/editor/common/modes';
@@ -144,7 +144,7 @@ class WordBasedCompletionItemProvider implements modes.CompletionItemProvider {
 		this._modelService = modelService;
 	}
 
-	provideCompletionItems(model: ITextModel, position: Position): Promise<modes.CompletionList | null> | undefined {
+	async provideCompletionItems(model: ITextModel, position: Position): Promise<modes.CompletionList | undefined> {
 		const { wordBasedSuggestions } = this._configurationService.getValue<{ wordBasedSuggestions?: boolean }>(model.uri, position, 'editor');
 		if (!wordBasedSuggestions) {
 			return undefined;
@@ -152,7 +152,27 @@ class WordBasedCompletionItemProvider implements modes.CompletionItemProvider {
 		if (!canSyncModel(this._modelService, model.uri)) {
 			return undefined; // File too large
 		}
-		return this._workerManager.withWorker().then(client => client.textualSuggest(model.uri, position));
+
+		const word = model.getWordAtPosition(position);
+		const replace = !word ? Range.fromPositions(position) : new Range(position.lineNumber, word.startColumn, position.lineNumber, word.endColumn);
+		const insert = replace.setEndPosition(position.lineNumber, position.column);
+
+		const client = await this._workerManager.withWorker();
+		const words = await client.textualSuggest(model.uri, position);
+		if (!words) {
+			return undefined;
+		}
+
+		return {
+			suggestions: words.map((word): modes.CompletionItem => {
+				return {
+					kind: modes.CompletionItemKind.Text,
+					label: word,
+					insertText: word,
+					range: { insert, replace }
+				};
+			})
+		};
 	}
 }
 
@@ -433,7 +453,7 @@ export class EditorWorkerClient extends Disposable {
 		});
 	}
 
-	public textualSuggest(resource: URI, position: IPosition): Promise<modes.CompletionList | null> {
+	public textualSuggest(resource: URI, position: IPosition): Promise<string[] | null> {
 		return this._withSyncedResources([resource]).then(proxy => {
 			let model = this._modelService.getModel(resource);
 			if (!model) {

src/vs/editor/common/services/markerDecorationsServiceImpl.ts

@@ -6,7 +6,7 @@
 import { IMarkerService, IMarker, MarkerSeverity, MarkerTag } from 'vs/platform/markers/common/markers';
 import { Disposable, toDisposable } from 'vs/base/common/lifecycle';
 import { URI } from 'vs/base/common/uri';
-import { IModelDeltaDecoration, ITextModel, IModelDecorationOptions, TrackedRangeStickiness, OverviewRulerLane, IModelDecoration } from 'vs/editor/common/model';
+import { IModelDeltaDecoration, ITextModel, IModelDecorationOptions, TrackedRangeStickiness, OverviewRulerLane, IModelDecoration, MinimapPosition, IModelDecorationMinimapOptions } from 'vs/editor/common/model';
 import { ClassName } from 'vs/editor/common/model/intervalTree';
 import { themeColorFromId, ThemeColor } from 'vs/platform/theme/common/themeService';
 import { overviewRulerWarning, overviewRulerInfo, overviewRulerError } from 'vs/editor/common/view/editorColorRegistry';
@@ -17,6 +17,7 @@ import { IMarkerDecorationsService } from 'vs/editor/common/services/markersDeco
 import { Schemas } from 'vs/base/common/network';
 import { Emitter, Event } from 'vs/base/common/event';
 import { withUndefinedAsNull } from 'vs/base/common/types';
+import { minimapWarning, minimapError } from 'vs/platform/theme/common/colorRegistry';
 
 function MODEL_ID(resource: URI): string {
 	return resource.toString();
@@ -189,6 +190,7 @@ export class MarkerDecorationsService extends Disposable implements IMarkerDecor
 		let color: ThemeColor | undefined = undefined;
 		let zIndex: number;
 		let inlineClassName: string | undefined = undefined;
+		let minimap: IModelDecorationMinimapOptions | undefined;
 
 		switch (marker.severity) {
 			case MarkerSeverity.Hint:
@@ -203,6 +205,10 @@ export class MarkerDecorationsService extends Disposable implements IMarkerDecor
 				className = ClassName.EditorWarningDecoration;
 				color = themeColorFromId(overviewRulerWarning);
 				zIndex = 20;
+				minimap = {
+					color: themeColorFromId(minimapWarning),
+					position: MinimapPosition.Inline
+				};
 				break;
 			case MarkerSeverity.Info:
 				className = ClassName.EditorInfoDecoration;
@@ -214,6 +220,10 @@ export class MarkerDecorationsService extends Disposable implements IMarkerDecor
 				className = ClassName.EditorErrorDecoration;
 				color = themeColorFromId(overviewRulerError);
 				zIndex = 30;
+				minimap = {
+					color: themeColorFromId(minimapError),
+					position: MinimapPosition.Inline
+				};
 				break;
 		}
 
@@ -234,6 +244,7 @@ export class MarkerDecorationsService extends Disposable implements IMarkerDecor
 				color,
 				position: OverviewRulerLane.Right
 			},
+			minimap,
 			zIndex,
 			inlineClassName,
 		};

src/vs/editor/common/services/modelServiceImpl.ts

@@ -6,19 +6,24 @@
 import { Emitter, Event } from 'vs/base/common/event';
 import { Disposable, IDisposable, DisposableStore } from 'vs/base/common/lifecycle';
 import * as platform from 'vs/base/common/platform';
+import * as errors from 'vs/base/common/errors';
 import { URI } from 'vs/base/common/uri';
 import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/config/editorOptions';
 import { EditOperation } from 'vs/editor/common/core/editOperation';
 import { Range } from 'vs/editor/common/core/range';
 import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model';
 import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel';
-import { IModelLanguageChangedEvent } from 'vs/editor/common/model/textModelEvents';
-import { LanguageIdentifier } from 'vs/editor/common/modes';
+import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
+import { LanguageIdentifier, SemanticTokensProviderRegistry, SemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits } from 'vs/editor/common/modes';
 import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry';
 import { ILanguageSelection } from 'vs/editor/common/services/modeService';
 import { IModelService } from 'vs/editor/common/services/modelService';
 import { ITextResourcePropertiesService } from 'vs/editor/common/services/resourceConfiguration';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
+import { RunOnceScheduler } from 'vs/base/common/async';
+import { CancellationTokenSource } from 'vs/base/common/cancellation';
+import { SparseEncodedTokens, MultilineTokens2 } from 'vs/editor/common/model/tokensStore';
+import { IThemeService } from 'vs/platform/theme/common/themeService';
 
 function MODEL_ID(resource: URI): string {
 	return resource.toString();
@@ -114,7 +119,8 @@ export class ModelServiceImpl extends Disposable implements IModelService {
 
 	constructor(
 		@IConfigurationService configurationService: IConfigurationService,
-		@ITextResourcePropertiesService resourcePropertiesService: ITextResourcePropertiesService
+		@ITextResourcePropertiesService resourcePropertiesService: ITextResourcePropertiesService,
+		@IThemeService themeService: IThemeService
 	) {
 		super();
 		this._configurationService = configurationService;
@@ -124,6 +130,8 @@ export class ModelServiceImpl extends Disposable implements IModelService {
 
 		this._configurationServiceSubscription = this._configurationService.onDidChangeConfiguration(e => this._updateModelOptions());
 		this._updateModelOptions();
+
+		this._register(new SemanticColoringFeature(this, themeService));
 	}
 
 	private static _readModelOptions(config: IRawConfig, isForSimpleWidget: boolean): ITextModelCreationOptions {
@@ -430,3 +438,467 @@ export class ModelServiceImpl extends Disposable implements IModelService {
 export interface ILineSequence {
 	getLineContent(lineNumber: number): string;
 }
+
+class SemanticColoringFeature extends Disposable {
+	private _watchers: Record<string, ModelSemanticColoring>;
+	private _semanticStyling: SemanticStyling;
+
+	constructor(modelService: IModelService, themeService: IThemeService) {
+		super();
+		this._watchers = Object.create(null);
+		this._semanticStyling = this._register(new SemanticStyling(themeService));
+		this._register(modelService.onModelAdded((model) => {
+			this._watchers[model.uri.toString()] = new ModelSemanticColoring(model, themeService, this._semanticStyling);
+		}));
+		this._register(modelService.onModelRemoved((model) => {
+			this._watchers[model.uri.toString()].dispose();
+			delete this._watchers[model.uri.toString()];
+		}));
+	}
+}
+
+class SemanticStyling extends Disposable {
+
+	private _caches: WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>;
+
+	constructor(
+		private readonly _themeService: IThemeService
+	) {
+		super();
+		this._caches = new WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>();
+		if (this._themeService) {
+			// workaround for tests which use undefined... :/
+			this._register(this._themeService.onThemeChange(() => {
+				this._caches = new WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>();
+			}));
+		}
+	}
+
+	public get(provider: SemanticTokensProvider): SemanticColoringProviderStyling {
+		if (!this._caches.has(provider)) {
+			this._caches.set(provider, new SemanticColoringProviderStyling(provider.getLegend(), this._themeService));
+		}
+		return this._caches.get(provider)!;
+	}
+}
+
+const enum Constants {
+	NO_STYLING = 0b01111111111111111111111111111111
+}
+
+class HashTableEntry {
+	public readonly tokenTypeIndex: number;
+	public readonly tokenModifierSet: number;
+	public readonly metadata: number;
+	public next: HashTableEntry | null;
+
+	constructor(tokenTypeIndex: number, tokenModifierSet: number, metadata: number) {
+		this.tokenTypeIndex = tokenTypeIndex;
+		this.tokenModifierSet = tokenModifierSet;
+		this.metadata = metadata;
+		this.next = null;
+	}
+}
+
+class HashTable {
+
+	private static _SIZES = [3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, 2097143];
+
+	private _elementsCount: number;
+	private _currentLengthIndex: number;
+	private _currentLength: number;
+	private _growCount: number;
+	private _elements: (HashTableEntry | null)[];
+
+	constructor() {
+		this._elementsCount = 0;
+		this._currentLengthIndex = 0;
+		this._currentLength = HashTable._SIZES[this._currentLengthIndex];
+		this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
+		this._elements = [];
+		HashTable._nullOutEntries(this._elements, this._currentLength);
+	}
+
+	private static _nullOutEntries(entries: (HashTableEntry | null)[], length: number): void {
+		for (let i = 0; i < length; i++) {
+			entries[i] = null;
+		}
+	}
+
+	private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number): number {
+		return ((((tokenTypeIndex << 5) - tokenTypeIndex) + tokenModifierSet) | 0) % this._currentLength;  // tokenTypeIndex * 31 + tokenModifierSet, keep as int32
+	}
+
+	public get(tokenTypeIndex: number, tokenModifierSet: number): HashTableEntry | null {
+		const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet);
+
+		let p = this._elements[hash];
+		while (p) {
+			if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet) {
+				return p;
+			}
+			p = p.next;
+		}
+
+		return null;
+	}
+
+	public add(tokenTypeIndex: number, tokenModifierSet: number, metadata: number): void {
+		this._elementsCount++;
+		if (this._growCount !== 0 && this._elementsCount >= this._growCount) {
+			// expand!
+			const oldElements = this._elements;
+
+			this._currentLengthIndex++;
+			this._currentLength = HashTable._SIZES[this._currentLengthIndex];
+			this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
+			this._elements = [];
+			HashTable._nullOutEntries(this._elements, this._currentLength);
+
+			for (const first of oldElements) {
+				let p = first;
+				while (p) {
+					const oldNext = p.next;
+					p.next = null;
+					this._add(p);
+					p = oldNext;
+				}
+			}
+		}
+		this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, metadata));
+	}
+
+	private _add(element: HashTableEntry): void {
+		const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet);
+		element.next = this._elements[hash];
+		this._elements[hash] = element;
+	}
+}
+
+class SemanticColoringProviderStyling {
+
+	private readonly _hashTable: HashTable;
+
+	constructor(
+		private readonly _legend: SemanticTokensLegend,
+		private readonly _themeService: IThemeService
+	) {
+		this._hashTable = new HashTable();
+	}
+
+	public getMetadata(tokenTypeIndex: number, tokenModifierSet: number): number {
+		const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet);
+		if (entry) {
+			return entry.metadata;
+		}
+
+		const tokenType = this._legend.tokenTypes[tokenTypeIndex];
+		const tokenModifiers: string[] = [];
+		for (let modifierIndex = 0; tokenModifierSet !== 0 && modifierIndex < this._legend.tokenModifiers.length; modifierIndex++) {
+			if (tokenModifierSet & 1) {
+				tokenModifiers.push(this._legend.tokenModifiers[modifierIndex]);
+			}
+			tokenModifierSet = tokenModifierSet >> 1;
+		}
+
+		let metadata = this._themeService.getTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
+		if (typeof metadata === 'undefined') {
+			metadata = Constants.NO_STYLING;
+		}
+
+		this._hashTable.add(tokenTypeIndex, tokenModifierSet, metadata);
+		return metadata;
+	}
+}
+
+const enum SemanticColoringConstants {
+	/**
+	 * Let's aim at having 8KB buffers if possible...
+	 * So that would be 8192 / (5 * 4) = 409.6 tokens per area
+	 */
+	DesiredTokensPerArea = 400,
+
+	/**
+	 * Try to keep the total number of areas under 1024 if possible,
+	 * simply compensate by having more tokens per area...
+	 */
+	DesiredMaxAreas = 1024,
+}
+
+class SemanticTokensResponse {
+	constructor(
+		private readonly _provider: SemanticTokensProvider,
+		public readonly resultId: string | undefined,
+		public readonly data: Uint32Array
+	) { }
+
+	public dispose(): void {
+		this._provider.releaseSemanticTokens(this.resultId);
+	}
+}
+
+class ModelSemanticColoring extends Disposable {
+
+	private _isDisposed: boolean;
+	private readonly _model: ITextModel;
+	private readonly _semanticStyling: SemanticStyling;
+	private readonly _fetchSemanticTokens: RunOnceScheduler;
+	private _currentResponse: SemanticTokensResponse | null;
+	private _currentRequestCancellationTokenSource: CancellationTokenSource | null;
+
+	constructor(model: ITextModel, themeService: IThemeService, stylingProvider: SemanticStyling) {
+		super();
+
+		this._isDisposed = false;
+		this._model = model;
+		this._semanticStyling = stylingProvider;
+		this._fetchSemanticTokens = this._register(new RunOnceScheduler(() => this._fetchSemanticTokensNow(), 500));
+		this._currentResponse = null;
+		this._currentRequestCancellationTokenSource = null;
+
+		this._register(this._model.onDidChangeContent(e => this._fetchSemanticTokens.schedule()));
+		this._register(SemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule()));
+		if (themeService) {
+			// workaround for tests which use undefined... :/
+			this._register(themeService.onThemeChange(_ => {
+				// clear out existing tokens
+				this._setSemanticTokens(null, null, null, []);
+				this._fetchSemanticTokens.schedule();
+			}));
+		}
+		this._fetchSemanticTokens.schedule(0);
+	}
+
+	public dispose(): void {
+		this._isDisposed = true;
+		if (this._currentResponse) {
+			this._currentResponse.dispose();
+			this._currentResponse = null;
+		}
+		if (this._currentRequestCancellationTokenSource) {
+			this._currentRequestCancellationTokenSource.cancel();
+			this._currentRequestCancellationTokenSource = null;
+		}
+		super.dispose();
+	}
+
+	private _fetchSemanticTokensNow(): void {
+		if (this._currentRequestCancellationTokenSource) {
+			// there is already a request running, let it finish...
+			return;
+		}
+		const provider = this._getSemanticColoringProvider();
+		if (!provider) {
+			return;
+		}
+		this._currentRequestCancellationTokenSource = new CancellationTokenSource();
+
+		const pendingChanges: IModelContentChangedEvent[] = [];
+		const contentChangeListener = this._model.onDidChangeContent((e) => {
+			pendingChanges.push(e);
+		});
+
+		const styling = this._semanticStyling.get(provider);
+
+		const lastResultId = this._currentResponse ? this._currentResponse.resultId || null : null;
+		const request = Promise.resolve(provider.provideSemanticTokens(this._model, lastResultId, null, this._currentRequestCancellationTokenSource.token));
+
+		request.then((res) => {
+			this._currentRequestCancellationTokenSource = null;
+			contentChangeListener.dispose();
+			this._setSemanticTokens(provider, res || null, styling, pendingChanges);
+		}, (err) => {
+			errors.onUnexpectedError(err);
+			this._currentRequestCancellationTokenSource = null;
+			contentChangeListener.dispose();
+			this._setSemanticTokens(provider, null, styling, pendingChanges);
+		});
+	}
+
+	private static _isSemanticTokens(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokens {
+		return v && !!((<SemanticTokens>v).data);
+	}
+
+	private static _isSemanticTokensEdits(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokensEdits {
+		return v && Array.isArray((<SemanticTokensEdits>v).edits);
+	}
+
+	private static _copy(src: Uint32Array, srcOffset: number, dest: Uint32Array, destOffset: number, length: number): void {
+		for (let i = 0; i < length; i++) {
+			dest[destOffset + i] = src[srcOffset + i];
+		}
+	}
+
+	private _setSemanticTokens(provider: SemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void {
+		const currentResponse = this._currentResponse;
+		if (this._currentResponse) {
+			this._currentResponse.dispose();
+			this._currentResponse = null;
+		}
+		if (this._isDisposed) {
+			// disposed!
+			if (provider && tokens) {
+				provider.releaseSemanticTokens(tokens.resultId);
+			}
+			return;
+		}
+		if (!provider || !tokens || !styling) {
+			this._model.setSemanticTokens(null);
+			return;
+		}
+
+		if (ModelSemanticColoring._isSemanticTokensEdits(tokens)) {
+			if (!currentResponse) {
+				// not possible!
+				this._model.setSemanticTokens(null);
+				return;
+			}
+			if (tokens.edits.length === 0) {
+				// nothing to do!
+				tokens = {
+					resultId: tokens.resultId,
+					data: currentResponse.data
+				};
+			} else {
+				let deltaLength = 0;
+				for (const edit of tokens.edits) {
+					deltaLength += (edit.data ? edit.data.length : 0) - edit.deleteCount;
+				}
+
+				const srcData = currentResponse.data;
+				const destData = new Uint32Array(srcData.length + deltaLength);
+
+				let srcLastStart = srcData.length;
+				let destLastStart = destData.length;
+				for (let i = tokens.edits.length - 1; i >= 0; i--) {
+					const edit = tokens.edits[i];
+
+					const copyCount = srcLastStart - (edit.start + edit.deleteCount);
+					if (copyCount > 0) {
+						ModelSemanticColoring._copy(srcData, srcLastStart - copyCount, destData, destLastStart - copyCount, copyCount);
+						destLastStart -= copyCount;
+					}
+
+					if (edit.data) {
+						ModelSemanticColoring._copy(edit.data, 0, destData, destLastStart - edit.data.length, edit.data.length);
+						destLastStart -= edit.data.length;
+					}
+
+					srcLastStart = edit.start;
+				}
+
+				if (srcLastStart > 0) {
+					ModelSemanticColoring._copy(srcData, 0, destData, 0, srcLastStart);
+				}
+
+				tokens = {
+					resultId: tokens.resultId,
+					data: destData
+				};
+			}
+		}
+
+		if (ModelSemanticColoring._isSemanticTokens(tokens)) {
+
+			this._currentResponse = new SemanticTokensResponse(provider, tokens.resultId, tokens.data);
+
+			const srcData = tokens.data;
+			const tokenCount = (tokens.data.length / 5) | 0;
+			const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea);
+
+			const result: MultilineTokens2[] = [];
+
+			let tokenIndex = 0;
+			let lastLineNumber = 1;
+			let lastStartCharacter = 0;
+			while (tokenIndex < tokenCount) {
+				const tokenStartIndex = tokenIndex;
+				let tokenEndIndex = Math.min(tokenStartIndex + tokensPerArea, tokenCount);
+
+				// Keep tokens on the same line in the same area...
+				if (tokenEndIndex < tokenCount) {
+
+					let smallTokenEndIndex = tokenEndIndex;
+					while (smallTokenEndIndex - 1 > tokenStartIndex && srcData[5 * smallTokenEndIndex] === 0) {
+						smallTokenEndIndex--;
+					}
+
+					if (smallTokenEndIndex - 1 === tokenStartIndex) {
+						// there are so many tokens on this line that our area would be empty, we must now go right
+						let bigTokenEndIndex = tokenEndIndex;
+						while (bigTokenEndIndex + 1 < tokenCount && srcData[5 * bigTokenEndIndex] === 0) {
+							bigTokenEndIndex++;
+						}
+						tokenEndIndex = bigTokenEndIndex;
+					} else {
+						tokenEndIndex = smallTokenEndIndex;
+					}
+				}
+
+				let destData = new Uint32Array((tokenEndIndex - tokenStartIndex) * 4);
+				let destOffset = 0;
+				let areaLine = 0;
+				while (tokenIndex < tokenEndIndex) {
+					const srcOffset = 5 * tokenIndex;
+					const deltaLine = srcData[srcOffset];
+					const deltaCharacter = srcData[srcOffset + 1];
+					const lineNumber = lastLineNumber + deltaLine;
+					const startCharacter = (deltaLine === 0 ? lastStartCharacter + deltaCharacter : deltaCharacter);
+					const length = srcData[srcOffset + 2];
+					const tokenTypeIndex = srcData[srcOffset + 3];
+					const tokenModifierSet = srcData[srcOffset + 4];
+					const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet);
+
+					if (metadata !== Constants.NO_STYLING) {
+						if (areaLine === 0) {
+							areaLine = lineNumber;
+						}
+						destData[destOffset] = lineNumber - areaLine;
+						destData[destOffset + 1] = startCharacter;
+						destData[destOffset + 2] = startCharacter + length;
+						destData[destOffset + 3] = metadata;
+						destOffset += 4;
+					}
+
+					lastLineNumber = lineNumber;
+					lastStartCharacter = startCharacter;
+					tokenIndex++;
+				}
+
+				if (destOffset !== destData.length) {
+					destData = destData.subarray(0, destOffset);
+				}
+
+				const tokens = new MultilineTokens2(areaLine, new SparseEncodedTokens(destData));
+				result.push(tokens);
+			}
+
+			// Adjust incoming semantic tokens
+			if (pendingChanges.length > 0) {
+				// More changes occurred while the request was running
+				// We need to:
+				// 1. Adjust incoming semantic tokens
+				// 2. Request them again
+				for (const change of pendingChanges) {
+					for (const area of result) {
+						for (const singleChange of change.changes) {
+							area.applyEdit(singleChange.range, singleChange.text);
+						}
+					}
+				}
+
+				this._fetchSemanticTokens.schedule();
+			}
+
+			this._model.setSemanticTokens(result);
+			return;
+		}
+
+		this._model.setSemanticTokens(null);
+	}
+
+	private _getSemanticColoringProvider(): SemanticTokensProvider | null {
+		const result = SemanticTokensProviderRegistry.ordered(this._model);
+		return (result.length > 0 ? result[0] : null);
+	}
+}

src/vs/editor/common/services/resolverService.ts

@@ -52,6 +52,9 @@ export interface ITextEditorModel extends IEditorModel {
 	createSnapshot(this: IResolvedTextEditorModel): ITextSnapshot;
 	createSnapshot(this: ITextEditorModel): ITextSnapshot | null;
 
+	/**
+	 * Signals if this model is readonly or not. 
+	 */
 	isReadonly(): boolean;
 }