Merge from vscode a5cf1da01d5db3d2557132be8d30f89c38019f6c (#8525)

* Merge from vscode a5cf1da01d5db3d2557132be8d30f89c38019f6c

* remove files we don't want

* fix hygiene

* update distro

* update distro

* fix hygiene

* fix strict nulls

* distro

* distro

* fix tests

* fix tests

* add another edit

* fix viewlet icon

* fix azure dialog

* fix some padding

* fix more padding issues
This commit is contained in:
Anthony Dresser
2019-12-04 19:28:22 -08:00
committed by GitHub
parent a8818ab0df
commit f5ce7fb2a5
1507 changed files with 42813 additions and 27370 deletions

View File

@@ -6,19 +6,24 @@
import { Emitter, Event } from 'vs/base/common/event';
import { Disposable, IDisposable, DisposableStore } from 'vs/base/common/lifecycle';
import * as platform from 'vs/base/common/platform';
import * as errors from 'vs/base/common/errors';
import { URI } from 'vs/base/common/uri';
import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/config/editorOptions';
import { EditOperation } from 'vs/editor/common/core/editOperation';
import { Range } from 'vs/editor/common/core/range';
import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model';
import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel';
import { IModelLanguageChangedEvent } from 'vs/editor/common/model/textModelEvents';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
import { LanguageIdentifier, SemanticTokensProviderRegistry, SemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits } from 'vs/editor/common/modes';
import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry';
import { ILanguageSelection } from 'vs/editor/common/services/modeService';
import { IModelService } from 'vs/editor/common/services/modelService';
import { ITextResourcePropertiesService } from 'vs/editor/common/services/resourceConfiguration';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
import { RunOnceScheduler } from 'vs/base/common/async';
import { CancellationTokenSource } from 'vs/base/common/cancellation';
import { SparseEncodedTokens, MultilineTokens2 } from 'vs/editor/common/model/tokensStore';
import { IThemeService } from 'vs/platform/theme/common/themeService';
function MODEL_ID(resource: URI): string {
return resource.toString();
@@ -114,7 +119,8 @@ export class ModelServiceImpl extends Disposable implements IModelService {
constructor(
@IConfigurationService configurationService: IConfigurationService,
@ITextResourcePropertiesService resourcePropertiesService: ITextResourcePropertiesService
@ITextResourcePropertiesService resourcePropertiesService: ITextResourcePropertiesService,
@IThemeService themeService: IThemeService
) {
super();
this._configurationService = configurationService;
@@ -124,6 +130,8 @@ export class ModelServiceImpl extends Disposable implements IModelService {
this._configurationServiceSubscription = this._configurationService.onDidChangeConfiguration(e => this._updateModelOptions());
this._updateModelOptions();
this._register(new SemanticColoringFeature(this, themeService));
}
private static _readModelOptions(config: IRawConfig, isForSimpleWidget: boolean): ITextModelCreationOptions {
@@ -430,3 +438,467 @@ export class ModelServiceImpl extends Disposable implements IModelService {
export interface ILineSequence {
getLineContent(lineNumber: number): string;
}
class SemanticColoringFeature extends Disposable {
private _watchers: Record<string, ModelSemanticColoring>;
private _semanticStyling: SemanticStyling;
constructor(modelService: IModelService, themeService: IThemeService) {
super();
this._watchers = Object.create(null);
this._semanticStyling = this._register(new SemanticStyling(themeService));
this._register(modelService.onModelAdded((model) => {
this._watchers[model.uri.toString()] = new ModelSemanticColoring(model, themeService, this._semanticStyling);
}));
this._register(modelService.onModelRemoved((model) => {
this._watchers[model.uri.toString()].dispose();
delete this._watchers[model.uri.toString()];
}));
}
}
class SemanticStyling extends Disposable {
private _caches: WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>;
constructor(
private readonly _themeService: IThemeService
) {
super();
this._caches = new WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>();
if (this._themeService) {
// workaround for tests which use undefined... :/
this._register(this._themeService.onThemeChange(() => {
this._caches = new WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>();
}));
}
}
public get(provider: SemanticTokensProvider): SemanticColoringProviderStyling {
if (!this._caches.has(provider)) {
this._caches.set(provider, new SemanticColoringProviderStyling(provider.getLegend(), this._themeService));
}
return this._caches.get(provider)!;
}
}
const enum Constants {
NO_STYLING = 0b01111111111111111111111111111111
}
class HashTableEntry {
public readonly tokenTypeIndex: number;
public readonly tokenModifierSet: number;
public readonly metadata: number;
public next: HashTableEntry | null;
constructor(tokenTypeIndex: number, tokenModifierSet: number, metadata: number) {
this.tokenTypeIndex = tokenTypeIndex;
this.tokenModifierSet = tokenModifierSet;
this.metadata = metadata;
this.next = null;
}
}
class HashTable {
private static _SIZES = [3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, 2097143];
private _elementsCount: number;
private _currentLengthIndex: number;
private _currentLength: number;
private _growCount: number;
private _elements: (HashTableEntry | null)[];
constructor() {
this._elementsCount = 0;
this._currentLengthIndex = 0;
this._currentLength = HashTable._SIZES[this._currentLengthIndex];
this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
this._elements = [];
HashTable._nullOutEntries(this._elements, this._currentLength);
}
private static _nullOutEntries(entries: (HashTableEntry | null)[], length: number): void {
for (let i = 0; i < length; i++) {
entries[i] = null;
}
}
private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number): number {
return ((((tokenTypeIndex << 5) - tokenTypeIndex) + tokenModifierSet) | 0) % this._currentLength; // tokenTypeIndex * 31 + tokenModifierSet, keep as int32
}
public get(tokenTypeIndex: number, tokenModifierSet: number): HashTableEntry | null {
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet);
let p = this._elements[hash];
while (p) {
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet) {
return p;
}
p = p.next;
}
return null;
}
public add(tokenTypeIndex: number, tokenModifierSet: number, metadata: number): void {
this._elementsCount++;
if (this._growCount !== 0 && this._elementsCount >= this._growCount) {
// expand!
const oldElements = this._elements;
this._currentLengthIndex++;
this._currentLength = HashTable._SIZES[this._currentLengthIndex];
this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
this._elements = [];
HashTable._nullOutEntries(this._elements, this._currentLength);
for (const first of oldElements) {
let p = first;
while (p) {
const oldNext = p.next;
p.next = null;
this._add(p);
p = oldNext;
}
}
}
this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, metadata));
}
private _add(element: HashTableEntry): void {
const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet);
element.next = this._elements[hash];
this._elements[hash] = element;
}
}
class SemanticColoringProviderStyling {
private readonly _hashTable: HashTable;
constructor(
private readonly _legend: SemanticTokensLegend,
private readonly _themeService: IThemeService
) {
this._hashTable = new HashTable();
}
public getMetadata(tokenTypeIndex: number, tokenModifierSet: number): number {
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet);
if (entry) {
return entry.metadata;
}
const tokenType = this._legend.tokenTypes[tokenTypeIndex];
const tokenModifiers: string[] = [];
for (let modifierIndex = 0; tokenModifierSet !== 0 && modifierIndex < this._legend.tokenModifiers.length; modifierIndex++) {
if (tokenModifierSet & 1) {
tokenModifiers.push(this._legend.tokenModifiers[modifierIndex]);
}
tokenModifierSet = tokenModifierSet >> 1;
}
let metadata = this._themeService.getTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
if (typeof metadata === 'undefined') {
metadata = Constants.NO_STYLING;
}
this._hashTable.add(tokenTypeIndex, tokenModifierSet, metadata);
return metadata;
}
}
const enum SemanticColoringConstants {
/**
* Let's aim at having 8KB buffers if possible...
* So that would be 8192 / (5 * 4) = 409.6 tokens per area
*/
DesiredTokensPerArea = 400,
/**
* Try to keep the total number of areas under 1024 if possible,
* simply compensate by having more tokens per area...
*/
DesiredMaxAreas = 1024,
}
class SemanticTokensResponse {
constructor(
private readonly _provider: SemanticTokensProvider,
public readonly resultId: string | undefined,
public readonly data: Uint32Array
) { }
public dispose(): void {
this._provider.releaseSemanticTokens(this.resultId);
}
}
class ModelSemanticColoring extends Disposable {
private _isDisposed: boolean;
private readonly _model: ITextModel;
private readonly _semanticStyling: SemanticStyling;
private readonly _fetchSemanticTokens: RunOnceScheduler;
private _currentResponse: SemanticTokensResponse | null;
private _currentRequestCancellationTokenSource: CancellationTokenSource | null;
constructor(model: ITextModel, themeService: IThemeService, stylingProvider: SemanticStyling) {
super();
this._isDisposed = false;
this._model = model;
this._semanticStyling = stylingProvider;
this._fetchSemanticTokens = this._register(new RunOnceScheduler(() => this._fetchSemanticTokensNow(), 500));
this._currentResponse = null;
this._currentRequestCancellationTokenSource = null;
this._register(this._model.onDidChangeContent(e => this._fetchSemanticTokens.schedule()));
this._register(SemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule()));
if (themeService) {
// workaround for tests which use undefined... :/
this._register(themeService.onThemeChange(_ => {
// clear out existing tokens
this._setSemanticTokens(null, null, null, []);
this._fetchSemanticTokens.schedule();
}));
}
this._fetchSemanticTokens.schedule(0);
}
public dispose(): void {
this._isDisposed = true;
if (this._currentResponse) {
this._currentResponse.dispose();
this._currentResponse = null;
}
if (this._currentRequestCancellationTokenSource) {
this._currentRequestCancellationTokenSource.cancel();
this._currentRequestCancellationTokenSource = null;
}
super.dispose();
}
private _fetchSemanticTokensNow(): void {
if (this._currentRequestCancellationTokenSource) {
// there is already a request running, let it finish...
return;
}
const provider = this._getSemanticColoringProvider();
if (!provider) {
return;
}
this._currentRequestCancellationTokenSource = new CancellationTokenSource();
const pendingChanges: IModelContentChangedEvent[] = [];
const contentChangeListener = this._model.onDidChangeContent((e) => {
pendingChanges.push(e);
});
const styling = this._semanticStyling.get(provider);
const lastResultId = this._currentResponse ? this._currentResponse.resultId || null : null;
const request = Promise.resolve(provider.provideSemanticTokens(this._model, lastResultId, null, this._currentRequestCancellationTokenSource.token));
request.then((res) => {
this._currentRequestCancellationTokenSource = null;
contentChangeListener.dispose();
this._setSemanticTokens(provider, res || null, styling, pendingChanges);
}, (err) => {
errors.onUnexpectedError(err);
this._currentRequestCancellationTokenSource = null;
contentChangeListener.dispose();
this._setSemanticTokens(provider, null, styling, pendingChanges);
});
}
private static _isSemanticTokens(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokens {
return v && !!((<SemanticTokens>v).data);
}
private static _isSemanticTokensEdits(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokensEdits {
return v && Array.isArray((<SemanticTokensEdits>v).edits);
}
private static _copy(src: Uint32Array, srcOffset: number, dest: Uint32Array, destOffset: number, length: number): void {
for (let i = 0; i < length; i++) {
dest[destOffset + i] = src[srcOffset + i];
}
}
private _setSemanticTokens(provider: SemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void {
const currentResponse = this._currentResponse;
if (this._currentResponse) {
this._currentResponse.dispose();
this._currentResponse = null;
}
if (this._isDisposed) {
// disposed!
if (provider && tokens) {
provider.releaseSemanticTokens(tokens.resultId);
}
return;
}
if (!provider || !tokens || !styling) {
this._model.setSemanticTokens(null);
return;
}
if (ModelSemanticColoring._isSemanticTokensEdits(tokens)) {
if (!currentResponse) {
// not possible!
this._model.setSemanticTokens(null);
return;
}
if (tokens.edits.length === 0) {
// nothing to do!
tokens = {
resultId: tokens.resultId,
data: currentResponse.data
};
} else {
let deltaLength = 0;
for (const edit of tokens.edits) {
deltaLength += (edit.data ? edit.data.length : 0) - edit.deleteCount;
}
const srcData = currentResponse.data;
const destData = new Uint32Array(srcData.length + deltaLength);
let srcLastStart = srcData.length;
let destLastStart = destData.length;
for (let i = tokens.edits.length - 1; i >= 0; i--) {
const edit = tokens.edits[i];
const copyCount = srcLastStart - (edit.start + edit.deleteCount);
if (copyCount > 0) {
ModelSemanticColoring._copy(srcData, srcLastStart - copyCount, destData, destLastStart - copyCount, copyCount);
destLastStart -= copyCount;
}
if (edit.data) {
ModelSemanticColoring._copy(edit.data, 0, destData, destLastStart - edit.data.length, edit.data.length);
destLastStart -= edit.data.length;
}
srcLastStart = edit.start;
}
if (srcLastStart > 0) {
ModelSemanticColoring._copy(srcData, 0, destData, 0, srcLastStart);
}
tokens = {
resultId: tokens.resultId,
data: destData
};
}
}
if (ModelSemanticColoring._isSemanticTokens(tokens)) {
this._currentResponse = new SemanticTokensResponse(provider, tokens.resultId, tokens.data);
const srcData = tokens.data;
const tokenCount = (tokens.data.length / 5) | 0;
const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea);
const result: MultilineTokens2[] = [];
let tokenIndex = 0;
let lastLineNumber = 1;
let lastStartCharacter = 0;
while (tokenIndex < tokenCount) {
const tokenStartIndex = tokenIndex;
let tokenEndIndex = Math.min(tokenStartIndex + tokensPerArea, tokenCount);
// Keep tokens on the same line in the same area...
if (tokenEndIndex < tokenCount) {
let smallTokenEndIndex = tokenEndIndex;
while (smallTokenEndIndex - 1 > tokenStartIndex && srcData[5 * smallTokenEndIndex] === 0) {
smallTokenEndIndex--;
}
if (smallTokenEndIndex - 1 === tokenStartIndex) {
// there are so many tokens on this line that our area would be empty, we must now go right
let bigTokenEndIndex = tokenEndIndex;
while (bigTokenEndIndex + 1 < tokenCount && srcData[5 * bigTokenEndIndex] === 0) {
bigTokenEndIndex++;
}
tokenEndIndex = bigTokenEndIndex;
} else {
tokenEndIndex = smallTokenEndIndex;
}
}
let destData = new Uint32Array((tokenEndIndex - tokenStartIndex) * 4);
let destOffset = 0;
let areaLine = 0;
while (tokenIndex < tokenEndIndex) {
const srcOffset = 5 * tokenIndex;
const deltaLine = srcData[srcOffset];
const deltaCharacter = srcData[srcOffset + 1];
const lineNumber = lastLineNumber + deltaLine;
const startCharacter = (deltaLine === 0 ? lastStartCharacter + deltaCharacter : deltaCharacter);
const length = srcData[srcOffset + 2];
const tokenTypeIndex = srcData[srcOffset + 3];
const tokenModifierSet = srcData[srcOffset + 4];
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet);
if (metadata !== Constants.NO_STYLING) {
if (areaLine === 0) {
areaLine = lineNumber;
}
destData[destOffset] = lineNumber - areaLine;
destData[destOffset + 1] = startCharacter;
destData[destOffset + 2] = startCharacter + length;
destData[destOffset + 3] = metadata;
destOffset += 4;
}
lastLineNumber = lineNumber;
lastStartCharacter = startCharacter;
tokenIndex++;
}
if (destOffset !== destData.length) {
destData = destData.subarray(0, destOffset);
}
const tokens = new MultilineTokens2(areaLine, new SparseEncodedTokens(destData));
result.push(tokens);
}
// Adjust incoming semantic tokens
if (pendingChanges.length > 0) {
// More changes occurred while the request was running
// We need to:
// 1. Adjust incoming semantic tokens
// 2. Request them again
for (const change of pendingChanges) {
for (const area of result) {
for (const singleChange of change.changes) {
area.applyEdit(singleChange.range, singleChange.text);
}
}
}
this._fetchSemanticTokens.schedule();
}
this._model.setSemanticTokens(result);
return;
}
this._model.setSemanticTokens(null);
}
private _getSemanticColoringProvider(): SemanticTokensProvider | null {
const result = SemanticTokensProviderRegistry.ordered(this._model);
return (result.length > 0 ? result[0] : null);
}
}