Merge from vscode 8e0f348413f4f616c23a88ae30030efa85811973 (#6381)

* Merge from vscode 8e0f348413f4f616c23a88ae30030efa85811973

* disable strict null check
This commit is contained in:
Anthony Dresser
2019-07-15 22:35:46 -07:00
committed by GitHub
parent f720ec642f
commit 0b7e7ddbf9
2406 changed files with 59140 additions and 35464 deletions

View File

@@ -8,7 +8,6 @@ import { onUnexpectedError } from 'vs/base/common/errors';
import { Emitter, Event } from 'vs/base/common/event';
import { IMarkdownString } from 'vs/base/common/htmlContent';
import { Disposable, IDisposable } from 'vs/base/common/lifecycle';
import { StopWatch } from 'vs/base/common/stopwatch';
import * as strings from 'vs/base/common/strings';
import { URI } from 'vs/base/common/uri';
import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/config/editorOptions';
@@ -23,9 +22,9 @@ import { IntervalNode, IntervalTree, getNodeIsInOverviewRuler, recomputeMaxEnd }
import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder';
import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents';
import { SearchData, SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch';
import { ModelLinesTokens, ModelTokensChangedEventBuilder } from 'vs/editor/common/model/textModelTokens';
import { TextModelTokenization, countEOL } from 'vs/editor/common/model/textModelTokens';
import { getWordAtText } from 'vs/editor/common/model/wordHelper';
import { IState, LanguageId, LanguageIdentifier, TokenizationRegistry, FormattingOptions } from 'vs/editor/common/modes';
import { LanguageId, LanguageIdentifier, FormattingOptions } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { NULL_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/nullMode';
import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
@@ -33,8 +32,8 @@ import { BracketsUtils, RichEditBracket, RichEditBrackets } from 'vs/editor/comm
import { ITheme, ThemeColor } from 'vs/platform/theme/common/themeService';
import { withUndefinedAsNull } from 'vs/base/common/types';
import { VSBufferReadableStream, VSBuffer } from 'vs/base/common/buffer';
const CHEAP_TOKENIZATION_LENGTH_LIMIT = 2048;
import { TokensStore, MultilineTokens } from 'vs/editor/common/model/tokensStore';
import { Color } from 'vs/base/common/color';
function createTextBufferBuilder() {
return new PieceTreeTextBufferBuilder();
@@ -235,6 +234,9 @@ export class TextModel extends Disposable implements model.ITextModel {
private readonly _onDidChangeOptions: Emitter<IModelOptionsChangedEvent> = this._register(new Emitter<IModelOptionsChangedEvent>());
public readonly onDidChangeOptions: Event<IModelOptionsChangedEvent> = this._onDidChangeOptions.event;
private readonly _onDidChangeAttached: Emitter<void> = this._register(new Emitter<void>());
public readonly onDidChangeAttached: Event<void> = this._onDidChangeAttached.event;
private readonly _eventEmitter: DidChangeContentEmitter = this._register(new DidChangeContentEmitter());
public onDidChangeRawContentFast(listener: (e: ModelRawContentChangedEvent) => void): IDisposable {
return this._eventEmitter.fastEvent((e: InternalModelContentChangeEvent) => listener(e.rawContentChangedEvent));
@@ -242,6 +244,9 @@ export class TextModel extends Disposable implements model.ITextModel {
public onDidChangeRawContent(listener: (e: ModelRawContentChangedEvent) => void): IDisposable {
return this._eventEmitter.slowEvent((e: InternalModelContentChangeEvent) => listener(e.rawContentChangedEvent));
}
public onDidChangeContentFast(listener: (e: IModelContentChangedEvent) => void): IDisposable {
return this._eventEmitter.fastEvent((e: InternalModelContentChangeEvent) => listener(e.contentChangedEvent));
}
public onDidChangeContent(listener: (e: IModelContentChangedEvent) => void): IDisposable {
return this._eventEmitter.slowEvent((e: InternalModelContentChangeEvent) => listener(e.contentChangedEvent));
}
@@ -284,10 +289,9 @@ export class TextModel extends Disposable implements model.ITextModel {
//#region Tokenization
private _languageIdentifier: LanguageIdentifier;
private readonly _tokenizationListener: IDisposable;
private readonly _languageRegistryListener: IDisposable;
private _revalidateTokensTimeout: any;
/*private*/_tokens: ModelLinesTokens;
private readonly _tokens: TokensStore;
private readonly _tokenization: TextModelTokenization;
//#endregion
constructor(source: string | model.ITextBufferFactory, creationOptions: model.ITextModelCreationOptions, languageIdentifier: LanguageIdentifier | null, associatedResource: URI | null = null) {
@@ -330,31 +334,12 @@ export class TextModel extends Disposable implements model.ITextModel {
this._isDisposing = false;
this._languageIdentifier = languageIdentifier || NULL_LANGUAGE_IDENTIFIER;
this._tokenizationListener = TokenizationRegistry.onDidChange((e) => {
if (e.changedLanguages.indexOf(this._languageIdentifier.language) === -1) {
return;
}
this._resetTokenizationState();
this.emitModelTokensChangedEvent({
tokenizationSupportChanged: true,
ranges: [{
fromLineNumber: 1,
toLineNumber: this.getLineCount()
}]
});
if (this._shouldAutoTokenize()) {
this._warmUpTokens();
}
});
this._revalidateTokensTimeout = -1;
this._languageRegistryListener = LanguageConfigurationRegistry.onDidChange((e) => {
if (e.languageIdentifier.id === this._languageIdentifier.id) {
this._onDidChangeLanguageConfiguration.fire({});
}
});
this._resetTokenizationState();
this._instanceId = singleLetter(MODEL_ID);
this._lastDecorationId = 0;
@@ -365,16 +350,17 @@ export class TextModel extends Disposable implements model.ITextModel {
this._isUndoing = false;
this._isRedoing = false;
this._trimAutoWhitespaceLines = null;
this._tokens = new TokensStore();
this._tokenization = new TextModelTokenization(this);
}
public dispose(): void {
this._isDisposing = true;
this._onWillDispose.fire();
this._tokenizationListener.dispose();
this._languageRegistryListener.dispose();
this._clearTimers();
this._tokenization.dispose();
this._isDisposed = true;
// Null out members, such that any use of a disposed model will throw exceptions sooner rather than later
super.dispose();
this._isDisposing = false;
}
@@ -439,8 +425,8 @@ export class TextModel extends Disposable implements model.ITextModel {
this._buffer = textBuffer;
this._increaseVersionId();
// Cancel tokenization, clear all tokens and begin tokenizing
this._resetTokenizationState();
// Flush all tokens
this._tokens.flush();
// Destroy all my decorations
this._decorations = Object.create(null);
@@ -524,36 +510,18 @@ export class TextModel extends Disposable implements model.ITextModel {
}
}
private _resetTokenizationState(): void {
this._clearTimers();
let tokenizationSupport = (
this._isTooLargeForTokenization
? null
: TokenizationRegistry.get(this._languageIdentifier.language)
);
this._tokens = new ModelLinesTokens(this._languageIdentifier, tokenizationSupport);
this._beginBackgroundTokenization();
}
private _clearTimers(): void {
if (this._revalidateTokensTimeout !== -1) {
clearTimeout(this._revalidateTokensTimeout);
this._revalidateTokensTimeout = -1;
}
}
public onBeforeAttached(): void {
this._attachedEditorCount++;
// Warm up tokens for the editor
this._warmUpTokens();
if (this._attachedEditorCount === 1) {
this._onDidChangeAttached.fire(undefined);
}
}
public onBeforeDetached(): void {
this._attachedEditorCount--;
}
private _shouldAutoTokenize(): boolean {
return this.isAttachedToEditor();
if (this._attachedEditorCount === 0) {
this._onDidChangeAttached.fire(undefined);
}
}
public isAttachedToEditor(): boolean {
@@ -1292,36 +1260,6 @@ export class TextModel extends Disposable implements model.ITextModel {
}
}
private static _eolCount(text: string): [number, number] {
let eolCount = 0;
let firstLineLength = 0;
for (let i = 0, len = text.length; i < len; i++) {
const chr = text.charCodeAt(i);
if (chr === CharCode.CarriageReturn) {
if (eolCount === 0) {
firstLineLength = i;
}
eolCount++;
if (i + 1 < len && text.charCodeAt(i + 1) === CharCode.LineFeed) {
// \r\n... case
i++; // skip \n
} else {
// \r... case
}
} else if (chr === CharCode.LineFeed) {
if (eolCount === 0) {
firstLineLength = i;
}
eolCount++;
}
}
if (eolCount === 0) {
firstLineLength = text.length;
}
return [eolCount, firstLineLength];
}
private _applyEdits(rawOperations: model.IIdentifiedSingleEditOperation[]): model.IIdentifiedSingleEditOperation[] {
for (let i = 0, len = rawOperations.length; i < len; i++) {
rawOperations[i].range = this.validateRange(rawOperations[i].range);
@@ -1340,13 +1278,8 @@ export class TextModel extends Disposable implements model.ITextModel {
let lineCount = oldLineCount;
for (let i = 0, len = contentChanges.length; i < len; i++) {
const change = contentChanges[i];
const [eolCount, firstLineLength] = TextModel._eolCount(change.text);
try {
this._tokens.applyEdits(change.range, eolCount, firstLineLength);
} catch (err) {
// emergency recovery => reset tokens
this._tokens = new ModelLinesTokens(this._tokens.languageIdentifier, this._tokens.tokenizationSupport);
}
const [eolCount, firstLineLength] = countEOL(change.text);
this._tokens.applyEdits(change.range, eolCount, firstLineLength);
this._onDidChangeDecorations.fire();
this._decorationsTree.acceptReplace(change.rangeOffset, change.rangeLength, change.text.length, change.forceMoveMarkers);
@@ -1407,10 +1340,6 @@ export class TextModel extends Disposable implements model.ITextModel {
);
}
if (this._tokens.hasLinesToTokenize(this._buffer)) {
this._beginBackgroundTokenization();
}
return result.reverseEdits;
}
@@ -1775,93 +1704,60 @@ export class TextModel extends Disposable implements model.ITextModel {
//#region Tokenization
public setLineTokens(lineNumber: number, tokens: Uint32Array): void {
if (lineNumber < 1 || lineNumber > this.getLineCount()) {
throw new Error('Illegal value for lineNumber');
}
this._tokens.setTokens(this._languageIdentifier.id, lineNumber - 1, this._buffer.getLineLength(lineNumber), tokens);
}
public setTokens(tokens: MultilineTokens[]): void {
if (tokens.length === 0) {
return;
}
let ranges: { fromLineNumber: number; toLineNumber: number; }[] = [];
for (let i = 0, len = tokens.length; i < len; i++) {
const element = tokens[i];
ranges.push({ fromLineNumber: element.startLineNumber, toLineNumber: element.startLineNumber + element.tokens.length - 1 });
for (let j = 0, lenJ = element.tokens.length; j < lenJ; j++) {
this.setLineTokens(element.startLineNumber + j, element.tokens[j]);
}
}
this._emitModelTokensChangedEvent({
tokenizationSupportChanged: false,
ranges: ranges
});
}
public tokenizeViewport(startLineNumber: number, endLineNumber: number): void {
if (!this._tokens.tokenizationSupport) {
// nothing to do
return;
}
startLineNumber = Math.max(1, startLineNumber);
endLineNumber = Math.min(this.getLineCount(), endLineNumber);
endLineNumber = Math.min(this._buffer.getLineCount(), endLineNumber);
this._tokenization.tokenizeViewport(startLineNumber, endLineNumber);
}
if (endLineNumber <= this._tokens.inValidLineStartIndex) {
// nothing to do
return;
}
public clearTokens(): void {
this._tokens.flush();
this._emitModelTokensChangedEvent({
tokenizationSupportChanged: true,
ranges: [{
fromLineNumber: 1,
toLineNumber: this._buffer.getLineCount()
}]
});
}
if (startLineNumber <= this._tokens.inValidLineStartIndex) {
// tokenization has reached the viewport start...
this.forceTokenization(endLineNumber);
return;
}
let nonWhitespaceColumn = this.getLineFirstNonWhitespaceColumn(startLineNumber);
let fakeLines: string[] = [];
let initialState: IState | null = null;
for (let i = startLineNumber - 1; nonWhitespaceColumn > 0 && i >= 1; i--) {
let newNonWhitespaceIndex = this.getLineFirstNonWhitespaceColumn(i);
if (newNonWhitespaceIndex === 0) {
continue;
}
if (newNonWhitespaceIndex < nonWhitespaceColumn) {
initialState = this._tokens._getState(i - 1);
if (initialState) {
break;
}
fakeLines.push(this.getLineContent(i));
nonWhitespaceColumn = newNonWhitespaceIndex;
}
}
if (!initialState) {
initialState = this._tokens.tokenizationSupport.getInitialState();
}
let state = initialState.clone();
for (let i = fakeLines.length - 1; i >= 0; i--) {
let r = this._tokens._tokenizeText(this._buffer, fakeLines[i], state);
if (r) {
state = r.endState.clone();
} else {
state = initialState.clone();
}
}
const eventBuilder = new ModelTokensChangedEventBuilder();
for (let i = startLineNumber; i <= endLineNumber; i++) {
let text = this.getLineContent(i);
let r = this._tokens._tokenizeText(this._buffer, text, state);
if (r) {
this._tokens._setTokens(this._tokens.languageIdentifier.id, i - 1, text.length, r.tokens);
// We cannot trust these states/tokens to be valid!
// (see https://github.com/Microsoft/vscode/issues/67607)
this._tokens._setIsInvalid(i - 1, true);
this._tokens._setState(i - 1, state);
state = r.endState.clone();
eventBuilder.registerChangedTokens(i);
} else {
state = initialState.clone();
}
}
const e = eventBuilder.build();
if (e) {
private _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {
if (!this._isDisposing) {
this._onDidChangeTokens.fire(e);
}
}
public flushTokens(): void {
this._resetTokenizationState();
this.emitModelTokensChangedEvent({
tokenizationSupportChanged: false,
ranges: [{
fromLineNumber: 1,
toLineNumber: this.getLineCount()
}]
});
public resetTokenization(): void {
this._tokenization.reset();
}
public forceTokenization(lineNumber: number): void {
@@ -1869,30 +1765,11 @@ export class TextModel extends Disposable implements model.ITextModel {
throw new Error('Illegal value for lineNumber');
}
const eventBuilder = new ModelTokensChangedEventBuilder();
this._tokens._updateTokensUntilLine(this._buffer, eventBuilder, lineNumber);
const e = eventBuilder.build();
if (e) {
this._onDidChangeTokens.fire(e);
}
this._tokenization.forceTokenization(lineNumber);
}
public isCheapToTokenize(lineNumber: number): boolean {
if (!this._tokens.isCheapToTokenize(lineNumber)) {
return false;
}
if (lineNumber < this._tokens.inValidLineStartIndex + 1) {
return true;
}
if (this.getLineLength(lineNumber) < CHEAP_TOKENIZATION_LENGTH_LIMIT) {
return true;
}
return false;
return this._tokenization.isCheapToTokenize(lineNumber);
}
public tokenizeIfCheap(lineNumber: number): void {
@@ -1910,7 +1787,7 @@ export class TextModel extends Disposable implements model.ITextModel {
}
private _getLineTokens(lineNumber: number): LineTokens {
const lineText = this._buffer.getLineContent(lineNumber);
const lineText = this.getLineContent(lineNumber);
return this._tokens.getTokens(this._languageIdentifier.id, lineNumber - 1, lineText);
}
@@ -1935,81 +1812,14 @@ export class TextModel extends Disposable implements model.ITextModel {
this._languageIdentifier = languageIdentifier;
// Cancel tokenization, clear all tokens and begin tokenizing
this._resetTokenizationState();
this.emitModelTokensChangedEvent({
tokenizationSupportChanged: true,
ranges: [{
fromLineNumber: 1,
toLineNumber: this.getLineCount()
}]
});
this._onDidChangeLanguage.fire(e);
this._onDidChangeLanguageConfiguration.fire({});
}
public getLanguageIdAtPosition(_lineNumber: number, _column: number): LanguageId {
if (!this._tokens.tokenizationSupport) {
return this._languageIdentifier.id;
}
let { lineNumber, column } = this.validatePosition({ lineNumber: _lineNumber, column: _column });
let lineTokens = this._getLineTokens(lineNumber);
return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(column - 1));
}
private _beginBackgroundTokenization(): void {
if (this._shouldAutoTokenize() && this._revalidateTokensTimeout === -1) {
this._revalidateTokensTimeout = setTimeout(() => {
this._revalidateTokensTimeout = -1;
this._revalidateTokensNow();
}, 0);
}
}
_warmUpTokens(): void {
// Warm up first 100 lines (if it takes less than 50ms)
const maxLineNumber = Math.min(100, this.getLineCount());
this._revalidateTokensNow(maxLineNumber);
if (this._tokens.hasLinesToTokenize(this._buffer)) {
this._beginBackgroundTokenization();
}
}
private _revalidateTokensNow(toLineNumber: number = this._buffer.getLineCount()): void {
const MAX_ALLOWED_TIME = 20;
const eventBuilder = new ModelTokensChangedEventBuilder();
const sw = StopWatch.create(false);
while (this._tokens.hasLinesToTokenize(this._buffer)) {
if (sw.elapsed() > MAX_ALLOWED_TIME) {
// Stop if MAX_ALLOWED_TIME is reached
break;
}
const tokenizedLineNumber = this._tokens._tokenizeOneLine(this._buffer, eventBuilder);
if (tokenizedLineNumber >= toLineNumber) {
break;
}
}
if (this._tokens.hasLinesToTokenize(this._buffer)) {
this._beginBackgroundTokenization();
}
const e = eventBuilder.build();
if (e) {
this._onDidChangeTokens.fire(e);
}
}
private emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {
if (!this._isDisposing) {
this._onDidChangeTokens.fire(e);
}
public getLanguageIdAtPosition(lineNumber: number, column: number): LanguageId {
const position = this.validatePosition(new Position(lineNumber, column));
const lineTokens = this.getLineTokens(position.lineNumber);
return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));
}
// Having tokens allows implementing additional helper methods
@@ -2823,17 +2633,25 @@ function cleanClassName(className: string): string {
return className.replace(/[^a-z0-9\-_]/gi, ' ');
}
export class ModelDecorationOverviewRulerOptions implements model.IModelDecorationOverviewRulerOptions {
class DecorationOptions implements model.IDecorationOptions {
readonly color: string | ThemeColor;
readonly darkColor: string | ThemeColor;
constructor(options: model.IDecorationOptions) {
this.color = options.color || strings.empty;
this.darkColor = options.darkColor || strings.empty;
}
}
export class ModelDecorationOverviewRulerOptions extends DecorationOptions {
readonly position: model.OverviewRulerLane;
private _resolvedColor: string | null;
constructor(options: model.IModelDecorationOverviewRulerOptions) {
this.color = options.color || strings.empty;
this.darkColor = options.darkColor || strings.empty;
this.position = (typeof options.position === 'number' ? options.position : model.OverviewRulerLane.Center);
super(options);
this._resolvedColor = null;
this.position = (typeof options.position === 'number' ? options.position : model.OverviewRulerLane.Center);
}
public getColor(theme: ITheme): string {
@@ -2863,6 +2681,36 @@ export class ModelDecorationOverviewRulerOptions implements model.IModelDecorati
}
}
export class ModelDecorationMinimapOptions extends DecorationOptions {
readonly position: model.MinimapPosition;
private _resolvedColor: Color | undefined;
constructor(options: model.IModelDecorationMinimapOptions) {
super(options);
this.position = options.position;
}
public getColor(theme: ITheme): Color | undefined {
if (!this._resolvedColor) {
if (theme.type !== 'light' && this.darkColor) {
this._resolvedColor = this._resolveColor(this.darkColor, theme);
} else {
this._resolvedColor = this._resolveColor(this.color, theme);
}
}
return this._resolvedColor;
}
private _resolveColor(color: string | ThemeColor, theme: ITheme): Color | undefined {
if (typeof color === 'string') {
return Color.fromHex(color);
}
return theme.getColor(color.id);
}
}
export class ModelDecorationOptions implements model.IModelDecorationOptions {
public static EMPTY: ModelDecorationOptions;
@@ -2884,6 +2732,7 @@ export class ModelDecorationOptions implements model.IModelDecorationOptions {
readonly showIfCollapsed: boolean;
readonly collapseOnReplaceEdit: boolean;
readonly overviewRuler: ModelDecorationOverviewRulerOptions | null;
readonly minimap: ModelDecorationMinimapOptions | null;
readonly glyphMarginClassName: string | null;
readonly linesDecorationsClassName: string | null;
readonly marginClassName: string | null;
@@ -2902,6 +2751,7 @@ export class ModelDecorationOptions implements model.IModelDecorationOptions {
this.showIfCollapsed = options.showIfCollapsed || false;
this.collapseOnReplaceEdit = options.collapseOnReplaceEdit || false;
this.overviewRuler = options.overviewRuler ? new ModelDecorationOverviewRulerOptions(options.overviewRuler) : null;
this.minimap = options.minimap ? new ModelDecorationMinimapOptions(options.minimap) : null;
this.glyphMarginClassName = options.glyphMarginClassName ? cleanClassName(options.glyphMarginClassName) : null;
this.linesDecorationsClassName = options.linesDecorationsClassName ? cleanClassName(options.linesDecorationsClassName) : null;
this.marginClassName = options.marginClassName ? cleanClassName(options.marginClassName) : null;