ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-11 18:48:33 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge from vscode 6268feb42ba4f2e2fa15484e88c9af60d254998c (#6530)

Browse Source
This commit is contained in:
Anthony Dresser
2019-07-29 21:03:02 -07:00
committed by GitHub
parent 2c8a22bb0d
commit 6db84eefa3
104 changed files with 1797 additions and 3740 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/vs/editor/common/model/textModel.ts
View File

@@ -22,7 +22,7 @@ import { IntervalNode, IntervalTree, getNodeIsInOverviewRuler, recomputeMaxEnd }
import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder';
import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents';
import { SearchData, SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch';
import { TextModelTokenization, countEOL } from 'vs/editor/common/model/textModelTokens';
import { TextModelTokenization } from 'vs/editor/common/model/textModelTokens';
import { getWordAtText } from 'vs/editor/common/model/wordHelper';
import { LanguageId, LanguageIdentifier, FormattingOptions } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
@@ -32,7 +32,7 @@ import { BracketsUtils, RichEditBracket, RichEditBrackets } from 'vs/editor/comm
import { ITheme, ThemeColor } from 'vs/platform/theme/common/themeService';
import { withUndefinedAsNull } from 'vs/base/common/types';
import { VSBufferReadableStream, VSBuffer } from 'vs/base/common/buffer';
import { TokensStore, MultilineTokens } from 'vs/editor/common/model/tokensStore';
import { TokensStore, MultilineTokens, countEOL } from 'vs/editor/common/model/tokensStore';
import { Color } from 'vs/base/common/color';
function createTextBufferBuilder() {
@@ -1279,7 +1279,7 @@ export class TextModel extends Disposable implements model.ITextModel {
for (let i = 0, len = contentChanges.length; i < len; i++) {
const change = contentChanges[i];
const [eolCount, firstLineLength] = countEOL(change.text);
this._tokens.applyEdits(change.range, eolCount, firstLineLength);
this._tokens.acceptEdit(change.range, eolCount, firstLineLength);
this._onDidChangeDecorations.fire();
this._decorationsTree.acceptReplace(change.rangeOffset, change.rangeLength, change.text.length, change.forceMoveMarkers);
@@ -1704,7 +1704,7 @@ export class TextModel extends Disposable implements model.ITextModel {
//#region Tokenization
public setLineTokens(lineNumber: number, tokens: Uint32Array): void {
public setLineTokens(lineNumber: number, tokens: Uint32Array | ArrayBuffer | null): void {
if (lineNumber < 1 || lineNumber > this.getLineCount()) {
throw new Error('Illegal value for lineNumber');
}

36
src/vs/editor/common/model/textModelTokens.ts
View File

@@ -15,38 +15,7 @@ import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
import { TextModel } from 'vs/editor/common/model/textModel';
import { Disposable } from 'vs/base/common/lifecycle';
import { StopWatch } from 'vs/base/common/stopwatch';
import { CharCode } from 'vs/base/common/charCode';
import { MultilineTokensBuilder } from 'vs/editor/common/model/tokensStore';
export function countEOL(text: string): [number, number] {
let eolCount = 0;
let firstLineLength = 0;
for (let i = 0, len = text.length; i < len; i++) {
const chr = text.charCodeAt(i);
if (chr === CharCode.CarriageReturn) {
if (eolCount === 0) {
firstLineLength = i;
}
eolCount++;
if (i + 1 < len && text.charCodeAt(i + 1) === CharCode.LineFeed) {
// \r\n... case
i++; // skip \n
} else {
// \r... case
}
} else if (chr === CharCode.LineFeed) {
if (eolCount === 0) {
firstLineLength = i;
}
eolCount++;
}
}
if (eolCount === 0) {
firstLineLength = text.length;
}
return [eolCount, firstLineLength];
}
import { MultilineTokensBuilder, countEOL } from 'vs/editor/common/model/tokensStore';
const enum Constants {
CHEAP_TOKENIZATION_LENGTH_LIMIT = 2048
@@ -117,6 +86,9 @@ export class TokenizationStateStore {
if (deleteCount === 0) {
return;
}
if (start + deleteCount > this._len) {
deleteCount = this._len - start;
}
this._beginState.splice(start, deleteCount);
this._valid.splice(start, deleteCount);
this._len -= deleteCount;

301
src/vs/editor/common/model/tokensStore.ts
View File

@@ -8,6 +8,38 @@ import { LineTokens } from 'vs/editor/common/core/lineTokens';
import { Position } from 'vs/editor/common/core/position';
import { IRange } from 'vs/editor/common/core/range';
import { ColorId, FontStyle, LanguageId, MetadataConsts, StandardTokenType, TokenMetadata } from 'vs/editor/common/modes';
import { writeUInt32BE, readUInt32BE } from 'vs/base/common/buffer';
import { CharCode } from 'vs/base/common/charCode';
export function countEOL(text: string): [number, number] {
let eolCount = 0;
let firstLineLength = 0;
for (let i = 0, len = text.length; i < len; i++) {
const chr = text.charCodeAt(i);
if (chr === CharCode.CarriageReturn) {
if (eolCount === 0) {
firstLineLength = i;
}
eolCount++;
if (i + 1 < len && text.charCodeAt(i + 1) === CharCode.LineFeed) {
// \r\n... case
i++; // skip \n
} else {
// \r... case
}
} else if (chr === CharCode.LineFeed) {
if (eolCount === 0) {
firstLineLength = i;
}
eolCount++;
}
}
if (eolCount === 0) {
firstLineLength = text.length;
}
return [eolCount, firstLineLength];
}
function getDefaultMetadata(topLevelLanguageId: LanguageId): number {
return (
@@ -39,23 +71,226 @@ export class MultilineTokensBuilder {
return;
}
}
this.tokens.push(new MultilineTokens(lineNumber, lineTokens));
this.tokens.push(new MultilineTokens(lineNumber, [lineTokens]));
}
public static deserialize(buff: Uint8Array): MultilineTokens[] {
let offset = 0;
const count = readUInt32BE(buff, offset); offset += 4;
let result: MultilineTokens[] = [];
for (let i = 0; i < count; i++) {
offset = MultilineTokens.deserialize(buff, offset, result);
}
return result;
}
public serialize(): Uint8Array {
const size = this._serializeSize();
const result = new Uint8Array(size);
this._serialize(result);
return result;
}
private _serializeSize(): number {
let result = 0;
result += 4; // 4 bytes for the count
for (let i = 0; i < this.tokens.length; i++) {
result += this.tokens[i].serializeSize();
}
return result;
}
private _serialize(destination: Uint8Array): void {
let offset = 0;
writeUInt32BE(destination, this.tokens.length, offset); offset += 4;
for (let i = 0; i < this.tokens.length; i++) {
offset = this.tokens[i].serialize(destination, offset);
}
}
}
export class MultilineTokens {
public readonly startLineNumber: number;
public readonly tokens: Uint32Array[];
public startLineNumber: number;
public tokens: (Uint32Array | ArrayBuffer | null)[];
constructor(lineNumber: number, tokens: Uint32Array) {
this.startLineNumber = lineNumber;
this.tokens = [tokens];
constructor(startLineNumber: number, tokens: Uint32Array[]) {
this.startLineNumber = startLineNumber;
this.tokens = tokens;
}
public static deserialize(buff: Uint8Array, offset: number, result: MultilineTokens[]): number {
const view32 = new Uint32Array(buff.buffer);
const startLineNumber = readUInt32BE(buff, offset); offset += 4;
const count = readUInt32BE(buff, offset); offset += 4;
let tokens: Uint32Array[] = [];
for (let i = 0; i < count; i++) {
const byteCount = readUInt32BE(buff, offset); offset += 4;
tokens.push(view32.subarray(offset / 4, offset / 4 + byteCount / 4));
offset += byteCount;
}
result.push(new MultilineTokens(startLineNumber, tokens));
return offset;
}
public serializeSize(): number {
let result = 0;
result += 4; // 4 bytes for the start line number
result += 4; // 4 bytes for the line count
for (let i = 0; i < this.tokens.length; i++) {
const lineTokens = this.tokens[i];
if (!(lineTokens instanceof Uint32Array)) {
throw new Error(`Not supported!`);
}
result += 4; // 4 bytes for the byte count
result += lineTokens.byteLength;
}
return result;
}
public serialize(destination: Uint8Array, offset: number): number {
writeUInt32BE(destination, this.startLineNumber, offset); offset += 4;
writeUInt32BE(destination, this.tokens.length, offset); offset += 4;
for (let i = 0; i < this.tokens.length; i++) {
const lineTokens = this.tokens[i];
if (!(lineTokens instanceof Uint32Array)) {
throw new Error(`Not supported!`);
}
writeUInt32BE(destination, lineTokens.byteLength, offset); offset += 4;
destination.set(new Uint8Array(lineTokens.buffer), offset); offset += lineTokens.byteLength;
}
return offset;
}
public applyEdit(range: IRange, text: string): void {
const [eolCount, firstLineLength] = countEOL(text);
this._acceptDeleteRange(range);
this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength);
}
private _acceptDeleteRange(range: IRange): void {
if (range.startLineNumber === range.endLineNumber && range.startColumn === range.endColumn) {
// Nothing to delete
return;
}
const firstLineIndex = range.startLineNumber - this.startLineNumber;
const lastLineIndex = range.endLineNumber - this.startLineNumber;
if (lastLineIndex < 0) {
// this deletion occurs entirely before this block, so we only need to adjust line numbers
const deletedLinesCount = lastLineIndex - firstLineIndex;
this.startLineNumber -= deletedLinesCount;
return;
}
if (firstLineIndex >= this.tokens.length) {
// this deletion occurs entirely after this block, so there is nothing to do
return;
}
if (firstLineIndex < 0 && lastLineIndex >= this.tokens.length) {
// this deletion completely encompasses this block
this.startLineNumber = 0;
this.tokens = [];
}
if (firstLineIndex === lastLineIndex) {
// a delete on a single line
this.tokens[firstLineIndex] = TokensStore._delete(this.tokens[firstLineIndex], range.startColumn - 1, range.endColumn - 1);
return;
}
if (firstLineIndex >= 0) {
// The first line survives
this.tokens[firstLineIndex] = TokensStore._deleteEnding(this.tokens[firstLineIndex], range.startColumn - 1);
if (lastLineIndex < this.tokens.length) {
// The last line survives
const lastLineTokens = TokensStore._deleteBeginning(this.tokens[lastLineIndex], range.endColumn - 1);
// Take remaining text on last line and append it to remaining text on first line
this.tokens[firstLineIndex] = TokensStore._append(this.tokens[firstLineIndex], lastLineTokens);
// Delete middle lines
this.tokens.splice(firstLineIndex + 1, lastLineIndex - firstLineIndex);
} else {
// The last line does not survive
// Take remaining text on last line and append it to remaining text on first line
this.tokens[firstLineIndex] = TokensStore._append(this.tokens[firstLineIndex], null);
// Delete lines
this.tokens = this.tokens.slice(0, firstLineIndex + 1);
}
} else {
// The first line does not survive
const deletedBefore = -firstLineIndex;
this.startLineNumber -= deletedBefore;
// Remove beginning from last line
this.tokens[lastLineIndex] = TokensStore._deleteBeginning(this.tokens[lastLineIndex], range.endColumn - 1);
// Delete lines
this.tokens = this.tokens.slice(lastLineIndex);
}
}
private _acceptInsertText(position: Position, eolCount: number, firstLineLength: number): void {
if (eolCount === 0 && firstLineLength === 0) {
// Nothing to insert
return;
}
const lineIndex = position.lineNumber - this.startLineNumber;
if (lineIndex < 0) {
// this insertion occurs before this block, so we only need to adjust line numbers
this.startLineNumber += eolCount;
return;
}
if (lineIndex >= this.tokens.length) {
// this insertion occurs after this block, so there is nothing to do
return;
}
if (eolCount === 0) {
// Inserting text on one line
this.tokens[lineIndex] = TokensStore._insert(this.tokens[lineIndex], position.column - 1, firstLineLength);
return;
}
this.tokens[lineIndex] = TokensStore._deleteEnding(this.tokens[lineIndex], position.column - 1);
this.tokens[lineIndex] = TokensStore._insert(this.tokens[lineIndex], position.column - 1, firstLineLength);
this._insertLines(position.lineNumber, eolCount);
}
private _insertLines(insertIndex: number, insertCount: number): void {
if (insertCount === 0) {
return;
}
let lineTokens: (Uint32Array | ArrayBuffer | null)[] = [];
for (let i = 0; i < insertCount; i++) {
lineTokens[i] = null;
}
this.tokens = arrays.arrayInsert(this.tokens, insertIndex, lineTokens);
}
}
function toUint32Array(arr: Uint32Array | ArrayBuffer): Uint32Array {
if (arr instanceof Uint32Array) {
return arr;
} else {
return new Uint32Array(arr);
}
}
export class TokensStore {
private _lineTokens: (ArrayBuffer | null)[];
private _lineTokens: (Uint32Array | ArrayBuffer | null)[];
private _len: number;
constructor() {
@@ -69,13 +304,13 @@ export class TokensStore {
}
public getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens {
let rawLineTokens: ArrayBuffer | null = null;
let rawLineTokens: Uint32Array | ArrayBuffer | null = null;
if (lineIndex < this._len) {
rawLineTokens = this._lineTokens[lineIndex];
}
if (rawLineTokens !== null && rawLineTokens !== EMPTY_LINE_TOKENS) {
return new LineTokens(new Uint32Array(rawLineTokens), lineText);
return new LineTokens(toUint32Array(rawLineTokens), lineText);
}
let lineTokens = new Uint32Array(2);
@@ -84,7 +319,10 @@ export class TokensStore {
return new LineTokens(lineTokens, lineText);
}
private static _massageTokens(topLevelLanguageId: LanguageId, lineTextLength: number, tokens: Uint32Array): ArrayBuffer {
private static _massageTokens(topLevelLanguageId: LanguageId, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null): Uint32Array | ArrayBuffer {
const tokens = _tokens ? toUint32Array(_tokens) : null;
if (lineTextLength === 0) {
let hasDifferentLanguageId = false;
if (tokens && tokens.length > 1) {
@@ -97,12 +335,20 @@ export class TokensStore {
}
if (!tokens || tokens.length === 0) {
tokens = new Uint32Array(2);
const tokens = new Uint32Array(2);
tokens[0] = lineTextLength;
tokens[1] = getDefaultMetadata(topLevelLanguageId);
return tokens.buffer;
}
return tokens.buffer;
// Ensure the last token covers the end of the text
tokens[tokens.length - 2] = lineTextLength;
if (tokens.byteOffset === 0 && tokens.byteLength === tokens.buffer.byteLength) {
// Store directly the ArrayBuffer pointer to save an object
return tokens.buffer;
}
return tokens;
}
private _ensureLine(lineIndex: number): void {
@@ -116,6 +362,9 @@ export class TokensStore {
if (deleteCount === 0) {
return;
}
if (start + deleteCount > this._len) {
deleteCount = this._len - start;
}
this._lineTokens.splice(start, deleteCount);
this._len -= deleteCount;
}
@@ -124,7 +373,7 @@ export class TokensStore {
if (insertCount === 0) {
return;
}
let lineTokens: (ArrayBuffer | null)[] = [];
let lineTokens: (Uint32Array | ArrayBuffer | null)[] = [];
for (let i = 0; i < insertCount; i++) {
lineTokens[i] = null;
}
@@ -132,7 +381,7 @@ export class TokensStore {
this._len += insertCount;
}
public setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, _tokens: Uint32Array): void {
public setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null): void {
const tokens = TokensStore._massageTokens(topLevelLanguageId, lineTextLength, _tokens);
this._ensureLine(lineIndex);
this._lineTokens[lineIndex] = tokens;
@@ -140,7 +389,7 @@ export class TokensStore {
//#region Editing
public applyEdits(range: IRange, eolCount: number, firstLineLength: number): void {
public acceptEdit(range: IRange, eolCount: number, firstLineLength: number): void {
this._acceptDeleteRange(range);
this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength);
}
@@ -165,7 +414,7 @@ export class TokensStore {
this._lineTokens[firstLineIndex] = TokensStore._deleteEnding(this._lineTokens[firstLineIndex], range.startColumn - 1);
const lastLineIndex = range.endLineNumber - 1;
let lastLineTokens: ArrayBuffer | null = null;
let lastLineTokens: Uint32Array | ArrayBuffer | null = null;
if (lastLineIndex < this._len) {
lastLineTokens = TokensStore._deleteBeginning(this._lineTokens[lastLineIndex], range.endColumn - 1);
}
@@ -201,29 +450,29 @@ export class TokensStore {
this._insertLines(position.lineNumber, eolCount);
}
private static _deleteBeginning(lineTokens: ArrayBuffer | null, toChIndex: number): ArrayBuffer | null {
public static _deleteBeginning(lineTokens: Uint32Array | ArrayBuffer | null, toChIndex: number): Uint32Array | ArrayBuffer | null {
if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) {
return lineTokens;
}
return TokensStore._delete(lineTokens, 0, toChIndex);
}
private static _deleteEnding(lineTokens: ArrayBuffer | null, fromChIndex: number): ArrayBuffer | null {
public static _deleteEnding(lineTokens: Uint32Array | ArrayBuffer | null, fromChIndex: number): Uint32Array | ArrayBuffer | null {
if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) {
return lineTokens;
}
const tokens = new Uint32Array(lineTokens);
const tokens = toUint32Array(lineTokens);
const lineTextLength = tokens[tokens.length - 2];
return TokensStore._delete(lineTokens, fromChIndex, lineTextLength);
}
private static _delete(lineTokens: ArrayBuffer | null, fromChIndex: number, toChIndex: number): ArrayBuffer | null {
public static _delete(lineTokens: Uint32Array | ArrayBuffer | null, fromChIndex: number, toChIndex: number): Uint32Array | ArrayBuffer | null {
if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS || fromChIndex === toChIndex) {
return lineTokens;
}
const tokens = new Uint32Array(lineTokens);
const tokens = toUint32Array(lineTokens);
const tokensCount = (tokens.length >>> 1);
// special case: deleting everything
@@ -275,7 +524,7 @@ export class TokensStore {
return tmp.buffer;
}
private static _append(lineTokens: ArrayBuffer | null, _otherTokens: ArrayBuffer | null): ArrayBuffer | null {
public static _append(lineTokens: Uint32Array | ArrayBuffer | null, _otherTokens: Uint32Array | ArrayBuffer | null): Uint32Array | ArrayBuffer | null {
if (_otherTokens === EMPTY_LINE_TOKENS) {
return lineTokens;
}
@@ -289,8 +538,8 @@ export class TokensStore {
// cannot determine combined line length...
return null;
}
const myTokens = new Uint32Array(lineTokens);
const otherTokens = new Uint32Array(_otherTokens);
const myTokens = toUint32Array(lineTokens);
const otherTokens = toUint32Array(_otherTokens);
const otherTokensCount = (otherTokens.length >>> 1);
let result = new Uint32Array(myTokens.length + otherTokens.length);
@@ -304,13 +553,13 @@ export class TokensStore {
return result.buffer;
}
private static _insert(lineTokens: ArrayBuffer | null, chIndex: number, textLength: number): ArrayBuffer | null {
public static _insert(lineTokens: Uint32Array | ArrayBuffer | null, chIndex: number, textLength: number): Uint32Array | ArrayBuffer | null {
if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) {
// nothing to do
return lineTokens;
}
const tokens = new Uint32Array(lineTokens);
const tokens = toUint32Array(lineTokens);
const tokensCount = (tokens.length >>> 1);
let fromTokenIndex = LineTokens.findIndexInTokensArray(tokens, chIndex);