ckaczor/azuredatastudio
1
0
Fork 0
mirror of https://github.com/ckaczor/azuredatastudio.git synced 2026-02-11 02:32:35 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge VS Code 1.23.1 (#1520)

Browse Source
This commit is contained in:
Matt Irvine
2018-06-05 11:24:51 -07:00
committed by GitHub
parent e3baf5c443
commit 0c58f09e59
3651 changed files with 74249 additions and 48599 deletions
Download Patch File Download Diff File Expand all files Collapse all files

323
src/vs/editor/common/model/chunksTextBuffer/bufferPiece.ts
View File

@@ -1,323 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { CharCode } from 'vs/base/common/charCode';
export class LeafOffsetLenEdit {
constructor(
public readonly start: number,
public readonly length: number,
public readonly text: string
) { }
}
export class BufferPiece {
private readonly _str: string;
public get text(): string { return this._str; }
private readonly _lineStarts: Uint32Array;
constructor(str: string, lineStarts: Uint32Array = null) {
this._str = str;
if (lineStarts === null) {
this._lineStarts = createLineStartsFast(str);
} else {
this._lineStarts = lineStarts;
}
}
public length(): number {
return this._str.length;
}
public newLineCount(): number {
return this._lineStarts.length;
}
public lineStartFor(relativeLineIndex: number): number {
return this._lineStarts[relativeLineIndex];
}
public charCodeAt(index: number): number {
return this._str.charCodeAt(index);
}
public substr(from: number, length: number): string {
return this._str.substr(from, length);
}
public findLineStartBeforeOffset(offset: number): number {
if (this._lineStarts.length === 0 || offset < this._lineStarts[0]) {
return -1;
}
let low = 0, high = this._lineStarts.length - 1;
while (low < high) {
let mid = low + Math.ceil((high - low) / 2);
let lineStart = this._lineStarts[mid];
if (offset === lineStart) {
return mid;
} else if (offset < lineStart) {
high = mid - 1;
} else {
low = mid;
}
}
return low;
}
public findLineFirstNonWhitespaceIndex(searchStartOffset: number): number {
for (let i = searchStartOffset, len = this._str.length; i < len; i++) {
const chCode = this._str.charCodeAt(i);
if (chCode === CharCode.CarriageReturn || chCode === CharCode.LineFeed) {
// Reached EOL
return -2;
}
if (chCode !== CharCode.Space && chCode !== CharCode.Tab) {
return i;
}
}
return -1;
}
public findLineLastNonWhitespaceIndex(searchStartOffset: number): number {
for (let i = searchStartOffset - 1; i >= 0; i--) {
const chCode = this._str.charCodeAt(i);
if (chCode === CharCode.CarriageReturn || chCode === CharCode.LineFeed) {
// Reached EOL
return -2;
}
if (chCode !== CharCode.Space && chCode !== CharCode.Tab) {
return i;
}
}
return -1;
}
public static normalizeEOL(target: BufferPiece, eol: '\r\n' | '\n'): BufferPiece {
return new BufferPiece(target._str.replace(/\r\n|\r|\n/g, eol));
}
public static deleteLastChar(target: BufferPiece): BufferPiece {
const targetCharsLength = target.length();
const targetLineStartsLength = target.newLineCount();
const targetLineStarts = target._lineStarts;
let newLineStartsLength;
if (targetLineStartsLength > 0 && targetLineStarts[targetLineStartsLength - 1] === targetCharsLength) {
newLineStartsLength = targetLineStartsLength - 1;
} else {
newLineStartsLength = targetLineStartsLength;
}
let newLineStarts = new Uint32Array(newLineStartsLength);
newLineStarts.set(targetLineStarts);
return new BufferPiece(
target._str.substr(0, targetCharsLength - 1),
newLineStarts
);
}
public static insertFirstChar(target: BufferPiece, character: number): BufferPiece {
const targetLineStartsLength = target.newLineCount();
const targetLineStarts = target._lineStarts;
const insertLineStart = ((character === CharCode.CarriageReturn && (targetLineStartsLength === 0 || targetLineStarts[0] !== 1 || target.charCodeAt(0) !== CharCode.LineFeed)) || (character === CharCode.LineFeed));
const newLineStartsLength = (insertLineStart ? targetLineStartsLength + 1 : targetLineStartsLength);
let newLineStarts = new Uint32Array(newLineStartsLength);
if (insertLineStart) {
newLineStarts[0] = 1;
for (let i = 0; i < targetLineStartsLength; i++) {
newLineStarts[i + 1] = targetLineStarts[i] + 1;
}
} else {
for (let i = 0; i < targetLineStartsLength; i++) {
newLineStarts[i] = targetLineStarts[i] + 1;
}
}
return new BufferPiece(
String.fromCharCode(character) + target._str,
newLineStarts
);
}
public static join(first: BufferPiece, second: BufferPiece): BufferPiece {
const firstCharsLength = first._str.length;
const firstLineStartsLength = first._lineStarts.length;
const secondLineStartsLength = second._lineStarts.length;
const firstLineStarts = first._lineStarts;
const secondLineStarts = second._lineStarts;
const newLineStartsLength = firstLineStartsLength + secondLineStartsLength;
let newLineStarts = new Uint32Array(newLineStartsLength);
newLineStarts.set(firstLineStarts, 0);
for (let i = 0; i < secondLineStartsLength; i++) {
newLineStarts[i + firstLineStartsLength] = secondLineStarts[i] + firstCharsLength;
}
return new BufferPiece(first._str + second._str, newLineStarts);
}
public static replaceOffsetLen(target: BufferPiece, edits: LeafOffsetLenEdit[], idealLeafLength: number, maxLeafLength: number, result: BufferPiece[]): void {
const editsSize = edits.length;
const originalCharsLength = target.length();
if (editsSize === 1 && edits[0].text.length === 0 && edits[0].start === 0 && edits[0].length === originalCharsLength) {
// special case => deleting everything
return;
}
let pieces: string[] = new Array<string>(2 * editsSize + 1);
let originalFromIndex = 0;
let piecesTextLength = 0;
for (let i = 0; i < editsSize; i++) {
const edit = edits[i];
const originalText = target._str.substr(originalFromIndex, edit.start - originalFromIndex);
pieces[2 * i] = originalText;
piecesTextLength += originalText.length;
originalFromIndex = edit.start + edit.length;
pieces[2 * i + 1] = edit.text;
piecesTextLength += edit.text.length;
}
// maintain the chars that survive to the right of the last edit
let text = target._str.substr(originalFromIndex, originalCharsLength - originalFromIndex);
pieces[2 * editsSize] = text;
piecesTextLength += text.length;
let targetDataLength = piecesTextLength > maxLeafLength ? idealLeafLength : piecesTextLength;
let targetDataOffset = 0;
let data: string = '';
for (let pieceIndex = 0, pieceCount = pieces.length; pieceIndex < pieceCount; pieceIndex++) {
const pieceText = pieces[pieceIndex];
const pieceLength = pieceText.length;
if (pieceLength === 0) {
continue;
}
let pieceOffset = 0;
while (pieceOffset < pieceLength) {
if (targetDataOffset >= targetDataLength) {
result.push(new BufferPiece(data));
targetDataLength = piecesTextLength > maxLeafLength ? idealLeafLength : piecesTextLength;
targetDataOffset = 0;
data = '';
}
let writingCnt = min(pieceLength - pieceOffset, targetDataLength - targetDataOffset);
data += pieceText.substr(pieceOffset, writingCnt);
pieceOffset += writingCnt;
targetDataOffset += writingCnt;
piecesTextLength -= writingCnt;
// check that the buffer piece does not end in a \r or high surrogate
if (targetDataOffset === targetDataLength && piecesTextLength > 0) {
const lastChar = data.charCodeAt(targetDataLength - 1);
if (lastChar === CharCode.CarriageReturn || (0xD800 <= lastChar && lastChar <= 0xDBFF)) {
// move lastChar over to next buffer piece
targetDataLength -= 1;
pieceOffset -= 1;
targetDataOffset -= 1;
piecesTextLength += 1;
data = data.substr(0, data.length - 1);
}
}
}
}
result.push(new BufferPiece(data));
}
}
function min(a: number, b: number): number {
return (a < b ? a : b);
}
export function createUint32Array(arr: number[]): Uint32Array {
let r = new Uint32Array(arr.length);
r.set(arr, 0);
return r;
}
export class LineStarts {
constructor(
public readonly lineStarts: Uint32Array,
public readonly cr: number,
public readonly lf: number,
public readonly crlf: number,
public readonly isBasicASCII: boolean
) { }
}
export function createLineStartsFast(str: string): Uint32Array {
let r: number[] = [], rLength = 0;
for (let i = 0, len = str.length; i < len; i++) {
const chr = str.charCodeAt(i);
if (chr === CharCode.CarriageReturn) {
if (i + 1 < len && str.charCodeAt(i + 1) === CharCode.LineFeed) {
// \r\n... case
r[rLength++] = i + 2;
i++; // skip \n
} else {
// \r... case
r[rLength++] = i + 1;
}
} else if (chr === CharCode.LineFeed) {
r[rLength++] = i + 1;
}
}
return createUint32Array(r);
}
export function createLineStarts(r: number[], str: string): LineStarts {
r.length = 0;
let rLength = 0;
let cr = 0, lf = 0, crlf = 0;
let isBasicASCII = true;
for (let i = 0, len = str.length; i < len; i++) {
const chr = str.charCodeAt(i);
if (chr === CharCode.CarriageReturn) {
if (i + 1 < len && str.charCodeAt(i + 1) === CharCode.LineFeed) {
// \r\n... case
crlf++;
r[rLength++] = i + 2;
i++; // skip \n
} else {
cr++;
// \r... case
r[rLength++] = i + 1;
}
} else if (chr === CharCode.LineFeed) {
lf++;
r[rLength++] = i + 1;
} else {
if (isBasicASCII) {
if (chr !== CharCode.Tab && (chr < 32 || chr > 126)) {
isBasicASCII = false;
}
}
}
}
const result = new LineStarts(createUint32Array(r), cr, lf, crlf, isBasicASCII);
r.length = 0;
return result;
}

1526
src/vs/editor/common/model/chunksTextBuffer/chunksTextBuffer.ts
View File

File diff suppressed because it is too large Load Diff

186
src/vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder.ts
View File

@@ -1,186 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as strings from 'vs/base/common/strings';
import { ITextBufferBuilder, ITextBufferFactory, ITextBuffer, DefaultEndOfLine } from 'vs/editor/common/model';
import { BufferPiece, createLineStarts } from 'vs/editor/common/model/chunksTextBuffer/bufferPiece';
import { ChunksTextBuffer } from 'vs/editor/common/model/chunksTextBuffer/chunksTextBuffer';
import { CharCode } from 'vs/base/common/charCode';
export class TextBufferFactory implements ITextBufferFactory {
constructor(
private readonly _pieces: BufferPiece[],
private readonly _averageChunkSize: number,
private readonly _BOM: string,
private readonly _cr: number,
private readonly _lf: number,
private readonly _crlf: number,
private readonly _containsRTL: boolean,
private readonly _isBasicASCII: boolean,
) {
}
/**
* if text source is empty or with precisely one line, returns null. No end of line is detected.
* if text source contains more lines ending with '\r\n', returns '\r\n'.
* Otherwise returns '\n'. More lines end with '\n'.
*/
private _getEOL(defaultEOL: DefaultEndOfLine): '\r\n' | '\n' {
const totalEOLCount = this._cr + this._lf + this._crlf;
const totalCRCount = this._cr + this._crlf;
if (totalEOLCount === 0) {
// This is an empty file or a file with precisely one line
return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n');
}
if (totalCRCount > totalEOLCount / 2) {
// More than half of the file contains \r\n ending lines
return '\r\n';
}
// At least one line more ends in \n
return '\n';
}
public create(defaultEOL: DefaultEndOfLine): ITextBuffer {
const eol = this._getEOL(defaultEOL);
let pieces = this._pieces;
if (
(eol === '\r\n' && (this._cr > 0 || this._lf > 0))
|| (eol === '\n' && (this._cr > 0 || this._crlf > 0))
) {
// Normalize pieces
for (let i = 0, len = pieces.length; i < len; i++) {
pieces[i] = BufferPiece.normalizeEOL(pieces[i], eol);
}
}
return new ChunksTextBuffer(pieces, this._averageChunkSize, this._BOM, eol, this._containsRTL, this._isBasicASCII);
}
public getFirstLineText(lengthLimit: number): string {
const firstPiece = this._pieces[0];
if (firstPiece.newLineCount() === 0) {
return firstPiece.substr(0, lengthLimit);
}
const firstEOLOffset = firstPiece.lineStartFor(0);
return firstPiece.substr(0, Math.min(lengthLimit, firstEOLOffset));
}
}
export class ChunksTextBufferBuilder implements ITextBufferBuilder {
private _rawPieces: BufferPiece[];
private _hasPreviousChar: boolean;
private _previousChar: number;
private _averageChunkSize: number;
private _tmpLineStarts: number[];
private BOM: string;
private cr: number;
private lf: number;
private crlf: number;
private containsRTL: boolean;
private isBasicASCII: boolean;
constructor() {
this._rawPieces = [];
this._hasPreviousChar = false;
this._previousChar = 0;
this._averageChunkSize = 0;
this._tmpLineStarts = [];
this.BOM = '';
this.cr = 0;
this.lf = 0;
this.crlf = 0;
this.containsRTL = false;
this.isBasicASCII = true;
}
public acceptChunk(chunk: string): void {
if (chunk.length === 0) {
return;
}
if (this._rawPieces.length === 0) {
if (strings.startsWithUTF8BOM(chunk)) {
this.BOM = strings.UTF8_BOM_CHARACTER;
chunk = chunk.substr(1);
}
}
this._averageChunkSize = (this._averageChunkSize * this._rawPieces.length + chunk.length) / (this._rawPieces.length + 1);
const lastChar = chunk.charCodeAt(chunk.length - 1);
if (lastChar === CharCode.CarriageReturn || (lastChar >= 0xd800 && lastChar <= 0xdbff)) {
// last character is \r or a high surrogate => keep it back
this._acceptChunk1(chunk.substr(0, chunk.length - 1), false);
this._hasPreviousChar = true;
this._previousChar = lastChar;
} else {
this._acceptChunk1(chunk, false);
this._hasPreviousChar = false;
this._previousChar = lastChar;
}
}
private _acceptChunk1(chunk: string, allowEmptyStrings: boolean): void {
if (!allowEmptyStrings && chunk.length === 0) {
// Nothing to do
return;
}
if (this._hasPreviousChar) {
this._acceptChunk2(chunk + String.fromCharCode(this._previousChar));
} else {
this._acceptChunk2(chunk);
}
}
private _acceptChunk2(chunk: string): void {
const lineStarts = createLineStarts(this._tmpLineStarts, chunk);
this._rawPieces.push(new BufferPiece(chunk, lineStarts.lineStarts));
this.cr += lineStarts.cr;
this.lf += lineStarts.lf;
this.crlf += lineStarts.crlf;
if (this.isBasicASCII) {
this.isBasicASCII = lineStarts.isBasicASCII;
}
if (!this.isBasicASCII && !this.containsRTL) {
// No need to check if is basic ASCII
this.containsRTL = strings.containsRTL(chunk);
}
}
public finish(): TextBufferFactory {
this._finish();
return new TextBufferFactory(this._rawPieces, this._averageChunkSize, this.BOM, this.cr, this.lf, this.crlf, this.containsRTL, this.isBasicASCII);
}
private _finish(): void {
if (this._rawPieces.length === 0) {
// no chunks => forcefully go through accept chunk
this._acceptChunk1('', true);
return;
}
if (this._hasPreviousChar) {
this._hasPreviousChar = false;
// recreate last chunk
const lastPiece = this._rawPieces[this._rawPieces.length - 1];
const tmp = new BufferPiece(String.fromCharCode(this._previousChar));
const newLastPiece = BufferPiece.join(lastPiece, tmp);
this._rawPieces[this._rawPieces.length - 1] = newLastPiece;
if (this._previousChar === CharCode.CarriageReturn) {
this.cr++;
}
}
}
}

10
src/vs/editor/common/model/intervalTree.ts
View File

@@ -12,13 +12,11 @@ import { IModelDecoration } from 'vs/editor/common/model';
// The red-black tree is based on the "Introduction to Algorithms" by Cormen, Leiserson and Rivest.
//
/**
* The class name sort order must match the severity order. Highest severity last.
*/
export const ClassName = {
EditorInfoDecoration: 'squiggly-a-info',
EditorWarningDecoration: 'squiggly-b-warning',
EditorErrorDecoration: 'squiggly-c-error'
EditorHintDecoration: 'squiggly-hint',
EditorInfoDecoration: 'squiggly-info',
EditorWarningDecoration: 'squiggly-warning',
EditorErrorDecoration: 'squiggly-error'
};
/**

660
src/vs/editor/common/model/linesTextBuffer/linesTextBuffer.ts
View File

@@ -1,660 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { Range } from 'vs/editor/common/core/range';
import { Position } from 'vs/editor/common/core/position';
import * as strings from 'vs/base/common/strings';
import * as arrays from 'vs/base/common/arrays';
import { PrefixSumComputer } from 'vs/editor/common/viewModel/prefixSumComputer';
import { ISingleEditOperationIdentifier, IIdentifiedSingleEditOperation, EndOfLinePreference, ITextBuffer, ApplyEditsResult, IInternalModelContentChange } from 'vs/editor/common/model';
import { ITextSnapshot } from 'vs/platform/files/common/files';
export interface IValidatedEditOperation {
sortIndex: number;
identifier: ISingleEditOperationIdentifier;
range: Range;
rangeOffset: number;
rangeLength: number;
lines: string[];
forceMoveMarkers: boolean;
isAutoWhitespaceEdit: boolean;
}
/**
* A processed string with its EOL resolved ready to be turned into an editor model.
*/
export interface ITextSource {
/**
* The text split into lines.
*/
readonly lines: string[];
/**
* The BOM (leading character sequence of the file).
*/
readonly BOM: string;
/**
* The end of line sequence.
*/
readonly EOL: string;
/**
* The text contains Unicode characters classified as "R" or "AL".
*/
readonly containsRTL: boolean;
/**
* The text contains only characters inside the ASCII range 32-126 or \t \r \n
*/
readonly isBasicASCII: boolean;
}
class LinesTextBufferSnapshot implements ITextSnapshot {
private readonly _lines: string[];
private readonly _linesLength: number;
private readonly _eol: string;
private readonly _bom: string;
private _lineIndex: number;
constructor(lines: string[], eol: string, bom: string) {
this._lines = lines;
this._linesLength = this._lines.length;
this._eol = eol;
this._bom = bom;
this._lineIndex = 0;
}
public read(): string {
if (this._lineIndex >= this._linesLength) {
return null;
}
let result: string = null;
if (this._lineIndex === 0) {
result = this._bom + this._lines[this._lineIndex];
} else {
result = this._lines[this._lineIndex];
}
this._lineIndex++;
if (this._lineIndex < this._linesLength) {
result += this._eol;
}
return result;
}
}
export class LinesTextBuffer implements ITextBuffer {
private _lines: string[];
private _BOM: string;
private _EOL: string;
private _mightContainRTL: boolean;
private _mightContainNonBasicASCII: boolean;
private _lineStarts: PrefixSumComputer;
constructor(textSource: ITextSource) {
this._lines = textSource.lines.slice(0);
this._BOM = textSource.BOM;
this._EOL = textSource.EOL;
this._mightContainRTL = textSource.containsRTL;
this._mightContainNonBasicASCII = !textSource.isBasicASCII;
this._constructLineStarts();
}
private _constructLineStarts(): void {
const eolLength = this._EOL.length;
const linesLength = this._lines.length;
const lineStartValues = new Uint32Array(linesLength);
for (let i = 0; i < linesLength; i++) {
lineStartValues[i] = this._lines[i].length + eolLength;
}
this._lineStarts = new PrefixSumComputer(lineStartValues);
}
public equals(other: ITextBuffer): boolean {
if (!(other instanceof LinesTextBuffer)) {
return false;
}
if (this._BOM !== other._BOM) {
return false;
}
if (this._EOL !== other._EOL) {
return false;
}
if (this._lines.length !== other._lines.length) {
return false;
}
for (let i = 0, len = this._lines.length; i < len; i++) {
if (this._lines[i] !== other._lines[i]) {
return false;
}
}
return true;
}
public mightContainRTL(): boolean {
return this._mightContainRTL;
}
public mightContainNonBasicASCII(): boolean {
return this._mightContainNonBasicASCII;
}
public getBOM(): string {
return this._BOM;
}
public getEOL(): string {
return this._EOL;
}
public getOffsetAt(lineNumber: number, column: number): number {
return this._lineStarts.getAccumulatedValue(lineNumber - 2) + column - 1;
}
public getPositionAt(offset: number): Position {
offset = Math.floor(offset);
offset = Math.max(0, offset);
let out = this._lineStarts.getIndexOf(offset);
let lineLength = this._lines[out.index].length;
// Ensure we return a valid position
return new Position(out.index + 1, Math.min(out.remainder + 1, lineLength + 1));
}
public getRangeAt(offset: number, length: number): Range {
const startResult = this._lineStarts.getIndexOf(offset);
const startLineLength = this._lines[startResult.index].length;
const startColumn = Math.min(startResult.remainder + 1, startLineLength + 1);
const endResult = this._lineStarts.getIndexOf(offset + length);
const endLineLength = this._lines[endResult.index].length;
const endColumn = Math.min(endResult.remainder + 1, endLineLength + 1);
return new Range(startResult.index + 1, startColumn, endResult.index + 1, endColumn);
}
private _getEndOfLine(eol: EndOfLinePreference): string {
switch (eol) {
case EndOfLinePreference.LF:
return '\n';
case EndOfLinePreference.CRLF:
return '\r\n';
case EndOfLinePreference.TextDefined:
return this.getEOL();
}
throw new Error('Unknown EOL preference');
}
public getValueInRange(range: Range, eol: EndOfLinePreference): string {
if (range.isEmpty()) {
return '';
}
if (range.startLineNumber === range.endLineNumber) {
return this._lines[range.startLineNumber - 1].substring(range.startColumn - 1, range.endColumn - 1);
}
const lineEnding = this._getEndOfLine(eol);
const startLineIndex = range.startLineNumber - 1;
const endLineIndex = range.endLineNumber - 1;
let resultLines: string[] = [];
resultLines.push(this._lines[startLineIndex].substring(range.startColumn - 1));
for (let i = startLineIndex + 1; i < endLineIndex; i++) {
resultLines.push(this._lines[i]);
}
resultLines.push(this._lines[endLineIndex].substring(0, range.endColumn - 1));
return resultLines.join(lineEnding);
}
public createSnapshot(preserveBOM: boolean): ITextSnapshot {
return new LinesTextBufferSnapshot(this._lines.slice(0), this._EOL, preserveBOM ? this._BOM : '');
}
public getValueLengthInRange(range: Range, eol: EndOfLinePreference): number {
if (range.isEmpty()) {
return 0;
}
if (range.startLineNumber === range.endLineNumber) {
return (range.endColumn - range.startColumn);
}
let startOffset = this.getOffsetAt(range.startLineNumber, range.startColumn);
let endOffset = this.getOffsetAt(range.endLineNumber, range.endColumn);
return endOffset - startOffset;
}
public getLineCount(): number {
return this._lines.length;
}
public getLinesContent(): string[] {
return this._lines.slice(0);
}
public getLength(): number {
return this._lineStarts.getTotalValue();
}
public getLineContent(lineNumber: number): string {
return this._lines[lineNumber - 1];
}
public getLineCharCode(lineNumber: number, index: number): number {
return this._lines[lineNumber - 1].charCodeAt(index);
}
public getLineLength(lineNumber: number): number {
return this._lines[lineNumber - 1].length;
}
public getLineFirstNonWhitespaceColumn(lineNumber: number): number {
const result = strings.firstNonWhitespaceIndex(this._lines[lineNumber - 1]);
if (result === -1) {
return 0;
}
return result + 1;
}
public getLineLastNonWhitespaceColumn(lineNumber: number): number {
const result = strings.lastNonWhitespaceIndex(this._lines[lineNumber - 1]);
if (result === -1) {
return 0;
}
return result + 2;
}
//#region Editing
public setEOL(newEOL: '\r\n' | '\n'): void {
this._EOL = newEOL;
this._constructLineStarts();
}
private static _sortOpsAscending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
let r = Range.compareRangesUsingEnds(a.range, b.range);
if (r === 0) {
return a.sortIndex - b.sortIndex;
}
return r;
}
private static _sortOpsDescending(a: IValidatedEditOperation, b: IValidatedEditOperation): number {
let r = Range.compareRangesUsingEnds(a.range, b.range);
if (r === 0) {
return b.sortIndex - a.sortIndex;
}
return -r;
}
public applyEdits(rawOperations: IIdentifiedSingleEditOperation[], recordTrimAutoWhitespace: boolean): ApplyEditsResult {
if (rawOperations.length === 0) {
return new ApplyEditsResult([], [], []);
}
let mightContainRTL = this._mightContainRTL;
let mightContainNonBasicASCII = this._mightContainNonBasicASCII;
let canReduceOperations = true;
let operations: IValidatedEditOperation[] = [];
for (let i = 0; i < rawOperations.length; i++) {
let op = rawOperations[i];
if (canReduceOperations && op._isTracked) {
canReduceOperations = false;
}
let validatedRange = op.range;
if (!mightContainRTL && op.text) {
// check if the new inserted text contains RTL
mightContainRTL = strings.containsRTL(op.text);
}
if (!mightContainNonBasicASCII && op.text) {
mightContainNonBasicASCII = !strings.isBasicASCII(op.text);
}
operations[i] = {
sortIndex: i,
identifier: op.identifier || null,
range: validatedRange,
rangeOffset: this.getOffsetAt(validatedRange.startLineNumber, validatedRange.startColumn),
rangeLength: this.getValueLengthInRange(validatedRange, EndOfLinePreference.TextDefined),
lines: op.text ? op.text.split(/\r\n|\r|\n/) : null,
forceMoveMarkers: op.forceMoveMarkers || false,
isAutoWhitespaceEdit: op.isAutoWhitespaceEdit || false
};
}
// Sort operations ascending
operations.sort(LinesTextBuffer._sortOpsAscending);
for (let i = 0, count = operations.length - 1; i < count; i++) {
let rangeEnd = operations[i].range.getEndPosition();
let nextRangeStart = operations[i + 1].range.getStartPosition();
if (nextRangeStart.isBefore(rangeEnd)) {
// overlapping ranges
throw new Error('Overlapping ranges are not allowed!');
}
}
if (canReduceOperations) {
operations = this._reduceOperations(operations);
}
// Delta encode operations
let reverseRanges = LinesTextBuffer._getInverseEditRanges(operations);
let newTrimAutoWhitespaceCandidates: { lineNumber: number, oldContent: string }[] = [];
for (let i = 0; i < operations.length; i++) {
let op = operations[i];
let reverseRange = reverseRanges[i];
if (recordTrimAutoWhitespace && op.isAutoWhitespaceEdit && op.range.isEmpty()) {
// Record already the future line numbers that might be auto whitespace removal candidates on next edit
for (let lineNumber = reverseRange.startLineNumber; lineNumber <= reverseRange.endLineNumber; lineNumber++) {
let currentLineContent = '';
if (lineNumber === reverseRange.startLineNumber) {
currentLineContent = this.getLineContent(op.range.startLineNumber);
if (strings.firstNonWhitespaceIndex(currentLineContent) !== -1) {
continue;
}
}
newTrimAutoWhitespaceCandidates.push({ lineNumber: lineNumber, oldContent: currentLineContent });
}
}
}
let reverseOperations: IIdentifiedSingleEditOperation[] = [];
for (let i = 0; i < operations.length; i++) {
let op = operations[i];
let reverseRange = reverseRanges[i];
reverseOperations[i] = {
identifier: op.identifier,
range: reverseRange,
text: this.getValueInRange(op.range, EndOfLinePreference.TextDefined),
forceMoveMarkers: op.forceMoveMarkers
};
}
this._mightContainRTL = mightContainRTL;
this._mightContainNonBasicASCII = mightContainNonBasicASCII;
const contentChanges = this._doApplyEdits(operations);
let trimAutoWhitespaceLineNumbers: number[] = null;
if (recordTrimAutoWhitespace && newTrimAutoWhitespaceCandidates.length > 0) {
// sort line numbers auto whitespace removal candidates for next edit descending
newTrimAutoWhitespaceCandidates.sort((a, b) => b.lineNumber - a.lineNumber);
trimAutoWhitespaceLineNumbers = [];
for (let i = 0, len = newTrimAutoWhitespaceCandidates.length; i < len; i++) {
let lineNumber = newTrimAutoWhitespaceCandidates[i].lineNumber;
if (i > 0 && newTrimAutoWhitespaceCandidates[i - 1].lineNumber === lineNumber) {
// Do not have the same line number twice
continue;
}
let prevContent = newTrimAutoWhitespaceCandidates[i].oldContent;
let lineContent = this.getLineContent(lineNumber);
if (lineContent.length === 0 || lineContent === prevContent || strings.firstNonWhitespaceIndex(lineContent) !== -1) {
continue;
}
trimAutoWhitespaceLineNumbers.push(lineNumber);
}
}
return new ApplyEditsResult(
reverseOperations,
contentChanges,
trimAutoWhitespaceLineNumbers
);
}
/**
* Transform operations such that they represent the same logic edit,
* but that they also do not cause OOM crashes.
*/
private _reduceOperations(operations: IValidatedEditOperation[]): IValidatedEditOperation[] {
if (operations.length < 1000) {
// We know from empirical testing that a thousand edits work fine regardless of their shape.
return operations;
}
// At one point, due to how events are emitted and how each operation is handled,
// some operations can trigger a high ammount of temporary string allocations,
// that will immediately get edited again.
// e.g. a formatter inserting ridiculous ammounts of \n on a model with a single line
// Therefore, the strategy is to collapse all the operations into a huge single edit operation
return [this._toSingleEditOperation(operations)];
}
_toSingleEditOperation(operations: IValidatedEditOperation[]): IValidatedEditOperation {
let forceMoveMarkers = false,
firstEditRange = operations[0].range,
lastEditRange = operations[operations.length - 1].range,
entireEditRange = new Range(firstEditRange.startLineNumber, firstEditRange.startColumn, lastEditRange.endLineNumber, lastEditRange.endColumn),
lastEndLineNumber = firstEditRange.startLineNumber,
lastEndColumn = firstEditRange.startColumn,
result: string[] = [];
for (let i = 0, len = operations.length; i < len; i++) {
let operation = operations[i],
range = operation.range;
forceMoveMarkers = forceMoveMarkers || operation.forceMoveMarkers;
// (1) -- Push old text
for (let lineNumber = lastEndLineNumber; lineNumber < range.startLineNumber; lineNumber++) {
if (lineNumber === lastEndLineNumber) {
result.push(this._lines[lineNumber - 1].substring(lastEndColumn - 1));
} else {
result.push('\n');
result.push(this._lines[lineNumber - 1]);
}
}
if (range.startLineNumber === lastEndLineNumber) {
result.push(this._lines[range.startLineNumber - 1].substring(lastEndColumn - 1, range.startColumn - 1));
} else {
result.push('\n');
result.push(this._lines[range.startLineNumber - 1].substring(0, range.startColumn - 1));
}
// (2) -- Push new text
if (operation.lines) {
for (let j = 0, lenJ = operation.lines.length; j < lenJ; j++) {
if (j !== 0) {
result.push('\n');
}
result.push(operation.lines[j]);
}
}
lastEndLineNumber = operation.range.endLineNumber;
lastEndColumn = operation.range.endColumn;
}
return {
sortIndex: 0,
identifier: operations[0].identifier,
range: entireEditRange,
rangeOffset: this.getOffsetAt(entireEditRange.startLineNumber, entireEditRange.startColumn),
rangeLength: this.getValueLengthInRange(entireEditRange, EndOfLinePreference.TextDefined),
lines: result.join('').split('\n'),
forceMoveMarkers: forceMoveMarkers,
isAutoWhitespaceEdit: false
};
}
private _setLineContent(lineNumber: number, content: string): void {
this._lines[lineNumber - 1] = content;
this._lineStarts.changeValue(lineNumber - 1, content.length + this._EOL.length);
}
private _doApplyEdits(operations: IValidatedEditOperation[]): IInternalModelContentChange[] {
// Sort operations descending
operations.sort(LinesTextBuffer._sortOpsDescending);
let contentChanges: IInternalModelContentChange[] = [];
for (let i = 0, len = operations.length; i < len; i++) {
const op = operations[i];
const startLineNumber = op.range.startLineNumber;
const startColumn = op.range.startColumn;
const endLineNumber = op.range.endLineNumber;
const endColumn = op.range.endColumn;
if (startLineNumber === endLineNumber && startColumn === endColumn && (!op.lines || op.lines.length === 0)) {
// no-op
continue;
}
const deletingLinesCnt = endLineNumber - startLineNumber;
const insertingLinesCnt = (op.lines ? op.lines.length - 1 : 0);
const editingLinesCnt = Math.min(deletingLinesCnt, insertingLinesCnt);
for (let j = editingLinesCnt; j >= 0; j--) {
const editLineNumber = startLineNumber + j;
let editText = (op.lines ? op.lines[j] : '');
if (editLineNumber === startLineNumber || editLineNumber === endLineNumber) {
const editStartColumn = (editLineNumber === startLineNumber ? startColumn : 1);
const editEndColumn = (editLineNumber === endLineNumber ? endColumn : this.getLineLength(editLineNumber) + 1);
editText = (
this._lines[editLineNumber - 1].substring(0, editStartColumn - 1)
+ editText
+ this._lines[editLineNumber - 1].substring(editEndColumn - 1)
);
}
this._setLineContent(editLineNumber, editText);
}
if (editingLinesCnt < deletingLinesCnt) {
// Must delete some lines
const spliceStartLineNumber = startLineNumber + editingLinesCnt;
const endLineRemains = this._lines[endLineNumber - 1].substring(endColumn - 1);
// Reconstruct first line
this._setLineContent(spliceStartLineNumber, this._lines[spliceStartLineNumber - 1] + endLineRemains);
this._lines.splice(spliceStartLineNumber, endLineNumber - spliceStartLineNumber);
this._lineStarts.removeValues(spliceStartLineNumber, endLineNumber - spliceStartLineNumber);
}
if (editingLinesCnt < insertingLinesCnt) {
// Must insert some lines
const spliceLineNumber = startLineNumber + editingLinesCnt;
let spliceColumn = (spliceLineNumber === startLineNumber ? startColumn : 1);
if (op.lines) {
spliceColumn += op.lines[editingLinesCnt].length;
}
// Split last line
const leftoverLine = this._lines[spliceLineNumber - 1].substring(spliceColumn - 1);
this._setLineContent(spliceLineNumber, this._lines[spliceLineNumber - 1].substring(0, spliceColumn - 1));
// Lines in the middle
let newLines: string[] = new Array<string>(insertingLinesCnt - editingLinesCnt);
let newLinesLengths = new Uint32Array(insertingLinesCnt - editingLinesCnt);
for (let j = editingLinesCnt + 1; j <= insertingLinesCnt; j++) {
newLines[j - editingLinesCnt - 1] = op.lines[j];
newLinesLengths[j - editingLinesCnt - 1] = op.lines[j].length + this._EOL.length;
}
newLines[newLines.length - 1] += leftoverLine;
newLinesLengths[newLines.length - 1] += leftoverLine.length;
this._lines = arrays.arrayInsert(this._lines, startLineNumber + editingLinesCnt, newLines);
this._lineStarts.insertValues(startLineNumber + editingLinesCnt, newLinesLengths);
}
const contentChangeRange = new Range(startLineNumber, startColumn, endLineNumber, endColumn);
const text = (op.lines ? op.lines.join(this.getEOL()) : '');
contentChanges.push({
range: contentChangeRange,
rangeLength: op.rangeLength,
text: text,
rangeOffset: op.rangeOffset,
forceMoveMarkers: op.forceMoveMarkers
});
}
return contentChanges;
}
/**
* Assumes `operations` are validated and sorted ascending
*/
public static _getInverseEditRanges(operations: IValidatedEditOperation[]): Range[] {
let result: Range[] = [];
let prevOpEndLineNumber: number;
let prevOpEndColumn: number;
let prevOp: IValidatedEditOperation = null;
for (let i = 0, len = operations.length; i < len; i++) {
let op = operations[i];
let startLineNumber: number;
let startColumn: number;
if (prevOp) {
if (prevOp.range.endLineNumber === op.range.startLineNumber) {
startLineNumber = prevOpEndLineNumber;
startColumn = prevOpEndColumn + (op.range.startColumn - prevOp.range.endColumn);
} else {
startLineNumber = prevOpEndLineNumber + (op.range.startLineNumber - prevOp.range.endLineNumber);
startColumn = op.range.startColumn;
}
} else {
startLineNumber = op.range.startLineNumber;
startColumn = op.range.startColumn;
}
let resultRange: Range;
if (op.lines && op.lines.length > 0) {
// the operation inserts something
let lineCount = op.lines.length;
let firstLine = op.lines[0];
let lastLine = op.lines[lineCount - 1];
if (lineCount === 1) {
// single line insert
resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn + firstLine.length);
} else {
// multi line insert
resultRange = new Range(startLineNumber, startColumn, startLineNumber + lineCount - 1, lastLine.length + 1);
}
} else {
// There is nothing to insert
resultRange = new Range(startLineNumber, startColumn, startLineNumber, startColumn);
}
prevOpEndLineNumber = resultRange.endLineNumber;
prevOpEndColumn = resultRange.endColumn;
result.push(resultRange);
prevOp = op;
}
return result;
}
//#endregion
}

139
src/vs/editor/common/model/linesTextBuffer/linesTextBufferBuilder.ts
View File

@@ -1,139 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as strings from 'vs/base/common/strings';
import { CharCode } from 'vs/base/common/charCode';
import { ITextBufferBuilder, ITextBufferFactory, ITextBuffer, DefaultEndOfLine } from 'vs/editor/common/model';
import { IRawTextSource, TextSource } from 'vs/editor/common/model/linesTextBuffer/textSource';
import { LinesTextBuffer } from 'vs/editor/common/model/linesTextBuffer/linesTextBuffer';
export class TextBufferFactory implements ITextBufferFactory {
constructor(public readonly rawTextSource: IRawTextSource) {
}
public create(defaultEOL: DefaultEndOfLine): ITextBuffer {
const textSource = TextSource.fromRawTextSource(this.rawTextSource, defaultEOL);
return new LinesTextBuffer(textSource);
}
public getFirstLineText(lengthLimit: number): string {
return this.rawTextSource.lines[0].substr(0, lengthLimit);
}
}
class ModelLineBasedBuilder {
private BOM: string;
private lines: string[];
private currLineIndex: number;
constructor() {
this.BOM = '';
this.lines = [];
this.currLineIndex = 0;
}
public acceptLines(lines: string[]): void {
if (this.currLineIndex === 0) {
// Remove the BOM (if present)
if (strings.startsWithUTF8BOM(lines[0])) {
this.BOM = strings.UTF8_BOM_CHARACTER;
lines[0] = lines[0].substr(1);
}
}
for (let i = 0, len = lines.length; i < len; i++) {
this.lines[this.currLineIndex++] = lines[i];
}
}
public finish(carriageReturnCnt: number, containsRTL: boolean, isBasicASCII: boolean): TextBufferFactory {
return new TextBufferFactory({
BOM: this.BOM,
lines: this.lines,
containsRTL: containsRTL,
totalCRCount: carriageReturnCnt,
isBasicASCII,
});
}
}
export class LinesTextBufferBuilder implements ITextBufferBuilder {
private leftoverPrevChunk: string;
private leftoverEndsInCR: boolean;
private totalCRCount: number;
private lineBasedBuilder: ModelLineBasedBuilder;
private containsRTL: boolean;
private isBasicASCII: boolean;
constructor() {
this.leftoverPrevChunk = '';
this.leftoverEndsInCR = false;
this.totalCRCount = 0;
this.lineBasedBuilder = new ModelLineBasedBuilder();
this.containsRTL = false;
this.isBasicASCII = true;
}
private _updateCRCount(chunk: string): void {
// Count how many \r are present in chunk to determine the majority EOL sequence
let chunkCarriageReturnCnt = 0;
let lastCarriageReturnIndex = -1;
while ((lastCarriageReturnIndex = chunk.indexOf('\r', lastCarriageReturnIndex + 1)) !== -1) {
chunkCarriageReturnCnt++;
}
this.totalCRCount += chunkCarriageReturnCnt;
}
public acceptChunk(chunk: string): void {
if (chunk.length === 0) {
return;
}
this._updateCRCount(chunk);
if (!this.containsRTL) {
this.containsRTL = strings.containsRTL(chunk);
}
if (this.isBasicASCII) {
this.isBasicASCII = strings.isBasicASCII(chunk);
}
// Avoid dealing with a chunk that ends in \r (push the \r to the next chunk)
if (this.leftoverEndsInCR) {
chunk = '\r' + chunk;
}
if (chunk.charCodeAt(chunk.length - 1) === CharCode.CarriageReturn) {
this.leftoverEndsInCR = true;
chunk = chunk.substr(0, chunk.length - 1);
} else {
this.leftoverEndsInCR = false;
}
let lines = chunk.split(/\r\n|\r|\n/);
if (lines.length === 1) {
// no \r or \n encountered
this.leftoverPrevChunk += lines[0];
return;
}
lines[0] = this.leftoverPrevChunk + lines[0];
this.lineBasedBuilder.acceptLines(lines.slice(0, lines.length - 1));
this.leftoverPrevChunk = lines[lines.length - 1];
}
public finish(): TextBufferFactory {
let finalLines = [this.leftoverPrevChunk];
if (this.leftoverEndsInCR) {
finalLines.push('');
}
this.lineBasedBuilder.acceptLines(finalLines);
return this.lineBasedBuilder.finish(this.totalCRCount, this.containsRTL, this.isBasicASCII);
}
}

66
src/vs/editor/common/model/linesTextBuffer/textSource.ts
View File

@@ -1,66 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the Source EULA. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { DefaultEndOfLine } from 'vs/editor/common/model';
import { ITextSource } from 'vs/editor/common/model/linesTextBuffer/linesTextBuffer';
/**
* A processed string ready to be turned into an editor model.
*/
export interface IRawTextSource {
/**
* The text split into lines.
*/
readonly lines: string[];
/**
* The BOM (leading character sequence of the file).
*/
readonly BOM: string;
/**
* The number of lines ending with '\r\n'
*/
readonly totalCRCount: number;
/**
* The text contains Unicode characters classified as "R" or "AL".
*/
readonly containsRTL: boolean;
/**
* The text contains only characters inside the ASCII range 32-126 or \t \r \n
*/
readonly isBasicASCII: boolean;
}
export class TextSource {
/**
* if text source is empty or with precisely one line, returns null. No end of line is detected.
* if text source contains more lines ending with '\r\n', returns '\r\n'.
* Otherwise returns '\n'. More lines end with '\n'.
*/
private static _getEOL(rawTextSource: IRawTextSource, defaultEOL: DefaultEndOfLine): '\r\n' | '\n' {
const lineFeedCnt = rawTextSource.lines.length - 1;
if (lineFeedCnt === 0) {
// This is an empty file or a file with precisely one line
return (defaultEOL === DefaultEndOfLine.LF ? '\n' : '\r\n');
}
if (rawTextSource.totalCRCount > lineFeedCnt / 2) {
// More than half of the file contains \r\n ending lines
return '\r\n';
}
// At least one line more ends in \n
return '\n';
}
public static fromRawTextSource(rawTextSource: IRawTextSource, defaultEOL: DefaultEndOfLine): ITextSource {
return {
lines: rawTextSource.lines,
BOM: rawTextSource.BOM,
EOL: TextSource._getEOL(rawTextSource, defaultEOL),
containsRTL: rawTextSource.containsRTL,
isBasicASCII: rawTextSource.isBasicASCII,
};
}
}

284
src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase.ts
View File

@@ -9,8 +9,11 @@ import { CharCode } from 'vs/base/common/charCode';
import { Range } from 'vs/editor/common/core/range';
import { ITextSnapshot } from 'vs/platform/files/common/files';
import { leftest, righttest, updateTreeMetadata, rbDelete, fixInsert, NodeColor, SENTINEL, TreeNode } from 'vs/editor/common/model/pieceTreeTextBuffer/rbTreeBase';
import { SearchData, isValidMatch, Searcher, createFindMatch } from 'vs/editor/common/model/textModelSearch';
import { FindMatch } from 'vs/editor/common/model';
// const lfRegex = new RegExp(/\r\n|\r|\n/g);
export const AverageBufferSize = 65535;
export function createUintArray(arr: number[]): Uint32Array | Uint16Array {
let r;
@@ -33,7 +36,7 @@ export class LineStarts {
) { }
}
export function createLineStartsFast(str: string, readonly: boolean = true): Uint32Array | number[] {
export function createLineStartsFast(str: string, readonly: boolean = true): Uint32Array | Uint16Array | number[] {
let r: number[] = [0], rLength = 1;
for (let i = 0, len = str.length; i < len; i++) {
@@ -309,7 +312,7 @@ export class PieceTreeBase {
}
normalizeEOL(eol: '\r\n' | '\n') {
let averageBufferSize = 65536;
let averageBufferSize = AverageBufferSize;
let min = averageBufferSize - Math.floor(averageBufferSize / 3);
let max = min * 2;
@@ -446,7 +449,7 @@ export class PieceTreeBase {
return new Position(1, 1);
}
public getValueInRange(range: Range): string {
public getValueInRange(range: Range, eol?: string): string {
if (range.startLineNumber === range.endLineNumber && range.startColumn === range.endColumn) {
return '';
}
@@ -454,7 +457,21 @@ export class PieceTreeBase {
let startPosition = this.nodeAt2(range.startLineNumber, range.startColumn);
let endPosition = this.nodeAt2(range.endLineNumber, range.endColumn);
return this.getValueInRange2(startPosition, endPosition);
let value = this.getValueInRange2(startPosition, endPosition);
if (eol) {
if (eol !== this._EOL || !this._EOLNormalized) {
return value.replace(/\r\n|\r|\n/g, eol);
}
if (eol === this.getEOL() && this._EOLNormalized) {
if (eol === '\r\n') {
}
return value;
}
return value.replace(/\r\n|\r|\n/g, eol);
}
return value;
}
public getValueInRange2(startPosition: NodePosition, endPosition: NodePosition): string {
@@ -520,11 +537,23 @@ export class PieceTreeBase {
public getLineCharCode(lineNumber: number, index: number): number {
let nodePos = this.nodeAt2(lineNumber, index + 1);
let buffer = this._buffers[nodePos.node.piece.bufferIndex];
let startOffset = this.offsetInBuffer(nodePos.node.piece.bufferIndex, nodePos.node.piece.start);
let targetOffset = startOffset + index;
if (nodePos.remainder === nodePos.node.piece.length) {
// the char we want to fetch is at the head of next node.
let matchingNode = nodePos.node.next();
if (!matchingNode) {
return 0;
}
return buffer.buffer.charCodeAt(targetOffset);
let buffer = this._buffers[matchingNode.piece.bufferIndex];
let startOffset = this.offsetInBuffer(matchingNode.piece.bufferIndex, matchingNode.piece.start);
return buffer.buffer.charCodeAt(startOffset);
} else {
let buffer = this._buffers[nodePos.node.piece.bufferIndex];
let startOffset = this.offsetInBuffer(nodePos.node.piece.bufferIndex, nodePos.node.piece.start);
let targetOffset = startOffset + nodePos.remainder;
return buffer.buffer.charCodeAt(targetOffset);
}
}
public getLineLength(lineNumber: number): number {
@@ -535,6 +564,151 @@ export class PieceTreeBase {
return this.getOffsetAt(lineNumber + 1, 1) - this.getOffsetAt(lineNumber, 1) - this._EOLLength;
}
public findMatchesInNode(node: TreeNode, searcher: Searcher, startLineNumber: number, startColumn: number, startCursor: BufferCursor, endCursor: BufferCursor, searchData: SearchData, captureMatches: boolean, limitResultCount: number, resultLen: number, result: FindMatch[]) {
let buffer = this._buffers[node.piece.bufferIndex];
let startOffsetInBuffer = this.offsetInBuffer(node.piece.bufferIndex, node.piece.start);
let start = this.offsetInBuffer(node.piece.bufferIndex, startCursor);
let end = this.offsetInBuffer(node.piece.bufferIndex, endCursor);
let m: RegExpExecArray;
// Reset regex to search from the beginning
searcher.reset(start);
let ret: BufferCursor = { line: 0, column: 0 };
do {
m = searcher.next(buffer.buffer);
if (m) {
if (m.index >= end) {
return resultLen;
}
this.positionInBuffer(node, m.index - startOffsetInBuffer, ret);
let lineFeedCnt = this.getLineFeedCnt(node.piece.bufferIndex, startCursor, ret);
let retStartColumn = ret.line === startCursor.line ? ret.column - startCursor.column + startColumn : ret.column + 1;
let retEndColumn = retStartColumn + m[0].length;
result[resultLen++] = createFindMatch(new Range(startLineNumber + lineFeedCnt, retStartColumn, startLineNumber + lineFeedCnt, retEndColumn), m, captureMatches);
if (m.index + m[0].length >= end) {
return resultLen;
}
if (resultLen >= limitResultCount) {
return resultLen;
}
}
} while (m);
return resultLen;
}
public findMatchesLineByLine(searchRange: Range, searchData: SearchData, captureMatches: boolean, limitResultCount: number): FindMatch[] {
const result: FindMatch[] = [];
let resultLen = 0;
const searcher = new Searcher(searchData.wordSeparators, searchData.regex);
let startPostion = this.nodeAt2(searchRange.startLineNumber, searchRange.startColumn);
if (startPostion === null) {
return [];
}
let endPosition = this.nodeAt2(searchRange.endLineNumber, searchRange.endColumn);
if (endPosition === null) {
return [];
}
let start = this.positionInBuffer(startPostion.node, startPostion.remainder);
let end = this.positionInBuffer(endPosition.node, endPosition.remainder);
if (startPostion.node === endPosition.node) {
this.findMatchesInNode(startPostion.node, searcher, searchRange.startLineNumber, searchRange.startColumn, start, end, searchData, captureMatches, limitResultCount, resultLen, result);
return result;
}
let startLineNumber = searchRange.startLineNumber;
let currentNode = startPostion.node;
while (currentNode !== endPosition.node) {
let lineBreakCnt = this.getLineFeedCnt(currentNode.piece.bufferIndex, start, currentNode.piece.end);
if (lineBreakCnt >= 1) {
// last line break position
let lineStarts = this._buffers[currentNode.piece.bufferIndex].lineStarts;
let startOffsetInBuffer = this.offsetInBuffer(currentNode.piece.bufferIndex, currentNode.piece.start);
let nextLineStartOffset = lineStarts[start.line + lineBreakCnt];
let startColumn = startLineNumber === searchRange.startLineNumber ? searchRange.startColumn : 1;
resultLen = this.findMatchesInNode(currentNode, searcher, startLineNumber, startColumn, start, this.positionInBuffer(currentNode, nextLineStartOffset - startOffsetInBuffer), searchData, captureMatches, limitResultCount, resultLen, result);
if (resultLen >= limitResultCount) {
return result;
}
startLineNumber += lineBreakCnt;
}
let startColumn = startLineNumber === searchRange.startLineNumber ? searchRange.startColumn - 1 : 0;
// search for the remaining content
if (startLineNumber === searchRange.endLineNumber) {
const text = this.getLineContent(startLineNumber).substring(startColumn, searchRange.endColumn - 1);
resultLen = this._findMatchesInLine(searchData, searcher, text, searchRange.endLineNumber, startColumn, resultLen, result, captureMatches, limitResultCount);
return result;
}
resultLen = this._findMatchesInLine(searchData, searcher, this.getLineContent(startLineNumber).substr(startColumn), startLineNumber, startColumn, resultLen, result, captureMatches, limitResultCount);
if (resultLen >= limitResultCount) {
return result;
}
startLineNumber++;
startPostion = this.nodeAt2(startLineNumber, 1);
currentNode = startPostion.node;
start = this.positionInBuffer(startPostion.node, startPostion.remainder);
}
if (startLineNumber === searchRange.endLineNumber) {
let startColumn = startLineNumber === searchRange.startLineNumber ? searchRange.startColumn - 1 : 0;
const text = this.getLineContent(startLineNumber).substring(startColumn, searchRange.endColumn - 1);
resultLen = this._findMatchesInLine(searchData, searcher, text, searchRange.endLineNumber, startColumn, resultLen, result, captureMatches, limitResultCount);
return result;
}
let startColumn = startLineNumber === searchRange.startLineNumber ? searchRange.startColumn : 1;
resultLen = this.findMatchesInNode(endPosition.node, searcher, startLineNumber, startColumn, start, end, searchData, captureMatches, limitResultCount, resultLen, result);
return result;
}
private _findMatchesInLine(searchData: SearchData, searcher: Searcher, text: string, lineNumber: number, deltaOffset: number, resultLen: number, result: FindMatch[], captureMatches: boolean, limitResultCount: number): number {
const wordSeparators = searchData.wordSeparators;
if (!captureMatches && searchData.simpleSearch) {
const searchString = searchData.simpleSearch;
const searchStringLen = searchString.length;
const textLength = text.length;
let lastMatchIndex = -searchStringLen;
while ((lastMatchIndex = text.indexOf(searchString, lastMatchIndex + searchStringLen)) !== -1) {
if (!wordSeparators || isValidMatch(wordSeparators, text, textLength, lastMatchIndex, searchStringLen)) {
result[resultLen++] = new FindMatch(new Range(lineNumber, lastMatchIndex + 1 + deltaOffset, lineNumber, lastMatchIndex + 1 + searchStringLen + deltaOffset), null);
if (resultLen >= limitResultCount) {
return resultLen;
}
}
}
return resultLen;
}
let m: RegExpExecArray;
// Reset regex to search from the beginning
searcher.reset(0);
do {
m = searcher.next(text);
if (m) {
result[resultLen++] = createFindMatch(new Range(lineNumber, m.index + 1 + deltaOffset, lineNumber, m.index + 1 + m[0].length + deltaOffset), m, captureMatches);
if (resultLen >= limitResultCount) {
return resultLen;
}
}
} while (m);
return resultLen;
}
// #endregion
// #region Piece Table
@@ -551,7 +725,8 @@ export class PieceTreeBase {
if (node.piece.bufferIndex === 0 &&
piece.end.line === this._lastChangeBufferPos.line &&
piece.end.column === this._lastChangeBufferPos.column &&
(nodeStartOffset + piece.length === offset)
(nodeStartOffset + piece.length === offset) &&
value.length < AverageBufferSize
) {
// changed buffer
this.appendToNode(node, value);
@@ -608,19 +783,27 @@ export class PieceTreeBase {
this.deleteNodeTail(node, insertPosInBuffer);
}
let newPiece = this.createNewPiece(value);
let newPieces = this.createNewPieces(value);
if (newRightPiece.length > 0) {
this.rbInsertRight(node, newRightPiece);
}
this.rbInsertRight(node, newPiece);
let tmpNode = node;
for (let k = 0; k < newPieces.length; k++) {
tmpNode = this.rbInsertRight(tmpNode, newPieces[k]);
}
this.deleteNodes(nodesToDel);
} else {
this.insertContentToNodeRight(value, node);
}
} else {
// insert new node
let piece = this.createNewPiece(value);
this.rbInsertLeft(null, piece);
let pieces = this.createNewPieces(value);
let node = this.rbInsertLeft(null, pieces[0]);
for (let k = 1; k < pieces.length; k++) {
node = this.rbInsertRight(node, pieces[k]);
}
}
// todo, this is too brutal. Total line feed count should be updated the same way as lf_left.
@@ -726,8 +909,11 @@ export class PieceTreeBase {
}
}
let newPiece = this.createNewPiece(value);
let newNode = this.rbInsertLeft(node, newPiece);
let newPieces = this.createNewPieces(value);
let newNode = this.rbInsertLeft(node, newPieces[newPieces.length - 1]);
for (let k = newPieces.length - 2; k >= 0; k--) {
newNode = this.rbInsertLeft(newNode, newPieces[k]);
}
this.validateCRLFWithPrevNode(newNode);
this.deleteNodes(nodesToDel);
}
@@ -739,12 +925,18 @@ export class PieceTreeBase {
value += '\n';
}
let newPiece = this.createNewPiece(value);
let newNode = this.rbInsertRight(node, newPiece);
let newPieces = this.createNewPieces(value);
let newNode = this.rbInsertRight(node, newPieces[0]);
let tmpNode = newNode;
for (let k = 1; k < newPieces.length; k++) {
tmpNode = this.rbInsertRight(tmpNode, newPieces[k]);
}
this.validateCRLFWithPrevNode(newNode);
}
positionInBuffer(node: TreeNode, remainder: number): BufferCursor {
positionInBuffer(node: TreeNode, remainder: number, ret?: BufferCursor): BufferCursor {
let piece = node.piece;
let bufferIndex = node.piece.bufferIndex;
let lineStarts = this._buffers[bufferIndex].lineStarts;
@@ -780,6 +972,12 @@ export class PieceTreeBase {
}
}
if (ret) {
ret.line = mid;
ret.column = offset - midStart;
return null;
}
return {
line: mid,
column: offset - midStart
@@ -827,7 +1025,47 @@ export class PieceTreeBase {
}
}
createNewPiece(text: string): Piece {
createNewPieces(text: string): Piece[] {
if (text.length > AverageBufferSize) {
// the content is large, operations like substring, charCode becomes slow
// so here we split it into smaller chunks, just like what we did for CR/LF normalization
let newPieces = [];
while (text.length > AverageBufferSize) {
const lastChar = text.charCodeAt(AverageBufferSize - 1);
let splitText;
if (lastChar === CharCode.CarriageReturn || (lastChar >= 0xd800 && lastChar <= 0xdbff)) {
// last character is \r or a high surrogate => keep it back
splitText = text.substring(0, AverageBufferSize - 1);
text = text.substring(AverageBufferSize - 1);
} else {
splitText = text.substring(0, AverageBufferSize);
text = text.substring(AverageBufferSize);
}
let lineStarts = createLineStartsFast(splitText);
newPieces.push(new Piece(
this._buffers.length, /* buffer index */
{ line: 0, column: 0 },
{ line: lineStarts.length - 1, column: splitText.length - lineStarts[lineStarts.length - 1] },
lineStarts.length - 1,
splitText.length
));
this._buffers.push(new StringBuffer(splitText, lineStarts));
}
let lineStarts = createLineStartsFast(text);
newPieces.push(new Piece(
this._buffers.length, /* buffer index */
{ line: 0, column: 0 },
{ line: lineStarts.length - 1, column: text.length - lineStarts[lineStarts.length - 1] },
lineStarts.length - 1,
text.length
));
this._buffers.push(new StringBuffer(text, lineStarts));
return newPieces;
}
let startOffset = this._buffers[0].buffer.length;
const lineStarts = createLineStartsFast(text, false);
@@ -862,14 +1100,14 @@ export class PieceTreeBase {
let endColumn = endOffset - this._buffers[0].lineStarts[endIndex];
let endPos = { line: endIndex, column: endColumn };
let newPiece = new Piece(
0,
0, /** todo */
start,
endPos,
this.getLineFeedCnt(0, start, endPos),
endOffset - startOffset
);
this._lastChangeBufferPos = endPos;
return newPiece;
return [newPiece];
}
getLinesRawContent(): string {
@@ -1352,8 +1590,8 @@ export class PieceTreeBase {
}
// create new piece which contains \r\n
let piece = this.createNewPiece('\r\n');
this.rbInsertRight(prev, piece);
let pieces = this.createNewPieces('\r\n');
this.rbInsertRight(prev, pieces[0]);
// delete empty nodes
for (let i = 0; i < nodesToDel.length; i++) {

50
src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts
View File

@@ -7,10 +7,25 @@
import { Range } from 'vs/editor/common/core/range';
import { Position } from 'vs/editor/common/core/position';
import * as strings from 'vs/base/common/strings';
import { IValidatedEditOperation } from 'vs/editor/common/model/linesTextBuffer/linesTextBuffer';
import { PieceTreeBase, StringBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
import { IIdentifiedSingleEditOperation, EndOfLinePreference, ITextBuffer, ApplyEditsResult, IInternalModelContentChange } from 'vs/editor/common/model';
import { IIdentifiedSingleEditOperation, EndOfLinePreference, ITextBuffer, ApplyEditsResult, IInternalModelContentChange, FindMatch, ISingleEditOperationIdentifier } from 'vs/editor/common/model';
import { ITextSnapshot } from 'vs/platform/files/common/files';
import { SearchData } from 'vs/editor/common/model/textModelSearch';
export interface IValidatedEditOperation {
sortIndex: number;
identifier: ISingleEditOperationIdentifier;
range: Range;
rangeOffset: number;
rangeLength: number;
lines: string[];
forceMoveMarkers: boolean;
isAutoWhitespaceEdit: boolean;
}
export interface IReverseSingleEditOperation extends IIdentifiedSingleEditOperation {
sortIndex: number;
}
export class PieceTreeTextBuffer implements ITextBuffer {
private _pieceTree: PieceTreeBase;
@@ -76,8 +91,7 @@ export class PieceTreeTextBuffer implements ITextBuffer {
}
const lineEnding = this._getEndOfLine(eol);
const text = this._pieceTree.getValueInRange(range);
return text.replace(/\r\n|\r|\n/g, lineEnding);
return this._pieceTree.getValueInRange(range, lineEnding);
}
public getValueLengthInRange(range: Range, eol: EndOfLinePreference = EndOfLinePreference.TextDefined): number {
@@ -192,13 +206,17 @@ export class PieceTreeTextBuffer implements ITextBuffer {
// Sort operations ascending
operations.sort(PieceTreeTextBuffer._sortOpsAscending);
let hasTouchingRanges = false;
for (let i = 0, count = operations.length - 1; i < count; i++) {
let rangeEnd = operations[i].range.getEndPosition();
let nextRangeStart = operations[i + 1].range.getStartPosition();
if (nextRangeStart.isBefore(rangeEnd)) {
// overlapping ranges
throw new Error('Overlapping ranges are not allowed!');
if (nextRangeStart.isBeforeOrEqual(rangeEnd)) {
if (nextRangeStart.isBefore(rangeEnd)) {
// overlapping ranges
throw new Error('Overlapping ranges are not allowed!');
}
hasTouchingRanges = true;
}
}
@@ -229,12 +247,13 @@ export class PieceTreeTextBuffer implements ITextBuffer {
}
}
let reverseOperations: IIdentifiedSingleEditOperation[] = [];
let reverseOperations: IReverseSingleEditOperation[] = [];
for (let i = 0; i < operations.length; i++) {
let op = operations[i];
let reverseRange = reverseRanges[i];
reverseOperations[i] = {
sortIndex: op.sortIndex,
identifier: op.identifier,
range: reverseRange,
text: this.getValueInRange(op.range),
@@ -242,6 +261,11 @@ export class PieceTreeTextBuffer implements ITextBuffer {
};
}
// Can only sort reverse operations when the order is not significant
if (!hasTouchingRanges) {
reverseOperations.sort((a, b) => a.sortIndex - b.sortIndex);
}
this._mightContainRTL = mightContainRTL;
this._mightContainNonBasicASCII = mightContainNonBasicASCII;
@@ -279,9 +303,9 @@ export class PieceTreeTextBuffer implements ITextBuffer {
}
/**
* Transform operations such that they represent the same logic edit,
* but that they also do not cause OOM crashes.
*/
* Transform operations such that they represent the same logic edit,
* but that they also do not cause OOM crashes.
*/
private _reduceOperations(operations: IValidatedEditOperation[]): IValidatedEditOperation[] {
if (operations.length < 1000) {
// We know from empirical testing that a thousand edits work fine regardless of their shape.
@@ -410,6 +434,10 @@ export class PieceTreeTextBuffer implements ITextBuffer {
return contentChanges;
}
findMatchesLineByLine(searchRange: Range, searchData: SearchData, captureMatches: boolean, limitResultCount: number): FindMatch[] {
return this._pieceTree.findMatchesLineByLine(searchRange, searchData, captureMatches, limitResultCount);
}
// #endregion
// #region helper

607
src/vs/editor/common/model/textModel.ts
View File

@@ -5,7 +5,7 @@
'use strict';
import URI from 'vs/base/common/uri';
import Event, { Emitter } from 'vs/base/common/event';
import { Event, Emitter } from 'vs/base/common/event';
import * as model from 'vs/editor/common/model';
import { LanguageIdentifier, TokenizationRegistry, LanguageId } from 'vs/editor/common/modes';
import { EditStack } from 'vs/editor/common/model/editStack';
@@ -30,31 +30,13 @@ import { getWordAtText } from 'vs/editor/common/model/wordHelper';
import { ModelLinesTokens, ModelTokensChangedEventBuilder } from 'vs/editor/common/model/textModelTokens';
import { guessIndentation } from 'vs/editor/common/model/indentationGuesser';
import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/config/editorOptions';
import { TextModelSearch, SearchParams } from 'vs/editor/common/model/textModelSearch';
import { TextModelSearch, SearchParams, SearchData } from 'vs/editor/common/model/textModelSearch';
import { TPromise } from 'vs/base/common/winjs.base';
import { IStringStream, ITextSnapshot } from 'vs/platform/files/common/files';
import { LinesTextBufferBuilder } from 'vs/editor/common/model/linesTextBuffer/linesTextBufferBuilder';
import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder';
import { ChunksTextBufferBuilder } from 'vs/editor/common/model/chunksTextBuffer/chunksTextBufferBuilder';
export enum TextBufferType {
LinesArray,
PieceTree,
Chunks
}
// Here is the master switch for the text buffer implementation:
export const OPTIONS = {
TEXT_BUFFER_IMPLEMENTATION: TextBufferType.PieceTree
};
function createTextBufferBuilder() {
if (OPTIONS.TEXT_BUFFER_IMPLEMENTATION === TextBufferType.PieceTree) {
return new PieceTreeTextBufferBuilder();
}
if (OPTIONS.TEXT_BUFFER_IMPLEMENTATION === TextBufferType.Chunks) {
return new ChunksTextBufferBuilder();
}
return new LinesTextBufferBuilder();
return new PieceTreeTextBufferBuilder();
}
export function createTextBufferFactory(text: string): model.ITextBufferFactory {
@@ -173,15 +155,17 @@ class TextModelSnapshot implements ITextSnapshot {
export class TextModel extends Disposable implements model.ITextModel {
private static readonly MODEL_SYNC_LIMIT = 50 * 1024 * 1024; // 50 MB
private static readonly MODEL_TOKENIZATION_LIMIT = 20 * 1024 * 1024; // 20 MB
private static readonly MANY_MANY_LINES = 300 * 1000; // 300K lines
private static readonly LARGE_FILE_SIZE_THRESHOLD = 20 * 1024 * 1024; // 20 MB;
private static readonly LARGE_FILE_LINE_COUNT_THRESHOLD = 300 * 1000; // 300K lines
public static DEFAULT_CREATION_OPTIONS: model.ITextModelCreationOptions = {
isForSimpleWidget: false,
tabSize: EDITOR_MODEL_DEFAULTS.tabSize,
insertSpaces: EDITOR_MODEL_DEFAULTS.insertSpaces,
detectIndentation: false,
defaultEOL: model.DefaultEndOfLine.LF,
trimAutoWhitespace: EDITOR_MODEL_DEFAULTS.trimAutoWhitespace,
largeFileOptimizations: EDITOR_MODEL_DEFAULTS.largeFileOptimizations,
};
public static createFromString(text: string, options: model.ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageIdentifier: LanguageIdentifier = null, uri: URI = null): TextModel {
@@ -228,15 +212,19 @@ export class TextModel extends Disposable implements model.ITextModel {
public readonly onDidChangeOptions: Event<IModelOptionsChangedEvent> = this._onDidChangeOptions.event;
private readonly _eventEmitter: DidChangeContentEmitter = this._register(new DidChangeContentEmitter());
public onDidChangeRawContentFast(listener: (e: ModelRawContentChangedEvent) => void): IDisposable {
return this._eventEmitter.fastEvent((e: InternalModelContentChangeEvent) => listener(e.rawContentChangedEvent));
}
public onDidChangeRawContent(listener: (e: ModelRawContentChangedEvent) => void): IDisposable {
return this._eventEmitter.event((e: InternalModelContentChangeEvent) => listener(e.rawContentChangedEvent));
return this._eventEmitter.slowEvent((e: InternalModelContentChangeEvent) => listener(e.rawContentChangedEvent));
}
public onDidChangeContent(listener: (e: IModelContentChangedEvent) => void): IDisposable {
return this._eventEmitter.event((e: InternalModelContentChangeEvent) => listener(e.contentChangedEvent));
return this._eventEmitter.slowEvent((e: InternalModelContentChangeEvent) => listener(e.contentChangedEvent));
}
//#endregion
public readonly id: string;
public readonly isForSimpleWidget: boolean;
private readonly _associatedResource: URI;
private _attachedEditorCount: number;
private _buffer: model.ITextBuffer;
@@ -249,7 +237,7 @@ export class TextModel extends Disposable implements model.ITextModel {
* Unlike, versionId, this can go down (via undo) or go to previous values (via redo)
*/
private _alternativeVersionId: number;
private readonly _shouldSimplifyMode: boolean;
private readonly _isTooLargeForSyncing: boolean;
private readonly _isTooLargeForTokenization: boolean;
//#region Editing
@@ -284,6 +272,7 @@ export class TextModel extends Disposable implements model.ITextModel {
// Generate a new unique model id
MODEL_ID++;
this.id = '$model' + MODEL_ID;
this.isForSimpleWidget = creationOptions.isForSimpleWidget;
if (typeof associatedResource === 'undefined' || associatedResource === null) {
this._associatedResource = URI.parse('inmemory://model/' + MODEL_ID);
} else {
@@ -297,18 +286,20 @@ export class TextModel extends Disposable implements model.ITextModel {
const bufferLineCount = this._buffer.getLineCount();
const bufferTextLength = this._buffer.getValueLengthInRange(new Range(1, 1, bufferLineCount, this._buffer.getLineLength(bufferLineCount) + 1), model.EndOfLinePreference.TextDefined);
// !!! Make a decision in the ctor and permanently respect this decision !!!
// If a model is too large at construction time, it will never get tokenized,
// under no circumstances.
this._isTooLargeForTokenization = (
(bufferTextLength > TextModel.MODEL_TOKENIZATION_LIMIT)
|| (bufferLineCount > TextModel.MANY_MANY_LINES)
);
if (creationOptions.largeFileOptimizations) {
this._isTooLargeForTokenization = (
(bufferTextLength > TextModel.LARGE_FILE_SIZE_THRESHOLD)
|| (bufferLineCount > TextModel.LARGE_FILE_LINE_COUNT_THRESHOLD)
);
} else {
this._isTooLargeForTokenization = false;
}
this._shouldSimplifyMode = (
this._isTooLargeForTokenization
|| (bufferTextLength > TextModel.MODEL_SYNC_LIMIT)
);
this._isTooLargeForSyncing = (bufferTextLength > TextModel.MODEL_SYNC_LIMIT);
this._setVersionId(1);
this._isDisposed = false;
@@ -398,10 +389,11 @@ export class TextModel extends Disposable implements model.ITextModel {
this.setValueFromTextBuffer(textBuffer);
}
private _createContentChanged2(startLineNumber: number, startColumn: number, endLineNumber: number, endColumn: number, rangeLength: number, text: string, isUndoing: boolean, isRedoing: boolean, isFlush: boolean): IModelContentChangedEvent {
private _createContentChanged2(range: Range, rangeOffset: number, rangeLength: number, text: string, isUndoing: boolean, isRedoing: boolean, isFlush: boolean): IModelContentChangedEvent {
return {
changes: [{
range: new Range(startLineNumber, startColumn, endLineNumber, endColumn),
range: range,
rangeOffset: rangeOffset,
rangeLength: rangeLength,
text: text,
}],
@@ -447,7 +439,7 @@ export class TextModel extends Disposable implements model.ITextModel {
false,
false
),
this._createContentChanged2(1, 1, endLineNumber, endColumn, oldModelValueLength, this.getValue(), false, false, true)
this._createContentChanged2(new Range(1, 1, endLineNumber, endColumn), 0, oldModelValueLength, this.getValue(), false, false, true)
);
}
@@ -478,7 +470,7 @@ export class TextModel extends Disposable implements model.ITextModel {
false,
false
),
this._createContentChanged2(1, 1, endLineNumber, endColumn, oldModelValueLength, this.getValue(), false, false, false)
this._createContentChanged2(new Range(1, 1, endLineNumber, endColumn), 0, oldModelValueLength, this.getValue(), false, false, false)
);
}
@@ -548,8 +540,12 @@ export class TextModel extends Disposable implements model.ITextModel {
return this._attachedEditorCount > 0;
}
public isTooLargeForHavingARichMode(): boolean {
return this._shouldSimplifyMode;
public getAttachedEditorCount(): number {
return this._attachedEditorCount;
}
public isTooLargeForSyncing(): boolean {
return this._isTooLargeForSyncing;
}
public isTooLargeForTokenization(): boolean {
@@ -779,6 +775,15 @@ export class TextModel extends Disposable implements model.ITextModel {
return this._buffer.getLineContent(lineNumber);
}
public getLineLength(lineNumber: number): number {
this._assertNotDisposed();
if (lineNumber < 1 || lineNumber > this.getLineCount()) {
throw new Error('Illegal value for lineNumber');
}
return this._buffer.getLineLength(lineNumber);
}
public getLinesContent(): string[] {
this._assertNotDisposed();
return this._buffer.getLinesContent();
@@ -889,6 +894,41 @@ export class TextModel extends Disposable implements model.ITextModel {
return new Range(startLineNumber, startColumn, endLineNumber, endColumn);
}
/**
* @param strict Do NOT allow a position inside a high-low surrogate pair
*/
private _isValidPosition(lineNumber: number, column: number, strict: boolean): boolean {
if (lineNumber < 1) {
return false;
}
const lineCount = this._buffer.getLineCount();
if (lineNumber > lineCount) {
return false;
}
if (column < 1) {
return false;
}
const maxColumn = this.getLineMaxColumn(lineNumber);
if (column > maxColumn) {
return false;
}
if (strict) {
if (column > 1) {
const charCodeBefore = this._buffer.getLineCharCode(lineNumber, column - 2);
if (strings.isHighSurrogate(charCodeBefore)) {
return false;
}
}
}
return true;
}
/**
* @param strict Do NOT allow a position inside a high-low surrogate pair
*/
@@ -929,11 +969,60 @@ export class TextModel extends Disposable implements model.ITextModel {
public validatePosition(position: IPosition): Position {
this._assertNotDisposed();
// Avoid object allocation and cover most likely case
if (position instanceof Position) {
if (this._isValidPosition(position.lineNumber, position.column, true)) {
return position;
}
}
return this._validatePosition(position.lineNumber, position.column, true);
}
/**
* @param strict Do NOT allow a range to have its boundaries inside a high-low surrogate pair
*/
private _isValidRange(range: Range, strict: boolean): boolean {
const startLineNumber = range.startLineNumber;
const startColumn = range.startColumn;
const endLineNumber = range.endLineNumber;
const endColumn = range.endColumn;
if (!this._isValidPosition(startLineNumber, startColumn, false)) {
return false;
}
if (!this._isValidPosition(endLineNumber, endColumn, false)) {
return false;
}
if (strict) {
const charCodeBeforeStart = (startColumn > 1 ? this._buffer.getLineCharCode(startLineNumber, startColumn - 2) : 0);
const charCodeBeforeEnd = (endColumn > 1 && endColumn <= this._buffer.getLineLength(endLineNumber) ? this._buffer.getLineCharCode(endLineNumber, endColumn - 2) : 0);
const startInsideSurrogatePair = strings.isHighSurrogate(charCodeBeforeStart);
const endInsideSurrogatePair = strings.isHighSurrogate(charCodeBeforeEnd);
if (!startInsideSurrogatePair && !endInsideSurrogatePair) {
return true;
}
return false;
}
return true;
}
public validateRange(_range: IRange): Range {
this._assertNotDisposed();
// Avoid object allocation and cover most likely case
if ((_range instanceof Range) && !(_range instanceof Selection)) {
if (this._isValidRange(_range, true)) {
return _range;
}
}
const start = this._validatePosition(_range.startLineNumber, _range.startColumn, false);
const end = this._validatePosition(_range.endLineNumber, _range.endColumn, false);
@@ -983,6 +1072,10 @@ export class TextModel extends Disposable implements model.ITextModel {
return new Range(1, 1, lineCount, this.getLineMaxColumn(lineCount));
}
private findMatchesLineByLine(searchRange: Range, searchData: SearchData, captureMatches: boolean, limitResultCount: number): model.FindMatch[] {
return this._buffer.findMatchesLineByLine(searchRange, searchData, captureMatches, limitResultCount);
}
public findMatches(searchString: string, rawSearchScope: any, isRegex: boolean, matchCase: boolean, wordSeparators: string, captureMatches: boolean, limitResultCount: number = LIMIT_FIND_COUNT): model.FindMatch[] {
this._assertNotDisposed();
@@ -993,12 +1086,46 @@ export class TextModel extends Disposable implements model.ITextModel {
searchRange = this.getFullModelRange();
}
if (!isRegex && searchString.indexOf('\n') < 0) {
// not regex, not multi line
const searchParams = new SearchParams(searchString, isRegex, matchCase, wordSeparators);
const searchData = searchParams.parseSearchRequest();
if (!searchData) {
return [];
}
return this.findMatchesLineByLine(searchRange, searchData, captureMatches, limitResultCount);
}
return TextModelSearch.findMatches(this, new SearchParams(searchString, isRegex, matchCase, wordSeparators), searchRange, captureMatches, limitResultCount);
}
public findNextMatch(searchString: string, rawSearchStart: IPosition, isRegex: boolean, matchCase: boolean, wordSeparators: string, captureMatches: boolean): model.FindMatch {
this._assertNotDisposed();
const searchStart = this.validatePosition(rawSearchStart);
if (!isRegex && searchString.indexOf('\n') < 0) {
const searchParams = new SearchParams(searchString, isRegex, matchCase, wordSeparators);
const searchData = searchParams.parseSearchRequest();
const lineCount = this.getLineCount();
let searchRange = new Range(searchStart.lineNumber, searchStart.column, lineCount, this.getLineMaxColumn(lineCount));
let ret = this.findMatchesLineByLine(searchRange, searchData, captureMatches, 1);
TextModelSearch.findNextMatch(this, new SearchParams(searchString, isRegex, matchCase, wordSeparators), searchStart, captureMatches);
if (ret.length > 0) {
return ret[0];
}
searchRange = new Range(1, 1, searchStart.lineNumber, this.getLineMaxColumn(searchStart.lineNumber));
ret = this.findMatchesLineByLine(searchRange, searchData, captureMatches, 1);
if (ret.length > 0) {
return ret[0];
}
return null;
}
return TextModelSearch.findNextMatch(this, new SearchParams(searchString, isRegex, matchCase, wordSeparators), searchStart, captureMatches);
}
@@ -1584,6 +1711,88 @@ export class TextModel extends Disposable implements model.ITextModel {
//#region Tokenization
public tokenizeViewport(startLineNumber: number, endLineNumber: number): void {
if (!this._tokens.tokenizationSupport) {
return;
}
// we tokenize `this._tokens.inValidLineStartIndex` lines in around 20ms so it's a good baseline.
const contextBefore = Math.floor(this._tokens.inValidLineStartIndex * 0.3);
startLineNumber = Math.max(1, startLineNumber - contextBefore);
if (startLineNumber <= this._tokens.inValidLineStartIndex) {
this.forceTokenization(endLineNumber);
return;
}
const eventBuilder = new ModelTokensChangedEventBuilder();
let nonWhitespaceColumn = this.getLineFirstNonWhitespaceColumn(startLineNumber);
let fakeLines = [];
let i = startLineNumber - 1;
let initialState = null;
if (nonWhitespaceColumn > 0) {
while (nonWhitespaceColumn > 0 && i >= 1) {
let newNonWhitespaceIndex = this.getLineFirstNonWhitespaceColumn(i);
if (newNonWhitespaceIndex === 0) {
i--;
continue;
}
if (newNonWhitespaceIndex < nonWhitespaceColumn) {
initialState = this._tokens._getState(i - 1);
if (initialState) {
break;
}
fakeLines.push(this.getLineContent(i));
nonWhitespaceColumn = newNonWhitespaceIndex;
}
i--;
}
}
if (!initialState) {
initialState = this._tokens.tokenizationSupport.getInitialState();
}
let state = initialState.clone();
for (let i = fakeLines.length - 1; i >= 0; i--) {
let r = this._tokens._tokenizeText(this._buffer, fakeLines[i], state);
if (r) {
state = r.endState.clone();
} else {
state = initialState.clone();
}
}
const contextAfter = Math.floor(this._tokens.inValidLineStartIndex * 0.4);
endLineNumber = Math.min(this.getLineCount(), endLineNumber + contextAfter);
for (let i = startLineNumber; i <= endLineNumber; i++) {
let text = this.getLineContent(i);
let r = this._tokens._tokenizeText(this._buffer, text, state);
if (r) {
this._tokens._setTokens(this._tokens.languageIdentifier.id, i - 1, text.length, r.tokens);
/*
* we think it's valid and give it a state but we don't update `_invalidLineStartIndex` then the top-to-bottom tokenization
* goes through the viewport, it can skip them if they already have correct tokens and state, and the lines after the viewport
* can still be tokenized.
*/
this._tokens._setIsInvalid(i - 1, false);
this._tokens._setState(i - 1, state);
state = r.endState.clone();
eventBuilder.registerChangedTokens(i);
} else {
state = initialState.clone();
}
}
const e = eventBuilder.build();
if (e) {
this._onDidChangeTokens.fire(e);
}
}
public forceTokenization(lineNumber: number): void {
if (lineNumber < 1 || lineNumber > this.getLineCount()) {
throw new Error('Illegal value for lineNumber');
@@ -1726,8 +1935,39 @@ export class TextModel extends Disposable implements model.ITextModel {
const position = this.validatePosition(_position);
const lineContent = this.getLineContent(position.lineNumber);
const lineTokens = this._getLineTokens(position.lineNumber);
const offset = position.column - 1;
const tokenIndex = lineTokens.findTokenIndexAtOffset(offset);
const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
// (1). First try checking right biased word
const [rbStartOffset, rbEndOffset] = TextModel._findLanguageBoundaries(lineTokens, tokenIndex);
const rightBiasedWord = getWordAtText(
position.column,
LanguageConfigurationRegistry.getWordDefinition(lineTokens.getLanguageId(tokenIndex)),
lineContent.substring(rbStartOffset, rbEndOffset),
rbStartOffset
);
if (rightBiasedWord) {
return rightBiasedWord;
}
// (2). Else, if we were at a language boundary, check the left biased word
if (tokenIndex > 0 && rbStartOffset === position.column - 1) {
// edge case, where `position` sits between two tokens belonging to two different languages
const [lbStartOffset, lbEndOffset] = TextModel._findLanguageBoundaries(lineTokens, tokenIndex - 1);
const leftBiasedWord = getWordAtText(
position.column,
LanguageConfigurationRegistry.getWordDefinition(lineTokens.getLanguageId(tokenIndex - 1)),
lineContent.substring(lbStartOffset, lbEndOffset),
lbStartOffset
);
if (leftBiasedWord) {
return leftBiasedWord;
}
}
return null;
}
private static _findLanguageBoundaries(lineTokens: LineTokens, tokenIndex: number): [number, number] {
const languageId = lineTokens.getLanguageId(tokenIndex);
// go left until a different language is hit
@@ -1742,12 +1982,7 @@ export class TextModel extends Disposable implements model.ITextModel {
endOffset = lineTokens.getEndOffset(i);
}
return getWordAtText(
position.column,
LanguageConfigurationRegistry.getWordDefinition(languageId),
lineContent.substring(startOffset, endOffset),
startOffset
);
return [startOffset, endOffset];
}
public getWordUntilPosition(position: IPosition): model.IWordAtPosition {
@@ -1809,25 +2044,13 @@ export class TextModel extends Disposable implements model.ITextModel {
// limit search to not go after `maxBracketLength`
const searchEndOffset = Math.min(lineTokens.getEndOffset(tokenIndex), position.column - 1 + currentModeBrackets.maxBracketLength);
// first, check if there is a bracket to the right of `position`
let foundBracket = BracketsUtils.findNextBracketInToken(currentModeBrackets.forwardRegex, lineNumber, lineText, position.column - 1, searchEndOffset);
if (foundBracket && foundBracket.startColumn === position.column) {
let foundBracketText = lineText.substring(foundBracket.startColumn - 1, foundBracket.endColumn - 1);
foundBracketText = foundBracketText.toLowerCase();
let r = this._matchFoundBracket(foundBracket, currentModeBrackets.textIsBracket[foundBracketText], currentModeBrackets.textIsOpenBracket[foundBracketText]);
// check that we can actually match this bracket
if (r) {
return r;
}
}
// it might still be the case that [currentTokenStart -> currentTokenEnd] contains multiple brackets
// it might be the case that [currentTokenStart -> currentTokenEnd] contains multiple brackets
// `bestResult` will contain the most right-side result
let bestResult: [Range, Range] = null;
while (true) {
let foundBracket = BracketsUtils.findNextBracketInToken(currentModeBrackets.forwardRegex, lineNumber, lineText, searchStartOffset, searchEndOffset);
if (!foundBracket) {
// there are no brackets in this text
// there are no more brackets in this text
break;
}
@@ -1840,12 +2063,16 @@ export class TextModel extends Disposable implements model.ITextModel {
// check that we can actually match this bracket
if (r) {
return r;
bestResult = r;
}
}
searchStartOffset = foundBracket.endColumn - 1;
}
if (bestResult) {
return bestResult;
}
}
// If position is in between two tokens, try also looking in the previous token
@@ -1879,6 +2106,10 @@ export class TextModel extends Disposable implements model.ITextModel {
}
private _matchFoundBracket(foundBracket: Range, data: RichEditBracket, isOpen: boolean): [Range, Range] {
if (!data) {
return null;
}
if (isOpen) {
let matched = this._findMatchingBracketDown(data, foundBracket.getEndPosition());
if (matched) {
@@ -2158,6 +2389,173 @@ export class TextModel extends Disposable implements model.ITextModel {
return TextModel.computeIndentLevel(this._buffer.getLineContent(lineIndex + 1), this._options.tabSize);
}
public getActiveIndentGuide(lineNumber: number, minLineNumber: number, maxLineNumber: number): model.IActiveIndentGuideInfo {
this._assertNotDisposed();
const lineCount = this.getLineCount();
if (lineNumber < 1 || lineNumber > lineCount) {
throw new Error('Illegal value for lineNumber');
}
const foldingRules = LanguageConfigurationRegistry.getFoldingRules(this._languageIdentifier.id);
const offSide = foldingRules && foldingRules.offSide;
let up_aboveContentLineIndex = -2; /* -2 is a marker for not having computed it */
let up_aboveContentLineIndent = -1;
let up_belowContentLineIndex = -2; /* -2 is a marker for not having computed it */
let up_belowContentLineIndent = -1;
const up_resolveIndents = (lineNumber: number) => {
if (up_aboveContentLineIndex !== -1 && (up_aboveContentLineIndex === -2 || up_aboveContentLineIndex > lineNumber - 1)) {
up_aboveContentLineIndex = -1;
up_aboveContentLineIndent = -1;
// must find previous line with content
for (let lineIndex = lineNumber - 2; lineIndex >= 0; lineIndex--) {
let indent = this._computeIndentLevel(lineIndex);
if (indent >= 0) {
up_aboveContentLineIndex = lineIndex;
up_aboveContentLineIndent = indent;
break;
}
}
}
if (up_belowContentLineIndex === -2) {
up_belowContentLineIndex = -1;
up_belowContentLineIndent = -1;
// must find next line with content
for (let lineIndex = lineNumber; lineIndex < lineCount; lineIndex++) {
let indent = this._computeIndentLevel(lineIndex);
if (indent >= 0) {
up_belowContentLineIndex = lineIndex;
up_belowContentLineIndent = indent;
break;
}
}
}
};
let down_aboveContentLineIndex = -2; /* -2 is a marker for not having computed it */
let down_aboveContentLineIndent = -1;
let down_belowContentLineIndex = -2; /* -2 is a marker for not having computed it */
let down_belowContentLineIndent = -1;
const down_resolveIndents = (lineNumber: number) => {
if (down_aboveContentLineIndex === -2) {
down_aboveContentLineIndex = -1;
down_aboveContentLineIndent = -1;
// must find previous line with content
for (let lineIndex = lineNumber - 2; lineIndex >= 0; lineIndex--) {
let indent = this._computeIndentLevel(lineIndex);
if (indent >= 0) {
down_aboveContentLineIndex = lineIndex;
down_aboveContentLineIndent = indent;
break;
}
}
}
if (down_belowContentLineIndex !== -1 && (down_belowContentLineIndex === -2 || down_belowContentLineIndex < lineNumber - 1)) {
down_belowContentLineIndex = -1;
down_belowContentLineIndent = -1;
// must find next line with content
for (let lineIndex = lineNumber; lineIndex < lineCount; lineIndex++) {
let indent = this._computeIndentLevel(lineIndex);
if (indent >= 0) {
down_belowContentLineIndex = lineIndex;
down_belowContentLineIndent = indent;
break;
}
}
}
};
let startLineNumber = 0;
let goUp = true;
let endLineNumber = 0;
let goDown = true;
let indent = 0;
for (let distance = 0; goUp || goDown; distance++) {
const upLineNumber = lineNumber - distance;
const downLineNumber = lineNumber + distance;
if (upLineNumber < 1 || upLineNumber < minLineNumber) {
goUp = false;
}
if (downLineNumber > lineCount || downLineNumber > maxLineNumber) {
goDown = false;
}
if (distance > 50000) {
// stop processing
goUp = false;
goDown = false;
}
if (goUp) {
// compute indent level going up
let upLineIndentLevel: number;
const currentIndent = this._computeIndentLevel(upLineNumber - 1);
if (currentIndent >= 0) {
// This line has content (besides whitespace)
// Use the line's indent
up_belowContentLineIndex = upLineNumber - 1;
up_belowContentLineIndent = currentIndent;
upLineIndentLevel = Math.ceil(currentIndent / this._options.tabSize);
} else {
up_resolveIndents(upLineNumber);
upLineIndentLevel = this._getIndentLevelForWhitespaceLine(offSide, up_aboveContentLineIndent, up_belowContentLineIndent);
}
if (distance === 0) {
// This is the initial line number
startLineNumber = upLineNumber;
endLineNumber = downLineNumber;
indent = upLineIndentLevel;
if (indent === 0) {
// No need to continue
return { startLineNumber, endLineNumber, indent };
}
continue;
}
if (upLineIndentLevel >= indent) {
startLineNumber = upLineNumber;
} else {
goUp = false;
}
}
if (goDown) {
// compute indent level going down
let downLineIndentLevel: number;
const currentIndent = this._computeIndentLevel(downLineNumber - 1);
if (currentIndent >= 0) {
// This line has content (besides whitespace)
// Use the line's indent
down_aboveContentLineIndex = downLineNumber - 1;
down_aboveContentLineIndent = currentIndent;
downLineIndentLevel = Math.ceil(currentIndent / this._options.tabSize);
} else {
down_resolveIndents(downLineNumber);
downLineIndentLevel = this._getIndentLevelForWhitespaceLine(offSide, down_aboveContentLineIndent, down_belowContentLineIndent);
}
if (downLineIndentLevel >= indent) {
endLineNumber = downLineNumber;
} else {
goDown = false;
}
}
}
return { startLineNumber, endLineNumber, indent };
}
public getLinesIndentGuides(startLineNumber: number, endLineNumber: number): number[] {
this._assertNotDisposed();
const lineCount = this.getLineCount();
@@ -2223,32 +2621,38 @@ export class TextModel extends Disposable implements model.ITextModel {
}
}
if (aboveContentLineIndent === -1 || belowContentLineIndent === -1) {
// At the top or bottom of the file
result[resultIndex] = 0;
result[resultIndex] = this._getIndentLevelForWhitespaceLine(offSide, aboveContentLineIndent, belowContentLineIndent);
} else if (aboveContentLineIndent < belowContentLineIndent) {
// we are inside the region above
result[resultIndex] = (1 + Math.floor(aboveContentLineIndent / this._options.tabSize));
} else if (aboveContentLineIndent === belowContentLineIndent) {
// we are in between two regions
result[resultIndex] = Math.ceil(belowContentLineIndent / this._options.tabSize);
} else {
if (offSide) {
// same level as region below
result[resultIndex] = Math.ceil(belowContentLineIndent / this._options.tabSize);
} else {
// we are inside the region that ends below
result[resultIndex] = (1 + Math.floor(belowContentLineIndent / this._options.tabSize));
}
}
}
return result;
}
private _getIndentLevelForWhitespaceLine(offSide: boolean, aboveContentLineIndent: number, belowContentLineIndent: number): number {
if (aboveContentLineIndent === -1 || belowContentLineIndent === -1) {
// At the top or bottom of the file
return 0;
} else if (aboveContentLineIndent < belowContentLineIndent) {
// we are inside the region above
return (1 + Math.floor(aboveContentLineIndent / this._options.tabSize));
} else if (aboveContentLineIndent === belowContentLineIndent) {
// we are in between two regions
return Math.ceil(belowContentLineIndent / this._options.tabSize);
} else {
if (offSide) {
// same level as region below
return Math.ceil(belowContentLineIndent / this._options.tabSize);
} else {
// we are inside the region that ends below
return (1 + Math.floor(belowContentLineIndent / this._options.tabSize));
}
}
}
//#endregion
}
@@ -2363,22 +2767,20 @@ export class ModelDecorationOverviewRulerOptions implements model.IModelDecorati
}
}
let lastStaticId = 0;
export class ModelDecorationOptions implements model.IModelDecorationOptions {
public static EMPTY: ModelDecorationOptions;
public static register(options: model.IModelDecorationOptions): ModelDecorationOptions {
return new ModelDecorationOptions(++lastStaticId, options);
return new ModelDecorationOptions(options);
}
public static createDynamic(options: model.IModelDecorationOptions): ModelDecorationOptions {
return new ModelDecorationOptions(0, options);
return new ModelDecorationOptions(options);
}
readonly staticId: number;
readonly stickiness: model.TrackedRangeStickiness;
readonly zIndex: number;
readonly className: string;
readonly hoverMessage: IMarkdownString | IMarkdownString[];
readonly glyphMarginHoverMessage: IMarkdownString | IMarkdownString[];
@@ -2389,12 +2791,13 @@ export class ModelDecorationOptions implements model.IModelDecorationOptions {
readonly linesDecorationsClassName: string;
readonly marginClassName: string;
readonly inlineClassName: string;
readonly inlineClassNameAffectsLetterSpacing: boolean;
readonly beforeContentClassName: string;
readonly afterContentClassName: string;
private constructor(staticId: number, options: model.IModelDecorationOptions) {
this.staticId = staticId;
private constructor(options: model.IModelDecorationOptions) {
this.stickiness = options.stickiness || model.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges;
this.zIndex = options.zIndex || 0;
this.className = options.className ? cleanClassName(options.className) : strings.empty;
this.hoverMessage = options.hoverMessage || [];
this.glyphMarginHoverMessage = options.glyphMarginHoverMessage || [];
@@ -2405,6 +2808,7 @@ export class ModelDecorationOptions implements model.IModelDecorationOptions {
this.linesDecorationsClassName = options.linesDecorationsClassName ? cleanClassName(options.linesDecorationsClassName) : strings.empty;
this.marginClassName = options.marginClassName ? cleanClassName(options.marginClassName) : strings.empty;
this.inlineClassName = options.inlineClassName ? cleanClassName(options.inlineClassName) : strings.empty;
this.inlineClassNameAffectsLetterSpacing = options.inlineClassNameAffectsLetterSpacing || false;
this.beforeContentClassName = options.beforeContentClassName ? cleanClassName(options.beforeContentClassName) : strings.empty;
this.afterContentClassName = options.afterContentClassName ? cleanClassName(options.afterContentClassName) : strings.empty;
}
@@ -2465,8 +2869,13 @@ export class DidChangeDecorationsEmitter extends Disposable {
export class DidChangeContentEmitter extends Disposable {
private readonly _actual: Emitter<InternalModelContentChangeEvent> = this._register(new Emitter<InternalModelContentChangeEvent>());
public readonly event: Event<InternalModelContentChangeEvent> = this._actual.event;
/**
* Both `fastEvent` and `slowEvent` work the same way and contain the same events, but first we invoke `fastEvent` and then `slowEvent`.
*/
private readonly _fastEmitter: Emitter<InternalModelContentChangeEvent> = this._register(new Emitter<InternalModelContentChangeEvent>());
public readonly fastEvent: Event<InternalModelContentChangeEvent> = this._fastEmitter.event;
private readonly _slowEmitter: Emitter<InternalModelContentChangeEvent> = this._register(new Emitter<InternalModelContentChangeEvent>());
public readonly slowEvent: Event<InternalModelContentChangeEvent> = this._slowEmitter.event;
private _deferredCnt: number;
private _deferredEvent: InternalModelContentChangeEvent;
@@ -2487,7 +2896,8 @@ export class DidChangeContentEmitter extends Disposable {
if (this._deferredEvent !== null) {
const e = this._deferredEvent;
this._deferredEvent = null;
this._actual.fire(e);
this._fastEmitter.fire(e);
this._slowEmitter.fire(e);
}
}
}
@@ -2501,6 +2911,7 @@ export class DidChangeContentEmitter extends Disposable {
}
return;
}
this._actual.fire(e);
this._fastEmitter.fire(e);
this._slowEmitter.fire(e);
}
}

4
src/vs/editor/common/model/textModelEvents.ts
View File

@@ -32,6 +32,10 @@ export interface IModelContentChange {
* The range that got replaced.
*/
readonly range: IRange;
/**
* The offset of the range that got replaced.
*/
readonly rangeOffset: number;
/**
* The length of the range that got replaced.
*/

16
src/vs/editor/common/model/textModelSearch.ts
View File

@@ -116,7 +116,7 @@ export class SearchData {
}
}
function createFindMatch(range: Range, rawMatches: RegExpExecArray, captureMatches: boolean): FindMatch {
export function createFindMatch(range: Range, rawMatches: RegExpExecArray, captureMatches: boolean): FindMatch {
if (!captureMatches) {
return new FindMatch(range, null);
}
@@ -434,6 +434,11 @@ function leftIsWordBounday(wordSeparators: WordCharacterClassifier, text: string
return true;
}
if (charBefore === CharCode.CarriageReturn || charBefore === CharCode.LineFeed) {
// The character before the match is line break or carriage return.
return true;
}
if (matchLength > 0) {
const firstCharInMatch = text.charCodeAt(matchStartIndex);
if (wordSeparators.get(firstCharInMatch) !== WordCharacterClass.Regular) {
@@ -457,6 +462,11 @@ function rightIsWordBounday(wordSeparators: WordCharacterClassifier, text: strin
return true;
}
if (charAfter === CharCode.CarriageReturn || charAfter === CharCode.LineFeed) {
// The character after the match is line break or carriage return.
return true;
}
if (matchLength > 0) {
const lastCharInMatch = text.charCodeAt(matchStartIndex + matchLength - 1);
if (wordSeparators.get(lastCharInMatch) !== WordCharacterClass.Regular) {
@@ -468,14 +478,14 @@ function rightIsWordBounday(wordSeparators: WordCharacterClassifier, text: strin
return false;
}
function isValidMatch(wordSeparators: WordCharacterClassifier, text: string, textLength: number, matchStartIndex: number, matchLength: number): boolean {
export function isValidMatch(wordSeparators: WordCharacterClassifier, text: string, textLength: number, matchStartIndex: number, matchLength: number): boolean {
return (
leftIsWordBounday(wordSeparators, text, textLength, matchStartIndex, matchLength)
&& rightIsWordBounday(wordSeparators, text, textLength, matchStartIndex, matchLength)
);
}
class Searcher {
export class Searcher {
private _wordSeparators: WordCharacterClassifier;
private _searchRegex: RegExp;
private _prevMatchStartIndex: number;

37
src/vs/editor/common/model/textModelTokens.ts
View File

@@ -25,7 +25,7 @@ function getDefaultMetadata(topLevelLanguageId: LanguageId): number {
) >>> 0;
}
const EMPTY_LINE_TOKENS = new Uint32Array(0);
const EMPTY_LINE_TOKENS = (new Uint32Array(0)).buffer;
class ModelLineTokens {
_state: IState;
@@ -196,9 +196,13 @@ export class ModelLinesTokens {
this._lastState = null;
}
public get inValidLineStartIndex() {
return this._invalidLineStartIndex;
}
public getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens {
let rawLineTokens: ArrayBuffer = null;
if (lineIndex < this._tokens.length) {
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
rawLineTokens = this._tokens[lineIndex]._lineTokens;
}
@@ -229,21 +233,21 @@ export class ModelLinesTokens {
}
}
private _setIsInvalid(lineIndex: number, invalid: boolean): void {
if (lineIndex < this._tokens.length) {
_setIsInvalid(lineIndex: number, invalid: boolean): void {
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
this._tokens[lineIndex]._invalid = invalid;
}
}
_isInvalid(lineIndex: number): boolean {
if (lineIndex < this._tokens.length) {
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
return this._tokens[lineIndex]._invalid;
}
return true;
}
_getState(lineIndex: number): IState {
if (lineIndex < this._tokens.length) {
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
return this._tokens[lineIndex]._state;
}
return null;
@@ -251,7 +255,7 @@ export class ModelLinesTokens {
_setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
let target: ModelLineTokens;
if (lineIndex < this._tokens.length) {
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
target = this._tokens[lineIndex];
} else {
target = new ModelLineTokens(null);
@@ -274,8 +278,8 @@ export class ModelLinesTokens {
target._lineTokens = tokens.buffer;
}
private _setState(lineIndex: number, state: IState): void {
if (lineIndex < this._tokens.length) {
_setState(lineIndex: number, state: IState): void {
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
this._tokens[lineIndex]._state = state;
} else {
const tmp = new ModelLineTokens(state);
@@ -376,6 +380,21 @@ export class ModelLinesTokens {
return lineNumber;
}
public _tokenizeText(buffer: ITextBuffer, text: string, state: IState): TokenizationResult2 {
let r: TokenizationResult2 = null;
try {
r = this.tokenizationSupport.tokenize2(text, state, 0);
} catch (e) {
onUnexpectedError(e);
}
if (!r) {
r = nullTokenize2(this.languageIdentifier.id, text, state, 0);
}
return r;
}
public _updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
if (!this.tokenizationSupport) {
this._invalidLineStartIndex = buffer.getLineCount();