Merge from vscode c58aaab8a1cc22a7139b761166a0d4f37d41e998 (#7880)

* Merge from vscode c58aaab8a1cc22a7139b761166a0d4f37d41e998

* fix pipelines

* fix strict-null-checks

* add missing files

src/vs/editor/common/modes/languageConfiguration.ts

@@ -247,7 +247,7 @@ export class StandardAutoClosingPairConditional {
 
 		if (Array.isArray(source.notIn)) {
 			for (let i = 0, len = source.notIn.length; i < len; i++) {
-				let notIn = source.notIn[i];
+				const notIn: string = source.notIn[i];
 				switch (notIn) {
 					case 'string':
 						this._standardTokenMask |= StandardTokenType.String;

src/vs/editor/common/modes/linkComputer.ts

@@ -5,7 +5,6 @@
 
 import { CharCode } from 'vs/base/common/charCode';
 import { CharacterClassifier } from 'vs/editor/common/core/characterClassifier';
-import { Uint8Matrix } from 'vs/editor/common/core/uint';
 import { ILink } from 'vs/editor/common/modes';
 
 export interface ILinkComputerTarget {
@@ -33,6 +32,32 @@ export const enum State {
 
 export type Edge = [State, number, State];
 
+export class Uint8Matrix {
+
+	private readonly _data: Uint8Array;
+	public readonly rows: number;
+	public readonly cols: number;
+
+	constructor(rows: number, cols: number, defaultValue: number) {
+		const data = new Uint8Array(rows * cols);
+		for (let i = 0, len = rows * cols; i < len; i++) {
+			data[i] = defaultValue;
+		}
+
+		this._data = data;
+		this.rows = rows;
+		this.cols = cols;
+	}
+
+	public get(row: number, col: number): number {
+		return this._data[row * this.cols + col];
+	}
+
+	public set(row: number, col: number, value: number): void {
+		this._data[row * this.cols + col] = value;
+	}
+}
+
 export class StateMachine {
 
 	private readonly _states: Uint8Matrix;

src/vs/editor/common/modes/modesRegistry.ts

@@ -60,5 +60,17 @@ LanguageConfigurationRegistry.register(PLAINTEXT_LANGUAGE_IDENTIFIER, {
 		['(', ')'],
 		['[', ']'],
 		['{', '}'],
-	]
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '<', close: '>' },
+		{ open: '\"', close: '\"' },
+		{ open: '\'', close: '\'' },
+		{ open: '`', close: '`' },
+	],
+	folding: {
+		offSide: true
+	}
 });

src/vs/editor/common/modes/supports/electricCharacter.ts

@@ -28,10 +28,11 @@ export class BracketElectricCharacterSupport {
 		let result: string[] = [];
 
 		if (this._richEditBrackets) {
-			for (let i = 0, len = this._richEditBrackets.brackets.length; i < len; i++) {
-				let bracketPair = this._richEditBrackets.brackets[i];
-				let lastChar = bracketPair.close.charAt(bracketPair.close.length - 1);
-				result.push(lastChar);
+			for (const bracket of this._richEditBrackets.brackets) {
+				for (const close of bracket.close) {
+					const lastChar = close.charAt(close.length - 1);
+					result.push(lastChar);
+				}
 			}
 		}
 
@@ -56,7 +57,7 @@ export class BracketElectricCharacterSupport {
 		let reversedBracketRegex = this._richEditBrackets.reversedRegex;
 		let text = context.getLineContent().substring(0, column - 1) + character;
 
-		let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, 1, text, 0, text.length);
+		let r = BracketsUtils.findPrevBracketInRange(reversedBracketRegex, 1, text, 0, text.length);
 		if (!r) {
 			return null;
 		}

src/vs/editor/common/modes/supports/richEditBrackets.ts

@@ -8,27 +8,107 @@ import { Range } from 'vs/editor/common/core/range';
 import { LanguageIdentifier } from 'vs/editor/common/modes';
 import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration';
 
-interface ISimpleInternalBracket {
-	open: string;
-	close: string;
+interface InternalBracket {
+	open: string[];
+	close: string[];
 }
 
 export class RichEditBracket {
 	_richEditBracketBrand: void;
 
 	readonly languageIdentifier: LanguageIdentifier;
-	readonly open: string;
-	readonly close: string;
+	readonly index: number;
+	readonly open: string[];
+	readonly close: string[];
 	readonly forwardRegex: RegExp;
 	readonly reversedRegex: RegExp;
+	private readonly _openSet: Set<string>;
+	private readonly _closeSet: Set<string>;
 
-	constructor(languageIdentifier: LanguageIdentifier, open: string, close: string, forwardRegex: RegExp, reversedRegex: RegExp) {
+	constructor(languageIdentifier: LanguageIdentifier, index: number, open: string[], close: string[], forwardRegex: RegExp, reversedRegex: RegExp) {
 		this.languageIdentifier = languageIdentifier;
+		this.index = index;
 		this.open = open;
 		this.close = close;
 		this.forwardRegex = forwardRegex;
 		this.reversedRegex = reversedRegex;
+		this._openSet = RichEditBracket._toSet(this.open);
+		this._closeSet = RichEditBracket._toSet(this.close);
 	}
+
+	public isOpen(text: string) {
+		return this._openSet.has(text);
+	}
+
+	public isClose(text: string) {
+		return this._closeSet.has(text);
+	}
+
+	private static _toSet(arr: string[]): Set<string> {
+		const result = new Set<string>();
+		for (const element of arr) {
+			result.add(element);
+		}
+		return result;
+	}
+}
+
+function groupFuzzyBrackets(brackets: CharacterPair[]): InternalBracket[] {
+	const N = brackets.length;
+
+	brackets = brackets.map(b => [b[0].toLowerCase(), b[1].toLowerCase()]);
+
+	const group: number[] = [];
+	for (let i = 0; i < N; i++) {
+		group[i] = i;
+	}
+
+	const areOverlapping = (a: CharacterPair, b: CharacterPair) => {
+		const [aOpen, aClose] = a;
+		const [bOpen, bClose] = b;
+		return (aOpen === bOpen || aOpen === bClose || aClose === bOpen || aClose === bClose);
+	};
+
+	const mergeGroups = (g1: number, g2: number) => {
+		const newG = Math.min(g1, g2);
+		const oldG = Math.max(g1, g2);
+		for (let i = 0; i < N; i++) {
+			if (group[i] === oldG) {
+				group[i] = newG;
+			}
+		}
+	};
+
+	// group together brackets that have the same open or the same close sequence
+	for (let i = 0; i < N; i++) {
+		const a = brackets[i];
+		for (let j = i + 1; j < N; j++) {
+			const b = brackets[j];
+			if (areOverlapping(a, b)) {
+				mergeGroups(group[i], group[j]);
+			}
+		}
+	}
+
+	const result: InternalBracket[] = [];
+	for (let g = 0; g < N; g++) {
+		let currentOpen: string[] = [];
+		let currentClose: string[] = [];
+		for (let i = 0; i < N; i++) {
+			if (group[i] === g) {
+				const [open, close] = brackets[i];
+				currentOpen.push(open);
+				currentClose.push(close);
+			}
+		}
+		if (currentOpen.length > 0) {
+			result.push({
+				open: currentOpen,
+				close: currentClose
+			});
+		}
+	}
+	return result;
 }
 
 export class RichEditBrackets {
@@ -41,87 +121,140 @@ export class RichEditBrackets {
 	public readonly textIsBracket: { [text: string]: RichEditBracket; };
 	public readonly textIsOpenBracket: { [text: string]: boolean; };
 
-	constructor(languageIdentifier: LanguageIdentifier, brackets: CharacterPair[]) {
-		this.brackets = brackets.map((b) => {
+	constructor(languageIdentifier: LanguageIdentifier, _brackets: CharacterPair[]) {
+		const brackets = groupFuzzyBrackets(_brackets);
+
+		this.brackets = brackets.map((b, index) => {
 			return new RichEditBracket(
 				languageIdentifier,
-				b[0],
-				b[1],
-				getRegexForBracketPair({ open: b[0], close: b[1] }),
-				getReversedRegexForBracketPair({ open: b[0], close: b[1] })
+				index,
+				b.open,
+				b.close,
+				getRegexForBracketPair(b.open, b.close, brackets, index),
+				getReversedRegexForBracketPair(b.open, b.close, brackets, index)
 			);
 		});
+
 		this.forwardRegex = getRegexForBrackets(this.brackets);
 		this.reversedRegex = getReversedRegexForBrackets(this.brackets);
 
 		this.textIsBracket = {};
 		this.textIsOpenBracket = {};
 
-		let maxBracketLength = 0;
-		this.brackets.forEach((b) => {
-			this.textIsBracket[b.open.toLowerCase()] = b;
-			this.textIsBracket[b.close.toLowerCase()] = b;
-			this.textIsOpenBracket[b.open.toLowerCase()] = true;
-			this.textIsOpenBracket[b.close.toLowerCase()] = false;
-			maxBracketLength = Math.max(maxBracketLength, b.open.length);
-			maxBracketLength = Math.max(maxBracketLength, b.close.length);
-		});
-		this.maxBracketLength = maxBracketLength;
-	}
-}
-
-function once<T, R>(keyFn: (input: T) => string, computeFn: (input: T) => R): (input: T) => R {
-	let cache: { [key: string]: R; } = {};
-	return (input: T): R => {
-		let key = keyFn(input);
-		if (!cache.hasOwnProperty(key)) {
-			cache[key] = computeFn(input);
+		this.maxBracketLength = 0;
+		for (const bracket of this.brackets) {
+			for (const open of bracket.open) {
+				this.textIsBracket[open] = bracket;
+				this.textIsOpenBracket[open] = true;
+				this.maxBracketLength = Math.max(this.maxBracketLength, open.length);
+			}
+			for (const close of bracket.close) {
+				this.textIsBracket[close] = bracket;
+				this.textIsOpenBracket[close] = false;
+				this.maxBracketLength = Math.max(this.maxBracketLength, close.length);
+			}
 		}
-		return cache[key];
-	};
+	}
 }
 
-const getRegexForBracketPair = once<ISimpleInternalBracket, RegExp>(
-	(input) => `${input.open};${input.close}`,
-	(input) => {
-		return createBracketOrRegExp([input.open, input.close]);
+function collectSuperstrings(str: string, brackets: InternalBracket[], currentIndex: number, dest: string[]): void {
+	for (let i = 0, len = brackets.length; i < len; i++) {
+		if (i === currentIndex) {
+			continue;
+		}
+		const bracket = brackets[i];
+		for (const open of bracket.open) {
+			if (open.indexOf(str) >= 0) {
+				dest.push(open);
+			}
+		}
+		for (const close of bracket.close) {
+			if (close.indexOf(str) >= 0) {
+				dest.push(close);
+			}
+		}
 	}
-);
+}
 
-const getReversedRegexForBracketPair = once<ISimpleInternalBracket, RegExp>(
-	(input) => `${input.open};${input.close}`,
-	(input) => {
-		return createBracketOrRegExp([toReversedString(input.open), toReversedString(input.close)]);
-	}
-);
+function lengthcmp(a: string, b: string) {
+	return a.length - b.length;
+}
 
-const getRegexForBrackets = once<ISimpleInternalBracket[], RegExp>(
-	(input) => input.map(b => `${b.open};${b.close}`).join(';'),
-	(input) => {
-		let pieces: string[] = [];
-		input.forEach((b) => {
-			pieces.push(b.open);
-			pieces.push(b.close);
-		});
-		return createBracketOrRegExp(pieces);
+function unique(arr: string[]): string[] {
+	if (arr.length <= 1) {
+		return arr;
 	}
-);
+	const result: string[] = [];
+	const seen = new Set<string>();
+	for (const element of arr) {
+		if (seen.has(element)) {
+			continue;
+		}
+		result.push(element);
+		seen.add(element);
+	}
+	return result;
+}
 
-const getReversedRegexForBrackets = once<ISimpleInternalBracket[], RegExp>(
-	(input) => input.map(b => `${b.open};${b.close}`).join(';'),
-	(input) => {
-		let pieces: string[] = [];
-		input.forEach((b) => {
-			pieces.push(toReversedString(b.open));
-			pieces.push(toReversedString(b.close));
-		});
-		return createBracketOrRegExp(pieces);
+function getRegexForBracketPair(open: string[], close: string[], brackets: InternalBracket[], currentIndex: number): RegExp {
+	// search in all brackets for other brackets that are a superstring of these brackets
+	let pieces: string[] = [];
+	pieces = pieces.concat(open);
+	pieces = pieces.concat(close);
+	for (let i = 0, len = pieces.length; i < len; i++) {
+		collectSuperstrings(pieces[i], brackets, currentIndex, pieces);
 	}
-);
+	pieces = unique(pieces);
+	pieces.sort(lengthcmp);
+	pieces.reverse();
+	return createBracketOrRegExp(pieces);
+}
+
+function getReversedRegexForBracketPair(open: string[], close: string[], brackets: InternalBracket[], currentIndex: number): RegExp {
+	// search in all brackets for other brackets that are a superstring of these brackets
+	let pieces: string[] = [];
+	pieces = pieces.concat(open);
+	pieces = pieces.concat(close);
+	for (let i = 0, len = pieces.length; i < len; i++) {
+		collectSuperstrings(pieces[i], brackets, currentIndex, pieces);
+	}
+	pieces = unique(pieces);
+	pieces.sort(lengthcmp);
+	pieces.reverse();
+	return createBracketOrRegExp(pieces.map(toReversedString));
+}
+
+function getRegexForBrackets(brackets: RichEditBracket[]): RegExp {
+	let pieces: string[] = [];
+	for (const bracket of brackets) {
+		for (const open of bracket.open) {
+			pieces.push(open);
+		}
+		for (const close of bracket.close) {
+			pieces.push(close);
+		}
+	}
+	pieces = unique(pieces);
+	return createBracketOrRegExp(pieces);
+}
+
+function getReversedRegexForBrackets(brackets: RichEditBracket[]): RegExp {
+	let pieces: string[] = [];
+	for (const bracket of brackets) {
+		for (const open of bracket.open) {
+			pieces.push(open);
+		}
+		for (const close of bracket.close) {
+			pieces.push(close);
+		}
+	}
+	pieces = unique(pieces);
+	return createBracketOrRegExp(pieces.map(toReversedString));
+}
 
 function prepareBracketForRegExp(str: string): string {
 	// This bracket pair uses letters like e.g. "begin" - "end"
-	const insertWordBoundaries = (/^[\w]+$/.test(str));
+	const insertWordBoundaries = (/^[\w ]+$/.test(str));
 	str = strings.escapeRegExpCharacters(str);
 	return (insertWordBoundaries ? `\\b${str}\\b` : str);
 }
@@ -168,12 +301,11 @@ export class BracketsUtils {
 		return new Range(lineNumber, absoluteMatchOffset - matchLength + 1, lineNumber, absoluteMatchOffset + 1);
 	}
 
-	public static findPrevBracketInToken(reversedBracketRegex: RegExp, lineNumber: number, lineText: string, currentTokenStart: number, currentTokenEnd: number): Range | null {
+	public static findPrevBracketInRange(reversedBracketRegex: RegExp, lineNumber: number, lineText: string, startOffset: number, endOffset: number): Range | null {
 		// Because JS does not support backwards regex search, we search forwards in a reversed string with a reversed regex ;)
-		let reversedLineText = toReversedString(lineText);
-		let reversedTokenText = reversedLineText.substring(lineText.length - currentTokenEnd, lineText.length - currentTokenStart);
-
-		return this._findPrevBracketInText(reversedBracketRegex, lineNumber, reversedTokenText, currentTokenStart);
+		const reversedLineText = toReversedString(lineText);
+		const reversedSubstr = reversedLineText.substring(lineText.length - endOffset, lineText.length - startOffset);
+		return this._findPrevBracketInText(reversedBracketRegex, lineNumber, reversedSubstr, startOffset);
 	}
 
 	public static findNextBracketInText(bracketRegex: RegExp, lineNumber: number, text: string, offset: number): Range | null {
@@ -193,10 +325,8 @@ export class BracketsUtils {
 		return new Range(lineNumber, absoluteMatchOffset + 1, lineNumber, absoluteMatchOffset + 1 + matchLength);
 	}
 
-	public static findNextBracketInToken(bracketRegex: RegExp, lineNumber: number, lineText: string, currentTokenStart: number, currentTokenEnd: number): Range | null {
-		let currentTokenText = lineText.substring(currentTokenStart, currentTokenEnd);
-
-		return this.findNextBracketInText(bracketRegex, lineNumber, currentTokenText, currentTokenStart);
+	public static findNextBracketInRange(bracketRegex: RegExp, lineNumber: number, lineText: string, startOffset: number, endOffset: number): Range | null {
+		const substr = lineText.substring(startOffset, endOffset);
+		return this.findNextBracketInText(bracketRegex, lineNumber, substr, startOffset);
 	}
-
 }

src/vs/editor/common/modes/textToHtmlTokenizer.ts

@@ -24,7 +24,7 @@ export function tokenizeToString(text: string, tokenizationSupport: IReducedToke
 	return _tokenizeToString(text, tokenizationSupport || fallback);
 }
 
-export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens, colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string {
+export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens, colorMap: string[], startOffset: number, endOffset: number, tabSize: number, useNbsp: boolean): string {
 	let result = `<div>`;
 	let charIndex = startOffset;
 	let tabsCharDelta = 0;
@@ -46,7 +46,7 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens
 					let insertSpacesCount = tabSize - (charIndex + tabsCharDelta) % tabSize;
 					tabsCharDelta += insertSpacesCount - 1;
 					while (insertSpacesCount > 0) {
-						partContent += '&nbsp;';
+						partContent += useNbsp ? '&nbsp;' : ' ';
 						insertSpacesCount--;
 					}
 					break;
@@ -78,7 +78,7 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens
 					break;
 
 				case CharCode.Space:
-					partContent += '&nbsp;';
+					partContent += useNbsp ? '&nbsp;' : ' ';
 					break;
 
 				default: