Merge from vscode 718331d6f3ebd1b571530ab499edb266ddd493d5

src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase.ts

@@ -670,7 +670,7 @@ export class PieceTreeBase {
 		if (searcher._wordSeparators) {
 			searchText = buffer.buffer.substring(start, end);
 			offsetInBuffer = (offset: number) => offset + start;
-			searcher.reset(-1);
+			searcher.reset(0);
 		} else {
 			searchText = buffer.buffer;
 			offsetInBuffer = (offset: number) => offset;

src/vs/editor/common/model/textModel.ts

@@ -2640,14 +2640,16 @@ export class TextModel extends Disposable implements model.ITextModel {
 		let goDown = true;
 		let indent = 0;
 
+		let initialIndent = 0;
+
 		for (let distance = 0; goUp || goDown; distance++) {
 			const upLineNumber = lineNumber - distance;
 			const downLineNumber = lineNumber + distance;
 
-			if (distance !== 0 && (upLineNumber < 1 || upLineNumber < minLineNumber)) {
+			if (distance > 1 && (upLineNumber < 1 || upLineNumber < minLineNumber)) {
 				goUp = false;
 			}
-			if (distance !== 0 && (downLineNumber > lineCount || downLineNumber > maxLineNumber)) {
+			if (distance > 1 && (downLineNumber > lineCount || downLineNumber > maxLineNumber)) {
 				goDown = false;
 			}
 			if (distance > 50000) {
@@ -2656,10 +2658,9 @@ export class TextModel extends Disposable implements model.ITextModel {
 				goDown = false;
 			}
 
+			let upLineIndentLevel: number = -1;
 			if (goUp) {
 				// compute indent level going up
-				let upLineIndentLevel: number;
-
 				const currentIndent = this._computeIndentLevel(upLineNumber - 1);
 				if (currentIndent >= 0) {
 					// This line has content (besides whitespace)
@@ -2671,30 +2672,11 @@ export class TextModel extends Disposable implements model.ITextModel {
 					up_resolveIndents(upLineNumber);
 					upLineIndentLevel = this._getIndentLevelForWhitespaceLine(offSide, up_aboveContentLineIndent, up_belowContentLineIndent);
 				}
-
-				if (distance === 0) {
-					// This is the initial line number
-					startLineNumber = upLineNumber;
-					endLineNumber = downLineNumber;
-					indent = upLineIndentLevel;
-					if (indent === 0) {
-						// No need to continue
-						return { startLineNumber, endLineNumber, indent };
-					}
-					continue;
-				}
-
-				if (upLineIndentLevel >= indent) {
-					startLineNumber = upLineNumber;
-				} else {
-					goUp = false;
-				}
 			}
 
+			let downLineIndentLevel = -1;
 			if (goDown) {
 				// compute indent level going down
-				let downLineIndentLevel: number;
-
 				const currentIndent = this._computeIndentLevel(downLineNumber - 1);
 				if (currentIndent >= 0) {
 					// This line has content (besides whitespace)
@@ -2706,7 +2688,50 @@ export class TextModel extends Disposable implements model.ITextModel {
 					down_resolveIndents(downLineNumber);
 					downLineIndentLevel = this._getIndentLevelForWhitespaceLine(offSide, down_aboveContentLineIndent, down_belowContentLineIndent);
 				}
+			}
 
+			if (distance === 0) {
+				initialIndent = upLineIndentLevel;
+				continue;
+			}
+
+			if (distance === 1) {
+				if (downLineNumber <= lineCount && downLineIndentLevel >= 0 && initialIndent + 1 === downLineIndentLevel) {
+					// This is the beginning of a scope, we have special handling here, since we want the
+					// child scope indent to be active, not the parent scope
+					goUp = false;
+					startLineNumber = downLineNumber;
+					endLineNumber = downLineNumber;
+					indent = downLineIndentLevel;
+					continue;
+				}
+
+				if (upLineNumber >= 1 && upLineIndentLevel >= 0 && upLineIndentLevel - 1 === initialIndent) {
+					// This is the end of a scope, just like above
+					goDown = false;
+					startLineNumber = upLineNumber;
+					endLineNumber = upLineNumber;
+					indent = upLineIndentLevel;
+					continue;
+				}
+
+				startLineNumber = lineNumber;
+				endLineNumber = lineNumber;
+				indent = initialIndent;
+				if (indent === 0) {
+					// No need to continue
+					return { startLineNumber, endLineNumber, indent };
+				}
+			}
+
+			if (goUp) {
+				if (upLineIndentLevel >= indent) {
+					startLineNumber = upLineNumber;
+				} else {
+					goUp = false;
+				}
+			}
+			if (goDown) {
 				if (downLineIndentLevel >= indent) {
 					endLineNumber = downLineNumber;
 				} else {

src/vs/editor/common/model/textModelEvents.ts

@@ -80,6 +80,7 @@ export interface IModelDecorationsChangedEvent {
 
 /**
  * An event describing that some ranges of lines have been tokenized (their tokens have changed).
+ * @internal
  */
 export interface IModelTokensChangedEvent {
 	readonly tokenizationSupportChanged: boolean;

src/vs/editor/common/model/tokensStore.ts

@@ -786,6 +786,15 @@ export class TokensStore2 {
 			const bEndCharacter = bTokens.getEndCharacter(bIndex);
 			const bMetadata = bTokens.getMetadata(bIndex);
 
+			const bMask = (
+				((bMetadata & MetadataConsts.SEMANTIC_USE_ITALIC) ? MetadataConsts.ITALIC_MASK : 0)
+				| ((bMetadata & MetadataConsts.SEMANTIC_USE_BOLD) ? MetadataConsts.BOLD_MASK : 0)
+				| ((bMetadata & MetadataConsts.SEMANTIC_USE_UNDERLINE) ? MetadataConsts.UNDERLINE_MASK : 0)
+				| ((bMetadata & MetadataConsts.SEMANTIC_USE_FOREGROUND) ? MetadataConsts.FOREGROUND_MASK : 0)
+				| ((bMetadata & MetadataConsts.SEMANTIC_USE_BACKGROUND) ? MetadataConsts.BACKGROUND_MASK : 0)
+			) >>> 0;
+			const aMask = (~bMask) >>> 0;
+
 			// push any token from `a` that is before `b`
 			while (aIndex < aLen && aTokens.getEndOffset(aIndex) <= bStartCharacter) {
 				result[resultLen++] = aTokens.getEndOffset(aIndex);
@@ -800,21 +809,24 @@ export class TokensStore2 {
 			}
 
 			// skip any tokens from `a` that are contained inside `b`
-			while (aIndex < aLen && aTokens.getEndOffset(aIndex) <= bEndCharacter) {
+			while (aIndex < aLen && aTokens.getEndOffset(aIndex) < bEndCharacter) {
+				result[resultLen++] = aTokens.getEndOffset(aIndex);
+				result[resultLen++] = (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask);
 				aIndex++;
 			}
 
-			const aMetadata = aTokens.getMetadata(Math.min(Math.max(0, aIndex - 1), aLen - 1));
-			const languageId = TokenMetadata.getLanguageId(aMetadata);
-			const tokenType = TokenMetadata.getTokenType(aMetadata);
+			if (aIndex < aLen && aTokens.getEndOffset(aIndex) === bEndCharacter) {
+				// `a` ends exactly at the same spot as `b`!
+				result[resultLen++] = aTokens.getEndOffset(aIndex);
+				result[resultLen++] = (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask);
+				aIndex++;
+			} else {
+				const aMergeIndex = Math.min(Math.max(0, aIndex - 1), aLen - 1);
 
-			// push the token from `b`
-			result[resultLen++] = bEndCharacter;
-			result[resultLen++] = (
-				(bMetadata & MetadataConsts.LANG_TTYPE_CMPL)
-				| ((languageId << MetadataConsts.LANGUAGEID_OFFSET) >>> 0)
-				| ((tokenType << MetadataConsts.TOKEN_TYPE_OFFSET) >>> 0)
-			);
+				// push the token from `b`
+				result[resultLen++] = bEndCharacter;
+				result[resultLen++] = (aTokens.getMetadata(aMergeIndex) & aMask) | (bMetadata & bMask);
+			}
 		}
 
 		// push the remaining tokens from `a`