Merge from vscode f5d3ffa6a0d655c87e1eb0e1e90773df58f7ff25 (#7929)

* Merge from vscode f5d3ffa6a0d655c87e1eb0e1e90773df58f7ff25

* fix launch script

* add missing files

src/vs/editor/common/controller/cursor.ts

@@ -173,6 +173,7 @@ export class Cursor extends viewEvents.ViewEventEmitter implements ICursors {
 
 	private _isHandling: boolean;
 	private _isDoingComposition: boolean;
+	private _selectionsWhenCompositionStarted: Selection[] | null;
 	private _columnSelectData: IColumnSelectData | null;
 	private _autoClosedActions: AutoClosedAction[];
 	private _prevEditOperationType: EditOperationType;
@@ -188,6 +189,7 @@ export class Cursor extends viewEvents.ViewEventEmitter implements ICursors {
 
 		this._isHandling = false;
 		this._isDoingComposition = false;
+		this._selectionsWhenCompositionStarted = null;
 		this._columnSelectData = null;
 		this._autoClosedActions = [];
 		this._prevEditOperationType = EditOperationType.Other;
@@ -667,6 +669,7 @@ export class Cursor extends viewEvents.ViewEventEmitter implements ICursors {
 
 		if (handlerId === H.CompositionStart) {
 			this._isDoingComposition = true;
+			this._selectionsWhenCompositionStarted = this.getSelections().slice(0);
 			return;
 		}
 
@@ -757,7 +760,8 @@ export class Cursor extends viewEvents.ViewEventEmitter implements ICursors {
 		if (!this._isDoingComposition && source === 'keyboard') {
 			// composition finishes, let's check if we need to auto complete if necessary.
 			const autoClosedCharacters = AutoClosedAction.getAllAutoClosedCharacters(this._autoClosedActions);
-			this._executeEditOperation(TypeOperations.compositionEndWithInterceptors(this._prevEditOperationType, this.context.config, this.context.model, this.getSelections(), autoClosedCharacters));
+			this._executeEditOperation(TypeOperations.compositionEndWithInterceptors(this._prevEditOperationType, this.context.config, this.context.model, this._selectionsWhenCompositionStarted, this.getSelections(), autoClosedCharacters));
+			this._selectionsWhenCompositionStarted = null;
 		}
 	}
 
@@ -765,19 +769,17 @@ export class Cursor extends viewEvents.ViewEventEmitter implements ICursors {
 		if (!this._isDoingComposition && source === 'keyboard') {
 			// If this event is coming straight from the keyboard, look for electric characters and enter
 
-			for (let i = 0, len = text.length; i < len; i++) {
-				let charCode = text.charCodeAt(i);
-				let chr: string;
-				if (strings.isHighSurrogate(charCode) && i + 1 < len) {
-					chr = text.charAt(i) + text.charAt(i + 1);
-					i++;
-				} else {
-					chr = text.charAt(i);
-				}
+			const len = text.length;
+			let offset = 0;
+			while (offset < len) {
+				const charLength = strings.nextCharLength(text, offset);
+				const chr = text.substr(offset, charLength);
 
 				// Here we must interpret each typed character individually
 				const autoClosedCharacters = AutoClosedAction.getAllAutoClosedCharacters(this._autoClosedActions);
 				this._executeEditOperation(TypeOperations.typeWithInterceptors(this._prevEditOperationType, this.context.config, this.context.model, this.getSelections(), autoClosedCharacters, chr));
+
+				offset += charLength;
 			}
 
 		} else {

src/vs/editor/common/controller/cursorCommon.ts

@@ -19,6 +19,7 @@ import { IAutoClosingPair, StandardAutoClosingPairConditional } from 'vs/editor/
 import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
 import { VerticalRevealType } from 'vs/editor/common/view/viewEvents';
 import { IViewModel } from 'vs/editor/common/viewModel/viewModel';
+import { Constants } from 'vs/base/common/uint';
 
 export interface IColumnSelectData {
 	isReal: boolean;
@@ -509,66 +510,53 @@ export class EditOperationResult {
  */
 export class CursorColumns {
 
-	public static isLowSurrogate(model: ICursorSimpleModel, lineNumber: number, charOffset: number): boolean {
-		let lineContent = model.getLineContent(lineNumber);
-		if (charOffset < 0 || charOffset >= lineContent.length) {
-			return false;
-		}
-		return strings.isLowSurrogate(lineContent.charCodeAt(charOffset));
-	}
-
-	public static isHighSurrogate(model: ICursorSimpleModel, lineNumber: number, charOffset: number): boolean {
-		let lineContent = model.getLineContent(lineNumber);
-		if (charOffset < 0 || charOffset >= lineContent.length) {
-			return false;
-		}
-		return strings.isHighSurrogate(lineContent.charCodeAt(charOffset));
-	}
-
-	public static isInsideSurrogatePair(model: ICursorSimpleModel, lineNumber: number, column: number): boolean {
-		return this.isHighSurrogate(model, lineNumber, column - 2);
-	}
-
 	public static visibleColumnFromColumn(lineContent: string, column: number, tabSize: number): number {
-		let endOffset = lineContent.length;
-		if (endOffset > column - 1) {
-			endOffset = column - 1;
-		}
+		const lineContentLength = lineContent.length;
+		const endOffset = column - 1 < lineContentLength ? column - 1 : lineContentLength;
 
 		let result = 0;
-		for (let i = 0; i < endOffset; i++) {
-			let charCode = lineContent.charCodeAt(i);
-			if (charCode === CharCode.Tab) {
-				result = this.nextRenderTabStop(result, tabSize);
-			} else if (strings.isFullWidthCharacter(charCode)) {
-				result = result + 2;
+		let i = 0;
+		while (i < endOffset) {
+			const codePoint = strings.getNextCodePoint(lineContent, endOffset, i);
+			i += (codePoint >= Constants.UNICODE_SUPPLEMENTARY_PLANE_BEGIN ? 2 : 1);
+
+			if (codePoint === CharCode.Tab) {
+				result = CursorColumns.nextRenderTabStop(result, tabSize);
 			} else {
-				result = result + 1;
+				while (i < endOffset) {
+					const nextCodePoint = strings.getNextCodePoint(lineContent, endOffset, i);
+					if (!strings.isUnicodeMark(nextCodePoint)) {
+						break;
+					}
+					i += (nextCodePoint >= Constants.UNICODE_SUPPLEMENTARY_PLANE_BEGIN ? 2 : 1);
+				}
+				if (strings.isFullWidthCharacter(codePoint) || strings.isEmojiImprecise(codePoint)) {
+					result = result + 2;
+				} else {
+					result = result + 1;
+				}
 			}
 		}
 		return result;
 	}
 
 	public static toStatusbarColumn(lineContent: string, column: number, tabSize: number): number {
-		let endOffset = lineContent.length;
-		if (endOffset > column - 1) {
-			endOffset = column - 1;
-		}
+		const lineContentLength = lineContent.length;
+		const endOffset = column - 1 < lineContentLength ? column - 1 : lineContentLength;
 
 		let result = 0;
-		for (let i = 0; i < endOffset; i++) {
-			let charCode = lineContent.charCodeAt(i);
-			if (charCode === CharCode.Tab) {
-				result = this.nextRenderTabStop(result, tabSize);
+		let i = 0;
+		while (i < endOffset) {
+			const codePoint = strings.getNextCodePoint(lineContent, endOffset, i);
+			i += (codePoint >= Constants.UNICODE_SUPPLEMENTARY_PLANE_BEGIN ? 2 : 1);
+
+			if (codePoint === CharCode.Tab) {
+				result = CursorColumns.nextRenderTabStop(result, tabSize);
 			} else {
-				if (strings.isHighSurrogate(charCode)) {
-					result = result + 1;
-					i = i + 1;
-				} else {
-					result = result + 1;
-				}
+				result = result + 1;
 			}
 		}
+
 		return result + 1;
 	}
 
@@ -584,29 +572,43 @@ export class CursorColumns {
 		const lineLength = lineContent.length;
 
 		let beforeVisibleColumn = 0;
-		for (let i = 0; i < lineLength; i++) {
-			let charCode = lineContent.charCodeAt(i);
+		let beforeColumn = 1;
+		let i = 0;
+		while (i < lineLength) {
+			const codePoint = strings.getNextCodePoint(lineContent, lineLength, i);
+			i += (codePoint >= Constants.UNICODE_SUPPLEMENTARY_PLANE_BEGIN ? 2 : 1);
 
 			let afterVisibleColumn: number;
-			if (charCode === CharCode.Tab) {
-				afterVisibleColumn = this.nextRenderTabStop(beforeVisibleColumn, tabSize);
-			} else if (strings.isFullWidthCharacter(charCode)) {
-				afterVisibleColumn = beforeVisibleColumn + 2;
+			if (codePoint === CharCode.Tab) {
+				afterVisibleColumn = CursorColumns.nextRenderTabStop(beforeVisibleColumn, tabSize);
 			} else {
-				afterVisibleColumn = beforeVisibleColumn + 1;
+				while (i < lineLength) {
+					const nextCodePoint = strings.getNextCodePoint(lineContent, lineLength, i);
+					if (!strings.isUnicodeMark(nextCodePoint)) {
+						break;
+					}
+					i += (nextCodePoint >= Constants.UNICODE_SUPPLEMENTARY_PLANE_BEGIN ? 2 : 1);
+				}
+				if (strings.isFullWidthCharacter(codePoint) || strings.isEmojiImprecise(codePoint)) {
+					afterVisibleColumn = beforeVisibleColumn + 2;
+				} else {
+					afterVisibleColumn = beforeVisibleColumn + 1;
+				}
 			}
+			const afterColumn = i + 1;
 
 			if (afterVisibleColumn >= visibleColumn) {
-				let prevDelta = visibleColumn - beforeVisibleColumn;
-				let afterDelta = afterVisibleColumn - visibleColumn;
-				if (afterDelta < prevDelta) {
-					return i + 2;
+				const beforeDelta = visibleColumn - beforeVisibleColumn;
+				const afterDelta = afterVisibleColumn - visibleColumn;
+				if (afterDelta < beforeDelta) {
+					return afterColumn;
 				} else {
-					return i + 1;
+					return beforeColumn;
 				}
 			}
 
 			beforeVisibleColumn = afterVisibleColumn;
+			beforeColumn = afterColumn;
 		}
 
 		// walked the entire string

src/vs/editor/common/controller/cursorMoveOperations.ts

@@ -6,6 +6,7 @@
 import { CursorColumns, CursorConfiguration, ICursorSimpleModel, SingleCursorState } from 'vs/editor/common/controller/cursorCommon';
 import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
+import * as strings from 'vs/base/common/strings';
 
 export class CursorPosition {
 	_cursorPositionBrand: void;
@@ -23,21 +24,19 @@ export class CursorPosition {
 
 export class MoveOperations {
 
-	public static left(config: CursorConfiguration, model: ICursorSimpleModel, lineNumber: number, column: number): CursorPosition {
-
+	public static leftPosition(model: ICursorSimpleModel, lineNumber: number, column: number): Position {
 		if (column > model.getLineMinColumn(lineNumber)) {
-			if (CursorColumns.isLowSurrogate(model, lineNumber, column - 2)) {
-				// character before column is a low surrogate
-				column = column - 2;
-			} else {
-				column = column - 1;
-			}
+			column = column - strings.prevCharLength(model.getLineContent(lineNumber), column - 1);
 		} else if (lineNumber > 1) {
 			lineNumber = lineNumber - 1;
 			column = model.getLineMaxColumn(lineNumber);
 		}
+		return new Position(lineNumber, column);
+	}
 
-		return new CursorPosition(lineNumber, column, 0);
+	public static left(config: CursorConfiguration, model: ICursorSimpleModel, lineNumber: number, column: number): CursorPosition {
+		const pos = MoveOperations.leftPosition(model, lineNumber, column);
+		return new CursorPosition(pos.lineNumber, pos.column, 0);
 	}
 
 	public static moveLeft(config: CursorConfiguration, model: ICursorSimpleModel, cursor: SingleCursorState, inSelectionMode: boolean, noOfColumns: number): SingleCursorState {
@@ -57,21 +56,19 @@ export class MoveOperations {
 		return cursor.move(inSelectionMode, lineNumber, column, 0);
 	}
 
-	public static right(config: CursorConfiguration, model: ICursorSimpleModel, lineNumber: number, column: number): CursorPosition {
-
+	public static rightPosition(model: ICursorSimpleModel, lineNumber: number, column: number): Position {
 		if (column < model.getLineMaxColumn(lineNumber)) {
-			if (CursorColumns.isHighSurrogate(model, lineNumber, column - 1)) {
-				// character after column is a high surrogate
-				column = column + 2;
-			} else {
-				column = column + 1;
-			}
+			column = column + strings.nextCharLength(model.getLineContent(lineNumber), column - 1);
 		} else if (lineNumber < model.getLineCount()) {
 			lineNumber = lineNumber + 1;
 			column = model.getLineMinColumn(lineNumber);
 		}
+		return new Position(lineNumber, column);
+	}
 
-		return new CursorPosition(lineNumber, column, 0);
+	public static right(config: CursorConfiguration, model: ICursorSimpleModel, lineNumber: number, column: number): CursorPosition {
+		const pos = MoveOperations.rightPosition(model, lineNumber, column);
+		return new CursorPosition(pos.lineNumber, pos.column, 0);
 	}
 
 	public static moveRight(config: CursorConfiguration, model: ICursorSimpleModel, cursor: SingleCursorState, inSelectionMode: boolean, noOfColumns: number): SingleCursorState {
@@ -102,15 +99,9 @@ export class MoveOperations {
 				column = model.getLineMaxColumn(lineNumber);
 			} else {
 				column = Math.min(model.getLineMaxColumn(lineNumber), column);
-				if (CursorColumns.isInsideSurrogatePair(model, lineNumber, column)) {
-					column = column - 1;
-				}
 			}
 		} else {
 			column = CursorColumns.columnFromVisibleColumn2(config, model, lineNumber, currentVisibleColumn);
-			if (CursorColumns.isInsideSurrogatePair(model, lineNumber, column)) {
-				column = column - 1;
-			}
 		}
 
 		leftoverVisibleColumns = currentVisibleColumn - CursorColumns.visibleColumnFromColumn(model.getLineContent(lineNumber), column, config.tabSize);
@@ -160,15 +151,9 @@ export class MoveOperations {
 				column = model.getLineMinColumn(lineNumber);
 			} else {
 				column = Math.min(model.getLineMaxColumn(lineNumber), column);
-				if (CursorColumns.isInsideSurrogatePair(model, lineNumber, column)) {
-					column = column - 1;
-				}
 			}
 		} else {
 			column = CursorColumns.columnFromVisibleColumn2(config, model, lineNumber, currentVisibleColumn);
-			if (CursorColumns.isInsideSurrogatePair(model, lineNumber, column)) {
-				column = column - 1;
-			}
 		}
 
 		leftoverVisibleColumns = currentVisibleColumn - CursorColumns.visibleColumnFromColumn(model.getLineContent(lineNumber), column, config.tabSize);

src/vs/editor/common/controller/cursorTypeOperations.ts

@@ -755,7 +755,12 @@ export class TypeOperations {
 	/**
 	 * This is very similar with typing, but the character is already in the text buffer!
 	 */
-	public static compositionEndWithInterceptors(prevEditOperationType: EditOperationType, config: CursorConfiguration, model: ITextModel, selections: Selection[], autoClosedCharacters: Range[]): EditOperationResult | null {
+	public static compositionEndWithInterceptors(prevEditOperationType: EditOperationType, config: CursorConfiguration, model: ITextModel, selectionsWhenCompositionStarted: Selection[] | null, selections: Selection[], autoClosedCharacters: Range[]): EditOperationResult | null {
+		if (!selectionsWhenCompositionStarted || Selection.selectionsArrEqual(selectionsWhenCompositionStarted, selections)) {
+			// no content was typed
+			return null;
+		}
+
 		let ch: string | null = null;
 		// extract last typed character
 		for (const selection of selections) {

src/vs/editor/common/model/textModel.ts

@@ -895,11 +895,9 @@ export class TextModel extends Disposable implements model.ITextModel {
 		}
 
 		if (strict) {
-			if (column > 1) {
-				const charCodeBefore = this._buffer.getLineCharCode(lineNumber, column - 2);
-				if (strings.isHighSurrogate(charCodeBefore)) {
-					return false;
-				}
+			const [charStartOffset,] = strings.getCharContainingOffset(this._buffer.getLineContent(lineNumber), column - 1);
+			if (column !== charStartOffset + 1) {
+				return false;
 			}
 		}
 
@@ -932,12 +930,9 @@ export class TextModel extends Disposable implements model.ITextModel {
 		}
 
 		if (strict) {
-			// If the position would end up in the middle of a high-low surrogate pair,
-			// we move it to before the pair
-			// !!At this point, column > 1
-			const charCodeBefore = this._buffer.getLineCharCode(lineNumber, column - 2);
-			if (strings.isHighSurrogate(charCodeBefore)) {
-				return new Position(lineNumber, column - 1);
+			const [charStartOffset,] = strings.getCharContainingOffset(this._buffer.getLineContent(lineNumber), column - 1);
+			if (column !== charStartOffset + 1) {
+				return new Position(lineNumber, charStartOffset + 1);
 			}
 		}
 
@@ -974,17 +969,23 @@ export class TextModel extends Disposable implements model.ITextModel {
 		}
 
 		if (strict) {
-			const charCodeBeforeStart = (startColumn > 1 ? this._buffer.getLineCharCode(startLineNumber, startColumn - 2) : 0);
-			const charCodeBeforeEnd = (endColumn > 1 && endColumn <= this._buffer.getLineLength(endLineNumber) ? this._buffer.getLineCharCode(endLineNumber, endColumn - 2) : 0);
-
-			const startInsideSurrogatePair = strings.isHighSurrogate(charCodeBeforeStart);
-			const endInsideSurrogatePair = strings.isHighSurrogate(charCodeBeforeEnd);
-
-			if (!startInsideSurrogatePair && !endInsideSurrogatePair) {
-				return true;
+			const startLineContent = this._buffer.getLineContent(startLineNumber);
+			if (startColumn < startLineContent.length + 1) {
+				const [charStartOffset,] = strings.getCharContainingOffset(startLineContent, startColumn - 1);
+				if (startColumn !== charStartOffset + 1) {
+					return false;
+				}
 			}
 
-			return false;
+			if (endColumn >= 2) {
+				const endLineContent = (endLineNumber === startLineNumber ? startLineContent : this._buffer.getLineContent(endLineNumber));
+				const [, charEndOffset] = strings.getCharContainingOffset(endLineContent, endColumn - 2);
+				if (endColumn !== charEndOffset + 1) {
+					return false;
+				}
+			}
+
+			return true;
 		}
 
 		return true;
@@ -1004,37 +1005,32 @@ export class TextModel extends Disposable implements model.ITextModel {
 		const end = this._validatePosition(_range.endLineNumber, _range.endColumn, false);
 
 		const startLineNumber = start.lineNumber;
-		const startColumn = start.column;
+		let startColumn = start.column;
 		const endLineNumber = end.lineNumber;
-		const endColumn = end.column;
+		let endColumn = end.column;
+		const isEmpty = (startLineNumber === endLineNumber && startColumn === endColumn);
 
-		const charCodeBeforeStart = (startColumn > 1 ? this._buffer.getLineCharCode(startLineNumber, startColumn - 2) : 0);
-		const charCodeBeforeEnd = (endColumn > 1 && endColumn <= this._buffer.getLineLength(endLineNumber) ? this._buffer.getLineCharCode(endLineNumber, endColumn - 2) : 0);
-
-		const startInsideSurrogatePair = strings.isHighSurrogate(charCodeBeforeStart);
-		const endInsideSurrogatePair = strings.isHighSurrogate(charCodeBeforeEnd);
-
-		if (!startInsideSurrogatePair && !endInsideSurrogatePair) {
-			return new Range(startLineNumber, startColumn, endLineNumber, endColumn);
+		const startLineContent = this._buffer.getLineContent(startLineNumber);
+		if (startColumn < startLineContent.length + 1) {
+			const [charStartOffset,] = strings.getCharContainingOffset(startLineContent, startColumn - 1);
+			if (startColumn !== charStartOffset + 1) {
+				if (isEmpty) {
+					// do not expand a collapsed range, simply move it to a valid location
+					return new Range(startLineNumber, charStartOffset + 1, startLineNumber, charStartOffset + 1);
+				}
+				startColumn = charStartOffset + 1;
+			}
 		}
 
-		if (startLineNumber === endLineNumber && startColumn === endColumn) {
-			// do not expand a collapsed range, simply move it to a valid location
-			return new Range(startLineNumber, startColumn - 1, endLineNumber, endColumn - 1);
+		if (endColumn >= 2) {
+			const endLineContent = (endLineNumber === startLineNumber ? startLineContent : this._buffer.getLineContent(endLineNumber));
+			const [, charEndOffset] = strings.getCharContainingOffset(endLineContent, endColumn - 2);
+			if (endColumn !== charEndOffset + 1) {
+				endColumn = charEndOffset + 1;
+			}
 		}
 
-		if (startInsideSurrogatePair && endInsideSurrogatePair) {
-			// expand range at both ends
-			return new Range(startLineNumber, startColumn - 1, endLineNumber, endColumn + 1);
-		}
-
-		if (startInsideSurrogatePair) {
-			// only expand range at the start
-			return new Range(startLineNumber, startColumn - 1, endLineNumber, endColumn);
-		}
-
-		// only expand range at the end
-		return new Range(startLineNumber, startColumn, endLineNumber, endColumn + 1);
+		return new Range(startLineNumber, startColumn, endLineNumber, endColumn);
 	}
 
 	public modifyPosition(rawPosition: IPosition, offset: number): Position {

src/vs/editor/common/modes.ts

@@ -954,7 +954,7 @@ export namespace SymbolKinds {
 	 * @internal
 	 */
 	export function toCssClassName(kind: SymbolKind, inline?: boolean): string {
-		return `symbol-icon ${inline ? 'inline' : 'block'} ${byKind.get(kind) || 'property'}`;
+		return `codicon ${inline ? 'inline' : 'block'} codicon-symbol-${byKind.get(kind) || 'property'}`;
 	}
 }