Merge from vscode 6268feb42ba4f2e2fa15484e88c9af60d254998c (#6530)

src/vs/editor/test/node/classification/typescript-test.ts

@@ -0,0 +1,71 @@
+///
+/* tslint:disable */
+const x01 = "string";
+///         ^^^^^^^^ string
+
+const x02 = '\'';
+///         ^^^^ string
+
+const x03 = '\n\'\t';
+///         ^^^^^^^^ string
+
+const x04 = 'this is\
+///         ^^^^^^^^^ string\
+a multiline string';
+/// <------------------- string
+
+const x05 = x01;// just some text
+///             ^^^^^^^^^^^^^^^^^ comment
+
+const x06 = x05;/* multi
+///             ^^^^^^^^ comment
+line *comment */
+/// <---------------- comment
+
+const x07 = 4 / 5;
+
+const x08 = `howdy`;
+///         ^^^^^^^ string
+
+const x09 = `\'\"\``;
+///         ^^^^^^^^ string
+
+const x10 = `$[]`;
+///         ^^^^^ string
+
+const x11 = `${x07 +/**/3}px`;
+///         ^^^ string
+///                 ^^^^ comment
+///                      ^^^^ string
+
+const x12 = `${x07 + (function () { return 5; })()/**/}px`;
+///         ^^^ string
+///                                               ^^^^ comment
+///                                                   ^^^^ string
+
+const x13 = /([\w\-]+)?(#([\w\-]+))?((.([\w\-]+))*)/;
+///         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ regex
+
+const x14 = /\./g;
+///         ^^^^^ regex
+
+
+const x15 = Math.abs(x07) / x07; // speed
+///                              ^^^^^^^^ comment
+
+const x16 = / x07; /.test('3');
+///         ^^^^^^^^ regex
+///                       ^^^ string
+
+const x17 = `.dialog-modal-block${true ? '.dimmed' : ''}`;
+///         ^^^^^^^^^^^^^^^^^^^^^^ string
+///                                      ^^^^^^^^^ string
+///                                                  ^^^^ string
+
+const x18 = Math.min((14 <= 0.5 ? 123 / (2 * 1) : ''.length / (2 - (2 * 1))), 1);
+///                                               ^^ string
+
+const x19 = `${3 / '5'.length} km/h)`;
+///         ^^^ string
+///                ^^^ string
+///                          ^^^^^^^ string

src/vs/editor/test/node/classification/typescript.test.ts

@@ -0,0 +1,139 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as assert from 'assert';
+import { StandardTokenType } from 'vs/editor/common/modes';
+import * as fs from 'fs';
+import { getPathFromAmdModule } from 'vs/base/common/amd';
+import { parse } from 'vs/editor/common/modes/tokenization/typescript';
+import { toStandardTokenType } from 'vs/editor/common/modes/supports/tokenization';
+
+interface IParseFunc {
+	(text: string): number[];
+}
+
+interface IAssertion {
+	testLineNumber: number;
+	startOffset: number;
+	length: number;
+	tokenType: StandardTokenType;
+}
+
+interface ITest {
+	content: string;
+	assertions: IAssertion[];
+}
+
+function parseTest(fileName: string): ITest {
+	interface ILineWithAssertions {
+		line: string;
+		assertions: ILineAssertion[];
+	}
+
+	interface ILineAssertion {
+		testLineNumber: number;
+		startOffset: number;
+		length: number;
+		expectedTokenType: StandardTokenType;
+	}
+
+	const testContents = fs.readFileSync(fileName).toString();
+	const lines = testContents.split(/\r\n|\n/);
+	const magicToken = lines[0];
+
+	let currentElement: ILineWithAssertions = {
+		line: lines[1],
+		assertions: []
+	};
+
+	let parsedTest: ILineWithAssertions[] = [];
+	for (let i = 2; i < lines.length; i++) {
+		let line = lines[i];
+		if (line.substr(0, magicToken.length) === magicToken) {
+			// this is an assertion line
+			let m1 = line.substr(magicToken.length).match(/^( +)([\^]+) (\w+)\\?$/);
+			if (m1) {
+				currentElement.assertions.push({
+					testLineNumber: i + 1,
+					startOffset: magicToken.length + m1[1].length,
+					length: m1[2].length,
+					expectedTokenType: toStandardTokenType(m1[3])
+				});
+			} else {
+				let m2 = line.substr(magicToken.length).match(/^( +)<(-+) (\w+)\\?$/);
+				if (m2) {
+					currentElement.assertions.push({
+						testLineNumber: i + 1,
+						startOffset: 0,
+						length: m2[2].length,
+						expectedTokenType: toStandardTokenType(m2[3])
+					});
+				} else {
+					throw new Error(`Invalid test line at line number ${i + 1}.`);
+				}
+			}
+		} else {
+			// this is a line to be parsed
+			parsedTest.push(currentElement);
+			currentElement = {
+				line: line,
+				assertions: []
+			};
+		}
+	}
+	parsedTest.push(currentElement);
+
+	let assertions: IAssertion[] = [];
+
+	let offset = 0;
+	for (let i = 0; i < parsedTest.length; i++) {
+		const parsedTestLine = parsedTest[i];
+		for (let j = 0; j < parsedTestLine.assertions.length; j++) {
+			const assertion = parsedTestLine.assertions[j];
+			assertions.push({
+				testLineNumber: assertion.testLineNumber,
+				startOffset: offset + assertion.startOffset,
+				length: assertion.length,
+				tokenType: assertion.expectedTokenType
+			});
+		}
+		offset += parsedTestLine.line.length + 1;
+	}
+
+	let content: string = parsedTest.map(parsedTestLine => parsedTestLine.line).join('\n');
+
+	return { content, assertions };
+}
+
+function executeTest(fileName: string, parseFunc: IParseFunc): void {
+	const { content, assertions } = parseTest(fileName);
+	const actual = parseFunc(content);
+
+	let actualIndex = 0, actualCount = actual.length / 3;
+	for (let i = 0; i < assertions.length; i++) {
+		const assertion = assertions[i];
+		while (actualIndex < actualCount && actual[3 * actualIndex] + actual[3 * actualIndex + 1] <= assertion.startOffset) {
+			actualIndex++;
+		}
+		assert.ok(
+			actual[3 * actualIndex] <= assertion.startOffset,
+			`Line ${assertion.testLineNumber} : startOffset : ${actual[3 * actualIndex]} <= ${assertion.startOffset}`
+		);
+		assert.ok(
+			actual[3 * actualIndex] + actual[3 * actualIndex + 1] >= assertion.startOffset + assertion.length,
+			`Line ${assertion.testLineNumber} : length : ${actual[3 * actualIndex]} + ${actual[3 * actualIndex + 1]} >= ${assertion.startOffset} + ${assertion.length}.`
+		);
+		assert.equal(
+			actual[3 * actualIndex + 2],
+			assertion.tokenType,
+			`Line ${assertion.testLineNumber} : tokenType`);
+	}
+}
+
+suite('Classification', () => {
+	test('TypeScript', () => {
+		executeTest(getPathFromAmdModule(require, 'vs/editor/test/node/classification/typescript-test.ts').replace(/\bout\b/, 'src'), parse);
+	});
+});