Alanren/integration test (#3657)

* add an extension for integration tests

* setup ads before running test

* test setup

* test cases

* bash script

* shorter temp folder name

* code cleanup

* add commented out original code

* fix test error

* test result path

* rename results file

* change file path

* report smoke test results

* test stablize

* test stablization and configurable test servers

* fix smoke test error

* connection provider

* simplify the integration test script

* add comment

* fix tslint error

* address PR comments

* add temp log to check whether the environment variable is already set

* remove temp log

* move api definition to testapi typing file

* exclude integration tests extension

* address comments

test/smoke/src/application.ts

@@ -62,8 +62,15 @@ export class Application {
 
 	async start(): Promise<any> {
 		await this._start();
+		//{{SQL CARBON EDIT}}
+		await this.code.waitForElement('.object-explorer-view');
+
+		//Original
+		/*
 		await this.code.waitForElement('.explorer-folders-view');
 		await this.code.waitForActiveElement(`.editor-instance[id="workbench.editor.walkThroughPart"] > div > div[tabIndex="0"]`);
+		 */
+		//{{END}}
 	}
 
 	async restart(options: { workspaceOrFolder?: string, extraArgs?: string[] }): Promise<any> {

test/smoke/src/areas/workbench/workbench.ts

@@ -20,6 +20,11 @@ import { Editors } from '../editor/editors';
 import { Code } from '../../vscode/code';
 import { Terminal } from '../terminal/terminal';
 
+// {{SQL CARBON EDIT}}
+import { ConnectionDialog } from '../../sql/connectionDialog/connectionDialog';
+import { Profiler } from '../../sql/profiler/profiler';
+// {{END}}
+
 export interface Commands {
 	runCommand(command: string): Promise<any>;
 }
@@ -42,6 +47,11 @@ export class Workbench {
 	readonly keybindingsEditor: KeybindingsEditor;
 	readonly terminal: Terminal;
 
+	// {{SQL CARBON EDIT}}
+	readonly connectionDialog: ConnectionDialog;
+	readonly profiler: Profiler;
+	// {{END}}
+
 	constructor(code: Code, userDataPath: string) {
 		this.editors = new Editors(code);
 		this.quickopen = new QuickOpen(code, this.editors);
@@ -58,6 +68,10 @@ export class Workbench {
 		this.settingsEditor = new SettingsEditor(code, userDataPath, this.editors, this.editor, this.quickopen);
 		this.keybindingsEditor = new KeybindingsEditor(code);
 		this.terminal = new Terminal(code);
+		// {{SQL CARBON EDIT}}
+		this.connectionDialog = new ConnectionDialog(code);
+		this.profiler = new Profiler(code, this.quickopen);
+		// {{END}}
 	}
 }
 

test/smoke/src/main.ts

@@ -12,7 +12,10 @@ import * as rimraf from 'rimraf';
 import * as mkdirp from 'mkdirp';
 import { ncp } from 'ncp';
 import { Application, Quality } from './application';
-
+//{{SQL CARBON EDIT}}
+import { setup as runProfilerTests } from './sql/profiler/profiler.test';
+//Original
+/*
 import { setup as setupDataMigrationTests } from './areas/workbench/data-migration.test';
 import { setup as setupDataLossTests } from './areas/workbench/data-loss.test';
 import { setup as setupDataExplorerTests } from './areas/explorer/explorer.test';
@@ -27,6 +30,8 @@ import { setup as setupDataExtensionTests } from './areas/extensions/extensions.
 import { setup as setupTerminalTests } from './areas/terminal/terminal.test';
 import { setup as setupDataMultirootTests } from './areas/multiroot/multiroot.test';
 import { setup as setupDataLocalizationTests } from './areas/workbench/localization.test';
+*/
+//{{END}}
 import { MultiLogger, Logger, ConsoleLogger, FileLogger } from './logger';
 
 const tmpDir = tmp.dirSync({ prefix: 't' }) as { name: string; removeCallback: Function; };
@@ -250,14 +255,32 @@ after(async function () {
 	await new Promise((c, e) => rimraf(testDataPath, { maxBusyTries: 10 }, err => err ? e(err) : c()));
 });
 
+//{{SQL CARBON EDIT}}
+/*
 describe('Data Migration', () => {
 	setupDataMigrationTests(userDataDir, createApp);
 });
+*/
+//{{END}}
 
-describe('Test', () => {
+describe('Smoke Test', () => {
 	before(async function () {
 		const app = createApp(quality);
 		await app!.start();
+		//{{SQL CARBON EDIT}}
+		const testExtLoadedText = 'Test Extension Loaded';
+		const testSetupCompletedText = 'Test Setup Completed';
+		const allExtensionsLoadedText = 'All Extensions Loaded';
+		const setupTestCommand = 'Test: Setup Integration Test';
+		const waitForExtensionsCommand = 'Test: Wait For Extensions To Load';
+		await app.workbench.statusbar.waitForStatusbarText(testExtLoadedText, testExtLoadedText);
+		await app.workbench.quickopen.runCommand(setupTestCommand);
+		await app.workbench.statusbar.waitForStatusbarText(testSetupCompletedText, testSetupCompletedText);
+		await app!.reload();
+		await app.workbench.statusbar.waitForStatusbarText(testExtLoadedText, testExtLoadedText);
+		await app.workbench.quickopen.runCommand(waitForExtensionsCommand);
+		await app.workbench.statusbar.waitForStatusbarText(allExtensionsLoadedText, allExtensionsLoadedText);
+		//{{END}}
 		this.app = app;
 	});
 
@@ -295,6 +318,10 @@ describe('Test', () => {
 		});
 	}
 
+	//{{SQL CARBON EDIT}}
+	runProfilerTests();
+	//Original
+	/*
 	setupDataLossTests();
 	setupDataExplorerTests();
 	setupDataPreferencesTests();
@@ -308,4 +335,6 @@ describe('Test', () => {
 	setupTerminalTests();
 	setupDataMultirootTests();
 	setupDataLocalizationTests();
+	*/
+	//{{END}}
 });

test/smoke/src/sql/connectionDialog/connectionDialog.ts

@@ -0,0 +1,51 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Code } from '../../vscode/code';
+import { waitForNewDialog, clickDialogButton } from '../sqlutils';
+import { TestServerProfile, AuthenticationType } from '../testConfig';
+
+const CONNECTION_DIALOG_TITLE = 'Connection';
+const CONNECTION_DIALOG_SELECTOR: string = '.modal-dialog .modal-content .modal-body .connection-dialog';
+const CONNECTION_DETAIL_CONTROL_SELECTOR: string = '.connection-type .connection-table .connection-input';
+
+const SERVER_INPUT_ARIA_LABEL = 'Server';
+const USERNAME_INPUT_ARIA_LABEL = 'User name';
+const PASSWORD_INPUT_ARIA_LABEL = 'Password';
+const AUTH_TYPE_ARIA_LABEL = 'Authentication type';
+
+const CONNECT_BUTTON_ARIA_LABEL = 'Connect';
+
+export class ConnectionDialog {
+
+	constructor(private code: Code) { }
+
+	async waitForConnectionDialog(): Promise<void> {
+		await waitForNewDialog(this.code, CONNECTION_DIALOG_TITLE);
+	}
+
+	async connect(profile: TestServerProfile): Promise<void> {
+		await this.code.waitForSetValue(this.getInputCssSelector(SERVER_INPUT_ARIA_LABEL), profile.serverName);
+		if (profile.authenticationType === AuthenticationType.SqlLogin) {
+			await this.code.waitAndClick(this.getSelectCssSelector(AUTH_TYPE_ARIA_LABEL));
+			await this.selectAuthType(profile.authenticationTypeDisplayName);
+			await this.code.waitForSetValue(this.getInputCssSelector(USERNAME_INPUT_ARIA_LABEL), profile.userName);
+			await this.code.waitForSetValue(this.getInputCssSelector(PASSWORD_INPUT_ARIA_LABEL), profile.password);
+		}
+		await clickDialogButton(this.code, CONNECT_BUTTON_ARIA_LABEL);
+	}
+
+	private getInputCssSelector(ariaLabel: string): string {
+		return `${CONNECTION_DIALOG_SELECTOR} ${CONNECTION_DETAIL_CONTROL_SELECTOR} .monaco-inputbox input[aria-label="${ariaLabel}"]`;
+	}
+
+	private getSelectCssSelector(ariaLabel: string): string {
+		return `${CONNECTION_DIALOG_SELECTOR} ${CONNECTION_DETAIL_CONTROL_SELECTOR} select[aria-label="${ariaLabel}"]`;
+	}
+
+	private async selectAuthType(authType: string) {
+		await this.code.waitAndClick(`.context-view.bottom.left .monaco-select-box-dropdown-container .select-box-dropdown-list-container .monaco-list .monaco-scrollable-element .monaco-list-rows div[aria-label="${authType}"][class*="monaco-list-row"]`);
+	}
+}

test/smoke/src/sql/profiler/profiler.test.ts

@@ -0,0 +1,19 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Application } from '../../application';
+import { getDefaultTestingServer } from '../testConfig';
+
+export function setup() {
+	describe('profiler test suite', () => {
+		it('Launch profiler test', async function () {
+			const app = this.app as Application;
+			await app.workbench.profiler.launchProfiler();
+			await app.workbench.connectionDialog.waitForConnectionDialog();
+			await app.workbench.connectionDialog.connect(await getDefaultTestingServer());
+			await app.workbench.profiler.waitForNewSessionDialogAndStart();
+		});
+	});
+}

test/smoke/src/sql/profiler/profiler.ts

@@ -0,0 +1,28 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Code } from '../../vscode/code';
+import { QuickOpen } from '../../areas/quickopen/quickopen';
+import { waitForNewDialog, clickDialogButton } from '../sqlutils';
+
+const NEW_SESSION_DIALOG_TITLE: string = 'Start New Profiler Session';
+
+export class Profiler {
+
+	constructor(private code: Code, private quickopen: QuickOpen) { }
+
+	async launchProfiler(): Promise<void> {
+		await this.quickopen.runCommand('Profiler: Launch Profiler');
+	}
+
+	async waitForNewSessionDialog() {
+		await waitForNewDialog(this.code, NEW_SESSION_DIALOG_TITLE);
+	}
+
+	async waitForNewSessionDialogAndStart() {
+		await this.waitForNewSessionDialog();
+		await clickDialogButton(this.code, 'Start');
+	}
+}

test/smoke/src/sql/sqlutils.ts

@@ -0,0 +1,9 @@
+import { Code } from '../vscode/code';
+
+export async function waitForNewDialog(code: Code, title: string) {
+	await code.waitForElement(`div[aria-label="${title}"][class="modal fade flyout-dialog"]`);
+}
+
+export async function clickDialogButton(code: Code, title: string) {
+	await code.waitAndClick(`.modal-dialog .modal-content .modal-footer .right-footer .footer-button a[aria-label="${title}"][aria-disabled="false"]`);
+}

test/smoke/src/sql/testConfig.ts

@@ -0,0 +1,92 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/*
+	TODO: Due to a runtime error, I duplicated this file at these 2 locations:
+	$/extensions/integration-test/src/testConfig.ts
+	$/test/smoke/src/sql/testConfig.ts
+	for now, make sure to keep both files in sync.
+*/
+
+interface ITestServerProfile {
+	serverName: string;
+	userName: string;
+	password: string;
+	authenticationType: AuthenticationType;
+	database: string;
+	provider: ConnectionProvider;
+	version: string;
+}
+
+interface INameDisplayNamePair {
+	name: string;
+	displayName: string;
+}
+
+export enum AuthenticationType {
+	Windows,
+	SqlLogin
+}
+
+export enum ConnectionProvider {
+	SQLServer
+}
+
+var connectionProviderMapping = {};
+var authenticationTypeMapping = {};
+connectionProviderMapping[ConnectionProvider.SQLServer] = { name: 'MSSQL', displayName: 'Microsoft SQL Server' };
+
+authenticationTypeMapping[AuthenticationType.SqlLogin] = { name: 'SqlLogin', displayName: 'SQL Login' };
+authenticationTypeMapping[AuthenticationType.Windows] = { name: 'Integrated', displayName: 'Windows Authentication' };
+
+export class TestServerProfile {
+	constructor(private _profile: ITestServerProfile) { }
+	public get serverName(): string { return this._profile.serverName; }
+	public get userName(): string { return this._profile.userName; }
+	public get password(): string { return this._profile.password; }
+	public get database(): string { return this._profile.database; }
+	public get version(): string { return this._profile.version; }
+	public get provider(): ConnectionProvider { return this._profile.provider; }
+	public get providerName(): string { return getEnumMappingEntry(connectionProviderMapping, this.provider).name; }
+	public get providerDisplayName(): string { return getEnumMappingEntry(connectionProviderMapping, this.provider).displayName; }
+	public get authenticationType(): AuthenticationType { return this._profile.authenticationType; }
+	public get authenticationTypeName(): string { return getEnumMappingEntry(authenticationTypeMapping, this.authenticationType).name; }
+	public get authenticationTypeDisplayName(): string { return getEnumMappingEntry(authenticationTypeMapping, this.authenticationType).displayName; }
+}
+
+var TestingServers: TestServerProfile[] = [
+	new TestServerProfile(
+		{
+			serverName: 'SQLTOOLS2017-3',
+			userName: '',
+			password: '',
+			authenticationType: AuthenticationType.Windows,
+			database: 'master',
+			provider: ConnectionProvider.SQLServer,
+			version: '2017'
+		})
+];
+
+function getEnumMappingEntry(mapping: any, enumValue: any): INameDisplayNamePair {
+	let entry = mapping[enumValue];
+	if (entry) {
+		return entry;
+	} else {
+		throw `Unknown enum type: ${enumValue.toString()}`;
+	}
+}
+
+export async function getDefaultTestingServer(): Promise<TestServerProfile> {
+	let servers = await getTestingServers();
+	return servers[0];
+}
+
+export async function getTestingServers(): Promise<TestServerProfile[]> {
+	let promise = new Promise<TestServerProfile[]>(resolve => {
+		resolve(TestingServers);
+	});
+	await promise;
+	return promise;
+}

test/smoke/src/vscode/code.ts

@@ -217,7 +217,9 @@ export class Code {
 	}
 
 	async waitForWindowIds(fn: (windowIds: number[]) => boolean): Promise<void> {
-		await poll(() => this.driver.getWindowIds(), fn, `get window ids`);
+		// {{SQL CARBON EDIT}}
+		await poll(() => this.driver.getWindowIds(), fn, `get window ids`, 600, 100);
+		// {{END}}
 	}
 
 	async dispatchKeybinding(keybinding: string): Promise<void> {

test/smoke/test/index.js

@@ -18,7 +18,9 @@ const opts = minimist(args, {
 
 const options = {
 	useColors: true,
-	timeout: 60000,
+	//{{SQL CARBON EDIT}}
+	timeout: 60000 * 2,
+	//{{END}}
 	slow: 30000,
 	grep: opts['f']
 };
@@ -36,4 +38,4 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 
 const mocha = new Mocha(options);
 mocha.addFile('out/main.js');
-mocha.run(failures => process.exit(failures ? -1 : 0));
+mocha.run(failures => process.exit(failures ? -1 : 0));