Alanren/integration test (#3657)

* add an extension for integration tests

* setup ads before running test

* test setup

* test cases

* bash script

* shorter temp folder name

* code cleanup

* add commented out original code

* fix test error

* test result path

* rename results file

* change file path

* report smoke test results

* test stablize

* test stablization and configurable test servers

* fix smoke test error

* connection provider

* simplify the integration test script

* add comment

* fix tslint error

* address PR comments

* add temp log to check whether the environment variable is already set

* remove temp log

* move api definition to testapi typing file

* exclude integration tests extension

* address comments

extensions/integration-tests/.vscode/launch.json

@@ -0,0 +1,17 @@
+// A launch configuration that compiles the extension and then opens it inside a new window
+{
+	"version": "0.1.0",
+	"configurations": [
+		{
+			"name": "Launch Tests",
+			"type": "extensionHost",
+			"request": "launch",
+			"runtimeExecutable": "${execPath}",
+			"args": ["${workspaceFolder}/../../", "${workspaceFolder}/test", "--extensionDevelopmentPath=${workspaceFolder}", "--extensionTestsPath=${workspaceFolder}/out" ],
+			"stopOnEntry": false,
+			"sourceMaps": true,
+			"outDir": "${workspaceFolder}/out",
+			"preLaunchTask": "npm"
+		}
+	]
+}

extensions/integration-tests/.vscode/tasks.json

@@ -0,0 +1,31 @@
+// Available variables which can be used inside of strings.
+// ${workspaceFolder}: the root folder of the team
+// ${file}: the current opened file
+// ${relativeFile}: the current opened file relative to cwd
+// ${fileBasename}: the current opened file's basename
+// ${fileDirname}: the current opened file's dirname
+// ${fileExtname}: the current opened file's extension
+// ${cwd}: the current working directory of the spawned process
+
+// A task runner that calls a custom npm script that compiles the extension.
+{
+	"version": "0.1.0",
+
+	// we want to run npm
+	"command": "npm",
+
+	// the command is a shell script
+	"isShellCommand": true,
+
+	// show the output window only if unrecognized errors occur.
+	"showOutput": "silent",
+
+	// we run the custom script "compile" as defined in package.json
+	"args": ["run", "compile", "--loglevel", "silent"],
+
+	// The tsc compiler is started in watching mode
+	"isWatching": true,
+
+	// use the standard tsc in watch mode problem matcher to find compile problems in the output.
+	"problemMatcher": "$tsc-watch"
+}

extensions/integration-tests/extensionInstallers/readme.md

@@ -0,0 +1 @@
+copy the extension installers to this folder

extensions/integration-tests/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "integration-tests",
+  "description": "Integration Tests",
+  "version": "0.0.1",
+  "publisher": "Microsoft",
+  "private": true,
+  "engines": {
+    "vscode": "*",
+    "sqlops": "*"
+  },
+  "activationEvents": [
+    "*"
+  ],
+  "main": "./out/main",
+  "extensionDependencies": [
+    "Microsoft.agent",
+    "Microsoft.import",
+    "Microsoft.profiler",
+    "Microsoft.mssql",
+    "Microsoft.notebook"
+  ],
+  "contributes": {
+    "configuration": {
+      "type": "object",
+      "title": "ADS Integration Test Configuration",
+      "properties": {
+        "test.testSetupCompleted": {
+          "type": "boolean",
+          "default": false,
+          "description": ""
+        }
+      }
+    },
+    "commands": [{
+        "command": "test.setupIntegrationTest",
+        "title": "Setup Integration Test",
+        "category": "Test"
+      },
+      {
+        "command": "test.waitForExtensionsToLoad",
+        "title": "Wait For Extensions To Load",
+        "category": "Test"
+      }
+    ]
+  },
+  "scripts": {
+    "vscode:prepublish": "node ../../node_modules/gulp/bin/gulp.js --gulpfile ../../build/gulpfile.extensions.js compile-extension:vscode-colorize-tests ./tsconfig.json",
+    "postinstall": "node ./node_modules/vscode/bin/install"
+  },
+  "devDependencies": {
+    "@types/node": "7.0.43",
+    "@types/chai": "3.4.34",
+    "mocha-junit-reporter": "^1.17.0",
+    "mocha-multi-reporters": "^1.1.7",
+    "vscode": "1.1.5",
+    "chai": "3.5.0"
+  }
+}

extensions/integration-tests/src/index.ts

@@ -0,0 +1,36 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import * as vscode from 'vscode';
+import { context } from './testContext';
+
+const path = require('path');
+const testRunner = require('vscode/lib/testrunner');
+
+const suite = 'Integration Tests';
+
+const options: any = {
+	ui: 'tdd',
+	useColors: true,
+	timeout: 600000
+};
+
+if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
+	options.reporter = 'mocha-multi-reporters';
+	options.reporterOptions = {
+		reporterEnabled: 'spec, mocha-junit-reporter',
+		mochaJunitReporterReporterOptions: {
+			testsuitesTitle: `${suite} ${process.platform}`,
+			mochaFile: path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY, `test-results/${process.platform}-${suite.toLowerCase().replace(/[^\w]/g, '-')}-results.xml`)
+		}
+	};
+}
+
+if (!vscode.workspace.getConfiguration('test')['testSetupCompleted']) {
+	context.RunTest = false;
+}
+
+testRunner.configure(options);
+
+export = testRunner;

extensions/integration-tests/src/main.ts

@@ -0,0 +1,77 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+import * as vscode from 'vscode';
+import * as sqlops from 'sqlops';
+import { normalize, join } from 'path';
+import * as fs from 'fs';
+
+const TEST_SETUP_COMPLETED_TEXT: string = 'Test Setup Completed';
+const EXTENSION_LOADED_TEXT: string = 'Test Extension Loaded';
+const ALL_EXTENSION_LOADED_TEXT: string = 'All Extensions Loaded';
+
+var statusBarItemTimer: NodeJS.Timer;
+
+export function activate(context: vscode.ExtensionContext) {
+	var statusBarItem = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left);
+	vscode.commands.registerCommand('test.setupIntegrationTest', async () => {
+		let extensionInstallersFolder = normalize(join(__dirname, '../extensionInstallers'));
+		let installers = fs.readdirSync(extensionInstallersFolder);
+		for (let i = 0; i < installers.length; i++) {
+			if (installers[i].endsWith('.vsix')) {
+				let installerFullPath = join(extensionInstallersFolder, installers[i]);
+				await sqlops.extensions.install(installerFullPath);
+			}
+		}
+		await setConfiguration('workbench.enablePreviewFeatures', true);
+		await setConfiguration('workbench.showConnectDialogOnStartup', false);
+		await setConfiguration('test.testSetupCompleted', true);
+		showStatusBarItem(statusBarItem, TEST_SETUP_COMPLETED_TEXT);
+	});
+
+	vscode.commands.registerCommand('test.waitForExtensionsToLoad', async () => {
+		let expectedExtensions = ['Microsoft.agent', 'Microsoft.import', 'Microsoft.mssql', 'Microsoft.profiler'];
+		do {
+			let extensions = vscode.extensions.all.filter(ext => { return expectedExtensions.indexOf(ext.id) !== -1; });
+
+			let isReady = true;
+			for (let i = 0; i < extensions.length; i++) {
+				let extension = extensions[i];
+				isReady = isReady && extension.isActive;
+				if (!isReady) {
+					break;
+				}
+			}
+
+			if (isReady) {
+				showStatusBarItem(statusBarItem, ALL_EXTENSION_LOADED_TEXT);
+				break;
+			} else {
+				await new Promise(resolve => { setTimeout(resolve, 1000); });
+			}
+		}
+		while (true);
+	});
+	showStatusBarItem(statusBarItem, EXTENSION_LOADED_TEXT);
+}
+
+function showStatusBarItem(statusBarItem: vscode.StatusBarItem, text: string) {
+	statusBarItem.text = text;
+	statusBarItem.tooltip = text;
+	statusBarItem.show();
+	clearTimeout(statusBarItemTimer);
+	statusBarItemTimer = setTimeout(function () {
+		statusBarItem.hide();
+	}, 5000);
+}
+
+// this method is called when your extension is deactivated
+export function deactivate(): void {
+
+}
+
+async function setConfiguration(name: string, value: any) {
+	await vscode.workspace.getConfiguration().update(name, value, true);
+}

extensions/integration-tests/src/objectExplorer.test.ts

@@ -0,0 +1,29 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import 'mocha';
+import * as sqlops from 'sqlops';
+import { context } from './testContext';
+import { getDefaultTestingServer } from './testConfig';
+import { connectToServer } from './utils';
+import assert = require('assert');
+
+if (context.RunTest) {
+	suite('Object Explorer integration test suite', () => {
+		test('context menu test', async function () {
+			await connectToServer(await getDefaultTestingServer());
+			let nodes = <sqlops.objectexplorer.ObjectExplorerNode[]>await sqlops.objectexplorer.getActiveConnectionNodes();
+			assert(nodes.length === 1, `expecting 1 active connection, actual: ${nodes.length}`);
+			let actions = await sqlops.objectexplorer.getNodeActions(nodes[0].connectionId, nodes[0].nodePath);
+			const expectedActions = ['Manage', 'New Query', 'Disconnect', 'Delete Connection', 'Refresh', 'Launch Profiler'];
+
+			const expectedString = expectedActions.join(',');
+			const actualString = actions.join(',');
+			assert(expectedActions.length === actions.length && expectedString === actualString, `Expected actions: "${expectedString}", Actual actions: "${actualString}"`);
+		});
+	});
+}

extensions/integration-tests/src/setup.test.ts

@@ -0,0 +1,21 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import 'mocha';
+import * as vscode from 'vscode';
+import { context } from './testContext';
+
+if (!context.RunTest) {
+	suite('integration test setup', () => {
+		test('test setup', async function () {
+			//Prepare the environment and make it ready for testing
+			await vscode.commands.executeCommand('test.setupIntegrationTest');
+			//Reload the window, this is required for some changes made by the 'test.setupIntegrationTest' to work
+			await vscode.commands.executeCommand('workbench.action.reloadWindow');
+		});
+	});
+}

extensions/integration-tests/src/testConfig.ts

@@ -0,0 +1,92 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/*
+	TODO: Due to a runtime error, I duplicated this file at these 2 locations:
+	$/extensions/integration-test/src/testConfig.ts
+	$/test/smoke/src/sql/testConfig.ts
+	for now, make sure to keep both files in sync.
+*/
+
+interface ITestServerProfile {
+	serverName: string;
+	userName: string;
+	password: string;
+	authenticationType: AuthenticationType;
+	database: string;
+	provider: ConnectionProvider;
+	version: string;
+}
+
+interface INameDisplayNamePair {
+	name: string;
+	displayName: string;
+}
+
+export enum AuthenticationType {
+	Windows,
+	SqlLogin
+}
+
+export enum ConnectionProvider {
+	SQLServer
+}
+
+var connectionProviderMapping = {};
+var authenticationTypeMapping = {};
+connectionProviderMapping[ConnectionProvider.SQLServer] = { name: 'MSSQL', displayName: 'Microsoft SQL Server' };
+
+authenticationTypeMapping[AuthenticationType.SqlLogin] = { name: 'SqlLogin', displayName: 'SQL Login' };
+authenticationTypeMapping[AuthenticationType.Windows] = { name: 'Integrated', displayName: 'Windows Authentication' };
+
+export class TestServerProfile {
+	constructor(private _profile: ITestServerProfile) { }
+	public get serverName(): string { return this._profile.serverName; }
+	public get userName(): string { return this._profile.userName; }
+	public get password(): string { return this._profile.password; }
+	public get database(): string { return this._profile.database; }
+	public get version(): string { return this._profile.version; }
+	public get provider(): ConnectionProvider { return this._profile.provider; }
+	public get providerName(): string { return getEnumMappingEntry(connectionProviderMapping, this.provider).name; }
+	public get providerDisplayName(): string { return getEnumMappingEntry(connectionProviderMapping, this.provider).displayName; }
+	public get authenticationType(): AuthenticationType { return this._profile.authenticationType; }
+	public get authenticationTypeName(): string { return getEnumMappingEntry(authenticationTypeMapping, this.authenticationType).name; }
+	public get authenticationTypeDisplayName(): string { return getEnumMappingEntry(authenticationTypeMapping, this.authenticationType).displayName; }
+}
+
+var TestingServers: TestServerProfile[] = [
+	new TestServerProfile(
+		{
+			serverName: 'SQLTOOLS2017-3',
+			userName: '',
+			password: '',
+			authenticationType: AuthenticationType.Windows,
+			database: 'master',
+			provider: ConnectionProvider.SQLServer,
+			version: '2017'
+		})
+];
+
+function getEnumMappingEntry(mapping: any, enumValue: any): INameDisplayNamePair {
+	let entry = mapping[enumValue];
+	if (entry) {
+		return entry;
+	} else {
+		throw `Unknown enum type: ${enumValue.toString()}`;
+	}
+}
+
+export async function getDefaultTestingServer(): Promise<TestServerProfile> {
+	let servers = await getTestingServers();
+	return servers[0];
+}
+
+export async function getTestingServers(): Promise<TestServerProfile[]> {
+	let promise = new Promise<TestServerProfile[]>(resolve => {
+		resolve(TestingServers);
+	});
+	await promise;
+	return promise;
+}

extensions/integration-tests/src/testContext.ts

@@ -0,0 +1,8 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+export var context = {
+	RunTest: true
+};

extensions/integration-tests/src/typings/ref.d.ts

@@ -0,0 +1,8 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+/// <reference path='../../../../src/sql/sqlops.d.ts'/>
+/// <reference path='../../../../src/sql/sqlops.proposed.d.ts'/>
+/// <reference path='../../../../src/sql/sqlops.test.d.ts'/>
+/// <reference types='@types/node'/>

extensions/integration-tests/src/utils.ts

@@ -0,0 +1,38 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert = require('assert');
+import * as sqlops from 'sqlops';
+import * as vscode from 'vscode';
+import { TestServerProfile } from './testConfig';
+
+export async function connectToServer(server: TestServerProfile) {
+	let connectionProfile: sqlops.IConnectionProfile = {
+		serverName: server.serverName,
+		databaseName: server.database,
+		authenticationType: server.authenticationTypeName,
+		providerName: server.providerName,
+		connectionName: '',
+		userName: server.userName,
+		password: server.password,
+		savePassword: false,
+		groupFullName: undefined,
+		saveProfile: true,
+		id: undefined,
+		groupId: undefined,
+		options: {}
+	};
+	await ensureConnectionViewOpened();
+	let result = <sqlops.ConnectionResult>await sqlops.connection.connect(connectionProfile);
+	assert(result.connected, `Failed to connect to "${connectionProfile.serverName}", error code: ${result.errorCode}, error message: ${result.errorMessage}`);
+
+	//workaround
+	//wait for OE to load
+	await new Promise(c => setTimeout(c, 3000));
+}
+
+export async function ensureConnectionViewOpened() {
+	await vscode.commands.executeCommand('workbench.view.connections');
+}

extensions/integration-tests/tsconfig.json

@@ -0,0 +1,15 @@
+{
+	"compilerOptions": {
+		"module": "commonjs",
+		"target": "ES5",
+		"outDir": "out",
+		"noUnusedLocals": true,
+		"lib": [
+			"es2015"
+		],
+		"sourceMap": true
+	},
+	"include": [
+		"src/**/*"
+	]
+}