web smoke tests (#11280)

* distro

* renable web smoke

* add missing script

* update node version

* update node version everywhere

* ensure playwright drivers are installed

* fix screenshot capture

* try this

* rewrite connection dialog code

* fix permissions

* more wip

* replace more $ with ^

* revert changes

* refactor and revert more changes

* add screen shot functionality to playwright

* fix compile

* fix profiler compile

* don't run new files for web

* continue on error for web

* continue on error for web not normal

* revert some changes

build/azure-pipelines/darwin/sql-product-build-darwin.yml

@@ -27,11 +27,11 @@ steps:
 
   - task: NodeTool@0
     inputs:
-      versionSpec: '10.15.3'
+      versionSpec: "12.13.0"
 
   - task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@3
     inputs:
-      versionSpec: '1.x'
+      versionSpec: "1.x"
 
   - task: AzureKeyVault@1
     displayName: 'Azure Key Vault: Get Secrets'
@@ -129,12 +129,14 @@ steps:
     displayName: Run smoke tests (Electron)
     condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
 
-  # - script: |
-  #     set -e
-  #     VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/azuredatastudio-reh-web-darwin" \
-  #     yarn smoketest --web --headless  --screenshots "$(build.artifactstagingdirectory)/smokeshots"
-  #   displayName: Run smoke tests (Browser)
-  #   condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
+  - script: |
+      set -e
+      node ./node_modules/playwright/install.js
+      VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/azuredatastudio-reh-web-darwin" \
+      yarn smoketest --web --headless  --screenshots "$(build.artifactstagingdirectory)/smokeshots"
+    displayName: Run smoke tests (Browser)
+    continueOnError: true
+    condition: and(succeeded(), eq(variables['RUN_TESTS'], 'true'))
 
   - script: |
       set -e

build/azure-pipelines/linux/sql-product-build-linux.yml

@@ -24,7 +24,7 @@ steps:
 
   - task: NodeTool@0
     inputs:
-      versionSpec: '10.15.1'
+      versionSpec: "12.13.0"
 
   - task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@3
     inputs:

build/azure-pipelines/win32/sql-product-build-win32.yml

@@ -21,7 +21,7 @@ steps:
 
   - task: NodeTool@0
     inputs:
-      versionSpec: "10.15.1"
+      versionSpec: "12.13.0"
 
   - task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@3
     inputs:

build/azure-pipelines/win32/sql-product-test-win32.yml

@@ -1,7 +1,7 @@
 steps:
 - task: NodeTool@0
   inputs:
-    versionSpec: "10.15.1"
+    versionSpec: "12.13.0"
 
 - task: geeklearningio.gl-vsts-tasks-yarn.yarn-installer-task.YarnInstaller@3
   inputs:

package.json

@@ -30,6 +30,7 @@
     "update-grammars": "node build/npm/update-all-grammars.js",
     "update-localization-extension": "node build/npm/update-localization-extension.js",
     "smoketest": "cd test/smoke && node test/index.js",
+    "download-builtin-extensions": "node build/lib/builtInExtensions.js",
     "monaco-compile-check": "tsc -p src/tsconfig.monaco.json --noEmit",
     "strict-vscode": "node --max_old_space_size=4095 node_modules/typescript/bin/tsc -p src/tsconfig.vscode.json",
     "strict-vscode-watch": "node --max_old_space_size=4095 node_modules/typescript/bin/tsc -p src/tsconfig.vscode.json --watch",

test/automation/src/code.ts

@@ -319,6 +319,11 @@ export class Code {
 		return await poll<IElement>(() => this.driver.getElements(windowId, selector).then(els => els[0]), accept, `get element '${selector}'`, retryCount);
 	}
 
+	async waitForElementGone(selector: string, accept: (result: IElement | undefined) => boolean = result => !result, retryCount: number = 200): Promise<IElement> {
+		const windowId = await this.getActiveWindowId();
+		return await poll<IElement>(() => this.driver.getElements(windowId, selector).then(els => els[0]), accept, `get element gone '${selector}'`, retryCount);
+	}
+
 	async waitForActiveElement(selector: string, retryCount: number = 200): Promise<void> {
 		const windowId = await this.getActiveWindowId();
 		await poll(() => this.driver.isActiveElement(windowId, selector), r => r, `is active element '${selector}'`, retryCount);

test/automation/src/index.ts

@@ -30,5 +30,3 @@ export * from './driver';
 export * from './sql/connectionDialog';
 export * from './sql/profiler';
 export * from './sql/queryEditors';
-export * from './sql/sqlutils';
-export * from './sql/testConfig';

test/automation/src/playwrightDriver.ts

@@ -35,7 +35,10 @@ function buildDriver(browser: playwright.Browser, page: playwright.Page): IDrive
 		getWindowIds: () => {
 			return Promise.resolve([1]);
 		},
-		capturePage: () => Promise.resolve(''),
+		capturePage: async () => {
+			const buffer = await page.screenshot();
+			return buffer.toString('base64');
+		},
 		reloadWindow: (windowId) => Promise.resolve(),
 		exitApplication: () => browser.close(),
 		dispatchKeybinding: async (windowId, keybinding) => {
@@ -143,7 +146,7 @@ function waitForEndpoint(): Promise<string> {
 export function connect(browserType: 'chromium' | 'webkit' | 'firefox' = 'chromium'): Promise<{ client: IDisposable, driver: IDriver }> {
 	return new Promise(async (c) => {
 		const browser = await playwright[browserType].launch({ headless: false });
-		const context = await browser.newContext();
+		const context = await browser.newContext({ permissions: ['clipboard-read'] }); // {{SQL CARBON EDIT}} avoid permissison request
 		const page = await context.newPage();
 		await page.setViewportSize({ width, height });
 		const payloadParam = `[["enableProposedApi",""]]`;

test/automation/src/sql/connectionDialog.ts

@@ -4,16 +4,18 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { Code } from '../code';
-import { waitForNewDialog } from './sqlutils';
+import { Dialog } from './dialog';
 
 const CONNECTION_DIALOG_TITLE = 'Connection';
 
-export class ConnectionDialog {
+export class ConnectionDialog extends Dialog {
 
-	constructor(private code: Code) { }
+	constructor(code: Code) {
+		super(CONNECTION_DIALOG_TITLE, code);
+	}
 
 	async waitForConnectionDialog(): Promise<void> {
-		await waitForNewDialog(this.code, CONNECTION_DIALOG_TITLE);
+		await this.waitForNewDialog();
 	}
 
 	private static readonly PROVIDER_SELECTOR = '.modal .modal-body select[aria-label="Connection type"]';
@@ -30,7 +32,6 @@ export class ConnectionDialog {
 	async connect(): Promise<void> {
 		await this.code.waitAndClick(ConnectionDialog.CONNECT_BUTTON_SELECTOR);
 
-		const selector = `.editor-instance .monaco-editor textarea`;
-		return this.code.waitForActiveElement(selector);
+		return this.waitForDialogGone();
 	}
 }

test/automation/src/sql/dialog.ts

@@ -0,0 +1,23 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the Source EULA. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Code } from '../code';
+
+export abstract class Dialog {
+	constructor(private readonly title: string, protected readonly code: Code) { }
+
+
+	protected async waitForNewDialog() {
+		await this.code.waitForElement(`div[aria-label="${this.title}"][class="modal fade flyout-dialog"]`);
+	}
+
+	protected async waitForDialogGone() {
+		await this.code.waitForElementGone(`div[aria-label="${this.title}"][class="modal fade flyout-dialog"]`);
+	}
+
+	protected async clickDialogButton(text: string) {
+		await this.code.waitAndClick(`.modal-dialog .modal-content .modal-footer .right-footer .footer-button a[aria-label="${text}"][aria-disabled="false"]`);
+	}
+}

test/automation/src/sql/profiler.ts

@@ -5,24 +5,26 @@
 
 import { Code } from '../code';
 import { QuickAccess } from '../quickaccess';
-import { waitForNewDialog, clickDialogButton } from './sqlutils';
+import { Dialog } from './dialog';
 
 const NEW_SESSION_DIALOG_TITLE: string = 'Start New Profiler Session';
 
-export class Profiler {
+export class Profiler extends Dialog {
 
-	constructor(private code: Code, private quickopen: QuickAccess) { }
+	constructor(code: Code, private quickopen: QuickAccess) {
+		super(NEW_SESSION_DIALOG_TITLE, code);
+	}
 
 	async launchProfiler(): Promise<void> {
 		await this.quickopen.runCommand('Profiler: Launch Profiler');
 	}
 
 	async waitForNewSessionDialog() {
-		await waitForNewDialog(this.code, NEW_SESSION_DIALOG_TITLE);
+		await this.waitForNewDialog();
 	}
 
 	async waitForNewSessionDialogAndStart() {
 		await this.waitForNewSessionDialog();
-		await clickDialogButton(this.code, 'Start');
+		await this.clickDialogButton('Start');
 	}
 }

test/automation/src/sql/sqlutils.ts

@@ -1,14 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { Code } from '../code';
-
-export async function waitForNewDialog(code: Code, title: string) {
-	await code.waitForElement(`div[aria-label="${title}"][class="modal fade flyout-dialog"]`);
-}
-
-export async function clickDialogButton(code: Code, title: string) {
-	await code.waitAndClick(`.modal-dialog .modal-content .modal-footer .right-footer .footer-button a[aria-label="${title}"][aria-disabled="false"]`);
-}

test/automation/src/sql/testConfig.ts

@@ -1,149 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the Source EULA. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-/*
-	TODO: Due to a runtime error, I duplicated this file at these 2 locations:
-	$/extensions/integration-test/src/testConfig.ts
-	$/test/smoke/src/sql/testConfig.ts
-	for now, make sure to keep both files in sync.
-*/
-
-interface ITestServerProfile {
-	serverName: string;
-	userName: string;
-	password: string;
-	authenticationType: AuthenticationType;
-	database: string;
-	provider: ConnectionProvider;
-	version: string;
-	engineType: EngineType;
-}
-
-interface INameDisplayNamePair {
-	name: string;
-	displayName: string;
-}
-
-export enum AuthenticationType {
-	Windows,
-	SqlLogin
-}
-
-export enum ConnectionProvider {
-	SQLServer
-}
-
-export enum EngineType {
-	Standalone,
-	Azure,
-	BigDataCluster
-}
-
-let connectionProviderMapping: { [key: string]: any } = {};
-let authenticationTypeMapping: { [key: string]: any } = {};
-connectionProviderMapping[ConnectionProvider.SQLServer] = { name: 'MSSQL', displayName: 'Microsoft SQL Server' };
-
-authenticationTypeMapping[AuthenticationType.SqlLogin] = { name: 'SqlLogin', displayName: 'SQL Login' };
-authenticationTypeMapping[AuthenticationType.Windows] = { name: 'Integrated', displayName: 'Windows Authentication' };
-
-export function getConfigValue(name: string): string {
-	let configValue = process.env[name];
-	return configValue ? configValue.toString() : '';
-}
-
-export const EnvironmentVariable_BDC_SERVER: string = 'BDC_BACKEND_HOSTNAME';
-export const EnvironmentVariable_BDC_USERNAME: string = 'BDC_BACKEND_USERNAME';
-export const EnvironmentVariable_BDC_PASSWORD: string = 'BDC_BACKEND_PWD';
-export const EnvironmentVariable_STANDALONE_SERVER: string = 'STANDALONE_SQL';
-export const EnvironmentVariable_STANDALONE_USERNAME: string = 'STANDALONE_SQL_USERNAME';
-export const EnvironmentVariable_STANDALONE_PASSWORD: string = 'STANDALONE_SQL_PWD';
-export const EnvironmentVariable_AZURE_SERVER: string = 'AZURE_SQL';
-export const EnvironmentVariable_AZURE_USERNAME: string = 'AZURE_SQL_USERNAME';
-export const EnvironmentVariable_AZURE_PASSWORD: string = 'AZURE_SQL_PWD';
-export const EnvironmentVariable_PYTHON_PATH: string = 'PYTHON_TEST_PATH';
-
-export class TestServerProfile {
-	constructor(private _profile: ITestServerProfile) { }
-	public get serverName(): string { return this._profile.serverName; }
-	public get userName(): string { return this._profile.userName; }
-	public get password(): string { return this._profile.password; }
-	public get database(): string { return this._profile.database; }
-	public get version(): string { return this._profile.version; }
-	public get provider(): ConnectionProvider { return this._profile.provider; }
-	public get providerName(): string { return getEnumMappingEntry(connectionProviderMapping, this.provider).name; }
-	public get providerDisplayName(): string { return getEnumMappingEntry(connectionProviderMapping, this.provider).displayName; }
-	public get authenticationType(): AuthenticationType { return this._profile.authenticationType; }
-	public get authenticationTypeName(): string { return getEnumMappingEntry(authenticationTypeMapping, this.authenticationType).name; }
-	public get authenticationTypeDisplayName(): string { return getEnumMappingEntry(authenticationTypeMapping, this.authenticationType).displayName; }
-	public get engineType(): EngineType { return this._profile.engineType; }
-}
-
-let TestingServers: TestServerProfile[] = [
-	new TestServerProfile(
-		{
-			serverName: getConfigValue(EnvironmentVariable_STANDALONE_SERVER),
-			userName: getConfigValue(EnvironmentVariable_STANDALONE_USERNAME),
-			password: getConfigValue(EnvironmentVariable_STANDALONE_PASSWORD),
-			authenticationType: AuthenticationType.SqlLogin,
-			database: 'master',
-			provider: ConnectionProvider.SQLServer,
-			version: '2017',
-			engineType: EngineType.Standalone
-		}),
-	new TestServerProfile(
-		{
-			serverName: getConfigValue(EnvironmentVariable_AZURE_SERVER),
-			userName: getConfigValue(EnvironmentVariable_AZURE_USERNAME),
-			password: getConfigValue(EnvironmentVariable_AZURE_PASSWORD),
-			authenticationType: AuthenticationType.SqlLogin,
-			database: 'master',
-			provider: ConnectionProvider.SQLServer,
-			version: '2012',
-			engineType: EngineType.Azure
-		}),
-	new TestServerProfile(
-		{
-			serverName: getConfigValue(EnvironmentVariable_BDC_SERVER),
-			userName: getConfigValue(EnvironmentVariable_BDC_USERNAME),
-			password: getConfigValue(EnvironmentVariable_BDC_PASSWORD),
-			authenticationType: AuthenticationType.SqlLogin,
-			database: 'master',
-			provider: ConnectionProvider.SQLServer,
-			version: '2019',
-			engineType: EngineType.BigDataCluster
-		})
-];
-
-function getEnumMappingEntry(mapping: any, enumValue: any): INameDisplayNamePair {
-	let entry = mapping[enumValue];
-	if (entry) {
-		return entry;
-	} else {
-		throw new Error(`Unknown enum type: ${enumValue.toString()}`);
-	}
-}
-
-export async function getAzureServer(): Promise<TestServerProfile> {
-	let servers = await getTestingServers();
-	return servers.filter(s => s.engineType === EngineType.Azure)[0];
-}
-
-export async function getStandaloneServer(): Promise<TestServerProfile> {
-	let servers = await getTestingServers();
-	return servers.filter(s => s.version === '2017' && s.engineType === EngineType.Standalone)[0];
-}
-
-export async function getBdcServer(): Promise<TestServerProfile> {
-	let servers = await getTestingServers();
-	return servers.filter(s => s.version === '2019' && s.engineType === EngineType.BigDataCluster)[0];
-}
-
-export async function getTestingServers(): Promise<TestServerProfile[]> {
-	let promise = new Promise<TestServerProfile[]>(resolve => {
-		resolve(TestingServers);
-	});
-	await promise;
-	return promise;
-}

test/smoke/src/main.ts

@@ -315,7 +315,7 @@ describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => {
 			await this.app.stop();
 		});
 
-		sqlMain();
+		sqlMain(opts.web);
 		/*if (!opts.web) { setupDataLossTests(); }
 		if (!opts.web) { setupDataPreferencesTests(); }
 		setupDataSearchTests();
@@ -329,4 +329,3 @@ describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => {
 		if (!opts.web) { setupLaunchTests(); }*/
 	});
 });
-

test/smoke/src/sql/areas/queryEditor/queryEditor.test.ts

@@ -7,19 +7,7 @@ import { Application } from '../../../../../automation';
 
 export function setup() {
 	describe('Query Editor', () => {
-
-		it('can open, connect and execute file', async function () {
-			const app = this.app as Application;
-			await app.workbench.quickaccess.openFile('test.sql');
-			await app.workbench.queryEditor.commandBar.connect();
-			await app.workbench.connectionDialog.waitForConnectionDialog();
-			await app.workbench.connectionDialog.setProvider('Sqlite');
-			await app.workbench.connectionDialog.setTarget('File', 'chinook.db');
-			await app.workbench.connectionDialog.connect();
-			await app.workbench.queryEditor.commandBar.run();
-			await app.workbench.queryEditor.waitForResults();
-			await app.workbench.quickaccess.runCommand('workbench.action.closeAllEditors');
-		});
+		setupWeb();
 
 		it('can new file, connect and execute', async function () {
 			const app = this.app as Application;
@@ -37,3 +25,18 @@ export function setup() {
 		});
 	});
 }
+
+export function setupWeb() {
+	it('can open, connect and execute file', async function () {
+		const app = this.app as Application;
+		await app.workbench.quickaccess.openFile('test.sql');
+		await app.workbench.queryEditor.commandBar.connect();
+		await app.workbench.connectionDialog.waitForConnectionDialog();
+		await app.workbench.connectionDialog.setProvider('Sqlite');
+		await app.workbench.connectionDialog.setTarget('File', 'chinook.db');
+		await app.workbench.connectionDialog.connect();
+		await app.workbench.queryEditor.commandBar.run();
+		await app.workbench.queryEditor.waitForResults();
+		await app.workbench.quickaccess.runCommand('workbench.action.closeAllEditors');
+	});
+}

test/smoke/src/sql/main.ts

@@ -3,7 +3,7 @@
  *  Licensed under the Source EULA. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { setup as setupQueryEditorTests } from './areas/queryEditor/queryEditor.test';
+import { setup as setupQueryEditorTests, setupWeb as setupQueryEditorWebTests } from './areas/queryEditor/queryEditor.test';
 import { setup as setupNotebookTests } from './areas/notebook/notebook.test';
 import { ApplicationOptions } from '../../../automation';
 import * as yazl from 'yauzl';
@@ -12,12 +12,17 @@ import * as path from 'path';
 import { request } from 'https';
 import * as mkdirp from 'mkdirp';
 
-export function main(): void {
-	setupQueryEditorTests();
+export function main(isWeb: boolean = false): void {
+	if (isWeb) {
+		setupQueryEditorWebTests();
+	} else {
+		setupQueryEditorTests();
+	}
 	setupNotebookTests();
 }
 
 /* eslint-disable no-sync */
+/* eslint-disable no-console */
 const PLATFORM = '${PLATFORM}';
 const RUNTIME = '${RUNTIME}';
 const VERSION = '${VERSION}';
@@ -25,6 +30,7 @@ const VERSION = '${VERSION}';
 const sqliteUrl = `https://github.com/anthonydresser/azuredatastudio-sqlite/releases/download/1.0.12/azuredatastudio-sqlite-${PLATFORM}-${RUNTIME}-${VERSION}.zip`;
 
 export async function setup(app: ApplicationOptions): Promise<void> {
+	console.log('*** Downloading test extensions');
 	const requestUrl = sqliteUrl.replace(PLATFORM, process.platform).replace(RUNTIME, getRuntime(app.web || app.remote || false)).replace(VERSION, getVersion(app.web || app.remote || false));
 	const zip = await fetch(requestUrl);
 	if (!zip) {