Unstable tests (#7487)

Setup unstable tests

extensions/integration-tests/setEnvironmentVariables.js

@@ -130,6 +130,10 @@ process.env[ENVAR_RUN_PYSPARK_TEST] = '0';
 
 const promises = [];
 
+// Open up a web browser to the login page so the user doesn't have to copy the URL manually
+const start = (process.platform == 'darwin'? 'open': process.platform == 'win32'? 'start': 'xdg-open');
+child_process.exec(`${start} https://microsoft.com/devicelogin`);
+
 // Fetch the values from AKV
 msrestAzure.interactiveLogin().then((credentials) => {
 	const client = new KeyVault.KeyVaultClient(credentials);

extensions/integration-tests/src/dacpac.test.ts

@@ -66,7 +66,7 @@ if (context.RunTest) {
 		// Tracking bug https://github.com/microsoft/azuredatastudio/issues/7323
 
 		const bacpac1: string = path.join(__dirname, '..', 'testData', 'Database1.bacpac');
-		test.skip('Import and export bacpac', async function () {
+		test('Import and export bacpac @UNSTABLE@', async function () {
 			const server = await getStandaloneServer();
 			await utils.connectToServer(server);
 

extensions/integration-tests/src/index.ts

@@ -27,26 +27,26 @@ if (suite === SuiteType.Stress) {
 	// both are set.
 	if (process.env.StressRuntime) {
 		options.timeout = (120 + 1.2 * parseInt(process.env.StressRuntime)) * 1000; // allow sufficient timeout based on StressRuntime setting
-		console.log(`setting options.timeout to:${options.timeout} based on process.env.StressRuntime value of ${process.env.StressRuntime} seconds`);
+		console.log(`setting options.timeout to: ${options.timeout} based on process.env.StressRuntime value of ${process.env.StressRuntime} seconds`);
 	}
 }
 
 // set relevant mocha options from the environment
 if (process.env.ADS_TEST_GREP) {
 	options.grep = process.env.ADS_TEST_GREP;
-	console.log(`setting options.grep to:${options.grep}`);
+	console.log(`setting options.grep to: ${options.grep}`);
 }
 if (process.env.ADS_TEST_INVERT_GREP) {
 	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
-	console.log(`setting options.invert to:${options.invert}`);
+	console.log(`setting options.invert to: ${options.invert}`);
 }
 if (process.env.ADS_TEST_TIMEOUT) {
 	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
-	console.log(`setting options.timeout to:${options.timeout}`);
+	console.log(`setting options.timeout to: ${options.timeout}`);
 }
 if (process.env.ADS_TEST_RETRIES) {
 	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
-	console.log(`setting options.retries to:${options.retries}`);
+	console.log(`setting options.retries to: ${options.retries}`);
 }
 
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {

extensions/integration-tests/src/notebook.test.ts

@@ -30,12 +30,12 @@ if (context.RunTest) {
 		});
 
 		// This test needs to be re-enabled once the SqlClient driver has been updated
-		test.skip('Sql NB test', async function () {
+		test('Sql NB test @UNSTABLE@', async function () {
 			await (new NotebookTester()).sqlNbTest(this.test.title);
 		});
 
 		// This test needs to be re-enabled once the SqlClient driver has been updated
-		test.skip('Sql NB multiple cells test', async function () {
+		test('Sql NB multiple cells test @UNSTABLE@', async function () {
 			await (new NotebookTester()).sqlNbMultipleCellsTest(this.test.title);
 		});
 
@@ -56,7 +56,7 @@ if (context.RunTest) {
 		});
 
 		// TODO: Need to make this test more reliable.
-		test.skip('should not be dirty after saving notebook test', async function () {
+		test('should not be dirty after saving notebook test @UNSTABLE@', async function () {
 			await (new NotebookTester().shouldNotBeDirtyAfterSavingNotebookTest(this.test.title));
 		});
 

extensions/integration-tests/src/objectExplorer.test.ts

@@ -21,7 +21,7 @@ if (context.RunTest) {
 		test('Standalone instance node label test', async function () {
 			return await (new ObjectExplorerTester()).standaloneNodeLabelTest();
 		});
-		test.skip('Azure SQL DB instance node label test', async function () {
+		test('Azure SQL DB instance node label test @UNSTABLE@', async function () {
 			return await (new ObjectExplorerTester()).sqlDbNodeLabelTest();
 		});
 		test('BDC instance context menu test', async function () {
@@ -30,7 +30,7 @@ if (context.RunTest) {
 		test('Azure SQL DB context menu test', async function () {
 			return await (new ObjectExplorerTester()).sqlDbContextMenuTest();
 		});
-		test.skip('Standalone database context menu test', async function () {
+		test('Standalone database context menu test @UNSTABLE@', async function () {
 			return await (new ObjectExplorerTester()).standaloneContextMenuTest();
 		});
 	});

extensions/integration-tests/src/schemaCompare.test.ts

@@ -46,7 +46,7 @@ if (context.RunTest) {
 		test('Schema compare dacpac to dacpac comparison and scmp', async function () {
 			await schemaCompareTester.SchemaCompareDacpacToDacpac();
 		});
-		test('Schema compare database to database comparison, script generation, and scmp', async function () {
+		test('Schema compare database to database comparison, script generation, and scmp @UNSTABLE@', async function () {
 			await schemaCompareTester.SchemaCompareDatabaseToDatabase();
 		});
 		// TODO: figure out why this is failing with Error: This editor is not connected to a database Parameter name: OwnerUri

scripts/sql-test-integration-unstable.bat

@@ -0,0 +1,9 @@
+setlocal
+
+set ADS_TEST_GREP=@UNSTABLE@
+
+echo Running unstable tests
+
+call %~dp0\sql-test-integration.bat
+
+endlocal

scripts/sql-test-integration-unstable.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+	realpath() { [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"; }
+	ROOT=$(dirname $(dirname $(realpath "$0")))
+else
+	ROOT=$(dirname $(dirname $(readlink -f $0)))
+fi
+
+cd $ROOT
+
+export ADS_TEST_GREP=@UNSTABLE@
+
+echo Running unstable tests
+
+./scripts/sql-test-integration.sh

scripts/sql-test-integration.bat

@@ -6,6 +6,13 @@ set VSCODEUSERDATADIR=%TMP%\adsuser-%RANDOM%-%TIME:~6,5%
 set VSCODEEXTENSIONSDIR=%TMP%\adsext-%RANDOM%-%TIME:~6,5%
 echo VSCODEUSERDATADIR=%VSCODEUSERDATADIR%
 echo VSCODEEXTENSIONSDIR=%VSCODEEXTENSIONSDIR%
+
+if "%ADS_TEST_GREP%" == "" (
+	echo Running stable tests only
+	set ADS_TEST_GREP=@UNSTABLE@
+	SET ADS_TEST_INVERT_GREP=1
+)
+
 @echo OFF
 
 if "%SKIP_PYTHON_INSTALL_TEST%" == "1" (

scripts/sql-test-integration.sh

@@ -12,6 +12,12 @@ else
 	VSCODEEXTDIR=`mktemp -d 2>/dev/null`
 fi
 
+if [[ "$ADS_TEST_GREP" == "" ]]; then
+	echo Running stable tests only
+	export ADS_TEST_GREP=@UNSTABLE@
+	export ADS_TEST_INVERT_GREP=1
+fi
+
 cd $ROOT
 echo VSCODEUSERDATADIR=$VSCODEUSERDATADIR
 echo VSCODEEXTDIR=$VSCODEEXTDIR