From 6582debd7388b1be512fc3a9b956d3e54f6115e7 Mon Sep 17 00:00:00 2001 From: Amir Omidi Date: Wed, 2 Oct 2019 22:01:17 -0700 Subject: [PATCH] Unstable tests (#7487) Setup unstable tests --- .../setEnvironmentVariables.js | 4 ++++ extensions/integration-tests/src/dacpac.test.ts | 2 +- extensions/integration-tests/src/index.ts | 10 +++++----- .../integration-tests/src/notebook.test.ts | 6 +++--- .../src/objectExplorer.test.ts | 4 ++-- .../integration-tests/src/schemaCompare.test.ts | 2 +- scripts/sql-test-integration-unstable.bat | 9 +++++++++ scripts/sql-test-integration-unstable.sh | 17 +++++++++++++++++ scripts/sql-test-integration.bat | 7 +++++++ scripts/sql-test-integration.sh | 6 ++++++ 10 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 scripts/sql-test-integration-unstable.bat create mode 100755 scripts/sql-test-integration-unstable.sh diff --git a/extensions/integration-tests/setEnvironmentVariables.js b/extensions/integration-tests/setEnvironmentVariables.js index ce5afb055a..b1a597aae5 100644 --- a/extensions/integration-tests/setEnvironmentVariables.js +++ b/extensions/integration-tests/setEnvironmentVariables.js @@ -130,6 +130,10 @@ process.env[ENVAR_RUN_PYSPARK_TEST] = '0'; const promises = []; +// Open up a web browser to the login page so the user doesn't have to copy the URL manually +const start = (process.platform == 'darwin'? 'open': process.platform == 'win32'? 'start': 'xdg-open'); +child_process.exec(`${start} https://microsoft.com/devicelogin`); + // Fetch the values from AKV msrestAzure.interactiveLogin().then((credentials) => { const client = new KeyVault.KeyVaultClient(credentials); diff --git a/extensions/integration-tests/src/dacpac.test.ts b/extensions/integration-tests/src/dacpac.test.ts index 1d66f8ab50..adba2931a3 100644 --- a/extensions/integration-tests/src/dacpac.test.ts +++ b/extensions/integration-tests/src/dacpac.test.ts @@ -66,7 +66,7 @@ if (context.RunTest) { // Tracking bug https://github.com/microsoft/azuredatastudio/issues/7323 const bacpac1: string = path.join(__dirname, '..', 'testData', 'Database1.bacpac'); - test.skip('Import and export bacpac', async function () { + test('Import and export bacpac @UNSTABLE@', async function () { const server = await getStandaloneServer(); await utils.connectToServer(server); diff --git a/extensions/integration-tests/src/index.ts b/extensions/integration-tests/src/index.ts index 51e57ea030..3e2f805351 100644 --- a/extensions/integration-tests/src/index.ts +++ b/extensions/integration-tests/src/index.ts @@ -27,26 +27,26 @@ if (suite === SuiteType.Stress) { // both are set. if (process.env.StressRuntime) { options.timeout = (120 + 1.2 * parseInt(process.env.StressRuntime)) * 1000; // allow sufficient timeout based on StressRuntime setting - console.log(`setting options.timeout to:${options.timeout} based on process.env.StressRuntime value of ${process.env.StressRuntime} seconds`); + console.log(`setting options.timeout to: ${options.timeout} based on process.env.StressRuntime value of ${process.env.StressRuntime} seconds`); } } // set relevant mocha options from the environment if (process.env.ADS_TEST_GREP) { options.grep = process.env.ADS_TEST_GREP; - console.log(`setting options.grep to:${options.grep}`); + console.log(`setting options.grep to: ${options.grep}`); } if (process.env.ADS_TEST_INVERT_GREP) { options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP); - console.log(`setting options.invert to:${options.invert}`); + console.log(`setting options.invert to: ${options.invert}`); } if (process.env.ADS_TEST_TIMEOUT) { options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT); - console.log(`setting options.timeout to:${options.timeout}`); + console.log(`setting options.timeout to: ${options.timeout}`); } if (process.env.ADS_TEST_RETRIES) { options.retries = parseInt(process.env.ADS_TEST_RETRIES); - console.log(`setting options.retries to:${options.retries}`); + console.log(`setting options.retries to: ${options.retries}`); } if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { diff --git a/extensions/integration-tests/src/notebook.test.ts b/extensions/integration-tests/src/notebook.test.ts index 44a7d79599..54325ad351 100644 --- a/extensions/integration-tests/src/notebook.test.ts +++ b/extensions/integration-tests/src/notebook.test.ts @@ -30,12 +30,12 @@ if (context.RunTest) { }); // This test needs to be re-enabled once the SqlClient driver has been updated - test.skip('Sql NB test', async function () { + test('Sql NB test @UNSTABLE@', async function () { await (new NotebookTester()).sqlNbTest(this.test.title); }); // This test needs to be re-enabled once the SqlClient driver has been updated - test.skip('Sql NB multiple cells test', async function () { + test('Sql NB multiple cells test @UNSTABLE@', async function () { await (new NotebookTester()).sqlNbMultipleCellsTest(this.test.title); }); @@ -56,7 +56,7 @@ if (context.RunTest) { }); // TODO: Need to make this test more reliable. - test.skip('should not be dirty after saving notebook test', async function () { + test('should not be dirty after saving notebook test @UNSTABLE@', async function () { await (new NotebookTester().shouldNotBeDirtyAfterSavingNotebookTest(this.test.title)); }); diff --git a/extensions/integration-tests/src/objectExplorer.test.ts b/extensions/integration-tests/src/objectExplorer.test.ts index 5b86fc2841..04ebb0d4f2 100644 --- a/extensions/integration-tests/src/objectExplorer.test.ts +++ b/extensions/integration-tests/src/objectExplorer.test.ts @@ -21,7 +21,7 @@ if (context.RunTest) { test('Standalone instance node label test', async function () { return await (new ObjectExplorerTester()).standaloneNodeLabelTest(); }); - test.skip('Azure SQL DB instance node label test', async function () { + test('Azure SQL DB instance node label test @UNSTABLE@', async function () { return await (new ObjectExplorerTester()).sqlDbNodeLabelTest(); }); test('BDC instance context menu test', async function () { @@ -30,7 +30,7 @@ if (context.RunTest) { test('Azure SQL DB context menu test', async function () { return await (new ObjectExplorerTester()).sqlDbContextMenuTest(); }); - test.skip('Standalone database context menu test', async function () { + test('Standalone database context menu test @UNSTABLE@', async function () { return await (new ObjectExplorerTester()).standaloneContextMenuTest(); }); }); diff --git a/extensions/integration-tests/src/schemaCompare.test.ts b/extensions/integration-tests/src/schemaCompare.test.ts index d966e9bef0..a81f600cb0 100644 --- a/extensions/integration-tests/src/schemaCompare.test.ts +++ b/extensions/integration-tests/src/schemaCompare.test.ts @@ -46,7 +46,7 @@ if (context.RunTest) { test('Schema compare dacpac to dacpac comparison and scmp', async function () { await schemaCompareTester.SchemaCompareDacpacToDacpac(); }); - test('Schema compare database to database comparison, script generation, and scmp', async function () { + test('Schema compare database to database comparison, script generation, and scmp @UNSTABLE@', async function () { await schemaCompareTester.SchemaCompareDatabaseToDatabase(); }); // TODO: figure out why this is failing with Error: This editor is not connected to a database Parameter name: OwnerUri diff --git a/scripts/sql-test-integration-unstable.bat b/scripts/sql-test-integration-unstable.bat new file mode 100644 index 0000000000..eab46eef8d --- /dev/null +++ b/scripts/sql-test-integration-unstable.bat @@ -0,0 +1,9 @@ +setlocal + +set ADS_TEST_GREP=@UNSTABLE@ + +echo Running unstable tests + +call %~dp0\sql-test-integration.bat + +endlocal diff --git a/scripts/sql-test-integration-unstable.sh b/scripts/sql-test-integration-unstable.sh new file mode 100755 index 0000000000..a8fb66e1aa --- /dev/null +++ b/scripts/sql-test-integration-unstable.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +if [[ "$OSTYPE" == "darwin"* ]]; then + realpath() { [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"; } + ROOT=$(dirname $(dirname $(realpath "$0"))) +else + ROOT=$(dirname $(dirname $(readlink -f $0))) +fi + +cd $ROOT + +export ADS_TEST_GREP=@UNSTABLE@ + +echo Running unstable tests + +./scripts/sql-test-integration.sh diff --git a/scripts/sql-test-integration.bat b/scripts/sql-test-integration.bat index 1146229909..7e078b90d5 100644 --- a/scripts/sql-test-integration.bat +++ b/scripts/sql-test-integration.bat @@ -6,6 +6,13 @@ set VSCODEUSERDATADIR=%TMP%\adsuser-%RANDOM%-%TIME:~6,5% set VSCODEEXTENSIONSDIR=%TMP%\adsext-%RANDOM%-%TIME:~6,5% echo VSCODEUSERDATADIR=%VSCODEUSERDATADIR% echo VSCODEEXTENSIONSDIR=%VSCODEEXTENSIONSDIR% + +if "%ADS_TEST_GREP%" == "" ( + echo Running stable tests only + set ADS_TEST_GREP=@UNSTABLE@ + SET ADS_TEST_INVERT_GREP=1 +) + @echo OFF if "%SKIP_PYTHON_INSTALL_TEST%" == "1" ( diff --git a/scripts/sql-test-integration.sh b/scripts/sql-test-integration.sh index 5988afafa3..d8e49e3c45 100755 --- a/scripts/sql-test-integration.sh +++ b/scripts/sql-test-integration.sh @@ -12,6 +12,12 @@ else VSCODEEXTDIR=`mktemp -d 2>/dev/null` fi +if [[ "$ADS_TEST_GREP" == "" ]]; then + echo Running stable tests only + export ADS_TEST_GREP=@UNSTABLE@ + export ADS_TEST_INVERT_GREP=1 +fi + cd $ROOT echo VSCODEUSERDATADIR=$VSCODEUSERDATADIR echo VSCODEEXTDIR=$VSCODEEXTDIR