From effa50a9bd107e1ce3c9a397f776dedce33244d9 Mon Sep 17 00:00:00 2001 From: Charles Gagnon Date: Mon, 7 Oct 2019 10:18:02 -0700 Subject: [PATCH] Add unstable test params for core and extension tests (#7513) * Add unstable test params for core and extension tests * unset invert options for unstable script runs * Fix copypasta * Add nogpu to dacpac tests --- .../admin-tool-ext-win/src/test/index.ts | 22 +++++++- extensions/agent/src/test/index.ts | 22 +++++++- extensions/azurecore/src/test/index.ts | 20 +++++++- extensions/cms/src/test/index.ts | 20 +++++++- extensions/dacpac/src/test/index.ts | 22 ++++++-- .../notebook/src/integrationTest/index.ts | 22 +++++++- .../notebook/src/test/book/book.test.ts | 2 +- extensions/notebook/src/test/index.ts | 22 +++++++- .../resource-deployment/src/test/index.ts | 20 +++++++- extensions/schema-compare/src/test/index.ts | 20 +++++++- scripts/sql-test-integration-unstable.bat | 3 +- scripts/sql-test-integration-unstable.sh | 3 +- scripts/sql-test-integration.bat | 1 + scripts/sql-test-integration.sh | 1 + scripts/test-extensions-unstable.bat | 12 +++++ scripts/test-extensions-unstable.sh | 17 +++++++ ...xtensions-unit.bat => test-extensions.bat} | 51 +++++++++++++++---- ...-extensions-unit.sh => test-extensions.sh} | 50 ++++++++++++++---- scripts/test-unstable.bat | 10 ++++ scripts/test-unstable.sh | 18 +++++++ scripts/test.bat | 17 ++++++- scripts/test.sh | 17 ++++++- test/electron/index.js | 1 + test/electron/renderer.js | 5 +- 24 files changed, 355 insertions(+), 43 deletions(-) create mode 100644 scripts/test-extensions-unstable.bat create mode 100644 scripts/test-extensions-unstable.sh rename scripts/{test-extensions-unit.bat => test-extensions.bat} (59%) rename scripts/{test-extensions-unit.sh => test-extensions.sh} (58%) mode change 100755 => 100644 create mode 100644 scripts/test-unstable.bat create mode 100644 scripts/test-unstable.sh diff --git a/extensions/admin-tool-ext-win/src/test/index.ts b/extensions/admin-tool-ext-win/src/test/index.ts index aa5127acdc..f7d29b4a09 100644 --- a/extensions/admin-tool-ext-win/src/test/index.ts +++ b/extensions/admin-tool-ext-win/src/test/index.ts @@ -6,7 +6,7 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Database Admin Tool Extensions for Windows'; +const suite = 'admin-tool-ext-win Extension Tests'; const options: any = { ui: 'bdd', @@ -14,6 +14,24 @@ const options: any = { timeout: 600000 }; +// set relevant mocha options from the environment +if (process.env.ADS_TEST_GREP) { + options.grep = process.env.ADS_TEST_GREP; + console.log(`setting options.grep to: ${options.grep}`); +} +if (process.env.ADS_TEST_INVERT_GREP) { + options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP); + console.log(`setting options.invert to: ${options.invert}`); +} +if (process.env.ADS_TEST_TIMEOUT) { + options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT); + console.log(`setting options.timeout to: ${options.timeout}`); +} +if (process.env.ADS_TEST_RETRIES) { + options.retries = parseInt(process.env.ADS_TEST_RETRIES); + console.log(`setting options.retries to: ${options.retries}`); +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { @@ -27,4 +45,4 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { testRunner.configure(options); -export = testRunner; \ No newline at end of file +export = testRunner; diff --git a/extensions/agent/src/test/index.ts b/extensions/agent/src/test/index.ts index ba3cd6e224..463566180a 100644 --- a/extensions/agent/src/test/index.ts +++ b/extensions/agent/src/test/index.ts @@ -6,7 +6,7 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Agent Tests'; +const suite = 'agent Extension Tests'; const options: any = { ui: 'bdd', @@ -14,6 +14,24 @@ const options: any = { timeout: 600000 }; +// set relevant mocha options from the environment +if (process.env.ADS_TEST_GREP) { + options.grep = process.env.ADS_TEST_GREP; + console.log(`setting options.grep to: ${options.grep}`); +} +if (process.env.ADS_TEST_INVERT_GREP) { + options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP); + console.log(`setting options.invert to: ${options.invert}`); +} +if (process.env.ADS_TEST_TIMEOUT) { + options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT); + console.log(`setting options.timeout to: ${options.timeout}`); +} +if (process.env.ADS_TEST_RETRIES) { + options.retries = parseInt(process.env.ADS_TEST_RETRIES); + console.log(`setting options.retries to: ${options.retries}`); +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { @@ -27,4 +45,4 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { testRunner.configure(options); -export = testRunner; \ No newline at end of file +export = testRunner; diff --git a/extensions/azurecore/src/test/index.ts b/extensions/azurecore/src/test/index.ts index a9f0ffbd82..9651794ab0 100644 --- a/extensions/azurecore/src/test/index.ts +++ b/extensions/azurecore/src/test/index.ts @@ -6,7 +6,7 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Integration Azure Tests'; +const suite = 'azurecore Extension Tests'; const options: any = { ui: 'bdd', @@ -14,6 +14,24 @@ const options: any = { timeout: 60000 }; +// set relevant mocha options from the environment +if (process.env.ADS_TEST_GREP) { + options.grep = process.env.ADS_TEST_GREP; + console.log(`setting options.grep to: ${options.grep}`); +} +if (process.env.ADS_TEST_INVERT_GREP) { + options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP); + console.log(`setting options.invert to: ${options.invert}`); +} +if (process.env.ADS_TEST_TIMEOUT) { + options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT); + console.log(`setting options.timeout to: ${options.timeout}`); +} +if (process.env.ADS_TEST_RETRIES) { + options.retries = parseInt(process.env.ADS_TEST_RETRIES); + console.log(`setting options.retries to: ${options.retries}`); +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/extensions/cms/src/test/index.ts b/extensions/cms/src/test/index.ts index f1fccb588a..94e41ca4d3 100644 --- a/extensions/cms/src/test/index.ts +++ b/extensions/cms/src/test/index.ts @@ -6,7 +6,7 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'CMS Unit Tests'; +const suite = 'cms Extension Tests'; const options: any = { ui: 'bdd', @@ -14,6 +14,24 @@ const options: any = { timeout: 60000 }; +// set relevant mocha options from the environment +if (process.env.ADS_TEST_GREP) { + options.grep = process.env.ADS_TEST_GREP; + console.log(`setting options.grep to: ${options.grep}`); +} +if (process.env.ADS_TEST_INVERT_GREP) { + options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP); + console.log(`setting options.invert to: ${options.invert}`); +} +if (process.env.ADS_TEST_TIMEOUT) { + options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT); + console.log(`setting options.timeout to: ${options.timeout}`); +} +if (process.env.ADS_TEST_RETRIES) { + options.retries = parseInt(process.env.ADS_TEST_RETRIES); + console.log(`setting options.retries to: ${options.retries}`); +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/extensions/dacpac/src/test/index.ts b/extensions/dacpac/src/test/index.ts index 043999ecac..0e683a6dfd 100644 --- a/extensions/dacpac/src/test/index.ts +++ b/extensions/dacpac/src/test/index.ts @@ -6,7 +6,7 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'DacFx Tests'; +const suite = 'dacpac Extension Tests'; const testOptions: any = { ui: 'bdd', @@ -14,9 +14,23 @@ const testOptions: any = { timeout: 60000 }; -// const coverageConfig: any = { -// coverConfig: '../../coverageConfig.json' -// }; +// set relevant mocha options from the environment +if (process.env.ADS_TEST_GREP) { + testOptions.grep = process.env.ADS_TEST_GREP; + console.log(`setting options.grep to: ${testOptions.grep}`); +} +if (process.env.ADS_TEST_INVERT_GREP) { + testOptions.invert = parseInt(process.env.ADS_TEST_INVERT_GREP); + console.log(`setting options.invert to: ${testOptions.invert}`); +} +if (process.env.ADS_TEST_TIMEOUT) { + testOptions.timeout = parseInt(process.env.ADS_TEST_TIMEOUT); + console.log(`setting options.timeout to: ${testOptions.timeout}`); +} +if (process.env.ADS_TEST_RETRIES) { + testOptions.retries = parseInt(process.env.ADS_TEST_RETRIES); + console.log(`setting options.retries to: ${testOptions.retries}`); +} if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { testOptions.reporter = 'mocha-multi-reporters'; diff --git a/extensions/notebook/src/integrationTest/index.ts b/extensions/notebook/src/integrationTest/index.ts index 5b7fe616a7..7e7a9b53f2 100644 --- a/extensions/notebook/src/integrationTest/index.ts +++ b/extensions/notebook/src/integrationTest/index.ts @@ -6,7 +6,7 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Notebook Extension Integration Tests'; +const suite = 'notebook Extension Integration Tests'; const options: any = { ui: 'bdd', @@ -14,6 +14,24 @@ const options: any = { timeout: 600000 }; +// set relevant mocha options from the environment +if (process.env.ADS_TEST_GREP) { + options.grep = process.env.ADS_TEST_GREP; + console.log(`setting options.grep to: ${options.grep}`); +} +if (process.env.ADS_TEST_INVERT_GREP) { + options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP); + console.log(`setting options.invert to: ${options.invert}`); +} +if (process.env.ADS_TEST_TIMEOUT) { + options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT); + console.log(`setting options.timeout to: ${options.timeout}`); +} +if (process.env.ADS_TEST_RETRIES) { + options.retries = parseInt(process.env.ADS_TEST_RETRIES); + console.log(`setting options.retries to: ${options.retries}`); +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { @@ -27,4 +45,4 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { testRunner.configure(options); -export = testRunner; \ No newline at end of file +export = testRunner; diff --git a/extensions/notebook/src/test/book/book.test.ts b/extensions/notebook/src/test/book/book.test.ts index b2c41725bf..204780ef96 100644 --- a/extensions/notebook/src/test/book/book.test.ts +++ b/extensions/notebook/src/test/book/book.test.ts @@ -34,7 +34,7 @@ export function equalBookItems(book: BookTreeItem, expectedBook: ExpectedBookIte } } -describe.skip('BookTreeViewProviderTests', function() { +describe('BookTreeViewProviderTests @UNSTABLE@', function() { describe('BookTreeViewProvider.getChildren', function (): void { let rootFolderPath: string; diff --git a/extensions/notebook/src/test/index.ts b/extensions/notebook/src/test/index.ts index 84a25723f2..948b7b7503 100644 --- a/extensions/notebook/src/test/index.ts +++ b/extensions/notebook/src/test/index.ts @@ -6,7 +6,7 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Notebook Tests'; +const suite = 'notebook Extension Tests'; const options: any = { ui: 'bdd', @@ -14,6 +14,24 @@ const options: any = { timeout: 600000 }; +// set relevant mocha options from the environment +if (process.env.ADS_TEST_GREP) { + options.grep = process.env.ADS_TEST_GREP; + console.log(`setting options.grep to: ${options.grep}`); +} +if (process.env.ADS_TEST_INVERT_GREP) { + options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP); + console.log(`setting options.invert to: ${options.invert}`); +} +if (process.env.ADS_TEST_TIMEOUT) { + options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT); + console.log(`setting options.timeout to: ${options.timeout}`); +} +if (process.env.ADS_TEST_RETRIES) { + options.retries = parseInt(process.env.ADS_TEST_RETRIES); + console.log(`setting options.retries to: ${options.retries}`); +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { @@ -27,4 +45,4 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { testRunner.configure(options); -export = testRunner; \ No newline at end of file +export = testRunner; diff --git a/extensions/resource-deployment/src/test/index.ts b/extensions/resource-deployment/src/test/index.ts index f525a8cf52..f51b28c5e1 100644 --- a/extensions/resource-deployment/src/test/index.ts +++ b/extensions/resource-deployment/src/test/index.ts @@ -6,7 +6,7 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Resource Deployment Unit Tests'; +const suite = 'resource-deployment Extension Tests'; const testOptions: any = { ui: 'tdd', @@ -14,6 +14,24 @@ const testOptions: any = { timeout: 60000 }; +// set relevant mocha options from the environment +if (process.env.ADS_TEST_GREP) { + testOptions.grep = process.env.ADS_TEST_GREP; + console.log(`setting options.grep to: ${testOptions.grep}`); +} +if (process.env.ADS_TEST_INVERT_GREP) { + testOptions.invert = parseInt(process.env.ADS_TEST_INVERT_GREP); + console.log(`setting options.invert to: ${testOptions.invert}`); +} +if (process.env.ADS_TEST_TIMEOUT) { + testOptions.timeout = parseInt(process.env.ADS_TEST_TIMEOUT); + console.log(`setting options.timeout to: ${testOptions.timeout}`); +} +if (process.env.ADS_TEST_RETRIES) { + testOptions.retries = parseInt(process.env.ADS_TEST_RETRIES); + console.log(`setting options.retries to: ${testOptions.retries}`); +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { testOptions.reporter = 'mocha-multi-reporters'; testOptions.reporterOptions = { diff --git a/extensions/schema-compare/src/test/index.ts b/extensions/schema-compare/src/test/index.ts index 1605846c44..23dc4a785c 100644 --- a/extensions/schema-compare/src/test/index.ts +++ b/extensions/schema-compare/src/test/index.ts @@ -6,7 +6,7 @@ const path = require('path'); const testRunner = require('vscode/lib/testrunner'); -const suite = 'Schema Compare Tests'; +const suite = 'schema-compare Extension Tests'; const options: any = { ui: 'bdd', @@ -14,6 +14,24 @@ const options: any = { timeout: 60000 }; +// set relevant mocha options from the environment +if (process.env.ADS_TEST_GREP) { + options.grep = process.env.ADS_TEST_GREP; + console.log(`setting options.grep to: ${options.grep}`); +} +if (process.env.ADS_TEST_INVERT_GREP) { + options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP); + console.log(`setting options.invert to: ${options.invert}`); +} +if (process.env.ADS_TEST_TIMEOUT) { + options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT); + console.log(`setting options.timeout to: ${options.timeout}`); +} +if (process.env.ADS_TEST_RETRIES) { + options.retries = parseInt(process.env.ADS_TEST_RETRIES); + console.log(`setting options.retries to: ${options.retries}`); +} + if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { options.reporter = 'mocha-multi-reporters'; options.reporterOptions = { diff --git a/scripts/sql-test-integration-unstable.bat b/scripts/sql-test-integration-unstable.bat index eab46eef8d..73bdc502b7 100644 --- a/scripts/sql-test-integration-unstable.bat +++ b/scripts/sql-test-integration-unstable.bat @@ -1,8 +1,9 @@ setlocal set ADS_TEST_GREP=@UNSTABLE@ +set ADS_TEST_INVERT_GREP= -echo Running unstable tests +echo Running UNSTABLE ADS Extension Integration tests call %~dp0\sql-test-integration.bat diff --git a/scripts/sql-test-integration-unstable.sh b/scripts/sql-test-integration-unstable.sh index a8fb66e1aa..7566457934 100755 --- a/scripts/sql-test-integration-unstable.sh +++ b/scripts/sql-test-integration-unstable.sh @@ -11,7 +11,8 @@ fi cd $ROOT export ADS_TEST_GREP=@UNSTABLE@ +export ADS_TEST_INVERT_GREP= -echo Running unstable tests +echo Running UNSTABLE ADS Extension Integration tests ./scripts/sql-test-integration.sh diff --git a/scripts/sql-test-integration.bat b/scripts/sql-test-integration.bat index 7e078b90d5..fae85b298c 100644 --- a/scripts/sql-test-integration.bat +++ b/scripts/sql-test-integration.bat @@ -7,6 +7,7 @@ set VSCODEEXTENSIONSDIR=%TMP%\adsext-%RANDOM%-%TIME:~6,5% echo VSCODEUSERDATADIR=%VSCODEUSERDATADIR% echo VSCODEEXTENSIONSDIR=%VSCODEEXTENSIONSDIR% +:: Default to only running stable tests if test grep isn't set if "%ADS_TEST_GREP%" == "" ( echo Running stable tests only set ADS_TEST_GREP=@UNSTABLE@ diff --git a/scripts/sql-test-integration.sh b/scripts/sql-test-integration.sh index 7083033359..f72e8ac935 100755 --- a/scripts/sql-test-integration.sh +++ b/scripts/sql-test-integration.sh @@ -12,6 +12,7 @@ else VSCODEEXTDIR=`mktemp -d 2>/dev/null` fi +# Default to only running stable tests if test grep isn't set if [[ "$ADS_TEST_GREP" == "" ]]; then echo Running stable tests only export ADS_TEST_GREP=@UNSTABLE@ diff --git a/scripts/test-extensions-unstable.bat b/scripts/test-extensions-unstable.bat new file mode 100644 index 0000000000..0e636a1f79 --- /dev/null +++ b/scripts/test-extensions-unstable.bat @@ -0,0 +1,12 @@ +:: Runs UNSTABLE Extension Tests + +setlocal + +set ADS_TEST_GREP=@UNSTABLE@ +set ADS_TEST_INVERT_GREP= + +echo Running UNSTABLE Extension Tests + +call %~dp0\test-extensions.bat + +endlocal diff --git a/scripts/test-extensions-unstable.sh b/scripts/test-extensions-unstable.sh new file mode 100644 index 0000000000..29a6b0892d --- /dev/null +++ b/scripts/test-extensions-unstable.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +if [[ "$OSTYPE" == "darwin"* ]]; then + realpath() { [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"; } + ROOT=$(dirname $(dirname $(realpath "$0"))) +else + ROOT=$(dirname $(dirname $(readlink -f $0))) +fi + +cd $ROOT + +export ADS_TEST_GREP=@UNSTABLE@ +export ADS_TEST_INVERT_GREP= +echo Running UNSTABLE Extension Tests + +./scripts/test-extensions.sh diff --git a/scripts/test-extensions-unit.bat b/scripts/test-extensions.bat similarity index 59% rename from scripts/test-extensions-unit.bat rename to scripts/test-extensions.bat index c217d14920..177e968000 100644 --- a/scripts/test-extensions-unit.bat +++ b/scripts/test-extensions.bat @@ -1,4 +1,4 @@ -:: Runs unit tests for Extensions +:: Runs Extension tests setlocal @@ -8,23 +8,54 @@ set VSCODEUSERDATADIR=%TMP%\adsuser-%RANDOM%-%TIME:~6,5% set VSCODEEXTENSIONSDIR=%TMP%\adsext-%RANDOM%-%TIME:~6,5% echo %VSCODEUSERDATADIR% echo %VSCODEEXTENSIONSDIR% + +:: Default to only running stable tests if test grep isn't set +if "%ADS_TEST_GREP%" == "" ( + echo Running stable tests only + set ADS_TEST_GREP=@UNSTABLE@ + SET ADS_TEST_INVERT_GREP=1 +) + @echo OFF -echo starting admin tool extension windows tests +echo *************************************************** +echo *** starting admin tool extension windows tests *** +echo *************************************************** call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\admin-tool-ext-win --extensionTestsPath=%~dp0\..\extensions\admin-tool-ext-win\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --disableExtensions --remote-debugging-port=9222 -echo starting agent tests + +echo **************************** +echo *** starting agent tests *** +echo **************************** call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\agent --extensionTestsPath=%~dp0\..\extensions\agent\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222 -echo starting azurecore tests + +echo ******************************** +echo *** starting azurecore tests *** +echo ******************************** call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\azurecore --extensionTestsPath=%~dp0\..\extensions\azurecore\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222 -echo starting cms tests + +echo ************************** +echo *** starting cms tests *** +echo ************************** call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\cms --extensionTestsPath=%~dp0\..\extensions\cms\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222 -echo starting dacpac tests -call .\scripts\code.bat --extensionDevelopmentPath=%~dp0\..\extensions\dacpac --extensionTestsPath=%~dp0\..\extensions\dacpac\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222 -echo starting schema compare tests + +echo ***************************** +echo *** starting dacpac tests *** +echo ***************************** +call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\dacpac --extensionTestsPath=%~dp0\..\extensions\dacpac\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222 + +echo ************************************* +echo *** starting schema compare tests *** +echo ************************************* call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\schema-compare --extensionTestsPath=%~dp0\..\extensions\schema-compare\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222 -echo starting notebook tests + +echo ******************************* +echo *** starting notebook tests *** +echo ******************************* call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\notebook --extensionTestsPath=%~dp0\..\extensions\notebook\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222 -echo starting resource deployment tests + +echo ****************************************** +echo *** starting resource deployment tests *** +echo ****************************************** call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\resource-deployment --extensionTestsPath=%~dp0\..\extensions\resource-deployment\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222 if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/scripts/test-extensions-unit.sh b/scripts/test-extensions.sh old mode 100755 new mode 100644 similarity index 58% rename from scripts/test-extensions-unit.sh rename to scripts/test-extensions.sh index 78ce6d4966..ffc29890a9 --- a/scripts/test-extensions-unit.sh +++ b/scripts/test-extensions.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Runs unit tests for Extensions +# Runs Extension Tests set -e if [[ "$OSTYPE" == "darwin"* ]]; then @@ -14,25 +14,55 @@ else VSCODEEXTDIR=`mktemp -d 2>/dev/null` fi +# Default to only running stable tests if test grep isn't set +if [[ "$ADS_TEST_GREP" == "" ]]; then + echo Running stable tests only + export ADS_TEST_GREP=@UNSTABLE@ + export ADS_TEST_INVERT_GREP=1 +fi + cd $ROOT echo $VSCODEUSERDATADIR echo $VSCODEEXTDIR -echo starting admin tool extension windows tests +echo *************************************************** +echo *** starting admin tool extension windows tests *** +echo *************************************************** ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/admin-tool-ext-win --extensionTestsPath=$ROOT/extensions/admin-tool-ext-win/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR -echo starting agent tests + +echo **************************** +echo *** starting agent tests *** +echo **************************** ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/agent --extensionTestsPath=$ROOT/extensions/agent/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR -echo starting azurecore tests + +echo ******************************** +echo *** starting azurecore tests *** +echo ******************************** ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/azurecore --extensionTestsPath=$ROOT/extensions/azurecore/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR -echo starting cms tests + +echo ************************** +echo *** starting cms tests *** +echo ************************** ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/cms --extensionTestsPath=$ROOT/extensions/cms/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR -echo starting dacpac tests -./scripts/code.sh --extensionDevelopmentPath=$ROOT/extensions/dacpac --extensionTestsPath=$ROOT/extensions/dacpac/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR -echo starting schema compare tests + +echo ***************************** +echo *** starting dacpac tests *** +echo ***************************** +./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/dacpac --extensionTestsPath=$ROOT/extensions/dacpac/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR + +echo ************************************* +echo *** starting schema compare tests *** +echo ************************************* ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/schema-compare --extensionTestsPath=$ROOT/extensions/schema-compare/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR -echo starting notebook tests + +echo ******************************* +echo *** starting notebook tests *** +echo ******************************* ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/notebook --extensionTestsPath=$ROOT/extensions/notebook/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR -echo starting resource deployment tests + +echo ****************************************** +echo *** starting resource deployment tests *** +echo ****************************************** ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/resource-deployment --extensionTestsPath=$ROOT/extensions/resource-deployment/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR rm -r $VSCODEUSERDATADIR diff --git a/scripts/test-unstable.bat b/scripts/test-unstable.bat new file mode 100644 index 0000000000..dbd64310cf --- /dev/null +++ b/scripts/test-unstable.bat @@ -0,0 +1,10 @@ +setlocal + +set ADS_TEST_GREP=@UNSTABLE@ +set ADS_TEST_INVERT_GREP= + +echo Running UNSTABLE ADS Core Tests + +call %~dp0\test.bat + +endlocal diff --git a/scripts/test-unstable.sh b/scripts/test-unstable.sh new file mode 100644 index 0000000000..2ecdf403c6 --- /dev/null +++ b/scripts/test-unstable.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +if [[ "$OSTYPE" == "darwin"* ]]; then + realpath() { [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"; } + ROOT=$(dirname $(dirname $(realpath "$0"))) +else + ROOT=$(dirname $(dirname $(readlink -f $0))) +fi + +cd $ROOT + +export ADS_TEST_GREP=@UNSTABLE@ +export ADS_TEST_INVERT_GREP= + +echo Running UNSTABLE ADS Core Tests + +./scripts/test.sh diff --git a/scripts/test.bat b/scripts/test.bat index 941b21d156..8cebfec3aa 100644 --- a/scripts/test.bat +++ b/scripts/test.bat @@ -15,9 +15,24 @@ set CODE=".build\electron\%NAMESHORT%" node build\lib\electron.js if %errorlevel% neq 0 node .\node_modules\gulp\bin\gulp.js electron +:: Default to only running stable tests if test grep isn't set +if "%ADS_TEST_GREP%" == "" ( + echo Running stable tests only + set ADS_TEST_GREP=@UNSTABLE@ + set ADS_TEST_INVERT_GREP=1 +) + +set CODE_ARGS=--grep %ADS_TEST_GREP% + +if "%ADS_TEST_INVERT_GREP%" == "1" ( + set CODE_ARGS=%CODE_ARGS% --invert +) else if "%ADS_TEST_INVERT_GREP%" == "true" ( + set CODE_ARGS=%CODE_ARGS% --invert +) + :: Run tests set ELECTRON_ENABLE_LOGGING=1 -%CODE% .\test\electron\index.js %* +%CODE% .\test\electron\index.js %CODE_ARGS% %* popd diff --git a/scripts/test.sh b/scripts/test.sh index 9b6a3b86b8..3ff498453f 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -18,6 +18,19 @@ else CODE=".build/electron/$NAME" fi +# Default to only running stable tests if test grep isn't set +if [[ "$ADS_TEST_GREP" == "" ]]; then + echo Running stable tests only + export ADS_TEST_GREP=@UNSTABLE@ + export ADS_TEST_INVERT_GREP=1 +fi + +CODE_ARGS=--grep %ADS_TEST_GREP% + +if [[ "$ADS_TEST_INVERT_GREP" == "1" ]] || [[ "$ADS_TEST_INVERT_GREP" == "true" ]]; then + set CODE_ARGS=$CODE_ARGS --invert +fi + # Node modules test -d node_modules || yarn @@ -29,10 +42,10 @@ if [[ "$OSTYPE" == "darwin"* ]] || [[ "$AGENT_OS" == "Darwin"* ]]; then cd $ROOT ; ulimit -n 4096 ; \ ELECTRON_ENABLE_LOGGING=1 \ "$CODE" \ - test/electron/index.js "$@" + test/electron/index.js $CODE_ARGS "$@" else cd $ROOT ; \ ELECTRON_ENABLE_LOGGING=1 \ "$CODE" \ - test/electron/index.js "$@" + test/electron/index.js $CODE_ARGS "$@" fi diff --git a/test/electron/index.js b/test/electron/index.js index 9e5e46c278..c5dc1a393f 100644 --- a/test/electron/index.js +++ b/test/electron/index.js @@ -16,6 +16,7 @@ const defaultReporterName = process.platform === 'win32' ? 'list' : 'spec'; const optimist = require('optimist') .describe('grep', 'only run tests matching ').alias('grep', 'g').alias('grep', 'f').string('grep') + .describe('invert', 'uses the inverse of the match specified by grep').alias('invert', 'i').string('invert') .describe('run', 'only run tests from ').string('run') .describe('runGlob', 'only run tests matching ').alias('runGlob', 'runGrep').string('runGlob') .describe('build', 'run with build output (out-build)').boolean('build') diff --git a/test/electron/renderer.js b/test/electron/renderer.js index 1e72144e6e..be19ac5a2e 100644 --- a/test/electron/renderer.js +++ b/test/electron/renderer.js @@ -206,7 +206,10 @@ function runTests(opts) { return loadTests(opts).then(() => { if (opts.grep) { - mocha.grep(opts.grep); + mocha.grep(new RegExp(opts.grep)); + if (opts.invert) { + mocha.invert(); + } } if (!opts.debug) {