Add unstable test params for core and extension tests (#7513)

* Add unstable test params for core and extension tests

* unset invert options for unstable script runs

* Fix copypasta

* Add nogpu to dacpac tests

extensions/admin-tool-ext-win/src/test/index.ts

@@ -6,7 +6,7 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Database Admin Tool Extensions for Windows';
+const suite = 'admin-tool-ext-win Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
@@ -14,6 +14,24 @@ const options: any = {
 	timeout: 600000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {

extensions/agent/src/test/index.ts

@@ -6,7 +6,7 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Agent Tests';
+const suite = 'agent Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
@@ -14,6 +14,24 @@ const options: any = {
 	timeout: 600000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {

extensions/azurecore/src/test/index.ts

@@ -6,7 +6,7 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Integration Azure Tests';
+const suite = 'azurecore Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
@@ -14,6 +14,24 @@ const options: any = {
 	timeout: 60000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {

extensions/cms/src/test/index.ts

@@ -6,7 +6,7 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'CMS Unit Tests';
+const suite = 'cms Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
@@ -14,6 +14,24 @@ const options: any = {
 	timeout: 60000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {

extensions/dacpac/src/test/index.ts

@@ -6,7 +6,7 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'DacFx Tests';
+const suite = 'dacpac Extension Tests';
 
 const testOptions: any = {
 	ui: 'bdd',
@@ -14,9 +14,23 @@ const testOptions: any = {
 	timeout: 60000
 };
 
-// const coverageConfig: any = {
-// 	coverConfig: '../../coverageConfig.json'
-// };
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	testOptions.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${testOptions.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	testOptions.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${testOptions.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	testOptions.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${testOptions.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	testOptions.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${testOptions.retries}`);
+}
 
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	testOptions.reporter = 'mocha-multi-reporters';

extensions/notebook/src/integrationTest/index.ts

@@ -6,7 +6,7 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Notebook Extension Integration Tests';
+const suite = 'notebook Extension Integration Tests';
 
 const options: any = {
 	ui: 'bdd',
@@ -14,6 +14,24 @@ const options: any = {
 	timeout: 600000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {

extensions/notebook/src/test/book/book.test.ts

@@ -34,7 +34,7 @@ export function equalBookItems(book: BookTreeItem, expectedBook: ExpectedBookIte
 	}
 }
 
-describe.skip('BookTreeViewProviderTests', function() {
+describe('BookTreeViewProviderTests @UNSTABLE@', function() {
 
 	describe('BookTreeViewProvider.getChildren', function (): void {
 		let rootFolderPath: string;

extensions/notebook/src/test/index.ts

@@ -6,7 +6,7 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Notebook Tests';
+const suite = 'notebook Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
@@ -14,6 +14,24 @@ const options: any = {
 	timeout: 600000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {

extensions/resource-deployment/src/test/index.ts

@@ -6,7 +6,7 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Resource Deployment Unit Tests';
+const suite = 'resource-deployment Extension Tests';
 
 const testOptions: any = {
 	ui: 'tdd',
@@ -14,6 +14,24 @@ const testOptions: any = {
 	timeout: 60000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	testOptions.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${testOptions.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	testOptions.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${testOptions.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	testOptions.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${testOptions.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	testOptions.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${testOptions.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	testOptions.reporter = 'mocha-multi-reporters';
 	testOptions.reporterOptions = {

extensions/schema-compare/src/test/index.ts

@@ -6,7 +6,7 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Schema Compare Tests';
+const suite = 'schema-compare Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
@@ -14,6 +14,24 @@ const options: any = {
 	timeout: 60000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {

scripts/sql-test-integration-unstable.bat

@@ -1,8 +1,9 @@
 setlocal
 
 set ADS_TEST_GREP=@UNSTABLE@
+set ADS_TEST_INVERT_GREP=
 
-echo Running unstable tests
+echo Running UNSTABLE ADS Extension Integration tests
 
 call %~dp0\sql-test-integration.bat
 

scripts/sql-test-integration-unstable.sh

@@ -11,7 +11,8 @@ fi
 cd $ROOT
 
 export ADS_TEST_GREP=@UNSTABLE@
+export ADS_TEST_INVERT_GREP=
 
-echo Running unstable tests
+echo Running UNSTABLE ADS Extension Integration tests
 
 ./scripts/sql-test-integration.sh

scripts/sql-test-integration.bat

@@ -7,6 +7,7 @@ set VSCODEEXTENSIONSDIR=%TMP%\adsext-%RANDOM%-%TIME:~6,5%
 echo VSCODEUSERDATADIR=%VSCODEUSERDATADIR%
 echo VSCODEEXTENSIONSDIR=%VSCODEEXTENSIONSDIR%
 
+:: Default to only running stable tests if test grep isn't set
 if "%ADS_TEST_GREP%" == "" (
 	echo Running stable tests only
 	set ADS_TEST_GREP=@UNSTABLE@

scripts/sql-test-integration.sh

@@ -12,6 +12,7 @@ else
 	VSCODEEXTDIR=`mktemp -d 2>/dev/null`
 fi
 
+# Default to only running stable tests if test grep isn't set
 if [[ "$ADS_TEST_GREP" == "" ]]; then
 	echo Running stable tests only
 	export ADS_TEST_GREP=@UNSTABLE@

scripts/test-extensions-unstable.bat

@@ -0,0 +1,12 @@
+:: Runs UNSTABLE Extension Tests
+
+setlocal
+
+set ADS_TEST_GREP=@UNSTABLE@
+set ADS_TEST_INVERT_GREP=
+
+echo Running UNSTABLE Extension Tests
+
+call %~dp0\test-extensions.bat
+
+endlocal

scripts/test-extensions-unstable.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+	realpath() { [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"; }
+	ROOT=$(dirname $(dirname $(realpath "$0")))
+else
+	ROOT=$(dirname $(dirname $(readlink -f $0)))
+fi
+
+cd $ROOT
+
+export ADS_TEST_GREP=@UNSTABLE@
+export ADS_TEST_INVERT_GREP=
+echo Running UNSTABLE Extension Tests
+
+./scripts/test-extensions.sh

scripts/test-extensions.bat

@@ -1,4 +1,4 @@
-:: Runs unit tests for Extensions
+:: Runs Extension tests
 
 setlocal
 
@@ -8,23 +8,54 @@ set VSCODEUSERDATADIR=%TMP%\adsuser-%RANDOM%-%TIME:~6,5%
 set VSCODEEXTENSIONSDIR=%TMP%\adsext-%RANDOM%-%TIME:~6,5%
 echo %VSCODEUSERDATADIR%
 echo %VSCODEEXTENSIONSDIR%
+
+:: Default to only running stable tests if test grep isn't set
+if "%ADS_TEST_GREP%" == "" (
+	echo Running stable tests only
+	set ADS_TEST_GREP=@UNSTABLE@
+	SET ADS_TEST_INVERT_GREP=1
+)
+
 @echo OFF
 
-echo starting admin tool extension windows tests
+echo ***************************************************
+echo *** starting admin tool extension windows tests ***
+echo ***************************************************
 call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\admin-tool-ext-win --extensionTestsPath=%~dp0\..\extensions\admin-tool-ext-win\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --disableExtensions --remote-debugging-port=9222
-echo starting agent tests
+
+echo ****************************
+echo *** starting agent tests ***
+echo ****************************
 call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\agent --extensionTestsPath=%~dp0\..\extensions\agent\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222
-echo starting azurecore tests
+
+echo ********************************
+echo *** starting azurecore tests ***
+echo ********************************
 call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\azurecore --extensionTestsPath=%~dp0\..\extensions\azurecore\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222
-echo starting cms tests
+
+echo **************************
+echo *** starting cms tests ***
+echo **************************
 call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\cms --extensionTestsPath=%~dp0\..\extensions\cms\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222
-echo starting dacpac tests
-call .\scripts\code.bat --extensionDevelopmentPath=%~dp0\..\extensions\dacpac --extensionTestsPath=%~dp0\..\extensions\dacpac\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222
-echo starting schema compare tests
+
+echo *****************************
+echo *** starting dacpac tests ***
+echo *****************************
+call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\dacpac --extensionTestsPath=%~dp0\..\extensions\dacpac\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222
+
+echo *************************************
+echo *** starting schema compare tests ***
+echo *************************************
 call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\schema-compare --extensionTestsPath=%~dp0\..\extensions\schema-compare\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222
-echo starting notebook tests
+
+echo *******************************
+echo *** starting notebook tests ***
+echo *******************************
 call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\notebook --extensionTestsPath=%~dp0\..\extensions\notebook\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222
-echo starting resource deployment tests
+
+echo ******************************************
+echo *** starting resource deployment tests ***
+echo ******************************************
 call .\scripts\code.bat --nogpu --extensionDevelopmentPath=%~dp0\..\extensions\resource-deployment --extensionTestsPath=%~dp0\..\extensions\resource-deployment\out\test --user-data-dir=%VSCODEUSERDATADIR% --extensions-dir=%VSCODEEXTENSIONSDIR% --remote-debugging-port=9222
 
 if %errorlevel% neq 0 exit /b %errorlevel%

scripts/test-extensions.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Runs unit tests for Extensions
+# Runs Extension Tests
 set -e
 
 if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -14,25 +14,55 @@ else
 	VSCODEEXTDIR=`mktemp -d 2>/dev/null`
 fi
 
+# Default to only running stable tests if test grep isn't set
+if [[ "$ADS_TEST_GREP" == "" ]]; then
+	echo Running stable tests only
+	export ADS_TEST_GREP=@UNSTABLE@
+	export ADS_TEST_INVERT_GREP=1
+fi
+
 cd $ROOT
 echo $VSCODEUSERDATADIR
 echo $VSCODEEXTDIR
 
-echo starting admin tool extension windows tests
+echo ***************************************************
+echo *** starting admin tool extension windows tests ***
+echo ***************************************************
 ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/admin-tool-ext-win --extensionTestsPath=$ROOT/extensions/admin-tool-ext-win/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR
-echo starting agent tests
+
+echo ****************************
+echo *** starting agent tests ***
+echo ****************************
 ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/agent --extensionTestsPath=$ROOT/extensions/agent/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR
-echo starting azurecore tests
+
+echo ********************************
+echo *** starting azurecore tests ***
+echo ********************************
 ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/azurecore --extensionTestsPath=$ROOT/extensions/azurecore/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR
-echo starting cms tests
+
+echo **************************
+echo *** starting cms tests ***
+echo **************************
 ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/cms --extensionTestsPath=$ROOT/extensions/cms/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR
-echo starting dacpac tests
-./scripts/code.sh --extensionDevelopmentPath=$ROOT/extensions/dacpac --extensionTestsPath=$ROOT/extensions/dacpac/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR
-echo starting schema compare tests
+
+echo *****************************
+echo *** starting dacpac tests ***
+echo *****************************
+./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/dacpac --extensionTestsPath=$ROOT/extensions/dacpac/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR
+
+echo *************************************
+echo *** starting schema compare tests ***
+echo *************************************
 ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/schema-compare --extensionTestsPath=$ROOT/extensions/schema-compare/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR
-echo starting notebook tests
+
+echo *******************************
+echo *** starting notebook tests ***
+echo *******************************
 ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/notebook --extensionTestsPath=$ROOT/extensions/notebook/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR
-echo starting resource deployment tests
+
+echo ******************************************
+echo *** starting resource deployment tests ***
+echo ******************************************
 ./scripts/code.sh --nogpu --extensionDevelopmentPath=$ROOT/extensions/resource-deployment --extensionTestsPath=$ROOT/extensions/resource-deployment/out/test --user-data-dir=$VSCODEUSERDATADIR --extensions-dir=$VSCODEEXTDIR
 
 rm -r $VSCODEUSERDATADIR

scripts/test-unstable.bat

@@ -0,0 +1,10 @@
+setlocal
+
+set ADS_TEST_GREP=@UNSTABLE@
+set ADS_TEST_INVERT_GREP=
+
+echo Running UNSTABLE ADS Core Tests
+
+call %~dp0\test.bat
+
+endlocal

scripts/test-unstable.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+	realpath() { [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"; }
+	ROOT=$(dirname $(dirname $(realpath "$0")))
+else
+	ROOT=$(dirname $(dirname $(readlink -f $0)))
+fi
+
+cd $ROOT
+
+export ADS_TEST_GREP=@UNSTABLE@
+export ADS_TEST_INVERT_GREP=
+
+echo Running UNSTABLE ADS Core Tests
+
+./scripts/test.sh

scripts/test.bat

@@ -15,9 +15,24 @@ set CODE=".build\electron\%NAMESHORT%"
 node build\lib\electron.js
 if %errorlevel% neq 0 node .\node_modules\gulp\bin\gulp.js electron
 
+:: Default to only running stable tests if test grep isn't set
+if "%ADS_TEST_GREP%" == "" (
+	echo Running stable tests only
+	set ADS_TEST_GREP=@UNSTABLE@
+	set ADS_TEST_INVERT_GREP=1
+)
+
+set CODE_ARGS=--grep %ADS_TEST_GREP%
+
+if "%ADS_TEST_INVERT_GREP%" == "1" (
+	set CODE_ARGS=%CODE_ARGS% --invert
+) else if "%ADS_TEST_INVERT_GREP%" == "true" (
+	set CODE_ARGS=%CODE_ARGS% --invert
+)
+
 :: Run tests
 set ELECTRON_ENABLE_LOGGING=1
-%CODE% .\test\electron\index.js %*
+%CODE% .\test\electron\index.js %CODE_ARGS% %*
 
 popd
 

scripts/test.sh

@@ -18,6 +18,19 @@ else
 	CODE=".build/electron/$NAME"
 fi
 
+# Default to only running stable tests if test grep isn't set
+if [[ "$ADS_TEST_GREP" == "" ]]; then
+	echo Running stable tests only
+	export ADS_TEST_GREP=@UNSTABLE@
+	export ADS_TEST_INVERT_GREP=1
+fi
+
+CODE_ARGS=--grep %ADS_TEST_GREP%
+
+if [[ "$ADS_TEST_INVERT_GREP" == "1" ]] || [[ "$ADS_TEST_INVERT_GREP" == "true" ]]; then
+	set CODE_ARGS=$CODE_ARGS --invert
+fi
+
 # Node modules
 test -d node_modules || yarn
 
@@ -29,10 +42,10 @@ if [[ "$OSTYPE" == "darwin"* ]] || [[ "$AGENT_OS" == "Darwin"* ]]; then
 	cd $ROOT ; ulimit -n 4096 ; \
 		ELECTRON_ENABLE_LOGGING=1 \
 		"$CODE" \
-		test/electron/index.js "$@"
+		test/electron/index.js $CODE_ARGS "$@"
 else
 	cd $ROOT ; \
 		ELECTRON_ENABLE_LOGGING=1 \
 		"$CODE" \
-		test/electron/index.js "$@"
+		test/electron/index.js $CODE_ARGS "$@"
 fi

test/electron/index.js

@@ -16,6 +16,7 @@ const defaultReporterName = process.platform === 'win32' ? 'list' : 'spec';
 
 const optimist = require('optimist')
 	.describe('grep', 'only run tests matching <pattern>').alias('grep', 'g').alias('grep', 'f').string('grep')
+	.describe('invert', 'uses the inverse of the match specified by grep').alias('invert', 'i').string('invert')
 	.describe('run', 'only run tests from <file>').string('run')
 	.describe('runGlob', 'only run tests matching <file_pattern>').alias('runGlob', 'runGrep').string('runGlob')
 	.describe('build', 'run with build output (out-build)').boolean('build')

test/electron/renderer.js

@@ -206,7 +206,10 @@ function runTests(opts) {
 	return loadTests(opts).then(() => {
 
 		if (opts.grep) {
-			mocha.grep(opts.grep);
+			mocha.grep(new RegExp(opts.grep));
+			if (opts.invert) {
+				mocha.invert();
+			}
 		}
 
 		if (!opts.debug) {