introduce unstable smoke test suite (#20639)

* allow mark tests as unstable

* fix

* fix invert

* stable and whole test suite

* docs and mark test cases as unstable

* a couple more unstable test cases

test/smoke/README.md

@@ -14,6 +14,9 @@ yarn --cwd test/smoke
 # Prepare OSS in repo*
 node build/lib/preLaunch.js
 
+# By default, only the stable test cases will be executed, to run all the test cases run the following script to set the 'RUN_UNSTABLE_TESTS' environment variable to true
+# export RUN_UNSTABLE_TESTS="true"
+
 # Dev (Electron)
 yarn smoketest
 
@@ -72,6 +75,12 @@ cd test/smoke
 yarn watch
 ```
 
+### Mark test case as unstable
+
+A test case should be marked as unstable if it is failing due to test reliability issue and a fix cannot be made immediately. Unstable test cases won't be picked up in the normal build pipelines, there is a dedicated build pipeline to run the unstable test cases.
+
+To mark a test case as unstable, simply append the tag "@UNSTABLE@" to the title of the test case. Similarly, remove the tag if you want to mark it as stable.
+
 ## Troubleshooting
 
 ### Error: Could not get a unique tmp filename, max tries reached

test/smoke/src/sql/areas/notebook/addRemoteBook.test.ts

@@ -19,7 +19,7 @@ export function setup(opts: minimist.ParsedArgs) {
 		beforeSuite(opts);
 		afterSuite(opts);
 
-		it.skip('can open remote book', async function () { // Skip until the rate limit issue can be fixed
+		it('can open remote book @UNSTABLE@', async function () { // Skip until the rate limit issue can be fixed
 			const app = this.app as Application;
 			await app.workbench.quickaccess.runCommand(AddRemoteBookCommand);
 			await app.workbench.addRemoteBookDialog.setLocation('GitHub');

test/smoke/src/sql/areas/notebook/createBook.test.ts

@@ -16,7 +16,7 @@ const CreateBookCommand = 'Jupyter Books: Create Jupyter Book';
 const bookName = 'my-book';
 
 export function setup(opts: minimist.ParsedArgs) {
-	describe.skip('CreateBookDialog', () => {
+	describe('CreateBookDialog @UNSTABLE@', () => {
 		beforeSuite(opts);
 		afterSuite(opts);
 

test/smoke/src/sql/areas/notebook/notebook.test.ts

@@ -115,7 +115,7 @@ export function setup(opts: minimist.ParsedArgs) {
 				await app.workbench.sqlNotebook.waitForActiveCellResults();
 			});
 
-			it('can add and remove new package from the Manage Packages wizard', async function () {
+			it('can add and remove new package from the Manage Packages wizard @UNSTABLE@', async function () {
 				// Use arrow package so that it's at the top of the packages list when uninstalling later
 				const testPackageName = 'arrow';
 
@@ -195,7 +195,7 @@ export function setup(opts: minimist.ParsedArgs) {
 		});
 
 		describe('Notebook keyboard navigation', async () => {
-			it.skip('can enter and exit edit mode and navigate using keyboard nav', async function () {
+			it('can enter and exit edit mode and navigate using keyboard nav @UNSTABLE@', async function () {
 				const app = this.app as Application;
 				await app.workbench.sqlNotebook.newUntitledNotebook();
 				await app.workbench.sqlNotebook.addCellFromPlaceholder('Code'); // add new code cell
@@ -273,7 +273,7 @@ export function setup(opts: minimist.ParsedArgs) {
 			});
 		});
 
-		describe.skip('Cell Toolbar Actions', function () {
+		describe('Cell Toolbar Actions @UNSTABLE@', function () {
 			async function verifyCellToolbarBehavior(app: Application, toolbarAction: () => Promise<void>, selector: string, checkIfGone: boolean = false): Promise<void> {
 				// Run the test for each of the default text editor modes
 				for (let editMode of ['Markdown', 'Split View']) {
@@ -506,7 +506,7 @@ export function setup(opts: minimist.ParsedArgs) {
 				await app.workbench.sqlNotebook.waitForActiveCellEditorContents(s => s.includes('- **_<u><mark>Markdown Test</mark></u>_**'));
 			});
 
-			it('can save and reopen WYSIWYG notebook', async function () {
+			it('can save and reopen WYSIWYG notebook @UNSTABLE@', async function () {
 				const app = this.app as Application;
 				const filename = 'emptyNotebook.ipynb';
 				await app.workbench.sqlNotebook.openFile(filename);

test/smoke/test/index.js

@@ -34,5 +34,21 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 }
 
 const mocha = new Mocha(options);
+
+// {{SQL CARBON EDIT}} - If grep option is specified, only run the matching test cases (local test case development/debug scenario),
+// otherwise the value of 'RUN_UNSTABLE_TESTS' environment variable will be used to determine whether to run the stable test cases or the whole test suite.
+// Unstable test cases have "@UNSTABLE@" in their full name (test suite name + test name).
+if (!options.grep) {
+	if (process.env.RUN_UNSTABLE_TESTS === 'true') {
+		console.info('running all test cases.');
+	} else {
+		console.info('running stable test cases.');
+		mocha.grep('@UNSTABLE@').invert();
+	}
+} else {
+	console.info('running test cases match the grep option.');
+}
+// {{SQL CARBON EDIT}} - end of edit.
+
 mocha.addFile('out/main.js');
 mocha.run(failures => process.exit(failures ? -1 : 0));