diff --git a/test/smoke/README.md b/test/smoke/README.md index cb97d25ce4..08bbdc6883 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -14,6 +14,9 @@ yarn --cwd test/smoke # Prepare OSS in repo* node build/lib/preLaunch.js +# By default, only the stable test cases will be executed, to run all the test cases run the following script to set the 'RUN_UNSTABLE_TESTS' environment variable to true +# export RUN_UNSTABLE_TESTS="true" + # Dev (Electron) yarn smoketest @@ -72,6 +75,12 @@ cd test/smoke yarn watch ``` +### Mark test case as unstable + +A test case should be marked as unstable if it is failing due to test reliability issue and a fix cannot be made immediately. Unstable test cases won't be picked up in the normal build pipelines, there is a dedicated build pipeline to run the unstable test cases. + +To mark a test case as unstable, simply append the tag "@UNSTABLE@" to the title of the test case. Similarly, remove the tag if you want to mark it as stable. + ## Troubleshooting ### Error: Could not get a unique tmp filename, max tries reached diff --git a/test/smoke/src/sql/areas/notebook/addRemoteBook.test.ts b/test/smoke/src/sql/areas/notebook/addRemoteBook.test.ts index 59510094c6..4e41702001 100644 --- a/test/smoke/src/sql/areas/notebook/addRemoteBook.test.ts +++ b/test/smoke/src/sql/areas/notebook/addRemoteBook.test.ts @@ -19,7 +19,7 @@ export function setup(opts: minimist.ParsedArgs) { beforeSuite(opts); afterSuite(opts); - it.skip('can open remote book', async function () { // Skip until the rate limit issue can be fixed + it('can open remote book @UNSTABLE@', async function () { // Skip until the rate limit issue can be fixed const app = this.app as Application; await app.workbench.quickaccess.runCommand(AddRemoteBookCommand); await app.workbench.addRemoteBookDialog.setLocation('GitHub'); diff --git a/test/smoke/src/sql/areas/notebook/createBook.test.ts b/test/smoke/src/sql/areas/notebook/createBook.test.ts index f357f38659..802d92d4ff 100644 --- a/test/smoke/src/sql/areas/notebook/createBook.test.ts +++ b/test/smoke/src/sql/areas/notebook/createBook.test.ts @@ -16,7 +16,7 @@ const CreateBookCommand = 'Jupyter Books: Create Jupyter Book'; const bookName = 'my-book'; export function setup(opts: minimist.ParsedArgs) { - describe.skip('CreateBookDialog', () => { + describe('CreateBookDialog @UNSTABLE@', () => { beforeSuite(opts); afterSuite(opts); diff --git a/test/smoke/src/sql/areas/notebook/notebook.test.ts b/test/smoke/src/sql/areas/notebook/notebook.test.ts index 28db5c3913..3989e20649 100644 --- a/test/smoke/src/sql/areas/notebook/notebook.test.ts +++ b/test/smoke/src/sql/areas/notebook/notebook.test.ts @@ -115,7 +115,7 @@ export function setup(opts: minimist.ParsedArgs) { await app.workbench.sqlNotebook.waitForActiveCellResults(); }); - it('can add and remove new package from the Manage Packages wizard', async function () { + it('can add and remove new package from the Manage Packages wizard @UNSTABLE@', async function () { // Use arrow package so that it's at the top of the packages list when uninstalling later const testPackageName = 'arrow'; @@ -195,7 +195,7 @@ export function setup(opts: minimist.ParsedArgs) { }); describe('Notebook keyboard navigation', async () => { - it.skip('can enter and exit edit mode and navigate using keyboard nav', async function () { + it('can enter and exit edit mode and navigate using keyboard nav @UNSTABLE@', async function () { const app = this.app as Application; await app.workbench.sqlNotebook.newUntitledNotebook(); await app.workbench.sqlNotebook.addCellFromPlaceholder('Code'); // add new code cell @@ -273,7 +273,7 @@ export function setup(opts: minimist.ParsedArgs) { }); }); - describe.skip('Cell Toolbar Actions', function () { + describe('Cell Toolbar Actions @UNSTABLE@', function () { async function verifyCellToolbarBehavior(app: Application, toolbarAction: () => Promise, selector: string, checkIfGone: boolean = false): Promise { // Run the test for each of the default text editor modes for (let editMode of ['Markdown', 'Split View']) { @@ -506,7 +506,7 @@ export function setup(opts: minimist.ParsedArgs) { await app.workbench.sqlNotebook.waitForActiveCellEditorContents(s => s.includes('- **_Markdown Test_**')); }); - it('can save and reopen WYSIWYG notebook', async function () { + it('can save and reopen WYSIWYG notebook @UNSTABLE@', async function () { const app = this.app as Application; const filename = 'emptyNotebook.ipynb'; await app.workbench.sqlNotebook.openFile(filename); diff --git a/test/smoke/test/index.js b/test/smoke/test/index.js index 8bbc07303e..346d0c1371 100644 --- a/test/smoke/test/index.js +++ b/test/smoke/test/index.js @@ -34,5 +34,21 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { } const mocha = new Mocha(options); + +// {{SQL CARBON EDIT}} - If grep option is specified, only run the matching test cases (local test case development/debug scenario), +// otherwise the value of 'RUN_UNSTABLE_TESTS' environment variable will be used to determine whether to run the stable test cases or the whole test suite. +// Unstable test cases have "@UNSTABLE@" in their full name (test suite name + test name). +if (!options.grep) { + if (process.env.RUN_UNSTABLE_TESTS === 'true') { + console.info('running all test cases.'); + } else { + console.info('running stable test cases.'); + mocha.grep('@UNSTABLE@').invert(); + } +} else { + console.info('running test cases match the grep option.'); +} +// {{SQL CARBON EDIT}} - end of edit. + mocha.addFile('out/main.js'); mocha.run(failures => process.exit(failures ? -1 : 0));