/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the Source EULA. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import { Suite, Context } from 'mocha'; import { dirname, join } from 'path'; import { Application, ApplicationOptions, Logger } from '../../automation'; export function describeRepeat(n: number, description: string, callback: (this: Suite) => void): void { for (let i = 0; i < n; i++) { describe(`${description} (iteration ${i})`, callback); } } export function itRepeat(n: number, description: string, callback: (this: Context) => any): void { for (let i = 0; i < n; i++) { it(`${description} (iteration ${i})`, callback); } } /** * Defines a test-case that will run but will be skips it if it throws an exception. This is useful * to get some runs in CI when trying to stabilize a flaky test, without failing the build. Note * that this only works if something inside the test throws, so a test's overall timeout won't work * but throwing due to a polling timeout will. * @param title The test-case title. * @param callback The test-case callback. */ export function itSkipOnFail(title: string, callback: (this: Context) => any): void { it(title, function () { return Promise.resolve().then(() => { return callback.apply(this, arguments); }).catch(e => { console.warn(`Test "${title}" failed but was marked as skip on fail:`, e); this.skip(); }); }); } export function installAllHandlers(logger: Logger, optionsTransform?: (opts: ApplicationOptions) => ApplicationOptions) { installDiagnosticsHandler(logger); installAppBeforeHandler(optionsTransform); installAppAfterHandler(); } export function installDiagnosticsHandler(logger: Logger, appFn?: () => Application | undefined) { // Before each suite before(async function () { const suiteTitle = this.currentTest?.parent?.title; logger.log(''); logger.log(`>>> Suite start: '${suiteTitle ?? 'unknown'}' <<<`); logger.log(''); }); // Before each test beforeEach(async function () { const testTitle = this.currentTest?.title; logger.log(''); logger.log(`>>> Test start: '${testTitle ?? 'unknown'}' <<<`); logger.log(''); const app: Application = appFn?.() ?? this.app; await app?.startTracing(testTitle ?? 'unknown'); }); // After each test afterEach(async function () { const currentTest = this.currentTest; if (!currentTest) { return; } const failed = currentTest.state === 'failed'; const testTitle = currentTest.title; logger.log(''); if (failed) { logger.log(`>>> !!! FAILURE !!! Test end: '${testTitle}' !!! FAILURE !!! <<<`); } else { logger.log(`>>> Test end: '${testTitle}' <<<`); } logger.log(''); const app: Application = appFn?.() ?? this.app; await app?.stopTracing(testTitle.replace(/[^a-z0-9\-]/ig, '_'), failed); }); } let logsCounter = 1; export function suiteLogsPath(options: ApplicationOptions, suiteName: string): string { return join(dirname(options.logsPath), `${logsCounter++}_suite_${suiteName.replace(/[^a-z0-9\-]/ig, '_')}`); } function installAppBeforeHandler(optionsTransform?: (opts: ApplicationOptions) => ApplicationOptions) { before(async function () { const suiteName = this.test?.parent?.title ?? 'unknown'; this.app = createApp({ ...this.defaultOptions, logsPath: suiteLogsPath(this.defaultOptions, suiteName) }, optionsTransform); await this.app.start(); }); } export function installAppAfterHandler(appFn?: () => Application | undefined, joinFn?: () => Promise) { after(async function () { const app: Application = appFn?.() ?? this.app; if (app) { await app.stop(); } if (joinFn) { await joinFn(); } }); } export function createApp(options: ApplicationOptions, optionsTransform?: (opts: ApplicationOptions) => ApplicationOptions): Application { if (optionsTransform) { options = optionsTransform({ ...options }); } const app = new Application({ ...options, userDataDir: getRandomUserDataDir(options) }); return app; } export function getRandomUserDataDir(options: ApplicationOptions): string { // Pick a random user data dir suffix that is not // too long to not run into max path length issues // https://github.com/microsoft/vscode/issues/34988 const userDataPathSuffix = [...Array(8)].map(() => Math.random().toString(36)[3]).join(''); return options.userDataDir.concat(`-${userDataPathSuffix}`); } export function installCommonAfterHandlers(opts: minimist.ParsedArgs, appFn?: () => Application | undefined, joinFn?: () => Promise) { afterEach(async function () { const app: Application = appFn?.() ?? this.app; if (this.currentTest?.state === 'failed' && opts.screenshots) { const name = this.currentTest!.fullTitle().replace(/[^a-z0-9\-]/ig, '_'); try { await app.captureScreenshot(name); } catch (error) { // ignore } } }); after(async function () { const app = this.app as Application; if (app) { await app.stop(); } if (joinFn) { await joinFn(); } }); afterEach(async function () { await this.app?.stopTracing(this.currentTest?.title, this.currentTest?.state === 'failed'); }); } export function timeout(i: number) { return new Promise(resolve => { setTimeout(() => { resolve(); }, i); }); } export async function retryWithRestart(app: Application, testFn: () => Promise, retries = 3, timeoutMs = 20000): Promise { let lastError: Error | undefined = undefined; for (let i = 0; i < retries; i++) { const result = await Promise.race([ testFn().then(() => true, error => { lastError = error; return false; }), timeout(timeoutMs).then(() => false) ]); if (result) { return; } await app.restart(); } throw lastError ?? new Error('retryWithRestart failed with an unknown error'); } export interface ITask { (): T; } export async function retry(task: ITask>, delay: number, retries: number, onBeforeRetry?: () => Promise): Promise { let lastError: Error | undefined; for (let i = 0; i < retries; i++) { try { if (i > 0 && typeof onBeforeRetry === 'function') { try { await onBeforeRetry(); } catch (error) { console.warn(`onBeforeRetry failed with: ${error}`); } } return await task(); } catch (error) { lastError = error; await timeout(delay); } } throw lastError; }