diff --git a/.changeset/pr-122.md b/.changeset/pr-122.md new file mode 100644 index 0000000..9d6aad0 --- /dev/null +++ b/.changeset/pr-122.md @@ -0,0 +1,5 @@ +--- +"@wdio/browserstack-service": minor +--- + +- Tests that never run because Mocha's `bail` halted a spec are now reported as `skipped`, instead of being left out of the report entirely — so a Test Run accounts for every test in that spec. diff --git a/packages/browserstack-service/README.md b/packages/browserstack-service/README.md index cd0127d..17f3216 100644 --- a/packages/browserstack-service/README.md +++ b/packages/browserstack-service/README.md @@ -76,6 +76,14 @@ export const config = { You can explore all the features of Test Reporting and Analytics in [this sandbox](https://automation.browserstack.com/) or read more about it [here](https://www.browserstack.com/docs/test-reporting-and-analytics/overview/what-is-test-observability). +#### Reporting of tests skipped by `bail` + +With `mochaOpts: { bail: true }`, Mocha halts a spec on its first failure and the remaining tests in that file never run. Those tests are now reported with the status `skipped`, so the report accounts for every test in the spec rather than silently omitting them. Tests in sibling `describe` blocks in the same file are covered too. + +Requires `mocha` as the framework and Test Reporting enabled. + +Note that WebdriverIO's own top-level `bail` option is a different setting: it counts failed *spec files* and stops scheduling further ones, rather than stopping tests within a spec. Tests in spec files that never start are not currently reported. + ### browserstackLocal Set this to true to enable routing connections from BrowserStack cloud through your computer. diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index b9e2bbe..04156b4 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -8,6 +8,7 @@ import { isBrowserstackSession, patchConsoleLogs, isTrue, + isFalse, getUniqueIdentifier, getHookType } from './util.js' @@ -87,6 +88,13 @@ export default class BrowserstackService implements Services.ServiceInstance { private _percyCaptureMode: string | undefined = undefined private _percyHandler?: PercyHandler private _turboScale + /** + * Only mocha's own bail drops the rest of a spec. wdio's top-level `bail` is a launcher + * spec-scheduling filter (it stops queueing further specs once N runners have failed) and + * never reaches mocha, so under it every test in the current spec still runs — enumerating + * on it would report tests as skipped that are about to execute. + */ + private _mochaBail: boolean = false constructor ( options: BrowserstackConfig & Options.Testrunner, @@ -104,6 +112,10 @@ export default class BrowserstackService implements Services.ServiceInstance { this._percy = isTrue(process.env.BROWSERSTACK_PERCY) this._percyCaptureMode = process.env.BROWSERSTACK_PERCY_CAPTURE_MODE this._turboScale = this._options.turboScale + // mirror mocha's own truthiness check on the option rather than isTrue(), which is a + // strict 'true' string compare and would miss a numeric `bail: 1` + const bailOpt: unknown = this._config?.mochaOpts?.bail + this._mochaBail = this._config?.framework === 'mocha' && Boolean(bailOpt) && !isFalse(bailOpt) PerformanceTester.startMonitoring('performance-report-service.csv') if (shouldProcessEventForTesthub('')) { @@ -542,6 +554,7 @@ export default class BrowserstackService implements Services.ServiceInstance { } await BrowserstackCLI.getInstance().getTestFramework()!.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, { test, result: results }) await BrowserstackCLI.getInstance().getTestFramework()!.trackEvent(TestFrameworkState.TEST, HookState.POST, { test, result: results, suiteTitle: this._suiteTitle }) + await this.reportBailSkippedTests(test, results) return } @@ -550,6 +563,35 @@ export default class BrowserstackService implements Services.ServiceInstance { await this._percyHandler?.afterTest() } + /** + * mocha's `bail` aborts the run on the first failure, so every test the spec had not reached + * yet is dropped without emitting any event and never appears on the dashboard. Report them + * as skipped — same cascade the failed-hook path uses, from the spec's root suite so sibling + * describes are covered too (bail kills the whole spec, not just the failing describe). + */ + private async reportBailSkippedTests(test: Frameworks.Test, results: Frameworks.TestResult) { + if (!this._mochaBail || results.passed || results.skipped) { + return + } + // a retry is still queued — mocha has not dropped anything yet + if (results.retries && results.retries.attempts < results.retries.limit) { + return + } + try { + const framework = BrowserstackCLI.getInstance().getTestFramework() + let suite = test.ctx?.test?.parent + if (!framework || !suite) { + return + } + while (suite.parent) { + suite = suite.parent + } + await reportSuiteSkipped(framework, suite) + } catch (err) { + BStackLogger.debug(`Failed reporting bail-skipped tests: ${err}`) + } + } + @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'after' }) async after (result: number) { try { diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index 95a2e32..5e6e1d4 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -2427,3 +2427,85 @@ describe('_isAppAutomate honors skipAppOverride', () => { expect(svc._isAppAutomate()).toBe(false) }) }) + +describe('afterTest bail skip cascade (SDK-7063)', () => { + let getInstanceSpy: ReturnType + + // reportSkippedTest de-dupes on `${parent} - ${title}` in a module-scope Set that outlives + // each test, so every case here needs its own titles. + const buildTree = (tag: string) => { + const root: any = { title: '', tests: [], suites: [], parent: undefined } + const suiteA: any = { title: `${tag} Suite A`, tests: [], suites: [], parent: root } + const suiteB: any = { title: `${tag} Suite B`, tests: [], suites: [], parent: root } + root.suites.push(suiteA, suiteB) + + const ran: any = { title: `${tag} A1`, state: 'passed', parent: suiteA, file: '/spec/a.js' } + const failing: any = { title: `${tag} A2`, state: 'failed', parent: suiteA, file: '/spec/a.js' } + const dropped: any = { title: `${tag} A3`, parent: suiteA, file: '/spec/a.js' } + suiteA.tests.push(ran, failing, dropped) + // sibling top-level describe — only reachable because the cascade walks up to root + suiteB.tests.push({ title: `${tag} B1`, parent: suiteB, file: '/spec/a.js' }) + + failing.ctx = { test: { parent: suiteA } } + return { failing, root } + } + + const makeService = (config: Record) => new BrowserstackService( + { testObservability: false } as any, + [] as any, + { user: 'foo', key: 'bar', ...config } as any + ) + + const runAfterTest = async (svc: BrowserstackService, failing: any, results: Record) => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => ({ trackEvent }) + } as any) + await svc.afterTest(failing, undefined as never, results as any) + return trackEvent + } + + afterEach(() => { + getInstanceSpy?.mockRestore() + }) + + it('reports un-run tests across sibling describes when mocha bail is on', async () => { + const { failing } = buildTree('bail1') + const svc = makeService({ framework: 'mocha', mochaOpts: { bail: true } }) + const trackEvent = await runAfterTest(svc, failing, { passed: false }) + + // 2 events close the failing test (LOG_REPORT/POST + TEST/POST), then 4 per skipped test. + // A3 (same describe) and B1 (SIBLING describe) => 2 skipped => 8. + expect(trackEvent).toHaveBeenCalledTimes(2 + 8) + }) + + it('does not cascade when only wdio-level bail is set', async () => { + // wdio's `bail` never halts a spec, so those tests still run — reporting them + // as skipped here would double-report them. + const { failing } = buildTree('bail2') + const svc = makeService({ framework: 'mocha', bail: 1 }) + const trackEvent = await runAfterTest(svc, failing, { passed: false }) + + expect(trackEvent).toHaveBeenCalledTimes(2) + }) + + it('does not cascade while a retry is still queued', async () => { + const { failing } = buildTree('bail3') + const svc = makeService({ framework: 'mocha', mochaOpts: { bail: true } }) + const trackEvent = await runAfterTest(svc, failing, { + passed: false, + retries: { attempts: 0, limit: 2 } + }) + + expect(trackEvent).toHaveBeenCalledTimes(2) + }) + + it('does not cascade when the test passed', async () => { + const { failing } = buildTree('bail4') + const svc = makeService({ framework: 'mocha', mochaOpts: { bail: true } }) + const trackEvent = await runAfterTest(svc, failing, { passed: true }) + + expect(trackEvent).toHaveBeenCalledTimes(2) + }) +})