From 91366db15f071762e4b0498a08a44e62cf90fc76 Mon Sep 17 00:00:00 2001 From: William Bezuidenhout Date: Thu, 18 Aug 2022 16:16:13 +0200 Subject: [PATCH] use custom mocha reporter to generate annotations (#40438) * initial commit of test reporter * better use of annotated cmd Co-authored-by: Valery Bugakov * review comments and add additional comments describing approach * rename reporter.js to something more descriptive * restore files changed for debugging * make eslint and prettier happy Co-authored-by: Valery Bugakov --- .mocharc.js | 1 + client/shared/dev/customMochaSpecReporter.js | 67 ++++++++++++++++++++ enterprise/dev/ci/internal/ci/operations.go | 8 ++- 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 client/shared/dev/customMochaSpecReporter.js diff --git a/.mocharc.js b/.mocharc.js index ff60d52b4ab..cb7ff7f3f19 100644 --- a/.mocharc.js +++ b/.mocharc.js @@ -5,6 +5,7 @@ module.exports = { __dirname + '/client/shared/dev/fetch', __dirname + '/client/shared/dev/suppressPollyErrors', ], + reporter: __dirname + '/client/shared/dev/customMochaSpecReporter.js', extension: ['js', 'ts'], // 1 minute test timeout. This must be greater than the default Puppeteer // command timeout of 30s in order to get the stack trace to point to the diff --git a/client/shared/dev/customMochaSpecReporter.js b/client/shared/dev/customMochaSpecReporter.js new file mode 100644 index 00000000000..f3612a10891 --- /dev/null +++ b/client/shared/dev/customMochaSpecReporter.js @@ -0,0 +1,67 @@ +const { Console } = require('console') +const fs = require('fs') + +const mocha = require('mocha') + +const originalConsoleLog = mocha.reporters.Base.consoleLog + +// SpecFileReporter is a custom Mocha reporter (https://mochajs.org/api/reporters_base.js.html) which behaves +// almost exactly like the stock standard mocha.reporters.Spec reporter, but it also writes the final report out +// to a file `./annotations/mocha-test-output-placeholder`. +// +// Note that this Reporter is *only* really useful in Buildkite itself, but the implementation can be adapted to +// other usecases such as when you want other Mocha reporters to output their results to disk. +// +// For more information see: +// - https://mochajs.org/api/reporters_base.js.html +// - https://mochajs.org/api/tutorial-custom-reporter.html +class SpecFileReporter extends mocha.reporters.Spec { + // TODO(burmudar): Allow one to specify a filename and target directory ? + constructor(runner, options) { + super(runner, options) + this.title = 'placeholder' + this.buildkite = false + + if ('BUILDKITE' in process.env) { + this.buildkite = true + } else { + console.info('Not in BUILDKITE. No annotation will be generated in ./annotations') + } + + if ('BUILDKITE_LABEL' in process.env) { + this.title = process.env.BUILDKIATE_LABEL + } + + if (this.buildkite === true && typeof process.env.BUILDKITE_LABEL === undefined) { + console.warn( + `In Buildkite but BUILDKITE_LABEL not found in environment. Using title '${this.title || 'placeholder'}'` + ) + } + } + + epilogue() { + // We first let mocha.reporters.Spec do it's usual reporting using the default console defined on Base + // Which means the report will be written to the terminal + super.epilogue() + + // We only output the epilogue to a file when we're in BUILDKITE and there are failures + if (this.buildkite === true && this.failures.length > 0) { + const customConsole = new Console({ + stdout: fs.createWriteStream(`./annotations/mocha-test-output-${this.title || 'placeholder'}`), + }) + // We now want the Spec reporter (aka epilogue) to be written to a file, but Spec uses the console defined on Base! + // So we swap out the consoleLog defined on Base with our customLog one + // https://sourcegraph.com/github.com/mochajs/mocha/-/blob/lib/reporters/base.js?L43:5 + // eslint-disable-next-line @typescript-eslint/unbound-method + mocha.reporters.Base.consoleLog = customConsole.log + // Generate report using custom logger + // https://mochajs.org/api/reporters_base.js.html#line367 + super.epilogue() + // The report has been written to a file, so now we swap the consoleLog back to the originalConsole logger + // eslint-disable-next-line @typescript-eslint/unbound-method + mocha.reporters.Base.consoleLog = originalConsoleLog + } + } +} + +module.exports = SpecFileReporter diff --git a/enterprise/dev/ci/internal/ci/operations.go b/enterprise/dev/ci/internal/ci/operations.go index 474b1deffe6..961b7d2e997 100644 --- a/enterprise/dev/ci/internal/ci/operations.go +++ b/enterprise/dev/ci/internal/ci/operations.go @@ -579,7 +579,9 @@ func serverE2E(candidateTag string) operations.Operation { bk.Env("TEST_USER_EMAIL", "test@sourcegraph.com"), bk.Env("TEST_USER_PASSWORD", "supersecurepassword"), bk.Env("INCLUDE_ADMIN_ONBOARDING", "false"), - bk.Cmd("dev/ci/integration/e2e/run.sh"), + bk.AnnotatedCmd("dev/ci/integration/e2e/run.sh", bk.AnnotatedCmdOpts{ + Annotations: &bk.AnnotationOpts{}, + }), bk.ArtifactPaths("./*.png", "./*.mp4", "./*.log")) } } @@ -598,7 +600,9 @@ func serverQA(candidateTag string) operations.Operation { bk.Env("TEST_USER_EMAIL", "test@sourcegraph.com"), bk.Env("TEST_USER_PASSWORD", "supersecurepassword"), bk.Env("INCLUDE_ADMIN_ONBOARDING", "false"), - bk.Cmd("dev/ci/integration/qa/run.sh"), + bk.AnnotatedCmd("dev/ci/integration/qa/run.sh", bk.AnnotatedCmdOpts{ + Annotations: &bk.AnnotationOpts{}, + }), bk.ArtifactPaths("./*.png", "./*.mp4", "./*.log")) } }