diff --git a/.gitignore b/.gitignore index 4816466e6f9..757d5fdd236 100644 --- a/.gitignore +++ b/.gitignore @@ -165,4 +165,4 @@ sitemap_query.db annotations/ # Buildkite analytics files -jest-junit.xml +test-reports/ diff --git a/dev/ci/go-test.sh b/dev/ci/go-test.sh index 1739a0c084b..ae28d9dcf85 100755 --- a/dev/ci/go-test.sh +++ b/dev/ci/go-test.sh @@ -31,52 +31,12 @@ function go_test() { -race \ -v \ $test_packages | tee "$tmpfile" | richgo testfilter - # Save the test exit code so we can return it after submitting the test run to the analytics. + # Save the test exit code so we can return it after saving the test report test_exit_code="${PIPESTATUS[0]}" set -eo pipefail # resume being strict about errors - local xml - xml=$(go-junit-report <"$tmpfile") - # escape xml output properly for JSON - local quoted_xml - quoted_xml="$(echo "$xml" | jq -R -s '.')" - - local data - data=$( - cat <>./test-reports/go-test-junit.xml return "$test_exit_code" } @@ -104,10 +64,6 @@ fi go install github.com/jstemmer/go-junit-report@latest asdf reshim golang -# TODO move to manifest -# https://github.com/sourcegraph/sourcegraph/issues/28469 -BUILDKITE_ANALYTICS_BACKEND_TEST_SUITE_API_KEY=$(gcloud secrets versions access latest --secret="BUILDKITE_ANALYTICS_BACKEND_TEST_SUITE_API_KEY" --project="sourcegraph-ci" --quiet) - # For searcher echo "--- comby install" ./dev/comby-install-or-upgrade.sh diff --git a/dev/ci/yarn-test.sh b/dev/ci/yarn-test.sh index 128f7820fe4..271087291f1 100755 --- a/dev/ci/yarn-test.sh +++ b/dev/ci/yarn-test.sh @@ -9,67 +9,17 @@ yarn --mutex network --frozen-lockfile --network-timeout 60000 echo "--- generate" yarn gulp generate +root_dir=$(pwd) cd "$1" echo "--- test" -function yarn_test() { - JEST_JUNIT_OUTPUT_NAME="jest-junit.xml" - export JEST_JUNIT_OUTPUT_NAME - JEST_JUNIT_OUTPUT_DIR=$(mktemp -d) - export JEST_JUNIT_OUTPUT_DIR - trap 'rm -Rf "$JEST_JUNIT_OUTPUT_DIR"' EXIT +JEST_JUNIT_OUTPUT_NAME="yarn-test-junit.xml" +export JEST_JUNIT_OUTPUT_NAME +JEST_JUNIT_OUTPUT_DIR="$root_dir/test-reports" +export JEST_JUNIT_OUTPUT_DIR +mkdir -p "$JEST_JUNIT_OUTPUT_DIR" - set +eo pipefail # so we still get the result if the test failed - local test_exit_code - - # Limit the number of workers to prevent the default of 1 worker per core from - # causing OOM on the buildkite nodes that have 96 CPUs. 4 matches the CPU limits - # in infrastructure/kubernetes/ci/buildkite/buildkite-agent/buildkite-agent.Deployment.yaml - yarn -s run test --maxWorkers 4 --verbose --testResultsProcessor jest-junit - - # Save the test exit code so we can return it after submitting the test run to the analytics. - test_exit_code="$?" - - set -eo pipefail # resume being strict about errors - - # escape xml output properly for JSON - local quoted_xml - quoted_xml="$(jq -R -s '.' "$JEST_JUNIT_OUTPUT_DIR/$JEST_JUNIT_OUTPUT_NAME")" - - local data - data=$( - cat < WARNING: The Buildkite API is not finalized and neither are the configuration options for `TestReportOpts`. +> To get started with Buildkite Analytics please reach out to the `#dev-experience` channel for assistance. ### Buildkite infrastructure diff --git a/enterprise/dev/ci/internal/buildkite/buildkite.go b/enterprise/dev/ci/internal/buildkite/buildkite.go index 85194f89acc..18c6f56e005 100644 --- a/enterprise/dev/ci/internal/buildkite/buildkite.go +++ b/enterprise/dev/ci/internal/buildkite/buildkite.go @@ -262,8 +262,7 @@ const ( AnnotationTypeError AnnotationType = "error" ) -// AnnotatedCmdOpts declares options for AnnotatedCmd. -type AnnotatedCmdOpts struct { +type AnnotationOpts struct { // Type indicates the type annotations from this command should be uploaded as. // Commands that upload annotations of different levels will create separate // annotations. @@ -286,46 +285,97 @@ type AnnotatedCmdOpts struct { MultiJobContext string } -// AnnotatedCmd runs the given command, picks up files left in the `./annotations` -// directory, and appends them to a shared annotation for this job. For example, to -// generate an annotation file on error: +type TestReportOpts struct { + // TestSuiteKeyVariableName is the name of the variable in gcloud secrets that holds + // the test suite key to upload to. + // + // TODO: This is not finalized, see https://github.com/sourcegraph/sourcegraph/issues/31971 + TestSuiteKeyVariableName string +} + +// AnnotatedCmdOpts declares options for AnnotatedCmd. +type AnnotatedCmdOpts struct { + // AnnotationOpts configures how AnnotatedCmd picks up files left in the + // `./annotations` directory and appends them to a shared annotation for this job. + // If nil, AnnotatedCmd will not look for annotations. + // + // To get started, generate an annotation file when you want to publish an annotation, + // typically on error, in the './annotations' directory: + // + // if [ $EXIT_CODE -ne 0 ]; then + // echo -e "$OUT" >./annotations/shfmt + // echo "^^^ +++" + // fi + // + // Make sure it has a sufficiently unique name, so as to avoid conflicts if multiple + // annotations are generated in a single job. + // + // Annotations can be formatted based on file extensions, for example: + // + // - './annotations/Job log.md' will have its contents appended as markdown + // - './annotations/shfmt' will have its contents formatted as terminal output + // + // Please be considerate about what generating annotations, since they can cause a lot + // of visual clutter in the Buildkite UI. When creating annotations: + // + // - keep them concise and short, to minimze the space they take up + // - ensure they are actionable: an annotation should enable you, the CI user, to + // know where to go and what to do next. + // + // DO NOT use 'buildkite-agent annotate' or 'annotate.sh' directly in scripts. + Annotations *AnnotationOpts + + // TestReports configures how AnnotatedCmd picks up files left in the `./test-reports` + // directory and uploads them to Buildkite Analytics. If nil, AnnotatedCmd will not + // look for test reports. + // + // To get started, generate a JUnit XML report for your tests in the './test-reports' + // directory. Make sure it has a sufficiently unique name, so as to avoid conflicts if + // multiple reports are generated in a single job. Consult your language's test + // tooling for more details. + // + // Use TestReportOpts to configure where to publish reports too. For more details, + // see https://buildkite.com/organizations/sourcegraph/analytics. + // + // DO NOT post directly to the Buildkite API or use 'upload-test-report.sh' directly + // in scripts. + TestReports *TestReportOpts +} + +// AnnotatedCmd runs the given command and picks up annotations generated by the command: // -// if [ $EXIT_CODE -ne 0 ]; then -// echo -e "$OUT" >./annotations/shfmt -// echo "^^^ +++" -// fi +// - annotations in `./annotations` +// - test reports in `./test-reports` // -// Annotations can be formatted based on file extensions, for example: -// -// - './annotations/Job log.md' will have its contents appended as markdown -// - './annotations/shfmt' will have its contents formatted as terminal output on append -// -// Please be considerate about what generating annotations, since they can cause a lot of -// visual clutter in the Buildkite UI. When creating annotations: -// -// - keep them concise and short, to minimze the space they take up -// - ensure they are actionable: an annotation should enable you, the CI user, to know -// where to go and what to do next. -// -// DO NOT use 'buildkite-agent annotate' or 'annotate.sh' directly in scripts. +// To learn more, see the AnnotatedCmdOpts docstrings. func AnnotatedCmd(command string, opts AnnotatedCmdOpts) StepOpt { + // Options for annotations var annotateOpts string - if opts.Type == "" { - annotateOpts += fmt.Sprintf(" -t %s", AnnotationTypeError) - } else { - annotateOpts += fmt.Sprintf(" -t %s", opts.Type) + if opts.Annotations != nil { + if opts.Annotations.Type == "" { + annotateOpts += fmt.Sprintf(" -t %s", AnnotationTypeError) + } else { + annotateOpts += fmt.Sprintf(" -t %s", opts.Annotations.Type) + } + if opts.Annotations.MultiJobContext != "" { + annotateOpts += fmt.Sprintf(" -c %q", opts.Annotations.MultiJobContext) + } + annotateOpts = fmt.Sprintf("%v %s", opts.Annotations.IncludeNames, strings.TrimSpace(annotateOpts)) } - if opts.MultiJobContext != "" { - annotateOpts += fmt.Sprintf(" -c %q", opts.MultiJobContext) + + // Options for test reports + var testReportOpts string + if opts.TestReports != nil { + testReportOpts += opts.TestReports.TestSuiteKeyVariableName } - annotateOpts = fmt.Sprintf("%v %s", opts.IncludeNames, strings.TrimSpace(annotateOpts)) // ./an is a symbolic link created by the .buildkite/hooks/post-checkout hook. // Its purpose is to keep the command excerpt in the buildkite UI clear enough to - // see the underlying command even if prefixed by the annotation script. + // see the underlying command even if prefixed by the annotation scraper. annotatedCmd := fmt.Sprintf("./an %q", tracedCmd(command)) return flattenStepOpts(RawCmd(annotatedCmd), - Env("ANNOTATE_OPTS", annotateOpts)) + Env("ANNOTATE_OPTS", annotateOpts), + Env("TEST_REPORT_OPTS", testReportOpts)) } func Async(async bool) StepOpt { diff --git a/enterprise/dev/ci/internal/ci/changed/diff.go b/enterprise/dev/ci/internal/ci/changed/diff.go index a234063b6d6..6f829e4fdc4 100644 --- a/enterprise/dev/ci/internal/ci/changed/diff.go +++ b/enterprise/dev/ci/internal/ci/changed/diff.go @@ -55,6 +55,9 @@ func ParseDiff(files []string) (diff Diff) { if !strings.HasSuffix(p, ".md") && (isRootClientFile(p) || strings.HasPrefix(p, "client/")) { diff |= Client } + if strings.HasSuffix(p, "dev/ci/yarn-test.sh") { + diff |= Client + } // Affects GraphQL if strings.HasSuffix(p, ".graphql") { diff --git a/enterprise/dev/ci/internal/ci/operations.go b/enterprise/dev/ci/internal/ci/operations.go index 5bd3b685aaf..21c6af6a17d 100644 --- a/enterprise/dev/ci/internal/ci/operations.go +++ b/enterprise/dev/ci/internal/ci/operations.go @@ -121,7 +121,9 @@ func addCIScriptsTests(pipeline *bk.Pipeline) { // Verifies the docs formatting and builds the `docsite` command. func addDocs(pipeline *bk.Pipeline) { pipeline.AddStep(":memo: Check and build docsite", - bk.AnnotatedCmd("./dev/check/docsite.sh", bk.AnnotatedCmdOpts{})) + bk.AnnotatedCmd("./dev/check/docsite.sh", bk.AnnotatedCmdOpts{ + Annotations: &bk.AnnotationOpts{}, + })) } // Adds the terraform scanner step. This executes very quickly ~6s @@ -136,7 +138,7 @@ func addCheck(pipeline *bk.Pipeline) { pipeline.AddStep(":clipboard: Misc linters", withYarnCache(), bk.AnnotatedCmd("./dev/check/all.sh", bk.AnnotatedCmdOpts{ - IncludeNames: true, + Annotations: &bk.AnnotationOpts{IncludeNames: true}, })) } @@ -200,7 +202,11 @@ func addWebApp(pipeline *bk.Pipeline) { // Webapp tests pipeline.AddStep(":jest::globe_with_meridians: Test", withYarnCache(), - bk.Cmd("dev/ci/yarn-test.sh client/web"), + bk.AnnotatedCmd("dev/ci/yarn-test.sh client/web", bk.AnnotatedCmdOpts{ + TestReports: &bk.TestReportOpts{ + TestSuiteKeyVariableName: "BUILDKITE_ANALYTICS_FRONTEND_UNIT_TEST_SUITE_API_KEY", + }, + }), bk.Cmd("dev/ci/codecov.sh -c -F typescript -F unit")) } @@ -338,7 +344,11 @@ func addGoTests(pipeline *bk.Pipeline) { buildGoTests(func(description, testSuffix string) { pipeline.AddStep( fmt.Sprintf(":go: Test (%s)", description), - bk.Cmd("./dev/ci/go-test.sh "+testSuffix), + bk.AnnotatedCmd("./dev/ci/go-test.sh "+testSuffix, bk.AnnotatedCmdOpts{ + TestReports: &bk.TestReportOpts{ + TestSuiteKeyVariableName: "BUILDKITE_ANALYTICS_BACKEND_TEST_SUITE_API_KEY", + }, + }), bk.Cmd("./dev/ci/codecov.sh -c -F go"), ) }) @@ -394,7 +404,7 @@ func addGoBuild(pipeline *bk.Pipeline) { func addDockerfileLint(pipeline *bk.Pipeline) { pipeline.AddStep(":docker: Docker linters", bk.AnnotatedCmd("go run ./dev/sg lint -annotations docker", bk.AnnotatedCmdOpts{ - IncludeNames: true, + Annotations: &bk.AnnotationOpts{IncludeNames: true}, })) } @@ -666,8 +676,10 @@ func trivyScanCandidateImage(app, tag string) operations.Operation { bk.SoftFail(vulnerabilityExitCode), bk.AnnotatedCmd("./dev/ci/trivy/trivy-scan-high-critical.sh", bk.AnnotatedCmdOpts{ - Type: bk.AnnotationTypeWarning, - MultiJobContext: "docker-security-scans", + Annotations: &bk.AnnotationOpts{ + Type: bk.AnnotationTypeWarning, + MultiJobContext: "docker-security-scans", + }, })) } } diff --git a/enterprise/dev/ci/scripts/annotated-command.sh b/enterprise/dev/ci/scripts/annotated-command.sh index 53d1107792d..87826a85cd3 100755 --- a/enterprise/dev/ci/scripts/annotated-command.sh +++ b/enterprise/dev/ci/scripts/annotated-command.sh @@ -11,6 +11,9 @@ cmd=$1 annotation_dir="./annotations" rm -rf $annotation_dir mkdir -p $annotation_dir +test_report_dir="./test-reports" +rm -rf $test_report_dir +mkdir -p $test_report_dir # Run the provided command eval "$cmd" @@ -54,4 +57,20 @@ if [ -n "${ANNOTATE_OPTS-''}" ]; then done fi +# Check for test reports left behind by the command +if [ -n "${TEST_REPORT_OPTS-''}" ]; then + test_report_opts="$TEST_REPORT_OPTS" + + echo "~~~ Uploading test reports" + echo "test_report_opts=$test_report_opts" + for file in "$test_report_dir"/*; do + if [ ! -f "$file" ]; then + continue + fi + + echo "handling $file" + eval "./enterprise/dev/ci/scripts/upload-test-report.sh $file $test_report_opts" + done +fi + exit "$exit_code" diff --git a/enterprise/dev/ci/scripts/upload-test-report.sh b/enterprise/dev/ci/scripts/upload-test-report.sh new file mode 100755 index 00000000000..5c18d1e2f03 --- /dev/null +++ b/enterprise/dev/ci/scripts/upload-test-report.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +xml_file=$1 +xml=$(cat "$xml_file") + +test_key_variable_name=$2 + +# escape xml output properly for JSON +quoted_xml="$(echo "$xml" | jq -R -s '.')" +data=$( + cat <