artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 14:51:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

dev/ci: integrate test reports with AnnotatedCmd (#31969)

Browse Source 
Integrates test reports scraping into AnnotatedCmd, removing a bunch of duplication and complexity from our test scripts and packaging test uploads into a neat pull-based API that allows local inspection of artefacts similarly to the annotations API.
This commit is contained in:
Robert Lin 2022-03-01 07:50:14 -08:00 committed by GitHub
parent d479784ed6
commit a4ea4b3004
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 210 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -165,4 +165,4 @@ sitemap_query.db
annotations/
# Buildkite analytics files
jest-junit.xml
test-reports/

50
dev/ci/go-test.sh
View File

@ -31,52 +31,12 @@ function go_test() {
-race \
-v \
$test_packages | tee "$tmpfile" | richgo testfilter
# Save the test exit code so we can return it after submitting the test run to the analytics.
# Save the test exit code so we can return it after saving the test report
test_exit_code="${PIPESTATUS[0]}"
set -eo pipefail # resume being strict about errors
local xml
xml=$(go-junit-report <"$tmpfile")
# escape xml output properly for JSON
local quoted_xml
quoted_xml="$(echo "$xml" | jq -R -s '.')"
local data
data=$(
cat <<EOF
{
"format": "junit",
"run_env": {
"CI": "buildkite",
"key": "$BUILDKITE_BUILD_ID",
"number": "$BUILDKITE_BUILD_NUMBER",
"job_id": "$BUILDKITE_JOB_ID",
"branch": "$BUILDKITE_BRANCH",
"commit_sha": "$BUILDKITE_COMMIT",
"message": "$BUILDKITE_MESSAGE",
"url": "$BUILDKITE_BUILD_URL"
},
"data": $quoted_xml
}
EOF
)
echo -e "\n--- :information_source: Uploading test results to Buildkite analytics"
set +e
echo "$data" | curl \
--fail \
--request POST \
--url https://analytics-api.buildkite.com/v1/uploads \
--header "Authorization: Token token=\"$BUILDKITE_ANALYTICS_BACKEND_TEST_SUITE_API_KEY\";" \
--header 'Content-Type: application/json' \
--data-binary @-
local curl_exit="$?"
if [ "$curl_exit" -eq 0 ]; then
echo -e "\n--- :information_source: Succesfully uploaded test results to Buildkite analytics"
else
echo -e "\n^^^ +++ :warning: Failed to upload test results to Buildkite analytics"
fi
set -e
mkdir -p './test-reports'
go-junit-report <"$tmpfile" >>./test-reports/go-test-junit.xml
return "$test_exit_code"
}
@ -104,10 +64,6 @@ fi
go install github.com/jstemmer/go-junit-report@latest
asdf reshim golang
# TODO move to manifest
# https://github.com/sourcegraph/sourcegraph/issues/28469
BUILDKITE_ANALYTICS_BACKEND_TEST_SUITE_API_KEY=$(gcloud secrets versions access latest --secret="BUILDKITE_ANALYTICS_BACKEND_TEST_SUITE_API_KEY" --project="sourcegraph-ci" --quiet)
# For searcher
echo "--- comby install"
./dev/comby-install-or-upgrade.sh

70
dev/ci/yarn-test.sh
View File

@ -9,67 +9,17 @@ yarn --mutex network --frozen-lockfile --network-timeout 60000
echo "--- generate"
yarn gulp generate
root_dir=$(pwd)
cd "$1"
echo "--- test"
function yarn_test() {
JEST_JUNIT_OUTPUT_NAME="jest-junit.xml"
export JEST_JUNIT_OUTPUT_NAME
JEST_JUNIT_OUTPUT_DIR=$(mktemp -d)
export JEST_JUNIT_OUTPUT_DIR
trap 'rm -Rf "$JEST_JUNIT_OUTPUT_DIR"' EXIT
JEST_JUNIT_OUTPUT_NAME="yarn-test-junit.xml"
export JEST_JUNIT_OUTPUT_NAME
JEST_JUNIT_OUTPUT_DIR="$root_dir/test-reports"
export JEST_JUNIT_OUTPUT_DIR
mkdir -p "$JEST_JUNIT_OUTPUT_DIR"
set +eo pipefail # so we still get the result if the test failed
local test_exit_code
# Limit the number of workers to prevent the default of 1 worker per core from
# causing OOM on the buildkite nodes that have 96 CPUs. 4 matches the CPU limits
# in infrastructure/kubernetes/ci/buildkite/buildkite-agent/buildkite-agent.Deployment.yaml
yarn -s run test --maxWorkers 4 --verbose --testResultsProcessor jest-junit
# Save the test exit code so we can return it after submitting the test run to the analytics.
test_exit_code="$?"
set -eo pipefail # resume being strict about errors
# escape xml output properly for JSON
local quoted_xml
quoted_xml="$(jq -R -s '.' "$JEST_JUNIT_OUTPUT_DIR/$JEST_JUNIT_OUTPUT_NAME")"
local data
data=$(
cat <<EOF
{
"format": "junit",
"run_env": {
"CI": "buildkite",
"key": "$BUILDKITE_BUILD_ID",
"number": "$BUILDKITE_BUILD_NUMBER",
"job_id": "$BUILDKITE_JOB_ID",
"branch": "$BUILDKITE_BRANCH",
"commit_sha": "$BUILDKITE_COMMIT",
"message": "$BUILDKITE_MESSAGE",
"url": "$BUILDKITE_BUILD_URL"
},
"data": $quoted_xml
}
EOF
)
echo "$data" | curl \
--request POST \
--url https://analytics-api.buildkite.com/v1/uploads \
--header "Authorization: Token token=\"$BUILDKITE_ANALYTICS_FRONTEND_UNIT_TEST_SUITE_API_KEY\";" \
--header 'Content-Type: application/json' \
--data-binary @-
echo -e "\n--- :information_source: Succesfully uploaded test results to Buildkite analytics"
unset JEST_JUNIT_OUTPUT_DIR
unset JEST_JUNIT_OUTPUT_NAME
return "$test_exit_code"
}
BUILDKITE_ANALYTICS_FRONTEND_UNIT_TEST_SUITE_API_KEY=$(gcloud secrets versions access latest --secret="BUILDKITE_ANALYTICS_FRONTEND_UNIT_TEST_SUITE_API_KEY" --project="sourcegraph-ci" --quiet)
yarn_test
# Limit the number of workers to prevent the default of 1 worker per core from
# causing OOM on the buildkite nodes that have 96 CPUs. 4 matches the CPU limits
# in infrastructure/kubernetes/ci/buildkite/buildkite-agent/buildkite-agent.Deployment.yaml
yarn -s run test --maxWorkers 4 --verbose --testResultsProcessor jest-junit

37
doc/dev/background-information/ci/index.md
View File

@ -198,14 +198,16 @@ The pipeline generator provides an API for this that, at a high level, works lik
fi
```
1. In your pipeline operation, replace the usual `bk.Cmd` with `bk.AnnotatedCmd`:
2. In your pipeline operation, replace the usual `bk.Cmd` with `bk.AnnotatedCmd`:
```go
pipeline.AddStep(":memo: Check and build docsite",
bk.AnnotatedCmd("./dev/check/docsite.sh", bk.AnnotatedCmdOpts{}))
bk.AnnotatedCmd("./dev/check/docsite.sh", bk.AnnotatedCmdOpts{
Annotations: &bk.AnnotationOpts{},
}))
```
1. That's it!
3. That's it!
For more details about best practices and additional features and capabilities, please refer to [the `bk.AnnotatedCmd` docstring](https://sourcegraph.com/search?q=context:global+repo:%5Egithub%5C.com/sourcegraph/sourcegraph%24+file:%5Eenterprise/dev/ci/internal/buildkite+AnnotatedCmd+type:symbol&patternType=literal).
@ -223,8 +225,7 @@ See the [Buildkite board on Honeycomb](https://ui.honeycomb.io/sourcegraph/board
Individual commands are tracked from the perspective of a given [step](#step-options):
```go
pipeline.AddStep(":memo: Check and build docsite",
bk.AnnotatedCmd("./dev/check/docsite.sh", bk.AnnotatedCmdOpts{}))
pipeline.AddStep(":memo: Check and build docsite", /* ... */)
```
Will result in a single trace span for the `./dev/check/docsite.sh` script. But the following will have individual trace spans for each `yarn` commands:
@ -243,12 +244,32 @@ Therefore, it's beneficial for tracing purposes to split the step in multiple co
##### Test analytics
Our test analytics is currently powered by a tool that Buildkite released in beta to analyse individual tests across builds called [Buildkite Analytics](https://buildkite.com/test-analytics).
This tool enables to observe the evolution of each individual tests on the following metrics: duration and flakiness.
Our test analytics is currently powered by a Buildkite beta feature for analysing individual tests across builds called [Buildkite Analytics](https://buildkite.com/test-analytics).
This tool enables us to observe the evolution of each individual test on the following metrics: duration and flakiness.
Browse the [dashboard](https://buildkite.com/organizations/sourcegraph/analytics) to explore the metrics and optionally set monitors that will alert if a given test or a test suite is deviating from its historical duration or flakiness.
In order to track a new test suite, the tests output must be converted to JUnit XML and then uploaded to Buildkite. You can find the instructions for the upload by creating a new Test Suite in the Buildkite Analytics UI.
In order to track a new test suite, test results must be converted to JUnit XML reports and uploaded to Buildkite.
The pipeline generator provides an API for this that, at a high level, works like this:
1. In your script, leave your JUnit XML test report in `./test-reports`
2. [Create a new Test Suite](https://buildkite.com/organizations/sourcegraph/analytics/suites/new) in the Buildkite Analytics UI.
3. In your pipeline operation, replace the usual `bk.Cmd` with `bk.AnnotatedCmd`:
```go
pipeline.AddStep(":jest::globe_with_meridians: Test",
withYarnCache(),
bk.AnnotatedCmd("dev/ci/yarn-test.sh client/web", bk.AnnotatedCmdOpts{
TestReports: &bk.TestReportOpts{/* ... */},
}),
```
4. That's it!
For more details about best practices and additional features and capabilities, please refer to [the `bk.AnnotatedCmd` docstring](https://sourcegraph.com/search?q=context:global+repo:%5Egithub%5C.com/sourcegraph/sourcegraph%24+file:%5Eenterprise/dev/ci/internal/buildkite+AnnotatedCmd+type:symbol&patternType=literal).
> WARNING: The Buildkite API is not finalized and neither are the configuration options for `TestReportOpts`.
> To get started with Buildkite Analytics please reach out to the `#dev-experience` channel for assistance.
### Buildkite infrastructure

112
enterprise/dev/ci/internal/buildkite/buildkite.go
View File

@ -262,8 +262,7 @@ const (
AnnotationTypeError AnnotationType = "error"
)
// AnnotatedCmdOpts declares options for AnnotatedCmd.
type AnnotatedCmdOpts struct {
type AnnotationOpts struct {
// Type indicates the type annotations from this command should be uploaded as.
// Commands that upload annotations of different levels will create separate
// annotations.
@ -286,46 +285,97 @@ type AnnotatedCmdOpts struct {
MultiJobContext string
}
// AnnotatedCmd runs the given command, picks up files left in the `./annotations`
// directory, and appends them to a shared annotation for this job. For example, to
// generate an annotation file on error:
type TestReportOpts struct {
// TestSuiteKeyVariableName is the name of the variable in gcloud secrets that holds
// the test suite key to upload to.
//
// TODO: This is not finalized, see https://github.com/sourcegraph/sourcegraph/issues/31971
TestSuiteKeyVariableName string
}
// AnnotatedCmdOpts declares options for AnnotatedCmd.
type AnnotatedCmdOpts struct {
// AnnotationOpts configures how AnnotatedCmd picks up files left in the
// `./annotations` directory and appends them to a shared annotation for this job.
// If nil, AnnotatedCmd will not look for annotations.
//
// To get started, generate an annotation file when you want to publish an annotation,
// typically on error, in the './annotations' directory:
//
// if [ $EXIT_CODE -ne 0 ]; then
// echo -e "$OUT" >./annotations/shfmt
// echo "^^^ +++"
// fi
//
// Make sure it has a sufficiently unique name, so as to avoid conflicts if multiple
// annotations are generated in a single job.
//
// Annotations can be formatted based on file extensions, for example:
//
// - './annotations/Job log.md' will have its contents appended as markdown
// - './annotations/shfmt' will have its contents formatted as terminal output
//
// Please be considerate about what generating annotations, since they can cause a lot
// of visual clutter in the Buildkite UI. When creating annotations:
//
// - keep them concise and short, to minimze the space they take up
// - ensure they are actionable: an annotation should enable you, the CI user, to
// know where to go and what to do next.
//
// DO NOT use 'buildkite-agent annotate' or 'annotate.sh' directly in scripts.
Annotations *AnnotationOpts
// TestReports configures how AnnotatedCmd picks up files left in the `./test-reports`
// directory and uploads them to Buildkite Analytics. If nil, AnnotatedCmd will not
// look for test reports.
//
// To get started, generate a JUnit XML report for your tests in the './test-reports'
// directory. Make sure it has a sufficiently unique name, so as to avoid conflicts if
// multiple reports are generated in a single job. Consult your language's test
// tooling for more details.
//
// Use TestReportOpts to configure where to publish reports too. For more details,
// see https://buildkite.com/organizations/sourcegraph/analytics.
//
// DO NOT post directly to the Buildkite API or use 'upload-test-report.sh' directly
// in scripts.
TestReports *TestReportOpts
}
// AnnotatedCmd runs the given command and picks up annotations generated by the command:
//
// if [ $EXIT_CODE -ne 0 ]; then
// echo -e "$OUT" >./annotations/shfmt
// echo "^^^ +++"
// fi
// - annotations in `./annotations`
// - test reports in `./test-reports`
//
// Annotations can be formatted based on file extensions, for example:
//
// - './annotations/Job log.md' will have its contents appended as markdown
// - './annotations/shfmt' will have its contents formatted as terminal output on append
//
// Please be considerate about what generating annotations, since they can cause a lot of
// visual clutter in the Buildkite UI. When creating annotations:
//
// - keep them concise and short, to minimze the space they take up
// - ensure they are actionable: an annotation should enable you, the CI user, to know
// where to go and what to do next.
//
// DO NOT use 'buildkite-agent annotate' or 'annotate.sh' directly in scripts.
// To learn more, see the AnnotatedCmdOpts docstrings.
func AnnotatedCmd(command string, opts AnnotatedCmdOpts) StepOpt {
// Options for annotations
var annotateOpts string
if opts.Type == "" {
annotateOpts += fmt.Sprintf(" -t %s", AnnotationTypeError)
} else {
annotateOpts += fmt.Sprintf(" -t %s", opts.Type)
if opts.Annotations != nil {
if opts.Annotations.Type == "" {
annotateOpts += fmt.Sprintf(" -t %s", AnnotationTypeError)
} else {
annotateOpts += fmt.Sprintf(" -t %s", opts.Annotations.Type)
}
if opts.Annotations.MultiJobContext != "" {
annotateOpts += fmt.Sprintf(" -c %q", opts.Annotations.MultiJobContext)
}
annotateOpts = fmt.Sprintf("%v %s", opts.Annotations.IncludeNames, strings.TrimSpace(annotateOpts))
}
if opts.MultiJobContext != "" {
annotateOpts += fmt.Sprintf(" -c %q", opts.MultiJobContext)
// Options for test reports
var testReportOpts string
if opts.TestReports != nil {
testReportOpts += opts.TestReports.TestSuiteKeyVariableName
}
annotateOpts = fmt.Sprintf("%v %s", opts.IncludeNames, strings.TrimSpace(annotateOpts))
// ./an is a symbolic link created by the .buildkite/hooks/post-checkout hook.
// Its purpose is to keep the command excerpt in the buildkite UI clear enough to
// see the underlying command even if prefixed by the annotation script.
// see the underlying command even if prefixed by the annotation scraper.
annotatedCmd := fmt.Sprintf("./an %q", tracedCmd(command))
return flattenStepOpts(RawCmd(annotatedCmd),
Env("ANNOTATE_OPTS", annotateOpts))
Env("ANNOTATE_OPTS", annotateOpts),
Env("TEST_REPORT_OPTS", testReportOpts))
}
func Async(async bool) StepOpt {

3
enterprise/dev/ci/internal/ci/changed/diff.go
View File

@ -55,6 +55,9 @@ func ParseDiff(files []string) (diff Diff) {
if !strings.HasSuffix(p, ".md") && (isRootClientFile(p) || strings.HasPrefix(p, "client/")) {
diff |= Client
}
if strings.HasSuffix(p, "dev/ci/yarn-test.sh") {
diff |= Client
}
// Affects GraphQL
if strings.HasSuffix(p, ".graphql") {

26
enterprise/dev/ci/internal/ci/operations.go
View File

@ -121,7 +121,9 @@ func addCIScriptsTests(pipeline *bk.Pipeline) {
// Verifies the docs formatting and builds the `docsite` command.
func addDocs(pipeline *bk.Pipeline) {
pipeline.AddStep(":memo: Check and build docsite",
bk.AnnotatedCmd("./dev/check/docsite.sh", bk.AnnotatedCmdOpts{}))
bk.AnnotatedCmd("./dev/check/docsite.sh", bk.AnnotatedCmdOpts{
Annotations: &bk.AnnotationOpts{},
}))
}
// Adds the terraform scanner step. This executes very quickly ~6s
@ -136,7 +138,7 @@ func addCheck(pipeline *bk.Pipeline) {
pipeline.AddStep(":clipboard: Misc linters",
withYarnCache(),
bk.AnnotatedCmd("./dev/check/all.sh", bk.AnnotatedCmdOpts{
IncludeNames: true,
Annotations: &bk.AnnotationOpts{IncludeNames: true},
}))
}
@ -200,7 +202,11 @@ func addWebApp(pipeline *bk.Pipeline) {
// Webapp tests
pipeline.AddStep(":jest::globe_with_meridians: Test",
withYarnCache(),
bk.Cmd("dev/ci/yarn-test.sh client/web"),
bk.AnnotatedCmd("dev/ci/yarn-test.sh client/web", bk.AnnotatedCmdOpts{
TestReports: &bk.TestReportOpts{
TestSuiteKeyVariableName: "BUILDKITE_ANALYTICS_FRONTEND_UNIT_TEST_SUITE_API_KEY",
},
}),
bk.Cmd("dev/ci/codecov.sh -c -F typescript -F unit"))
}
@ -338,7 +344,11 @@ func addGoTests(pipeline *bk.Pipeline) {
buildGoTests(func(description, testSuffix string) {
pipeline.AddStep(
fmt.Sprintf(":go: Test (%s)", description),
bk.Cmd("./dev/ci/go-test.sh "+testSuffix),
bk.AnnotatedCmd("./dev/ci/go-test.sh "+testSuffix, bk.AnnotatedCmdOpts{
TestReports: &bk.TestReportOpts{
TestSuiteKeyVariableName: "BUILDKITE_ANALYTICS_BACKEND_TEST_SUITE_API_KEY",
},
}),
bk.Cmd("./dev/ci/codecov.sh -c -F go"),
)
})
@ -394,7 +404,7 @@ func addGoBuild(pipeline *bk.Pipeline) {
func addDockerfileLint(pipeline *bk.Pipeline) {
pipeline.AddStep(":docker: Docker linters",
bk.AnnotatedCmd("go run ./dev/sg lint -annotations docker", bk.AnnotatedCmdOpts{
IncludeNames: true,
Annotations: &bk.AnnotationOpts{IncludeNames: true},
}))
}
@ -666,8 +676,10 @@ func trivyScanCandidateImage(app, tag string) operations.Operation {
bk.SoftFail(vulnerabilityExitCode),
bk.AnnotatedCmd("./dev/ci/trivy/trivy-scan-high-critical.sh", bk.AnnotatedCmdOpts{
Type: bk.AnnotationTypeWarning,
MultiJobContext: "docker-security-scans",
Annotations: &bk.AnnotationOpts{
Type: bk.AnnotationTypeWarning,
MultiJobContext: "docker-security-scans",
},
}))
}
}

19
enterprise/dev/ci/scripts/annotated-command.sh
View File

@ -11,6 +11,9 @@ cmd=$1
annotation_dir="./annotations"
rm -rf $annotation_dir
mkdir -p $annotation_dir
test_report_dir="./test-reports"
rm -rf $test_report_dir
mkdir -p $test_report_dir
# Run the provided command
eval "$cmd"
@ -54,4 +57,20 @@ if [ -n "${ANNOTATE_OPTS-''}" ]; then
done
fi
# Check for test reports left behind by the command
if [ -n "${TEST_REPORT_OPTS-''}" ]; then
test_report_opts="$TEST_REPORT_OPTS"
echo "~~~ Uploading test reports"
echo "test_report_opts=$test_report_opts"
for file in "$test_report_dir"/*; do
if [ ! -f "$file" ]; then
continue
fi
echo "handling $file"
eval "./enterprise/dev/ci/scripts/upload-test-report.sh $file $test_report_opts"
done
fi
exit "$exit_code"

45
enterprise/dev/ci/scripts/upload-test-report.sh Executable file
View File

@ -0,0 +1,45 @@
#!/bin/bash
xml_file=$1
xml=$(cat "$xml_file")
test_key_variable_name=$2
# escape xml output properly for JSON
quoted_xml="$(echo "$xml" | jq -R -s '.')"
data=$(
cat <<EOF
{
"format": "junit",
"run_env": {
"CI": "buildkite",
"key": "$BUILDKITE_BUILD_ID",
"number": "$BUILDKITE_BUILD_NUMBER",
"job_id": "$BUILDKITE_JOB_ID",
"branch": "$BUILDKITE_BRANCH",
"commit_sha": "$BUILDKITE_COMMIT",
"message": "$BUILDKITE_MESSAGE",
"url": "$BUILDKITE_BUILD_URL"
},
"data": $quoted_xml
}
EOF
)
TOKEN=$(gcloud secrets versions access latest --secret="$test_key_variable_name" --project="sourcegraph-ci" --quiet)
set +e
echo "$data" | curl \
--fail \
--request POST \
--url https://analytics-api.buildkite.com/v1/uploads \
--header "Authorization: Token token=\"$TOKEN\";" \
--header 'Content-Type: application/json' \
--data-binary @-
curl_exit="$?"
if [ "$curl_exit" -eq 0 ]; then
echo -e "\n:information_source: Succesfully uploaded test results to Buildkite analytics"
else
echo -e "\n^^^ +++ :warning: Failed to upload test results to Buildkite analytics"
fi
set -e