diff --git a/.aspect/bazelrc/ci.sourcegraph.bazelrc b/.aspect/bazelrc/ci.sourcegraph.bazelrc index 672a427ad7e..83e8e00713b 100644 --- a/.aspect/bazelrc/ci.sourcegraph.bazelrc +++ b/.aspect/bazelrc/ci.sourcegraph.bazelrc @@ -39,3 +39,6 @@ build --workspace_status_command=./dev/bazel_buildkite_stamp_vars.sh # temp build --test_env=INCLUDE_ADMIN_ONBOARDING=false + +# Used for container_structure_tests +build --test_env=DOCKER_HOST diff --git a/.bazelrc b/.bazelrc index ba39fad1feb..9e7025e7950 100644 --- a/.bazelrc +++ b/.bazelrc @@ -30,3 +30,8 @@ try-import %workspace%/user.bazelrc # Some special sauce for the special NixOS users in your life :) set by dev-shell shell-hook try-import %workspace%/.bazelrc-nix + +# Used to locally cross compile, when targeting docker images +build:darwin-docker --incompatible_enable_cc_toolchain_resolution +build:darwin-docker --platforms @zig_sdk//platform:linux_amd64 +build:darwin-docker --extra_toolchains @zig_sdk//toolchain:linux_amd64_gnu.2.31 diff --git a/BUILD.bazel b/BUILD.bazel index 362bbdcc2d8..0ec7482de17 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -9,6 +9,7 @@ load("@io_bazel_rules_go//proto/wkt:well_known_types.bzl", "WELL_KNOWN_TYPES_API load("@npm//:defs.bzl", "npm_link_all_packages") load("//dev/linters/staticcheck:analyzers.bzl", "STATIC_CHECK_ANALYZERS") load("@npm//:eslint/package_json.bzl", eslint_bin = "bin") +load("//:stamp_tags.bzl", "stamp_tags") load("//dev:eslint.bzl", "eslint_test_with_types") # Gazelle config @@ -323,3 +324,8 @@ exports_files([ # under certain conditions. See //ui/assets/... "CHANGELOG.md", ]) + +stamp_tags( + name = "tags", + remote_tags = ["""($stamp.STABLE_VERSION // "0.0.0")"""], +) diff --git a/WORKSPACE b/WORKSPACE index 1a638bde4f2..d4d594ea21d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -74,6 +74,56 @@ http_archive( urls = ["https://github.com/bazelbuild/rules_rust/releases/download/0.19.0/rules_rust-v0.19.0.tar.gz"], ) +# Container rules +http_archive( + name = "rules_oci", + sha256 = "db57efd706f01eb3ce771468366baa1614b5b25f4cce99757e2b8d942155b8ec", + strip_prefix = "rules_oci-1.0.0", + url = "https://github.com/bazel-contrib/rules_oci/releases/download/v1.0.0/rules_oci-v1.0.0.tar.gz", +) + +http_archive( + name = "rules_pkg", + sha256 = "8c20f74bca25d2d442b327ae26768c02cf3c99e93fad0381f32be9aab1967675", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.8.1/rules_pkg-0.8.1.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.8.1/rules_pkg-0.8.1.tar.gz", + ], +) + +SRC_CLI_VERSION = "5.0.3" +http_archive( + name = "src-cli-linux-amd64", + sha256 = "d125d732ad4c47ae6977c49574b01cc1b3c943b2a2108142267438e829538aa3", + url = "https://github.com/sourcegraph/src-cli/releases/download/{0}/src-cli_{0}_linux_amd64.tar.gz".format(SRC_CLI_VERSION), + build_file_content = """ +filegroup( + name = "src-cli-linux-amd64", + srcs = ["src"], + visibility = ["//visibility:public"], +) + """ +) + + +http_archive( + name = "container_structure_test", + sha256 = "42edb647b51710cb917b5850380cc18a6c925ad195986f16e3b716887267a2d7", + strip_prefix = "container-structure-test-104a53ede5f78fff72172639781ac52df9f5b18f", + urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/104a53ede5f78fff72172639781ac52df9f5b18f.zip"], +) + +# hermetic_cc_toolchain setup ================================ +HERMETIC_CC_TOOLCHAIN_VERSION = "v2.0.0-rc2" +http_archive( + name = "hermetic_cc_toolchain", + sha256 = "40dff82816735e631e8bd51ede3af1c4ed1ad4646928ffb6a0e53e228e55738c", + urls = [ + "https://mirror.bazel.build/github.com/uber/hermetic_cc_toolchain/releases/download/{0}/hermetic_cc_toolchain-{0}.tar.gz".format(HERMETIC_CC_TOOLCHAIN_VERSION), + "https://github.com/uber/hermetic_cc_toolchain/releases/download/{0}/hermetic_cc_toolchain-{0}.tar.gz".format(HERMETIC_CC_TOOLCHAIN_VERSION), + ], +) + # rules_js setup ================================ load("@aspect_rules_js//js:repositories.bzl", "rules_js_dependencies") @@ -292,17 +342,6 @@ load("@crate_index//:defs.bzl", "crate_repositories") crate_repositories() -BAZEL_ZIG_CC_VERSION = "v2.0.0-rc2" - -http_archive( - name = "hermetic_cc_toolchain", - sha256 = "40dff82816735e631e8bd51ede3af1c4ed1ad4646928ffb6a0e53e228e55738c", - urls = [ - "https://mirror.bazel.build/github.com/uber/hermetic_cc_toolchain/releases/download/{0}/hermetic_cc_toolchain-{0}.tar.gz".format(BAZEL_ZIG_CC_VERSION), - "https://github.com/uber/hermetic_cc_toolchain/releases/download/{0}/hermetic_cc_toolchain-{0}.tar.gz".format(BAZEL_ZIG_CC_VERSION), - ], -) - load("@hermetic_cc_toolchain//toolchain:defs.bzl", zig_toolchains = "toolchains") zig_toolchains() @@ -311,6 +350,32 @@ load("//dev/backcompat:defs.bzl", "back_compat_defs") back_compat_defs() +# containers steup =============================== +load("@rules_oci//oci:dependencies.bzl", "rules_oci_dependencies") + +rules_oci_dependencies() + +load("@rules_oci//oci:repositories.bzl", "LATEST_CRANE_VERSION", "LATEST_ZOT_VERSION", "oci_register_toolchains") + +oci_register_toolchains( + name = "oci", + crane_version = LATEST_CRANE_VERSION, + # Uncommenting the zot toolchain will cause it to be used instead of crane for some tasks. + # Note that it does not support docker-format images. + # zot_version = LATEST_ZOT_VERSION, +) + +# Optional, for oci_tarball rule +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") + +rules_pkg_dependencies() + +load("//dev:oci_deps.bzl", "oci_deps") +oci_deps() + load("//enterprise/cmd/embeddings/shared:assets.bzl", "embbedings_assets_deps") embbedings_assets_deps() + +load("@container_structure_test//:repositories.bzl", "container_structure_test_register_toolchain") +container_structure_test_register_toolchain(name = "cst") diff --git a/client/web/src/end-to-end/BUILD.bazel b/client/web/src/end-to-end/BUILD.bazel index 97f0a3ffe9e..134e6acac5f 100644 --- a/client/web/src/end-to-end/BUILD.bazel +++ b/client/web/src/end-to-end/BUILD.bazel @@ -63,5 +63,14 @@ mocha_test( "requires-network", ], tests = [test.replace(".ts", ".js") for test in glob(["**/*.test.ts"])], + visibility = ["//testing:__pkg__"], deps = [":end-to-end_tests"], ) + +# For some reason, we can't explicitly set a visibility on the target itself, +# it seems the esbuild rule doesn't pass along the visibility attribute properly. +alias( + name = "testing_e2e_bundle", + actual = ":e2e_bundle", + visibility = ["//testing:__pkg__"], +) diff --git a/client/web/src/integration/BUILD.bazel b/client/web/src/integration/BUILD.bazel index 158e862a118..a9525fcc604 100644 --- a/client/web/src/integration/BUILD.bazel +++ b/client/web/src/integration/BUILD.bazel @@ -115,6 +115,7 @@ mocha_test( flaky = True, is_percy_enabled = True, tags = [ + "manual", "no-sandbox", "requires-network", ], diff --git a/cmd/blobstore/BUILD.bazel b/cmd/blobstore/BUILD.bazel index b0f8ed60c68..7ebe0564eb7 100644 --- a/cmd/blobstore/BUILD.bazel +++ b/cmd/blobstore/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "blobstore_lib", @@ -21,3 +25,95 @@ go_binary( embed = [":blobstore_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_blobstore", + srcs = [":blobstore"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/blobstore", + ], + tars = [":tar_blobstore"], +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["blobstore:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +# The rules below covers the old blobstore, which is based on s3 proxy. We don't push the newer one, as it's still considered experimental. +oci_image( + name = "s3_proxy_image", + base = "@wolfi_s3proxy_base", + entrypoint = [ + "/sbin/tini", + "--", + "/opt/s3proxy/run-docker-container.sh", + ], + env = { + "LOG_LEVEL": "info", + "S3PROXY_AUTHORIZATION": "none", + "S3PROXY_ENDPOINT": "http://0.0.0.0:9000", + "S3PROXY_IDENTITY": "local-identity", + "S3PROXY_CREDENTIAL": "local-credential", + "S3PROXY_VIRTUALHOST": "", + "S3PROXY_CORS_ALLOW_ALL": "false", + "S3PROXY_CORS_ALLOW_ORIGINS": "", + "S3PROXY_CORS_ALLOW_METHODS": "", + "S3PROXY_CORS_ALLOW_HEADERS": "", + "S3PROXY_IGNORE_UNKNOWN_HEADERS": "false", + "S3PROXY_ENCRYPTED_BLOBSTORE": "", + "S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD": "", + "S3PROXY_ENCRYPTED_BLOBSTORE_SALT": "", + "S3PROXY_V4_MAX_NON_CHUNKED_REQ_SIZE": "33554432", + "JCLOUDS_PROVIDER": "filesystem", + "JCLOUDS_ENDPOINT": "", + "JCLOUDS_REGION": "", + "JCLOUDS_REGIONS": "us-east-1", + "JCLOUDS_IDENTITY": "remote-identity", + "JCLOUDS_CREDENTIAL": "remote-credential", + "JCLOUDS_KEYSTONE_VERSION": "", + "JCLOUDS_KEYSTONE_SCOPE": "", + "JCLOUDS_KEYSTONE_PROJECT_DOMAIN_NAME": "", + "JCLOUDS_FILESYSTEM_BASEDIR": "/data", + }, + user = "sourcegraph", +) + +container_structure_test( + name = "s3_proxy_image_test", + timeout = "short", + configs = ["s3_proxy_image_test.yaml"], + driver = "docker", + image = ":s3_proxy_image", + tags = ["requires-network"], +) + +oci_tarball( + name = "s3_proxy_image_tarball", + image = ":s3_proxy_image", + repo_tags = ["blobstore:candidate"], +) + +oci_push( + name = "s3_proxy_candidate_push", + image = ":s3_proxy_image", + remote_tags = "//:tags", + repository = image_repository("blobstore"), +) diff --git a/cmd/blobstore/image_test.yaml b/cmd/blobstore/image_test.yaml new file mode 100644 index 00000000000..145614567be --- /dev/null +++ b/cmd/blobstore/image_test.yaml @@ -0,0 +1,16 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/blobstore" + envVars: + - key: "SANITY_CHECK" + value: "true" + + # TODO(security): This container should not be running as root + # - name: "not running as root" + # command: "/usr/bin/id" + # args: + # - -u + # excludedOutput: ["^0"] + # exitCode: 0 diff --git a/cmd/blobstore/s3_proxy_image_test.yaml b/cmd/blobstore/s3_proxy_image_test.yaml new file mode 100644 index 00000000000..f6bb9e74611 --- /dev/null +++ b/cmd/blobstore/s3_proxy_image_test.yaml @@ -0,0 +1,16 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: +- name: 'Test for run-docker-container.sh' + path: '/opt/s3proxy/run-docker-container.sh' + shouldExist: true + uid: 0 + gid: 0 diff --git a/cmd/frontend/BUILD.bazel b/cmd/frontend/BUILD.bazel index a4b61106ad2..ce3c96ee0b6 100644 --- a/cmd/frontend/BUILD.bazel +++ b/cmd/frontend/BUILD.bazel @@ -1,4 +1,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") go_library( name = "frontend_lib", @@ -23,3 +26,49 @@ go_binary( "github.com/sourcegraph/sourcegraph/internal/version.timestamp": "{VERSION_TIMESTAMP}", }, ) + +pkg_tar( + name = "tar_frontend", + srcs = [":frontend"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/frontend", + ], + env = { + "CONFIGURATION_MODE": "server", + "PGDATABASE": "sg", + "PGHOST": "pgsql", + "PGPORT": "5432", + "PGSSLMODE": "disable", + "PGUSER": "sg", + "CODEINTEL_PGDATABASE": "sg", + "CODEINTEL_PGHOST": "codeintel-db", + "CODEINTEL_PGPORT": "5432", + "CODEINTEL_PGSSLMODE": "disable", + "CODEINTEL_PGUSER": "sg", + "PUBLIC_REPO_REDIRECTS": "true", + }, + tars = [":tar_frontend"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["frontend:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) diff --git a/cmd/frontend/image_test.yaml b/cmd/frontend/image_test.yaml new file mode 100644 index 00000000000..0e9b870136c --- /dev/null +++ b/cmd/frontend/image_test.yaml @@ -0,0 +1,28 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/frontend" + envVars: + - key: "SANITY_CHECK" + value: "true" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: +- name: '/mnt/cache/frontend' + path: '/mnt/cache/frontend' + shouldExist: true + uid: 100 + gid: 101 + +metadataTest: + envVars: + - key: PGHOST + value: .+ + isRegex: true diff --git a/cmd/github-proxy/BUILD.bazel b/cmd/github-proxy/BUILD.bazel index b83d7e95439..20f7023f797 100644 --- a/cmd/github-proxy/BUILD.bazel +++ b/cmd/github-proxy/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "github-proxy_lib", @@ -21,3 +25,45 @@ go_binary( embed = [":github-proxy_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_github-proxy", + srcs = [":github-proxy"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/github-proxy", + ], + env = { + "LOG_REQUEST": "true", + }, + tars = [":tar_github-proxy"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["github-proxy:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("github-proxy"), +) diff --git a/cmd/github-proxy/image_test.yaml b/cmd/github-proxy/image_test.yaml new file mode 100644 index 00000000000..b4ac3123203 --- /dev/null +++ b/cmd/github-proxy/image_test.yaml @@ -0,0 +1,20 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/github-proxy" + envVars: + - key: "SANITY_CHECK" + value: "true" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +metadataTest: + envVars: + - key: LOG_REQUEST + value: true diff --git a/cmd/gitserver/BUILD.bazel b/cmd/gitserver/BUILD.bazel index c7caa4f8387..e9b8a2ce06d 100644 --- a/cmd/gitserver/BUILD.bazel +++ b/cmd/gitserver/BUILD.bazel @@ -1,4 +1,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") go_library( name = "gitserver_lib", @@ -21,3 +24,52 @@ go_binary( embed = [":gitserver_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_gitserver", + srcs = [":gitserver"], +) + +pkg_tar( + name = "tar_p4_fusion_wrappers", + srcs = [ + "p4-fusion-wrapper-detect-kill.sh", + "process-stats-watcher.sh", + ], + package_dir = "/usr/local/bin", + remap_paths = { + "/p4-fusion-wrapper-detect-kill.sh": "/p4-fusion", + }, + visibility = ["//visibility:public"], +) + +oci_image( + name = "image", + base = "@wolfi_gitserver_base", + entrypoint = [ + "/sbin/tini", + "--", + "/gitserver", + ], + tars = [ + ":tar_gitserver", + ":tar_p4_fusion_wrappers", + ], + user = "sourcegraph", + workdir = "/", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["gitserver:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) diff --git a/cmd/gitserver/image_test.yaml b/cmd/gitserver/image_test.yaml new file mode 100644 index 00000000000..445fa88f14f --- /dev/null +++ b/cmd/gitserver/image_test.yaml @@ -0,0 +1,68 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/gitserver" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "git is runnable" + command: "git" + args: + - version + - name: "git-lfs is runnable" + command: "git-lfs" + args: + - version + - name: "git p4 is runnable" + command: "git" + args: + - p4 + expectedOutput: ["valid commands: submit"] + exitCode: 2 + - name: "ssh is runnable" + command: "ssh" + exitCode: 255 + - name: "python3 is runnable" + command: "python3" + args: + - --version + - name: "bash is runnable" + command: "bash" + args: + - --version + - name: "p4 is runnable" + command: "p4" + args: + - -h + - name: "coursier is runnable" + command: "coursier" + - name: "p4-fusion is runnable" + command: "p4-fusion-binary" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: +- name: '/data/repos' + path: '/data/repos' + shouldExist: true + uid: 100 + gid: 101 +# p4-fusion wrappers +- name: '/usr/local/bin/p4-fusion' + path: '/usr/local/bin/p4-fusion' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +- name: '/usr/local/bin/process-stats-watcher.sh' + path: '/usr/local/bin/process-stats-watcher.sh' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' diff --git a/cmd/loadtest/BUILD.bazel b/cmd/loadtest/BUILD.bazel index ef6f5ae4a9b..eb6bbdefebd 100644 --- a/cmd/loadtest/BUILD.bazel +++ b/cmd/loadtest/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "loadtest_lib", @@ -22,3 +26,41 @@ go_binary( embed = [":loadtest_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_loadtest", + srcs = [":loadtest"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/loadtest", + ], + tars = [":tar_loadtest"], +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["loadtest:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("loadtest"), +) diff --git a/cmd/loadtest/image_test.yaml b/cmd/loadtest/image_test.yaml new file mode 100644 index 00000000000..ac959c3ca9a --- /dev/null +++ b/cmd/loadtest/image_test.yaml @@ -0,0 +1,16 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/loadtest" + envVars: + - key: "SANITY_CHECK" + value: "true" + + # TODO(security): This container should not be running as root + # - name: "not running as root" + # command: "/usr/bin/id" + # args: + # - -u + # excludedOutput: ["^0"] + # exitCode: 0 diff --git a/cmd/migrator/BUILD.bazel b/cmd/migrator/BUILD.bazel index fe31f586b14..9d3eabe375e 100644 --- a/cmd/migrator/BUILD.bazel +++ b/cmd/migrator/BUILD.bazel @@ -1,4 +1,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") go_library( name = "migrator_lib", @@ -23,3 +26,315 @@ go_binary( embed = [":migrator_lib"], visibility = ["//visibility:public"], ) + +genrule( + name = "schema_descriptions", + srcs = ["generate.sh"], + outs = [ + "schema-descriptions/v3.20.0-internal_database_schema.json", + "schema-descriptions/v3.20.1-internal_database_schema.json", + "schema-descriptions/v3.21.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.21.0-internal_database_schema.json", + "schema-descriptions/v3.21.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.21.1-internal_database_schema.json", + "schema-descriptions/v3.21.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.21.2-internal_database_schema.json", + "schema-descriptions/v3.22.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.22.0-internal_database_schema.json", + "schema-descriptions/v3.22.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.22.1-internal_database_schema.json", + "schema-descriptions/v3.23.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.23.0-internal_database_schema.json", + "schema-descriptions/v3.24.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.24.0-internal_database_schema.json", + "schema-descriptions/v3.24.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.24.1-internal_database_schema.json", + "schema-descriptions/v3.25.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.25.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.25.0-internal_database_schema.json", + "schema-descriptions/v3.25.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.25.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.25.1-internal_database_schema.json", + "schema-descriptions/v3.25.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.25.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.25.2-internal_database_schema.json", + "schema-descriptions/v3.26.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.26.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.26.0-internal_database_schema.json", + "schema-descriptions/v3.26.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.26.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.26.1-internal_database_schema.json", + "schema-descriptions/v3.26.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.26.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.26.2-internal_database_schema.json", + "schema-descriptions/v3.26.3-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.26.3-internal_database_schema.codeintel.json", + "schema-descriptions/v3.26.3-internal_database_schema.json", + "schema-descriptions/v3.27.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.27.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.27.0-internal_database_schema.json", + "schema-descriptions/v3.27.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.27.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.27.1-internal_database_schema.json", + "schema-descriptions/v3.27.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.27.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.27.2-internal_database_schema.json", + "schema-descriptions/v3.27.3-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.27.3-internal_database_schema.codeintel.json", + "schema-descriptions/v3.27.3-internal_database_schema.json", + "schema-descriptions/v3.27.4-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.27.4-internal_database_schema.codeintel.json", + "schema-descriptions/v3.27.4-internal_database_schema.json", + "schema-descriptions/v3.27.5-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.27.5-internal_database_schema.codeintel.json", + "schema-descriptions/v3.27.5-internal_database_schema.json", + "schema-descriptions/v3.28.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.28.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.28.0-internal_database_schema.json", + "schema-descriptions/v3.29.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.29.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.29.0-internal_database_schema.json", + "schema-descriptions/v3.29.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.29.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.29.1-internal_database_schema.json", + "schema-descriptions/v3.30.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.30.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.30.0-internal_database_schema.json", + "schema-descriptions/v3.30.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.30.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.30.1-internal_database_schema.json", + "schema-descriptions/v3.30.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.30.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.30.2-internal_database_schema.json", + "schema-descriptions/v3.30.3-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.30.3-internal_database_schema.codeintel.json", + "schema-descriptions/v3.30.3-internal_database_schema.json", + "schema-descriptions/v3.30.4-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.30.4-internal_database_schema.codeintel.json", + "schema-descriptions/v3.30.4-internal_database_schema.json", + "schema-descriptions/v3.31.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.31.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.31.0-internal_database_schema.json", + "schema-descriptions/v3.31.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.31.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.31.1-internal_database_schema.json", + "schema-descriptions/v3.31.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.31.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.31.2-internal_database_schema.json", + "schema-descriptions/v3.32.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.32.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.32.0-internal_database_schema.json", + "schema-descriptions/v3.32.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.32.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.32.1-internal_database_schema.json", + "schema-descriptions/v3.33.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.33.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.33.0-internal_database_schema.json", + "schema-descriptions/v3.33.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.33.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.33.1-internal_database_schema.json", + "schema-descriptions/v3.33.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.33.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.33.2-internal_database_schema.json", + "schema-descriptions/v3.34.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.34.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.34.0-internal_database_schema.json", + "schema-descriptions/v3.34.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.34.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.34.1-internal_database_schema.json", + "schema-descriptions/v3.34.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.34.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.34.2-internal_database_schema.json", + "schema-descriptions/v3.35.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.35.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.35.0-internal_database_schema.json", + "schema-descriptions/v3.35.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.35.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.35.1-internal_database_schema.json", + "schema-descriptions/v3.35.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.35.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.35.2-internal_database_schema.json", + "schema-descriptions/v3.36.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.36.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.36.0-internal_database_schema.json", + "schema-descriptions/v3.36.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.36.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.36.1-internal_database_schema.json", + "schema-descriptions/v3.36.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.36.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.36.2-internal_database_schema.json", + "schema-descriptions/v3.36.3-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.36.3-internal_database_schema.codeintel.json", + "schema-descriptions/v3.36.3-internal_database_schema.json", + "schema-descriptions/v3.37.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.37.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.37.0-internal_database_schema.json", + "schema-descriptions/v3.38.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.38.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.38.0-internal_database_schema.json", + "schema-descriptions/v3.38.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.38.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.38.1-internal_database_schema.json", + "schema-descriptions/v3.39.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.39.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.39.0-internal_database_schema.json", + "schema-descriptions/v3.39.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.39.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.39.1-internal_database_schema.json", + "schema-descriptions/v3.40.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.40.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.40.0-internal_database_schema.json", + "schema-descriptions/v3.40.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.40.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.40.1-internal_database_schema.json", + "schema-descriptions/v3.40.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.40.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.40.2-internal_database_schema.json", + "schema-descriptions/v3.41.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.41.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.41.0-internal_database_schema.json", + "schema-descriptions/v3.41.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.41.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.41.1-internal_database_schema.json", + "schema-descriptions/v3.42.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.42.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.42.0-internal_database_schema.json", + "schema-descriptions/v3.42.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.42.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.42.1-internal_database_schema.json", + "schema-descriptions/v3.42.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.42.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.42.2-internal_database_schema.json", + "schema-descriptions/v3.43.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.43.0-internal_database_schema.codeintel.json", + "schema-descriptions/v3.43.0-internal_database_schema.json", + "schema-descriptions/v3.43.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.43.1-internal_database_schema.codeintel.json", + "schema-descriptions/v3.43.1-internal_database_schema.json", + "schema-descriptions/v3.43.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v3.43.2-internal_database_schema.codeintel.json", + "schema-descriptions/v3.43.2-internal_database_schema.json", + "schema-descriptions/v4.0.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.0.0-internal_database_schema.codeintel.json", + "schema-descriptions/v4.0.0-internal_database_schema.json", + "schema-descriptions/v4.0.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.0.1-internal_database_schema.codeintel.json", + "schema-descriptions/v4.0.1-internal_database_schema.json", + "schema-descriptions/v4.1.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.1.0-internal_database_schema.codeintel.json", + "schema-descriptions/v4.1.0-internal_database_schema.json", + "schema-descriptions/v4.1.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.1.1-internal_database_schema.codeintel.json", + "schema-descriptions/v4.1.1-internal_database_schema.json", + "schema-descriptions/v4.1.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.1.2-internal_database_schema.codeintel.json", + "schema-descriptions/v4.1.2-internal_database_schema.json", + "schema-descriptions/v4.1.3-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.1.3-internal_database_schema.codeintel.json", + "schema-descriptions/v4.1.3-internal_database_schema.json", + "schema-descriptions/v4.2.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.2.0-internal_database_schema.codeintel.json", + "schema-descriptions/v4.2.0-internal_database_schema.json", + "schema-descriptions/v4.2.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.2.1-internal_database_schema.codeintel.json", + "schema-descriptions/v4.2.1-internal_database_schema.json", + "schema-descriptions/v4.3.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.3.0-internal_database_schema.codeintel.json", + "schema-descriptions/v4.3.0-internal_database_schema.json", + "schema-descriptions/v4.3.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.3.1-internal_database_schema.codeintel.json", + "schema-descriptions/v4.3.1-internal_database_schema.json", + "schema-descriptions/v4.4.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.4.0-internal_database_schema.codeintel.json", + "schema-descriptions/v4.4.0-internal_database_schema.json", + "schema-descriptions/v4.4.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.4.1-internal_database_schema.codeintel.json", + "schema-descriptions/v4.4.1-internal_database_schema.json", + "schema-descriptions/v4.4.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.4.2-internal_database_schema.codeintel.json", + "schema-descriptions/v4.4.2-internal_database_schema.json", + "schema-descriptions/v4.5.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.5.0-internal_database_schema.codeintel.json", + "schema-descriptions/v4.5.0-internal_database_schema.json", + "schema-descriptions/v4.5.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v4.5.1-internal_database_schema.codeintel.json", + "schema-descriptions/v4.5.1-internal_database_schema.json", + "schema-descriptions/v5.0.0-internal_database_schema.codeinsights.json", + "schema-descriptions/v5.0.0-internal_database_schema.codeintel.json", + "schema-descriptions/v5.0.0-internal_database_schema.json", + "schema-descriptions/v5.0.1-internal_database_schema.codeinsights.json", + "schema-descriptions/v5.0.1-internal_database_schema.codeintel.json", + "schema-descriptions/v5.0.1-internal_database_schema.json", + "schema-descriptions/v5.0.2-internal_database_schema.codeinsights.json", + "schema-descriptions/v5.0.2-internal_database_schema.codeintel.json", + "schema-descriptions/v5.0.2-internal_database_schema.json", + "schema-descriptions/v5.0.3-internal_database_schema.codeinsights.json", + "schema-descriptions/v5.0.3-internal_database_schema.codeintel.json", + "schema-descriptions/v5.0.3-internal_database_schema.json", + "schema-descriptions/v5.0.4-internal_database_schema.codeinsights.json", + "schema-descriptions/v5.0.4-internal_database_schema.codeintel.json", + "schema-descriptions/v5.0.4-internal_database_schema.json", + ], + cmd = "$(location generate.sh) $(@D)", + tags = ["requires-network"], + visibility = ["//visibility:public"], +) + +sh_test( + name = "schema_descriptions_test", + size = "small", + srcs = [ + "schema_descriptions_test.sh", + ], + args = [ + "$(location generate.sh)", + "$(locations :schema_descriptions)", + ], + data = [ + "generate.sh", + ":schema_descriptions", + ], + tags = ["requires-network"], +) + +pkg_tar( + name = "tar_schema_descriptions", + srcs = [":schema_descriptions"], + package_dir = "schema-descriptions", + visibility = ["//enterprise/cmd/migrator:__pkg__"], +) + +pkg_tar( + name = "tar_migrator", + srcs = [":migrator"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/migrator", + ], + tars = [ + ":tar_migrator", + ":tar_schema_descriptions", + ], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["migrator:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) diff --git a/cmd/migrator/build.sh b/cmd/migrator/build.sh index b9dd4dbb524..87ad08e93fe 100755 --- a/cmd/migrator/build.sh +++ b/cmd/migrator/build.sh @@ -47,6 +47,7 @@ gcs_filenames=( function download_gcs() { outfile="${OUTPUT}/schema-descriptions/${1}-${2}" + echo "${outfile}" if ! curl -fsSL "https://storage.googleapis.com/sourcegraph-assets/migrations/drift/${1}-${2}" 2>/dev/null >"${outfile}"; then rm "${outfile}" fi diff --git a/cmd/migrator/generate.sh b/cmd/migrator/generate.sh new file mode 100755 index 00000000000..5b72c9cd497 --- /dev/null +++ b/cmd/migrator/generate.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# This script generates all the schema-descriptions files. + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." +set -eu + +OUTPUT="$1" + +echo "Compiling schema descriptions ..." +mkdir -p "${OUTPUT}/schema-descriptions" + +# See internal/database/migration/cliutil/drift-schemas/generate-all.sh +gcs_versions=( + v3.20.0 v3.20.1 + v3.21.0 v3.21.1 v3.21.2 + v3.22.0 v3.22.1 + v3.23.0 + v3.24.0 v3.24.1 + v3.25.0 v3.25.1 v3.25.2 + v3.26.0 v3.26.1 v3.26.2 v3.26.3 + v3.27.0 v3.27.1 v3.27.2 v3.27.3 v3.27.4 v3.27.5 + v3.28.0 + v3.29.0 v3.29.1 + v3.30.0 v3.30.1 v3.30.2 v3.30.3 v3.30.4 + v3.31.0 v3.31.1 v3.31.2 + v3.32.0 v3.32.1 + v3.33.0 v3.33.1 v3.33.2 + v3.34.0 v3.34.1 v3.34.2 + v3.35.0 v3.35.1 v3.35.2 + v3.36.0 v3.36.1 v3.36.2 v3.36.3 + v3.37.0 + v3.38.0 v3.38.1 + v3.39.0 v3.39.1 + v3.40.0 v3.40.1 v3.40.2 + v3.41.0 v3.41.1 +) +gcs_filenames=( + internal_database_schema.json + internal_database_schema.codeintel.json + internal_database_schema.codeinsights.json +) + +function download_gcs() { + outfile="${OUTPUT}/schema-descriptions/${1}-${2}" + # 3.20.0 is missing the codeintel and codeinsights schemas. + if ! curl -fsSL "https://storage.googleapis.com/sourcegraph-assets/migrations/drift/${1}-${2}" >"${outfile}"; then + rm "${outfile}" + fi +} + +for version in "${gcs_versions[@]}"; do + echo "Persisting schemas for ${version} from GCS..." + for filename in "${gcs_filenames[@]}"; do + download_gcs "${version}" "${filename}" + done +done + +function download_github() { + local version + version="$1" + local github_url + github_url="https://raw.githubusercontent.com/sourcegraph/sourcegraph/${version}/internal/database" + + curl -fsSL "$github_url/schema.json" >"${OUTPUT}/schema-descriptions/${version}-internal_database_schema.json" + curl -fsSL "$github_url/schema.codeintel.json" >"${OUTPUT}/schema-descriptions/${version}-internal_database_schema.codeintel.json" + curl -fsSL "$github_url/schema.codeinsights.json" >"${OUTPUT}/schema-descriptions/${version}-internal_database_schema.codeinsights.json" +} + +git_versions=( + v3.42.0 v3.42.1 v3.42.2 + v3.43.0 v3.43.1 v3.43.2 + v4.0.0 v4.0.1 + v4.1.0 v4.1.1 v4.1.2 v4.1.3 + v4.2.0 v4.2.1 + v4.3.0 v4.3.1 + v4.4.0 v4.4.1 v4.4.2 + v4.5.0 v4.5.1 + v5.0.0 v5.0.1 v5.0.2 v5.0.3 v5.0.4 +) + +for version in "${git_versions[@]}"; do + echo "Persisting schemas for ${version} from GitHub..." + download_github "${version}" +done + diff --git a/cmd/migrator/image_test.yaml b/cmd/migrator/image_test.yaml new file mode 100644 index 00000000000..b4e8b566352 --- /dev/null +++ b/cmd/migrator/image_test.yaml @@ -0,0 +1,62 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/migrator" + envVars: + - key: "SANITY_CHECK" + value: "true" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: + # Following files are fetched through GCS +- name: '/schema-descriptions 3.20.0 schema' + path: '/schema-descriptions/v3.20.0-internal_database_schema.json' + shouldExist: true + uid: 0 + gid: 0 +- name: '/schema-descriptions 3.20.0 schema.codeintel does not exists' + path: '/schema-descriptions/v3.20.0-internal_database_schema.codeintel.json' + shouldExist: false + uid: 0 + gid: 0 + +- name: '/schema-descriptions 3.21.0 schema' + path: '/schema-descriptions/v3.21.0-internal_database_schema.json' + shouldExist: true + uid: 0 + gid: 0 +- name: '/schema-descriptions 3.21.0 schema.codeintel' + path: '/schema-descriptions/v3.21.0-internal_database_schema.codeintel.json' + shouldExist: true + uid: 0 + gid: 0 +- name: '/schema-descriptions 3.21.0 schema.codeinsights does not exists' + # We don't have codeinsights for that version, there should not be a file + path: '/schema-descriptions/v3.21.0-internal_database_schema.codeinsights.json' + shouldExist: false + uid: 0 + gid: 0 + + # Following files are fetched through GitHub raw HTTP requests +- name: '/schema-descriptions 5.0.1 schema' + path: '/schema-descriptions/v5.0.1-internal_database_schema.json' + shouldExist: true + uid: 0 + gid: 0 +- name: '/schema-descriptions 5.0.1 schema.codeintel' + path: '/schema-descriptions/v5.0.1-internal_database_schema.codeintel.json' + shouldExist: true + uid: 0 + gid: 0 +- name: '/schema-descriptions 5.0.1 schema.codeinsights' + path: '/schema-descriptions/v5.0.1-internal_database_schema.codeinsights.json' + shouldExist: true + uid: 0 + gid: 0 diff --git a/cmd/migrator/schema_descriptions_test.sh b/cmd/migrator/schema_descriptions_test.sh new file mode 100755 index 00000000000..662cca97432 --- /dev/null +++ b/cmd/migrator/schema_descriptions_test.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Path to the schema_descriptions tool +generate_bin="$1" + +# Array of paths for each of the outputs from the :generate_config target. +# shellcheck disable=SC2124 +got_files="${@:2}" + +# Manually run the script again, so have a list of all the files +# we expect the :schema_descriptions target to output. +# +# We put them in the ./expected folder. +"$generate_bin" expected/ + +# Loop over all of them and check if we can find each of them in the +# outputs from :schema_descriptions target. +for file in expected/**/*; do + # Trim the "expected" part of the path + want="${file##expected}" + found="false" + + # Loop over all files we got. + # shellcheck disable=SC2068 + for got in ${got_files[@]}; do + # Trim the path from the "monitoring/output" prefix + # and test it against the expected file we're currently iterating with. + if [[ "${got##cmd/migrator}" == "$want" ]]; then + found="true" + break + fi + done + + # If we didn't find it, return an error. + if [[ $found == "false" ]]; then + echo "Couldn't find expected output $want, perhaps it's missing from the 'srcs' attribute?" + exit 1 + fi +done + diff --git a/cmd/repo-updater/BUILD.bazel b/cmd/repo-updater/BUILD.bazel index ecd18553125..63b51fec9e6 100644 --- a/cmd/repo-updater/BUILD.bazel +++ b/cmd/repo-updater/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "repo-updater_lib", @@ -21,3 +25,42 @@ go_binary( "github.com/sourcegraph/sourcegraph/internal/version.timestamp": "{VERSION_TIMESTAMP}", }, ) + +pkg_tar( + name = "tar_repo-updater", + srcs = [":repo-updater"], +) + +oci_image( + name = "image", + base = "@wolfi_repo_updater_base", + entrypoint = [ + "/sbin/tini", + "--", + "/repo-updater", + ], + tars = [":tar_repo-updater"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["repo-updater:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("repo-updater"), +) diff --git a/cmd/repo-updater/image_test.yaml b/cmd/repo-updater/image_test.yaml new file mode 100644 index 00000000000..6bf04e8b1ed --- /dev/null +++ b/cmd/repo-updater/image_test.yaml @@ -0,0 +1,21 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/repo-updater" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "p4 is runnable" + command: "p4" + args: + - -h + - name: "coursier is runnable" + command: "coursier" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/cmd/searcher/BUILD.bazel b/cmd/searcher/BUILD.bazel index 41ab6efed96..958d4c1756f 100644 --- a/cmd/searcher/BUILD.bazel +++ b/cmd/searcher/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "searcher_lib", @@ -21,3 +25,45 @@ go_binary( "github.com/sourcegraph/sourcegraph/internal/version.timestamp": "{VERSION_TIMESTAMP}", }, ) + +pkg_tar( + name = "tar_searcher", + srcs = [":searcher"], +) + +oci_image( + name = "image", + base = "@wolfi_searcher_base", + entrypoint = [ + "/sbin/tini", + "--", + "/searcher", + ], + env = { + "CACHE_DIR": "/mnt/cache/searcher", + }, + tars = [":tar_searcher"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["searcher:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("searcher"), +) diff --git a/cmd/searcher/Dockerfile.wolfi b/cmd/searcher/Dockerfile.wolfi index b9f977f2242..3586e3edcda 100644 --- a/cmd/searcher/Dockerfile.wolfi +++ b/cmd/searcher/Dockerfile.wolfi @@ -15,7 +15,6 @@ LABEL org.opencontainers.image.version=${VERSION} LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} ENV CACHE_DIR=/mnt/cache/searcher -RUN mkdir -p ${CACHE_DIR} && chown -R sourcegraph:sourcegraph ${CACHE_DIR} USER sourcegraph ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/searcher"] diff --git a/cmd/searcher/image_test.yaml b/cmd/searcher/image_test.yaml new file mode 100644 index 00000000000..17846315686 --- /dev/null +++ b/cmd/searcher/image_test.yaml @@ -0,0 +1,30 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/searcher" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "pcre is runnable" + command: "pcregrep" + args: + - --help + - name: "comby is runnable" + command: "comby" + args: + - -h + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: +- name: '/mnt/cache/searcher' + path: '/mnt/cache/searcher' + shouldExist: true + uid: 100 + gid: 101 diff --git a/cmd/server/BUILD.bazel b/cmd/server/BUILD.bazel index cc2b02a7f2d..dfe1790bde7 100644 --- a/cmd/server/BUILD.bazel +++ b/cmd/server/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("macro.bzl", "container_dependencies", "dependencies_tars") go_library( name = "server_lib", @@ -17,3 +21,133 @@ go_binary( embed = [":server_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_server", + srcs = [":server"], +) + +exports_files(["postgres_exporter.yaml"]) + +pkg_tar( + name = "tar_postgres_exporter_config", + srcs = ["postgres_exporter.yaml"], +) + +pkg_tar( + name = "tar_monitoring_config", + srcs = [ + "//dev/prometheus:prometheus_targets_linux", + "//docker-images/grafana/config", + "//docker-images/prometheus:startup_scripts", + "//docker-images/prometheus/config:base_config", + "//monitoring:generate_config", + ], + remap_paths = { + "monitoring/outputs/docs": "/sg_config_docs", + "monitoring/outputs/prometheus": "/sg_config_prometheus", + "monitoring/outputs/grafana": "/sg_config_grafana/provisioning/dashboards/sourcegraph", + "docker-images/grafana/config": "/sg_config_grafana", + "docker-images/prometheus/config": "/sg_config_prometheus", + "docker-images/prometheus": "", + "dev/prometheus/linux": "/sg_prometheus_add_ons", + }, + strip_prefix = ".", +) + +DEPS = [ + "//cmd/frontend", + "//cmd/github-proxy", + "//cmd/gitserver", + "//cmd/migrator", + "//cmd/repo-updater", + "//cmd/searcher", + "//cmd/symbols", + "//cmd/worker", +] + +ZOEKT_DEPS = [ + "@com_github_sourcegraph_zoekt//cmd/zoekt-archive-index", + "@com_github_sourcegraph_zoekt//cmd/zoekt-git-index", + "@com_github_sourcegraph_zoekt//cmd/zoekt-sourcegraph-indexserver", + "@com_github_sourcegraph_zoekt//cmd/zoekt-webserver", +] + +# Declares rules for building go_cross_binary + pkg_tar for each dep in DEPS +container_dependencies(DEPS) + +container_dependencies(ZOEKT_DEPS) + +# This one is a special case because inside server images, the procfile expects to find it +# under syntax-highlighter instead of syntect_server. +pkg_tar( + name = "tar_syntax-highlighter", + srcs = ["//docker-images/syntax-highlighter:syntect_server"], + remap_paths = {"/syntect_server": "/usr/local/bin/syntax_highlighter"}, +) + +pkg_tar( + name = "tar_scip-ctags", + srcs = ["//docker-images/syntax-highlighter:scip-ctags"], + package_dir = "/usr/local/bin", +) + +pkg_tar( + name = "tar_postgres_optimize", + srcs = ["//cmd/server/rootfs:postgres-optimize.sh"], +) + +# TODO(@jhchabran) prom-wrapper has to be in /bin while we're still +# building the old and new images, because this path is fed to the procfile +# by the server, and cannot be placed at two different places. +pkg_tar( + name = "tar_prom-wrapper", + srcs = ["//docker-images/prometheus/cmd/prom-wrapper"], + package_dir = "/bin", +) + +# Tip: to view exactly what gets built here, you can run: +# bazel cquery '//cmd/server:image' --output build +oci_image( + name = "image", + base = "@wolfi_server_base", + entrypoint = [ + "/sbin/tini", + "--", + "/server", + ], + env = { + "GO111MODULES": "on", + "LANG": "en_US.utf8", + "LC_ALL": "en_US.utf8", + # "PGHOST": "/var/run/postgresql", + }, + tars = [ + ":tar_server", + ":tar_postgres_exporter_config", + ":tar_monitoring_config", + ":tar_syntax-highlighter", + ":tar_scip-ctags", + ":tar_postgres_optimize", + ":tar_prom-wrapper", + "//cmd/gitserver:tar_p4_fusion_wrappers", + ] + dependencies_tars(DEPS) + dependencies_tars(ZOEKT_DEPS), + workdir = "/", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["server:candidate"], +) + +container_structure_test( + name = "image_test", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image_tarball", + tags = [ + "manual", # this test is broken, an init function is checking for the config before reaching the main func. + "requires-network", + ], +) diff --git a/cmd/server/Dockerfile.wolfi b/cmd/server/Dockerfile.wolfi index 90e1fd60bcc..c08b289ce1f 100644 --- a/cmd/server/Dockerfile.wolfi +++ b/cmd/server/Dockerfile.wolfi @@ -61,10 +61,7 @@ RUN env SANITY_CHECK=true /usr/local/bin/symbols WORKDIR / -# TODO: Nginx expects these directories but doesn't create them by default, figure out why -RUN mkdir /var/lib/nginx/tmp /var/run - -# TODO: Check all paths in script still line up ENV GO111MODULES=on -# ENV LANG=en_US.utf8 # TODO: Not setting this seems to fix a postgres startup issue +ENV LANG=en_US.utf8 +ENV PGHOST=/var/run/postgresql ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/server"] diff --git a/cmd/server/image_test.yaml b/cmd/server/image_test.yaml new file mode 100644 index 00000000000..83d8fb4f1a7 --- /dev/null +++ b/cmd/server/image_test.yaml @@ -0,0 +1,20 @@ +commandTests: + - name: "sanity check" + command: "/server" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "scip-ctags is runnable" + command: "/usr/local/bin/scip-ctags" + envVars: + - key: "SANITY_CHECK" + value: "true" + + + # TODO(security): This container should not be running as root + # - name: "not running as root" + # command: "/usr/bin/id" + # args: + # - -u + # excludedOutput: ["^0"] + # exitCode: 0 diff --git a/cmd/server/macro.bzl b/cmd/server/macro.bzl new file mode 100644 index 00000000000..cd7373e0471 --- /dev/null +++ b/cmd/server/macro.bzl @@ -0,0 +1,30 @@ +load("@rules_pkg//:pkg.bzl", "pkg_tar") + +def get_last_segment(path): + segments = path.split("/") + last_segment = segments[-1] + + s = last_segment.split(":") + if len(s) == 1: + return last_segment + else: + return s[-1] + +def container_dependencies(targets): + for target in targets: + name = get_last_segment(target) + + pkg_tar( + name = "tar_{}".format(name), + srcs = [target], + remap_paths = { "/{}".format(name): "/usr/local/bin/{}".format(name) } + ) + +def dependencies_tars(targets): + tars = [] + for target in targets: + name = get_last_segment(target) + tars.append(":tar_{}".format(name)) + + return tars + diff --git a/cmd/server/postgres_exporter.yaml b/cmd/server/postgres_exporter.yaml new file mode 100644 index 00000000000..795ab6fc948 --- /dev/null +++ b/cmd/server/postgres_exporter.yaml @@ -0,0 +1 @@ +auth_modules: diff --git a/cmd/server/rootfs/BUILD.bazel b/cmd/server/rootfs/BUILD.bazel new file mode 100644 index 00000000000..4781fbbce90 --- /dev/null +++ b/cmd/server/rootfs/BUILD.bazel @@ -0,0 +1,3 @@ +exports_files([ + "postgres-optimize.sh", +]) diff --git a/cmd/server/shared/shared.go b/cmd/server/shared/shared.go index 0bce1a50362..6022f3f383b 100644 --- a/cmd/server/shared/shared.go +++ b/cmd/server/shared/shared.go @@ -149,7 +149,7 @@ func Main() { log.Fatal("Failed to setup nginx:", err) } - postgresExporterLine := fmt.Sprintf(`postgres_exporter: env DATA_SOURCE_NAME="%s" postgres_exporter --log.level=%s`, postgresdsn.New("", "postgres", os.Getenv), convertLogLevel(os.Getenv("SRC_LOG_LEVEL"))) + postgresExporterLine := fmt.Sprintf(`postgres_exporter: env DATA_SOURCE_NAME="%s" postgres_exporter --config.file="/postgres_exporter.yaml" --log.level=%s`, postgresdsn.New("", "postgres", os.Getenv), convertLogLevel(os.Getenv("SRC_LOG_LEVEL"))) // TODO: This should be fixed properly. // Tell `gitserver` that its `hostname` is what the others think of as gitserver hostnames. diff --git a/cmd/symbols/BUILD.bazel b/cmd/symbols/BUILD.bazel index 9936f0db6dd..a7d4be912a3 100644 --- a/cmd/symbols/BUILD.bazel +++ b/cmd/symbols/BUILD.bazel @@ -1,4 +1,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") go_library( name = "symbols_lib", @@ -21,3 +24,46 @@ go_binary( "github.com/sourcegraph/sourcegraph/internal/version.timestamp": "{VERSION_TIMESTAMP}", }, ) + +pkg_tar( + name = "tar_symbols", + srcs = [":symbols"], +) + +pkg_tar( + name = "tar_scip-ctags", + srcs = ["//docker-images/syntax-highlighter:scip-ctags"], + package_dir = "/usr/local/bin", +) + +oci_image( + name = "image", + base = "@wolfi_symbols_base", + entrypoint = [ + "/sbin/tini", + "--", + "/symbols", + ], + env = { + "CACHE_DIR": "/mnt/cache/symbols", + }, + tars = [ + ":tar_symbols", + ":tar_scip-ctags", + ], +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["symbols:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) diff --git a/cmd/symbols/Dockerfile.wolfi b/cmd/symbols/Dockerfile.wolfi index 2a5aa8b903a..a60a784123b 100644 --- a/cmd/symbols/Dockerfile.wolfi +++ b/cmd/symbols/Dockerfile.wolfi @@ -63,6 +63,5 @@ COPY --from=symbols-build /symbols /usr/local/bin/symbols RUN env SANITY_CHECK=true /usr/local/bin/symbols ENV CACHE_DIR=/mnt/cache/symbols -RUN mkdir -p ${CACHE_DIR} EXPOSE 3184 ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/symbols"] diff --git a/cmd/symbols/image_test.yaml b/cmd/symbols/image_test.yaml new file mode 100644 index 00000000000..4c9e486a10f --- /dev/null +++ b/cmd/symbols/image_test.yaml @@ -0,0 +1,36 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/symbols" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "ctags is runnable" + command: "universal-ctags" + args: + - --version + - name: "scip-ctags is runnable" + command: "/usr/local/bin/scip-ctags" + envVars: + - key: "SANITY_CHECK" + value: "true" + + # TODO(security): This container should not be running as root + # - name: "not running as root" + # command: "/usr/bin/id" + # args: + # - -u + # excludedOutput: ["^0"] + # exitCode: 0 + +fileExistenceTests: +- name: '/mnt/cache/symbols' + path: '/mnt/cache/symbols' + shouldExist: true + uid: 100 + gid: 101 + permissions: 'drwxr-xr-x' +- name: 'jansson package' + path: 'usr/lib/libjansson.la' + shouldExist: true diff --git a/cmd/worker/BUILD.bazel b/cmd/worker/BUILD.bazel index bf66bd2da8b..9f0542e7a8c 100644 --- a/cmd/worker/BUILD.bazel +++ b/cmd/worker/BUILD.bazel @@ -1,4 +1,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") go_library( name = "worker_lib", @@ -21,3 +24,34 @@ go_binary( "github.com/sourcegraph/sourcegraph/internal/version.timestamp": "{VERSION_TIMESTAMP}", }, ) + +pkg_tar( + name = "tar_worker", + srcs = [":worker"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/worker", + ], + tars = [":tar_worker"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["worker:candidate"], +) + +container_structure_test( + name = "image_test", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) diff --git a/cmd/worker/image_test.yaml b/cmd/worker/image_test.yaml new file mode 100644 index 00000000000..d197fa98aeb --- /dev/null +++ b/cmd/worker/image_test.yaml @@ -0,0 +1,15 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/worker" + envVars: + - key: "SANITY_CHECK" + value: "true" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/cmd/worker/internal/migrations/version.go b/cmd/worker/internal/migrations/version.go index fcd6a878829..cc1209c013a 100644 --- a/cmd/worker/internal/migrations/version.go +++ b/cmd/worker/internal/migrations/version.go @@ -34,6 +34,13 @@ func currentVersion(logger log.Logger) (oobmigration.Version, error) { return version, nil } + // TODO: @jhchabran + // The infer mechanism doesn't work in CI, because we weren't expecting to run a container + // with a 0.0.0+dev version. This fixes it. We should come back to this. + if version.IsDev(version.Version()) && os.Getenv("BAZEL_SKIP_OOB_INFER_VERSION") != "" { + return oobmigration.NewVersion(5, 99), nil + } + version, err := inferNextReleaseVersion() if err != nil { return oobmigration.Version{}, err diff --git a/dev/authtest/BUILD.bazel b/dev/authtest/BUILD.bazel index 8792107bfc4..e82a2ba3bb6 100644 --- a/dev/authtest/BUILD.bazel +++ b/dev/authtest/BUILD.bazel @@ -10,6 +10,7 @@ go_test( "repository_test.go", "site_admin_test.go", ], + visibility = ["//testing:__subpackages__"], deps = [ "//internal/auth", "//internal/extsvc", diff --git a/dev/backcompat/flakes.bzl b/dev/backcompat/flakes.bzl index 472a7a9fbf8..87e02615469 100644 --- a/dev/backcompat/flakes.bzl +++ b/dev/backcompat/flakes.bzl @@ -22,6 +22,6 @@ FLAKES = { "path": "enterprise/internal/codeintel/ranking/internal/store", "prefix": "Test", "reason": "Shifting constraints on table; ranking is experimental" - } + }, ] } diff --git a/dev/bazel_stamp_vars.sh b/dev/bazel_stamp_vars.sh index 632efbe7137..2c37372f645 100755 --- a/dev/bazel_stamp_vars.sh +++ b/dev/bazel_stamp_vars.sh @@ -1,6 +1,16 @@ #!/usr/bin/env bash -stamp_version="${VERSION:-$(git rev-parse HEAD)}" +build_number="${BUILDKITE_BUILD_NUMBER:-000000}" +date_fragment="$(date +%y-%m-%d)" +latest_tag="5.0" +stamp_version="${build_number}_${date_fragment}_${latest_tag}-$(git rev-parse --short HEAD)-bazel-qa" echo STABLE_VERSION "$stamp_version" echo VERSION_TIMESTAMP "$(date +%s)" + +# Unstable Buildkite env vars +echo "BUILDKITE $BUILDKITE" +echo "BUILDKITE_COMMIT $BUILDKITE_COMMIT" +echo "BUILDKITE_BRANCH $BUILDKITE_BRANCH" +echo "BUILDKITE_PULL_REQUEST_REPO $BUILDKITE_PULL_REQUEST_REPO" +echo "BUILDKITE_PULL_REQUEST $BUILDKITE_PULL_REQUEST" diff --git a/dev/ci/integration/code-intel/BUILD.bazel b/dev/ci/integration/code-intel/BUILD.bazel new file mode 100644 index 00000000000..0ca12b35b63 --- /dev/null +++ b/dev/ci/integration/code-intel/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["repos.json"]) diff --git a/dev/ci/integration/e2e/test.sh b/dev/ci/integration/e2e/test.sh index c8d61a55f92..28da690af9c 100755 --- a/dev/ci/integration/e2e/test.sh +++ b/dev/ci/integration/e2e/test.sh @@ -3,12 +3,18 @@ cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." set -e -URL="${1:-"http://localhost:7080"}" +# URL="${1:-"http://localhost:7080"}" -echo "--- pnpm run test-e2e" -env SOURCEGRAPH_BASE_URL="$URL" pnpm run cover-e2e +echo "--- bazel test e2e" +bazel \ + --bazelrc=.bazelrc \ + --bazelrc=.aspect/bazelrc/ci.bazelrc \ + --bazelrc=.aspect/bazelrc/ci.sourcegraph.bazelrc \ + test \ + //client/web/src/end-to-end:e2e +# env SOURCEGRAPH_BASE_URL="$URL" pnpm run cover-e2e -echo "--- coverage" -pnpm nyc report -r json -# Upload the coverage under the "e2e" flag (toggleable in the CodeCov UI) -./dev/ci/codecov.sh -F e2e +# echo "--- coverage" +# pnpm nyc report -r json +# # Upload the coverage under the "e2e" flag (toggleable in the CodeCov UI) +# ./dev/ci/codecov.sh -F e2e diff --git a/dev/ci/integration/setup-deps.sh b/dev/ci/integration/setup-deps.sh index e69c4024a4d..ab6bf997705 100755 --- a/dev/ci/integration/setup-deps.sh +++ b/dev/ci/integration/setup-deps.sh @@ -2,8 +2,5 @@ set -euo pipefail -pnpm install --frozen-lockfile -pnpm generate - curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /usr/local/bin/src chmod +x /usr/local/bin/src diff --git a/dev/ci/integration/upgrade/run.sh b/dev/ci/integration/upgrade/run.sh index 4f3906540ad..41442954ac2 100755 --- a/dev/ci/integration/upgrade/run.sh +++ b/dev/ci/integration/upgrade/run.sh @@ -7,6 +7,10 @@ root_dir=$(pwd) set -ex +# Install dependencies for upgrade test script +pnpm install --frozen-lockfile +pnpm generate + dev/ci/integration/setup-deps.sh dev/ci/integration/setup-display.sh diff --git a/dev/codeintel-qa/cmd/download/main.go b/dev/codeintel-qa/cmd/download/main.go index e32abcf4ee6..b029190cbef 100644 --- a/dev/codeintel-qa/cmd/download/main.go +++ b/dev/codeintel-qa/cmd/download/main.go @@ -70,7 +70,13 @@ func getPaths(ctx context.Context, bucket *storage.BucketHandle) (paths []string func downloadAll(ctx context.Context, bucket *storage.BucketHandle, paths []string) error { repoRoot, err := root.RepositoryRoot() if err != nil { - return err + if err == root.ErrNotInsideSourcegraph && os.Getenv("BAZEL_TEST") != "" { + // If we're running inside Bazel, we do not have access to the repo root. + // In that case, we simply use CWD instead. + repoRoot = "." + } else { + return err + } } indexesDir := filepath.Join(repoRoot, relativeIndexesDir) diff --git a/dev/codeintel-qa/cmd/upload/main.go b/dev/codeintel-qa/cmd/upload/main.go index 3cadadbd774..c9e232f0818 100644 --- a/dev/codeintel-qa/cmd/upload/main.go +++ b/dev/codeintel-qa/cmd/upload/main.go @@ -17,6 +17,7 @@ var ( verbose bool pollInterval time.Duration timeout time.Duration + srcPath string start = time.Now() ) @@ -28,6 +29,7 @@ func init() { flag.BoolVar(&verbose, "verbose", false, "Display full state from graphql") flag.DurationVar(&pollInterval, "poll-interval", time.Second*5, "The time to wait between graphql requests") flag.DurationVar(&timeout, "timeout", 0, "The time it should take to upload and process all targets") + flag.StringVar(&srcPath, "src-path", "src", "Path to src-cli binary") } func main() { diff --git a/dev/codeintel-qa/cmd/upload/upload.go b/dev/codeintel-qa/cmd/upload/upload.go index 41e9ed677ea..80b9b5d0ec2 100644 --- a/dev/codeintel-qa/cmd/upload/upload.go +++ b/dev/codeintel-qa/cmd/upload/upload.go @@ -127,7 +127,7 @@ func upload(ctx context.Context, repoName, commit, file string) (string, error) return "", err } - cmd := exec.CommandContext(ctx, "src", append([]string{"lsif", "upload", "-json"}, args...)...) + cmd := exec.CommandContext(ctx, srcPath, append([]string{"lsif", "upload", "-json"}, args...)...) cmd.Dir = tempDir cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf("SRC_ENDPOINT=%s", internal.SourcegraphEndpoint)) diff --git a/dev/gqltest/BUILD.bazel b/dev/gqltest/BUILD.bazel index 4121a2f3bfe..8089a9e2838 100644 --- a/dev/gqltest/BUILD.bazel +++ b/dev/gqltest/BUILD.bazel @@ -18,6 +18,10 @@ go_test( "site_config_test.go", "sub_repo_permissions_test.go", ], + visibility = [ + "//testing:__pkg__", + "//testing:__subpackages__", + ], deps = [ "//internal/extsvc", "//internal/gqltestutil", diff --git a/dev/oci_defs.bzl b/dev/oci_defs.bzl new file mode 100644 index 00000000000..544f1e7bbfd --- /dev/null +++ b/dev/oci_defs.bzl @@ -0,0 +1,6 @@ +REGISTRY_REPOSITORY_PREFIX = "europe-west1-docker.pkg.dev/sourcegraph-security-logging/rules-oci-test/{}" +# REGISTRY_REPOSITORY_PREFIX = "us.gcr.io/sourcegraph-dev/{}" + +def image_repository(image): + return REGISTRY_REPOSITORY_PREFIX.format(image) + diff --git a/dev/oci_deps.bzl b/dev/oci_deps.bzl new file mode 100644 index 00000000000..4656a74d674 --- /dev/null +++ b/dev/oci_deps.bzl @@ -0,0 +1,173 @@ +load("@rules_oci//oci:pull.bzl", "oci_pull") + +# Quick script to get the latest tags for each of the base images: +# +# grep 'image = ' ./dev/oci_deps.bzl | while read -r str ; do +# str_no_spaces="${str#"${str%%[![:space:]]*}"}" # remove leading spaces +# url="${str_no_spaces#*\"}" # remove prefix until first quote +# url="${url%%\"*}" # remove suffix from first quote +# +# IMAGE_DETAILS=$(gcloud container images list-tags $url --limit=1 --sort-by=~timestamp --format=json) +# TAG=$(echo $IMAGE_DETAILS | jq -r '.[0].tags[0]') +# DIGEST=$(echo $IMAGE_DETAILS | jq -r '.[0].digest') +# +# echo $url +# echo $DIGEST +# done + +def oci_deps(): + oci_pull( + name = "wolfi_base", + digest = "sha256:a236182c8e16e23aafc2d96a0baea17414626064373e38b9abbd5056c3c4d990", + image = "us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base", + ) + + oci_pull( + name = "wolfi_cadvisor_base", + digest = "sha256:11ab7cbe533a01b31d8b8803ec8e6e52074f6d5ba20a96166f7f765db4e8819b", + image = "us.gcr.io/sourcegraph-dev/wolfi-cadvisor-base", + ) + + oci_pull( + name = "wolfi_symbols_base", + digest = "sha256:c1b84bd2c2840bbed25a65fa6ade38755be180f4ca0b60e45b461339073a5dc6", + image = "us.gcr.io/sourcegraph-dev/wolfi-symbols-base", + ) + + oci_pull( + name = "wolfi_server_base", + digest = "sha256:0cded0aabc7509d0ff02cc16f76374730e834d8aadd875f47bb4dc381d4105a4", + image = "us.gcr.io/sourcegraph-dev/wolfi-server-base", + ) + + oci_pull( + name = "wolfi_gitserver_base", + digest = "sha256:ac49e90f580bd7f729c78116e384feb25e8c2e1dc7f1ed69395901e68cfd82d1", + image = "us.gcr.io/sourcegraph-dev/wolfi-gitserver-base", + ) + + oci_pull( + name = "wolfi_grafana_base", + digest = "sha256:ec1049f35ff7e4ab6ff7b4cc6790996ad74d196b8dcee8ea5283fca759156637", + image = "us.gcr.io/sourcegraph-dev/wolfi-grafana", + ) + + oci_pull( + name = "wolfi_postgres_exporter_base", + digest = "sha256:544d4f8a44cd03c7110855654d17490d3d485d69198a8789a2bfa25029a66e09", + image = "us.gcr.io/sourcegraph-dev/wolfi-postgres-exporter-base", + ) + + oci_pull( + name = "wolfi_jaeger_all_in_one_base", + digest = "sha256:f8e416626dcdc7d14894876e0751f2bfa0653169b14c4a929b3834d30cea087d", + image = "us.gcr.io/sourcegraph-dev/wolfi-jaeger-all-in-one-base", + ) + + oci_pull( + name = "wolfi_jaeger_agent_base", + digest = "sha256:aa6ee947778196115d3027eab91bf0d0e0cc91a03d01f5c2854cd6ecf97b089f", + image = "us.gcr.io/sourcegraph-dev/wolfi-jaeger-agent-base", + ) + + oci_pull( + name = "wolfi_redis_base", + digest = "sha256:4945cd8307f1d835b9b9607a1168ecfb84cdc5a5c14eb7c4ba84c08c50741b7b", + image = "us.gcr.io/sourcegraph-dev/wolfi-redis-base", + ) + + oci_pull( + name = "wolfi_redis_exporter_base", + digest = "sha256:d1a3d302a4be9447f2b0863587cb042e69c5cceb4eaac5294c9632b58f285a64", + image = "us.gcr.io/sourcegraph-dev/wolfi-redis-exporter-base", + ) + + oci_pull( + name = "wolfi_syntax_highlighter_base", + digest = "sha256:0c777bb76c4e586702f5367f54e62881f2f0fa5a96a1bd519ebaff1e982d1ef1", + image = "us.gcr.io/sourcegraph-dev/wolfi-syntax-highlighter-base", + ) + + oci_pull( + name = "wolfi_search_indexer_base", + digest = "sha256:991f696b62c4afa2ced41b1071b15e44094a2c541d973a831b34d4a4db4c2131", + image = "us.gcr.io/sourcegraph-dev/wolfi-search-indexer-base", + ) + + oci_pull( + name = "wolfi_repo_updater_base", + digest = "sha256:74e478195b750c5547d6d240bc5d9e94b10d470d6bf2ef855dcd912f83550cdf", + image = "us.gcr.io/sourcegraph-dev/wolfi-repo-updater-base", + ) + + oci_pull( + name = "wolfi_searcher_base", + digest = "sha256:fba8f4cce1306463b03c0388eb830bac63f02f4b8941c4df8f9fde99390da32e", + image = "us.gcr.io/sourcegraph-dev/wolfi-searcher-base", + ) + + oci_pull( + name = "wolfi_executor_base", + digest = "sha256:0ab096b0ffae9054fa18fa8121b105acfda767c5f74fd8530f72f8fe87ef20c2", + image = "us.gcr.io/sourcegraph-dev/wolfi-executor-base", + ) + + oci_pull( + name = "wolfi_bundled_executor_base", + digest = "sha256:8941bfcf8db44c462c4fee057b4d2a128cd268d6fb385989086af4381a05137e", + image = "us.gcr.io/sourcegraph-dev/wolfi-bundled-executor-base", + ) + + oci_pull( + name = "wolfi_executor_kubernetes_base", + digest = "sha256:c2053b17cb8904a09773552049929b73215af24520ca22613cb5b16f96d8bcfa", + image = "us.gcr.io/sourcegraph-dev/wolfi-executor-kubernetes-base", + ) + + oci_pull( + name = "wolfi_batcheshelper_base", + digest = "sha256:2abe940a2d9e13a998d07e4faf072c7ba6e17243a0b9c56a3adf9878d9332f6a", + image = "us.gcr.io/sourcegraph-dev/wolfi-batcheshelper-base", + ) + + oci_pull( + name = "wolfi_prometheus_base", + digest = "sha256:11a84d7ae6a7f3a8954306f224391a2301e5518fbe2d6e8dfee4abe12ca91180", + image = "us.gcr.io/sourcegraph-dev/wolfi-prometheus-base", + ) + + oci_pull( + name = "wolfi_postgresql-12_base", + digest = "sha256:7c8bfb96038fb6b3980fb6b12f692a54ff8f1d1cebbd72d9706578be8d278cae", + image = "us.gcr.io/sourcegraph-dev/wolfi-postgresql-12-base", + ) + + oci_pull( + name = "wolfi_postgresql-12-codeinsights_base", + digest = "sha256:4d85ed245a6d3f22a42cf2fdf840a8e14590fb5216f597a19697e7f3025a0a26", + image = "us.gcr.io/sourcegraph-dev/wolfi-postgresql-12-codeinsights-base", + ) + + oci_pull( + name = "wolfi_node_exporter_base", + digest = "sha256:0a42810eafc6c81f95cb3295003de966d3a857d37dd79ef45a3be54c3b4c8e7c", + image = "us.gcr.io/sourcegraph-dev/wolfi-node-exporter-base", + ) + + oci_pull( + name = "wolfi_opentelemetry_collector_base", + digest = "sha256:d5d55fb77056422eea328b709795a38cf599e42f2a90787c9dd32c2f1a3654f3", + image = "us.gcr.io/sourcegraph-dev/wolfi-opentelemetry-collector-base", + ) + + oci_pull( + name = "wolfi_searcher_base", + digest = "sha256:7ae7d14bc055f5dbcdc727261025e9527558513a61093e956cb39dae7dbc0dcf", + image = "us.gcr.io/sourcegraph-dev/wolfi-searcher-base", + ) + + oci_pull( + name = "wolfi_s3proxy_base", + digest = "sha256:4076564aaa3bfc17a8e50822e9937f240f155a69f9cffcb039a49f892446d28c", + image = "us.gcr.io/sourcegraph-dev/wolfi-blobstore-base", + ) diff --git a/dev/prometheus/BUILD.bazel b/dev/prometheus/BUILD.bazel new file mode 100644 index 00000000000..4d465ea6f28 --- /dev/null +++ b/dev/prometheus/BUILD.bazel @@ -0,0 +1,15 @@ +filegroup( + name = "prometheus_targets_linux", + srcs = [ + "linux/prometheus_targets.yml", + ], + visibility = ["//visibility:public"], +) + +filegroup( + name = "prometheus_targets_all", + srcs = [ + "all/prometheus_targets.yml", + ], + visibility = ["//visibility:public"], +) diff --git a/dev/run-server-image.sh b/dev/run-server-image.sh index b63a2b7fad4..3e4f07960b1 100755 --- a/dev/run-server-image.sh +++ b/dev/run-server-image.sh @@ -9,6 +9,7 @@ URL="http://localhost:$PORT" DATA=${DATA:-"/tmp/sourcegraph-data"} SOURCEGRAPH_LICENSE_GENERATION_KEY=${SOURCEGRAPH_LICENSE_GENERATION_KEY:-""} SG_FEATURE_FLAG_GRPC=${SG_FEATURE_FLAG_GRPC:-"false"} +DB_STARTUP_TIMEOUT="10s" echo "--- Checking for existing Sourcegraph instance at $URL" if curl --output /dev/null --silent --head --fail "$URL"; then @@ -19,16 +20,16 @@ fi # shellcheck disable=SC2153 case "$CLEAN" in - "true") - clean=y - ;; - "false") - clean=n - ;; - *) - echo -n "Do you want to delete $DATA and start clean? [Y/n] " - read -r clean - ;; +"true") + clean=y + ;; +"false") + clean=n + ;; +*) + echo -n "Do you want to delete $DATA and start clean? [Y/n] " + read -r clean + ;; esac if [ "$clean" != "n" ] && [ "$clean" != "N" ]; then @@ -36,14 +37,19 @@ if [ "$clean" != "n" ] && [ "$clean" != "N" ]; then rm -rf "$DATA" fi +# WIP WIP +# -e DISABLE_BLOBSTORE=true \ +# -e DISABLE_OBSERVABILITY=true \ +# -it \ +# --entrypoint sh \ + echo "--- Starting server ${IMAGE} on port ${PORT}" docker run "$@" \ --publish "$PORT":7080 \ - -e SRC_LOG_LEVEL=dbug \ - -e DEBUG=t \ -e ALLOW_SINGLE_DOCKER_CODE_INSIGHTS=t \ -e SOURCEGRAPH_LICENSE_GENERATION_KEY="$SOURCEGRAPH_LICENSE_GENERATION_KEY" \ -e SG_FEATURE_FLAG_GRPC="$SG_FEATURE_FLAG_GRPC" \ + -e DB_STARTUP_TIMEOUT="$DB_STARTUP_TIMEOUT" \ --volume "$DATA/config:/etc/sourcegraph" \ --volume "$DATA/data:/var/opt/sourcegraph" \ "$IMAGE" diff --git a/doc/dev/background-information/ci/reference.md b/doc/dev/background-information/ci/reference.md index 01caba5428d..1532f4d6ba1 100644 --- a/doc/dev/background-information/ci/reference.md +++ b/doc/dev/background-information/ci/reference.md @@ -214,7 +214,7 @@ Base pipeline (more steps might be included based on branch changes): - **Client checks**: Upload Storybook to Chromatic, Enterprise build, Build (client/jetbrains), Tests for VS Code extension, Unit, integration, and E2E tests for the Cody VS Code extension, Stylelint (all) - **Integration tests**: Backend integration tests (gRPC), Backend integration tests, Code Intel QA - **End-to-end tests**: Executors E2E, Sourcegraph E2E, Sourcegraph Upgrade -- **Publish images**: server, executor, alpine-3.14, postgres-12-alpine, blobstore, cadvisor, codeinsights-db, codeintel-db, frontend, github-proxy, gitserver, grafana, indexed-searcher, migrator, node-exporter, opentelemetry-collector, postgres_exporter, precise-code-intel-worker, prometheus, prometheus-gcp, redis-cache, redis-store, redis_exporter, repo-updater, search-indexer, searcher, syntax-highlighter, worker, symbols, batcheshelper, blobstore2, bundled-executor, dind, embeddings, executor-kubernetes, executor-vm, jaeger-agent, jaeger-all-in-one, cody-gateway, sg, cody-slack, Publish executor image, Publish executor binary, Publish docker registry mirror image +- **Publish images**: server, executor, alpine-3.14, postgres-12-alpine, blobstore, cadvisor, codeinsights-db, codeintel-db, frontend, github-proxy, gitserver, grafana, indexed-searcher, migrator, node-exporter, opentelemetry-collector, postgres_exporter, precise-code-intel-worker, prometheus, prometheus-gcp, redis-cache, redis-store, redis_exporter, repo-updater, search-indexer, searcher, syntax-highlighter, worker, symbols, batcheshelper, blobstore2, bundled-executor, dind, embeddings, executor-kubernetes, executor-vm, jaeger-agent, jaeger-all-in-one, cody-gateway, sg, cody-slack, Publish executor image, Publish executor binary, Publish docker registry mirror image, Push OCI/Wolfi ### Release branch @@ -233,7 +233,7 @@ Base pipeline (more steps might be included based on branch changes): - **Client checks**: Upload Storybook to Chromatic, Enterprise build, Build (client/jetbrains), Tests for VS Code extension, Unit, integration, and E2E tests for the Cody VS Code extension, Stylelint (all) - **Integration tests**: Backend integration tests (gRPC), Backend integration tests, Code Intel QA - **End-to-end tests**: Executors E2E, Sourcegraph E2E, Sourcegraph Upgrade -- **Publish images**: server, executor, alpine-3.14, postgres-12-alpine, blobstore, cadvisor, codeinsights-db, codeintel-db, frontend, github-proxy, gitserver, grafana, indexed-searcher, migrator, node-exporter, opentelemetry-collector, postgres_exporter, precise-code-intel-worker, prometheus, prometheus-gcp, redis-cache, redis-store, redis_exporter, repo-updater, search-indexer, searcher, syntax-highlighter, worker, symbols, batcheshelper, blobstore2, bundled-executor, dind, embeddings, executor-kubernetes, executor-vm, jaeger-agent, jaeger-all-in-one, cody-gateway, sg, cody-slack +- **Publish images**: server, executor, alpine-3.14, postgres-12-alpine, blobstore, cadvisor, codeinsights-db, codeintel-db, frontend, github-proxy, gitserver, grafana, indexed-searcher, migrator, node-exporter, opentelemetry-collector, postgres_exporter, precise-code-intel-worker, prometheus, prometheus-gcp, redis-cache, redis-store, redis_exporter, repo-updater, search-indexer, searcher, syntax-highlighter, worker, symbols, batcheshelper, blobstore2, bundled-executor, dind, embeddings, executor-kubernetes, executor-vm, jaeger-agent, jaeger-all-in-one, cody-gateway, sg, cody-slack, Push OCI/Wolfi ### Browser extension release build @@ -293,7 +293,7 @@ Base pipeline (more steps might be included based on branch changes): - **Client checks**: Upload Storybook to Chromatic, Enterprise build, Build (client/jetbrains), Tests for VS Code extension, Unit, integration, and E2E tests for the Cody VS Code extension, Stylelint (all) - **Integration tests**: Backend integration tests (gRPC), Backend integration tests, Code Intel QA - **End-to-end tests**: Executors E2E, Sourcegraph E2E, Sourcegraph Upgrade -- **Publish images**: server, executor, alpine-3.14, postgres-12-alpine, blobstore, cadvisor, codeinsights-db, codeintel-db, frontend, github-proxy, gitserver, grafana, indexed-searcher, migrator, node-exporter, opentelemetry-collector, postgres_exporter, precise-code-intel-worker, prometheus, prometheus-gcp, redis-cache, redis-store, redis_exporter, repo-updater, search-indexer, searcher, syntax-highlighter, worker, symbols, batcheshelper, blobstore2, bundled-executor, dind, embeddings, executor-kubernetes, executor-vm, jaeger-agent, jaeger-all-in-one, cody-gateway, sg, cody-slack, Publish executor image, Publish executor binary +- **Publish images**: server, executor, alpine-3.14, postgres-12-alpine, blobstore, cadvisor, codeinsights-db, codeintel-db, frontend, github-proxy, gitserver, grafana, indexed-searcher, migrator, node-exporter, opentelemetry-collector, postgres_exporter, precise-code-intel-worker, prometheus, prometheus-gcp, redis-cache, redis-store, redis_exporter, repo-updater, search-indexer, searcher, syntax-highlighter, worker, symbols, batcheshelper, blobstore2, bundled-executor, dind, embeddings, executor-kubernetes, executor-vm, jaeger-agent, jaeger-all-in-one, cody-gateway, sg, cody-slack, Publish executor image, Publish executor binary, Push OCI/Wolfi ### Main dry run @@ -317,7 +317,7 @@ Base pipeline (more steps might be included based on branch changes): - **Client checks**: Upload Storybook to Chromatic, Enterprise build, Build (client/jetbrains), Tests for VS Code extension, Unit, integration, and E2E tests for the Cody VS Code extension, Stylelint (all) - **Integration tests**: Backend integration tests (gRPC), Backend integration tests, Code Intel QA - **End-to-end tests**: Executors E2E, Sourcegraph E2E, Sourcegraph Upgrade -- **Publish images**: server, executor, alpine-3.14, postgres-12-alpine, blobstore, cadvisor, codeinsights-db, codeintel-db, frontend, github-proxy, gitserver, grafana, indexed-searcher, migrator, node-exporter, opentelemetry-collector, postgres_exporter, precise-code-intel-worker, prometheus, prometheus-gcp, redis-cache, redis-store, redis_exporter, repo-updater, search-indexer, searcher, syntax-highlighter, worker, symbols, batcheshelper, blobstore2, bundled-executor, dind, embeddings, executor-kubernetes, executor-vm, jaeger-agent, jaeger-all-in-one, cody-gateway, sg, cody-slack +- **Publish images**: server, executor, alpine-3.14, postgres-12-alpine, blobstore, cadvisor, codeinsights-db, codeintel-db, frontend, github-proxy, gitserver, grafana, indexed-searcher, migrator, node-exporter, opentelemetry-collector, postgres_exporter, precise-code-intel-worker, prometheus, prometheus-gcp, redis-cache, redis-store, redis_exporter, repo-updater, search-indexer, searcher, syntax-highlighter, worker, symbols, batcheshelper, blobstore2, bundled-executor, dind, embeddings, executor-kubernetes, executor-vm, jaeger-agent, jaeger-all-in-one, cody-gateway, sg, cody-slack, Push OCI/Wolfi ### Patch image diff --git a/docker-images/cadvisor/BUILD.bazel b/docker-images/cadvisor/BUILD.bazel new file mode 100644 index 00000000000..31a80c92ba5 --- /dev/null +++ b/docker-images/cadvisor/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") + +oci_image( + name = "image", + base = "@wolfi_cadvisor_base", + cmd = ["--sampling.strategies-file=/etc/jaeger/sampling_strategies.json"], + entrypoint = [ + "/usr/bin/cadvisor", + "-logtostderr", + "-port=48080", + "-enable_metrics=cpu,diskIO,memory,network", + "-docker_only", + "-housekeeping_interval=10s", + "-max_housekeeping_interval=15s", + "-event_storage_event_limit=default=0", + "-v=3", + "-event_storage_age_limit=default=0", + "-containerd=/var/run/containerd/containerd.sock", + ], +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["cadvdisor:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("cadvisor"), +) diff --git a/docker-images/cadvisor/image_test.yaml b/docker-images/cadvisor/image_test.yaml new file mode 100644 index 00000000000..db8926c797c --- /dev/null +++ b/docker-images/cadvisor/image_test.yaml @@ -0,0 +1,15 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "cadvisor is runnable" + command: "cadvisor" + args: + - --version + +# TODO(security): Image runs as root + # - name: "not running as root" + # command: "/usr/bin/id" + # args: + # - -u + # excludedOutput: ["^0"] + # exitCode: 0 diff --git a/docker-images/codeinsights-db/BUILD.bazel b/docker-images/codeinsights-db/BUILD.bazel new file mode 100644 index 00000000000..e47f5b74f40 --- /dev/null +++ b/docker-images/codeinsights-db/BUILD.bazel @@ -0,0 +1,61 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//dev:oci_defs.bzl", "image_repository") + +filegroup( + name = "config", + srcs = glob( + ["rootfs/*"], + ["config/*"], + ), +) + +pkg_tar( + name = "config_tar", + srcs = [ + ":config", + ], + remap_paths = { + "/rootfs": "/", + "/config": "/usr/share/postgresql", + }, +) + +oci_image( + name = "image", + base = "@wolfi_postgresql-12-codeinsights_base", + entrypoint = ["/postgres-wolfi.sh"], + env = { + "POSTGRES_PASSWORD": "", + "POSTGRES_USER": "sg", + "POSTGRES_DB": "sg", + "PGDATA": "/var/lib/postgresql/pgdata", + "LANG": "en_US.utf8", + "PGHOST": "/var/run/postgresql", + }, + tars = [":config_tar"], + user = "postgres", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["codeinsights-db:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("codeinsights-db"), +) diff --git a/docker-images/codeinsights-db/config/postgresql.conf.sample b/docker-images/codeinsights-db/config/postgresql.conf.sample new file mode 100644 index 00000000000..a866656fcdc --- /dev/null +++ b/docker-images/codeinsights-db/config/postgresql.conf.sample @@ -0,0 +1,753 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, run "pg_ctl reload", or execute +# "SELECT pg_reload_conf()". Some parameters, which are marked below, +# require a server shutdown and restart to take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: B = bytes Time units: us = microseconds +# kB = kilobytes ms = milliseconds +# MB = megabytes s = seconds +# GB = gigabytes min = minutes +# TB = terabytes h = hours +# d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +#data_directory = 'ConfigDir' # use data in another directory + # (change requires restart) +#hba_file = 'ConfigDir/pg_hba.conf' # host-based authentication file + # (change requires restart) +#ident_file = 'ConfigDir/pg_ident.conf' # ident configuration file + # (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +#external_pid_file = '' # write an extra PID file + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +# Sourcegraph: Listen on all interfaces +listen_addresses = '*' # what IP address(es) to listen on; + # comma-separated list of addresses; + # defaults to 'localhost'; use '*' for all + # (change requires restart) +#port = 5432 # (change requires restart) +max_connections = 100 # (change requires restart) +#superuser_reserved_connections = 3 # (change requires restart) +unix_socket_directories = '/var/run/postgresql' # comma-separated list of directories + # (change requires restart) +#unix_socket_group = '' # (change requires restart) +#unix_socket_permissions = 0777 # begin with 0 to use octal notation + # (change requires restart) +#bonjour = off # advertise server via Bonjour + # (change requires restart) +#bonjour_name = '' # defaults to the computer name + # (change requires restart) + +# - TCP settings - +# see "man 7 tcp" for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; + # 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; + # 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; + # 0 selects the system default +#tcp_user_timeout = 0 # TCP_USER_TIMEOUT, in milliseconds; + # 0 selects the system default + +# - Authentication - + +#authentication_timeout = 1min # 1s-600s +#password_encryption = md5 # md5 or scram-sha-256 +#db_user_namespace = off + +# GSSAPI using Kerberos +#krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab' +#krb_caseins_users = off + +# - SSL - + +#ssl = off +#ssl_ca_file = '' +#ssl_cert_file = 'server.crt' +#ssl_crl_file = '' +#ssl_key_file = 'server.key' +#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +#ssl_prefer_server_ciphers = on +#ssl_ecdh_curve = 'prime256v1' +#ssl_min_protocol_version = 'TLSv1' +#ssl_max_protocol_version = '' +#ssl_dh_params_file = '' +#ssl_passphrase_command = '' +#ssl_passphrase_command_supports_reload = off + + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# - Memory - + +shared_buffers = 128MB # min 128kB + # (change requires restart) +#huge_pages = try # on, off, or try + # (change requires restart) +#temp_buffers = 8MB # min 800kB +#max_prepared_transactions = 0 # zero disables the feature + # (change requires restart) +# Caution: it is not advisable to set max_prepared_transactions nonzero unless +# you actively intend to use prepared transactions. +#work_mem = 4MB # min 64kB +#maintenance_work_mem = 64MB # min 1MB +#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +#max_stack_depth = 2MB # min 100kB +#shared_memory_type = mmap # the default is the first option + # supported by the operating system: + # mmap + # sysv + # windows + # (change requires restart) +dynamic_shared_memory_type = posix # the default is the first option + # supported by the operating system: + # posix + # sysv + # windows + # mmap + # (change requires restart) + +# - Disk - + +#temp_file_limit = -1 # limits per-process temp file space + # in kB, or -1 for no limit + +# - Kernel Resources - + +#max_files_per_process = 1000 # min 25 + # (change requires restart) + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-100 milliseconds (0 disables) +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 10 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 1-10000 credits + +# - Background Writer - + +#bgwriter_delay = 200ms # 10-10000ms between rounds +#bgwriter_lru_maxpages = 100 # max buffers written/round, 0 disables +#bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round +#bgwriter_flush_after = 512kB # measured in pages, 0 disables + +# - Asynchronous Behavior - + +#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching +#max_worker_processes = 8 # (change requires restart) +#max_parallel_maintenance_workers = 2 # taken from max_parallel_workers +#max_parallel_workers_per_gather = 2 # taken from max_parallel_workers +#parallel_leader_participation = on +#max_parallel_workers = 8 # maximum number of max_worker_processes that + # can be used in parallel operations +#old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate + # (change requires restart) +#backend_flush_after = 0 # measured in pages, 0 disables + + +#------------------------------------------------------------------------------ +# WRITE-AHEAD LOG +#------------------------------------------------------------------------------ + +# - Settings - + +#wal_level = replica # minimal, replica, or logical + # (change requires restart) +#fsync = on # flush data to disk for crash safety + # (turning this off can cause + # unrecoverable data corruption) +#synchronous_commit = on # synchronization level; + # off, local, remote_write, remote_apply, or on +#wal_sync_method = fsync # the default is the first option + # supported by the operating system: + # open_datasync + # fdatasync (default on Linux and FreeBSD) + # fsync + # fsync_writethrough + # open_sync +#full_page_writes = on # recover from partial page writes +#wal_compression = off # enable compression of full-page writes +#wal_log_hints = off # also do full page writes of non-critical updates + # (change requires restart) +#wal_init_zero = on # zero-fill new WAL files +#wal_recycle = on # recycle WAL files +#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers + # (change requires restart) +#wal_writer_delay = 200ms # 1-10000 milliseconds +#wal_writer_flush_after = 1MB # measured in pages, 0 disables + +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_timeout = 5min # range 30s-1d +max_wal_size = 1GB +min_wal_size = 80MB +#checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 +#checkpoint_flush_after = 256kB # measured in pages, 0 disables +#checkpoint_warning = 30s # 0 disables + +# - Archiving - + +#archive_mode = off # enables archiving; off, on, or always + # (change requires restart) +#archive_command = '' # command to use to archive a logfile segment + # placeholders: %p = path of file to archive + # %f = file name only + # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' +#archive_timeout = 0 # force a logfile segment switch after this + # number of seconds; 0 disables + +# - Archive Recovery - + +# These are only used in recovery mode. + +#restore_command = '' # command to use to restore an archived logfile segment + # placeholders: %p = path of file to restore + # %f = file name only + # e.g. 'cp /mnt/server/archivedir/%f %p' + # (change requires restart) +#archive_cleanup_command = '' # command to execute at every restartpoint +#recovery_end_command = '' # command to execute at completion of recovery + +# - Recovery Target - + +# Set these only when performing a targeted recovery. + +#recovery_target = '' # 'immediate' to end recovery as soon as a + # consistent state is reached + # (change requires restart) +#recovery_target_name = '' # the named restore point to which recovery will proceed + # (change requires restart) +#recovery_target_time = '' # the time stamp up to which recovery will proceed + # (change requires restart) +#recovery_target_xid = '' # the transaction ID up to which recovery will proceed + # (change requires restart) +#recovery_target_lsn = '' # the WAL LSN up to which recovery will proceed + # (change requires restart) +#recovery_target_inclusive = on # Specifies whether to stop: + # just after the specified recovery target (on) + # just before the recovery target (off) + # (change requires restart) +#recovery_target_timeline = 'latest' # 'current', 'latest', or timeline ID + # (change requires restart) +#recovery_target_action = 'pause' # 'pause', 'promote', 'shutdown' + # (change requires restart) + + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +# - Sending Servers - + +# Set these on the master and on any standby that will send replication data. + +#max_wal_senders = 10 # max number of walsender processes + # (change requires restart) +#wal_keep_segments = 0 # in logfile segments; 0 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables + +#max_replication_slots = 10 # max number of replication slots + # (change requires restart) +#track_commit_timestamp = off # collect timestamp of transaction commit + # (change requires restart) + +# - Master Server - + +# These settings are ignored on a standby server. + +#synchronous_standby_names = '' # standby servers that provide sync rep + # method to choose sync standbys, number of sync standbys, + # and comma-separated list of application_name + # from standby(s); '*' = all +#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed + +# - Standby Servers - + +# These settings are ignored on a master server. + +#primary_conninfo = '' # connection string to sending server + # (change requires restart) +#primary_slot_name = '' # replication slot on sending server + # (change requires restart) +#promote_trigger_file = '' # file name whose presence ends recovery +#hot_standby = on # "off" disallows queries during recovery + # (change requires restart) +#max_standby_archive_delay = 30s # max delay before canceling queries + # when reading WAL from archive; + # -1 allows indefinite delay +#max_standby_streaming_delay = 30s # max delay before canceling queries + # when reading streaming WAL; + # -1 allows indefinite delay +#wal_receiver_status_interval = 10s # send replies at least this often + # 0 disables +#hot_standby_feedback = off # send info from standby to prevent + # query conflicts +#wal_receiver_timeout = 60s # time that receiver waits for + # communication from master + # in milliseconds; 0 disables +#wal_retrieve_retry_interval = 5s # time to wait before retrying to + # retrieve WAL after a failed attempt +#recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery + +# - Subscribers - + +# These settings are ignored on a publisher. + +#max_logical_replication_workers = 4 # taken from max_worker_processes + # (change requires restart) +#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers + + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# - Planner Method Configuration - + +#enable_bitmapscan = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_indexscan = on +#enable_indexonlyscan = on +#enable_material = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_parallel_append = on +#enable_seqscan = on +#enable_sort = on +#enable_tidscan = on +#enable_partitionwise_join = off +#enable_partitionwise_aggregate = off +#enable_parallel_hash = on +#enable_partition_pruning = on + +# - Planner Cost Constants - + +#seq_page_cost = 1.0 # measured on an arbitrary scale +#random_page_cost = 4.0 # same scale as above +#cpu_tuple_cost = 0.01 # same scale as above +#cpu_index_tuple_cost = 0.005 # same scale as above +#cpu_operator_cost = 0.0025 # same scale as above +#parallel_tuple_cost = 0.1 # same scale as above +#parallel_setup_cost = 1000.0 # same scale as above + +#jit_above_cost = 100000 # perform JIT compilation if available + # and query more expensive than this; + # -1 disables +#jit_inline_above_cost = 500000 # inline small functions if query is + # more expensive than this; -1 disables +#jit_optimize_above_cost = 500000 # use expensive JIT optimizations if + # query is more expensive than this; + # -1 disables + +#min_parallel_table_scan_size = 8MB +#min_parallel_index_scan_size = 512kB +#effective_cache_size = 4GB + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 +#geqo_seed = 0.0 # range 0.0-1.0 + +# - Other Planner Options - + +#default_statistics_target = 100 # range 1-10000 +#constraint_exclusion = partition # on, off, or partition +#cursor_tuple_fraction = 0.1 # range 0.0-1.0 +#from_collapse_limit = 8 +#join_collapse_limit = 8 # 1 disables collapsing of explicit + # JOIN clauses +#force_parallel_mode = off +#jit = on # allow JIT compilation +#plan_cache_mode = auto # auto, force_generic_plan or + # force_custom_plan + + +#------------------------------------------------------------------------------ +# REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +# - Where to Log - + +#log_destination = 'stderr' # Valid values are combinations of + # stderr, csvlog, syslog, and eventlog, + # depending on platform. csvlog + # requires logging_collector to be on. + +# This is used when logging to stderr: +#logging_collector = off # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) + +# These are only used if logging_collector is on: +#log_directory = 'log' # directory where log files are written, + # can be absolute or relative to PGDATA +#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, + # can include strftime() escapes +#log_file_mode = 0600 # creation mode for log files, + # begin with 0 to use octal notation +#log_truncate_on_rotation = off # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. +#log_rotation_age = 1d # Automatic rotation of logfiles will + # happen after that time. 0 disables. +#log_rotation_size = 10MB # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' +#syslog_sequence_numbers = on +#syslog_split_messages = on + +# This is only relevant when logging to eventlog (win32): +# (change requires restart) +#event_source = 'PostgreSQL' + +# - When to Log - + +#log_min_messages = warning # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + +#log_min_error_statement = error # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic (effectively off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements + # and their durations, > 0 logs only + # statements running at least this number + # of milliseconds + +#log_transaction_sample_rate = 0.0 # Fraction of transactions whose statements + # are logged regardless of their duration. 1.0 logs all + # statements from all transactions, 0.0 never logs. + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = on +#log_checkpoints = off +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_error_verbosity = default # terse, default, or verbose messages +#log_hostname = off +#log_line_prefix = '%m [%p] ' # special values: + # %a = application name + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %p = process ID + # %t = timestamp without milliseconds + # %m = timestamp with milliseconds + # %n = timestamp with milliseconds (as a Unix epoch) + # %i = command tag + # %e = SQL state + # %c = session ID + # %l = session line number + # %s = session start timestamp + # %v = virtual transaction ID + # %x = transaction ID (0 if none) + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' +#log_lock_waits = off # log lock waits >= deadlock_timeout +#log_statement = 'none' # none, ddl, mod, all +#log_replication_commands = off +#log_temp_files = -1 # log temporary files equal or larger + # than the specified size in kilobytes; + # -1 disables, 0 logs all temp files +log_timezone = 'UTC' + +#------------------------------------------------------------------------------ +# PROCESS TITLE +#------------------------------------------------------------------------------ + +#cluster_name = '' # added to process titles if nonempty + # (change requires restart) +#update_process_title = on + + +#------------------------------------------------------------------------------ +# STATISTICS +#------------------------------------------------------------------------------ + +# - Query and Index Statistics Collector - + +#track_activities = on +#track_counts = on +#track_io_timing = off +#track_functions = none # none, pl, all +#track_activity_query_size = 1024 # (change requires restart) +#stats_temp_directory = 'pg_stat_tmp' + + +# - Monitoring - + +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off +#log_statement_stats = off + + +#------------------------------------------------------------------------------ +# AUTOVACUUM +#------------------------------------------------------------------------------ + +#autovacuum = on # Enable autovacuum subprocess? 'on' + # requires track_counts to also be on. +#log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and + # their durations, > 0 logs only + # actions running at least this number + # of milliseconds. +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses + # (change requires restart) +#autovacuum_naptime = 1min # time between autovacuum runs +#autovacuum_vacuum_threshold = 50 # min number of row updates before + # vacuum +#autovacuum_analyze_threshold = 50 # min number of row updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum +#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum + # (change requires restart) +#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age + # before forced vacuum + # (change requires restart) +#autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for + # autovacuum, in milliseconds; + # -1 means use vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovacuum, -1 means use + # vacuum_cost_limit + + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +# - Statement Behavior - + +#client_min_messages = notice # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error +#search_path = '"$user", public' # schema names +#row_security = on +#default_tablespace = '' # a tablespace name, '' uses the default +#temp_tablespaces = '' # a list of tablespace names, '' uses + # only default tablespace +#default_table_access_method = 'heap' +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#default_transaction_deferrable = off +#session_replication_role = 'origin' +#statement_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled +#idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled +#vacuum_freeze_min_age = 50000000 +#vacuum_freeze_table_age = 150000000 +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_freeze_table_age = 150000000 +#vacuum_cleanup_index_scale_factor = 0.1 # fraction of total number of tuples + # before index cleanup, 0 always performs + # index cleanup +#bytea_output = 'hex' # hex, escape +#xmlbinary = 'base64' +#xmloption = 'content' +#gin_fuzzy_search_limit = 0 +#gin_pending_list_limit = 4MB + +# - Locale and Formatting - + +datestyle = 'iso, mdy' +#intervalstyle = 'postgres' +timezone = 'UTC' +#timezone_abbreviations = 'Default' # Select the set of available time zone + # abbreviations. Currently, there are + # Default + # Australia (historical usage) + # India + # You can create your own file in + # share/timezonesets/. +#extra_float_digits = 1 # min -15, max 3; any value >0 actually + # selects precise output mode +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# Sourcegraph: Use 'en_US.utf8' locale +# These settings are initialized by initdb, but they can be changed. +lc_messages = 'en_US.utf8' # locale for system error message + # strings +lc_monetary = 'en_US.utf8' # locale for monetary formatting +lc_numeric = 'en_US.utf8' # locale for number formatting +lc_time = 'en_US.utf8' # locale for time formatting + +# default configuration for text search +default_text_search_config = 'pg_catalog.english' + +# - Shared Library Preloading - + +#shared_preload_libraries = '' # (change requires restart) +#local_preload_libraries = '' +#session_preload_libraries = '' +#jit_provider = 'llvmjit' # JIT library to use + +# - Other Defaults - + +#dynamic_library_path = '$libdir' + + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +#deadlock_timeout = 1s +#max_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_relation = -2 # negative values mean + # (max_pred_locks_per_transaction + # / -max_pred_locks_per_relation) - 1 +#max_pred_locks_per_page = 2 # min 0 + + +#------------------------------------------------------------------------------ +# VERSION AND PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# - Previous PostgreSQL Versions - + +#array_nulls = on +#backslash_quote = safe_encoding # on, off, or safe_encoding +#escape_string_warning = on +#lo_compat_privileges = off +#operator_precedence_warning = off +#quote_all_identifiers = off +#standard_conforming_strings = on +#synchronize_seqscans = on + +# - Other Platforms and Clients - + +#transform_null_equals = off + + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +#exit_on_error = off # terminate session on any error? +#restart_after_crash = on # reinitialize after backend crash? +#data_sync_retry = off # retry or panic on failure to fsync + # data? + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf. Note that these are directives, not variable +# assignments, so they can usefully be given more than once. + +#include_dir = '...' # include files ending in '.conf' from + # a directory, e.g., 'conf.d' +#include_if_exists = '...' # include file only if it exists +#include = '...' # include file + + +#------------------------------------------------------------------------------ +# CUSTOMIZED OPTIONS +#------------------------------------------------------------------------------ + +# Add settings for extensions here diff --git a/docker-images/codeinsights-db/image_test.yaml b/docker-images/codeinsights-db/image_test.yaml new file mode 100644 index 00000000000..e2c162ade91 --- /dev/null +++ b/docker-images/codeinsights-db/image_test.yaml @@ -0,0 +1,43 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "postgres is runnable" + command: "postgres" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + - name: "postgres user has correct uid" + command: "/usr/bin/id" + args: + - -u + expectedOutput: ["^70\n"] + exitCode: 0 + - name: "postgres user has correct gid" + command: "/usr/bin/id" + args: + - -g + expectedOutput: ["^70\n"] + exitCode: 0 + +fileExistenceTests: +- name: '/data/pgdata-12' + path: '/data/pgdata-12' + shouldExist: true + uid: 70 + gid: 70 + +metadataTest: + envVars: + - key: PGDATA + value: .+ + isRegex: true + - key: LANG + value: 'en_US.utf8' + - key: PGHOST + value: '/var/run/postgresql' diff --git a/docker-images/codeinsights-db/rootfs/conf.sh b/docker-images/codeinsights-db/rootfs/conf.sh new file mode 100755 index 00000000000..5aab7ab75d7 --- /dev/null +++ b/docker-images/codeinsights-db/rootfs/conf.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -x +# Copies postgresql.conf over from /conf/postgresql.conf if it exists + +if [ ! -d "/conf" ] || [ ! -f "/conf/postgresql.conf" ]; then + exit 0 +fi + +cp /conf/postgresql.conf "$PGDATA/postgresql.conf" + +# allow the container to be started with `--user` +if [ "$(id -u)" = '0' ]; then + chown postgres:postgres "$PGDATA/postgresql.conf" +fi diff --git a/docker-images/codeinsights-db/rootfs/env.sh b/docker-images/codeinsights-db/rootfs/env.sh new file mode 100755 index 00000000000..76662df443c --- /dev/null +++ b/docker-images/codeinsights-db/rootfs/env.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +export REINDEX_COMPLETED_FILE="${PGDATA}/3.31-reindex.completed" diff --git a/docker-images/codeinsights-db/rootfs/initdb.sh b/docker-images/codeinsights-db/rootfs/initdb.sh new file mode 100755 index 00000000000..f4a7b73ccc8 --- /dev/null +++ b/docker-images/codeinsights-db/rootfs/initdb.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash + +# Adapted from https://github.com/docker-library/postgres/blob/master/11/docker-entrypoint.sh +# to support running this separately from starting postgres. See postgres.sh for usage. + +set -Eexo pipefail +# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(<"${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# allow the container to be started with `--user` +if [ "$(id -u)" = '0' ]; then + mkdir -p "$PGDATA" + chown -R postgres "$PGDATA" + chmod 700 "$PGDATA" + + mkdir -p /var/run/postgresql + chown -R postgres /var/run/postgresql + chmod 775 /var/run/postgresql + + # Create the transaction log directory before initdb is run (below) so the directory is owned by the correct user + if [ "$POSTGRES_INITDB_WALDIR" ]; then + mkdir -p "$POSTGRES_INITDB_WALDIR" + chown -R postgres "$POSTGRES_INITDB_WALDIR" + chmod 700 "$POSTGRES_INITDB_WALDIR" + fi + + # TODO@davejrt is this fix what you meant? + su-exec postgres "${BASH_SOURCE[0]}" "$@" +fi + +mkdir -p "$PGDATA" +chown -R "$(id -u)" "$PGDATA" 2>/dev/null || : +chmod 700 "$PGDATA" 2>/dev/null || : + +# look specifically for PG_VERSION, as it is expected in the DB dir +if [ ! -s "$PGDATA/PG_VERSION" ]; then + # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary + # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html + if ! getent passwd "$(id -u)" &>/dev/null && [ -e /usr/lib/libnss_wrapper.so ]; then + export LD_PRELOAD='/usr/lib/libnss_wrapper.so' + + NSS_WRAPPER_PASSWD="$(mktemp)" + export NSS_WRAPPER_PASSWD + + NSS_WRAPPER_GROUP="$(mktemp)" + export NSS_WRAPPER_GROUP + + echo "postgres:x:$(id -u):$(id -g):PostgreSQL:$PGDATA:/bin/false" >"$NSS_WRAPPER_PASSWD" + echo "postgres:x:$(id -g):" >"$NSS_WRAPPER_GROUP" + fi + + file_env 'POSTGRES_USER' 'postgres' + file_env 'POSTGRES_PASSWORD' + + file_env 'POSTGRES_INITDB_ARGS' + if [ "$POSTGRES_INITDB_WALDIR" ]; then + export POSTGRES_INITDB_ARGS="$POSTGRES_INITDB_ARGS --waldir $POSTGRES_INITDB_WALDIR" + fi + eval 'initdb --username="$POSTGRES_USER" --pwfile=<(echo "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS" + + # unset/cleanup "nss_wrapper" bits + if [ "${LD_PRELOAD:-}" = '/usr/lib/libnss_wrapper.so' ]; then + rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" + unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + fi + + # check password first so we can output the warning before postgres + # messes it up + if [ -n "$POSTGRES_PASSWORD" ]; then + authMethod=md5 + + if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then + cat >&2 <<-'EOWARN' + WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. + + This will not work if used via PGPASSWORD with "psql". + + https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) + https://github.com/docker-library/postgres/issues/507 + + EOWARN + fi + else + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOWARN' + **************************************************** + WARNING: No password has been set for the database. + This will allow anyone with access to the + Postgres port to access your database. In + Docker's default configuration, this is + effectively any other container on the same + system. + + Use "-e POSTGRES_PASSWORD=password" to set + it in "docker run". + **************************************************** + EOWARN + + authMethod=trust + fi + + { + echo + echo "host all all all $authMethod" + } >>"$PGDATA/pg_hba.conf" + + # internal start of server in order to allow set-up using psql-client + # does not listen on external TCP/IP and waits until start finishes + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + pg_ctl -D "$PGDATA" \ + -o "-c listen_addresses='' -c unix_socket_directories=/var/run/postgresql" \ + -w start + + file_env 'POSTGRES_DB' "$POSTGRES_USER" + + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + psql=(psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --no-password) + + if [ "$POSTGRES_DB" != 'postgres' ]; then + "${psql[@]}" --dbname postgres --set db="$POSTGRES_DB" <<-'EOSQL' + CREATE DATABASE :"db" ; + EOSQL + echo + fi + psql+=(--dbname "$POSTGRES_DB") + + echo + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + # shellcheck source=/dev/null + . "$f" + fi + ;; + *.sql) + echo "$0: running $f" + "${psql[@]}" -f "$f" + echo + ;; + *.sql.gz) + echo "$0: running $f" + gunzip -c "$f" | "${psql[@]}" + echo + ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done + + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + pg_ctl -D "$PGDATA" -m fast -w stop + + unset PGPASSWORD + + echo + echo 'PostgreSQL init process complete; ready for start up.' + echo +fi diff --git a/docker-images/codeinsights-db/rootfs/liveness.sh b/docker-images/codeinsights-db/rootfs/liveness.sh new file mode 100755 index 00000000000..5723be50a6d --- /dev/null +++ b/docker-images/codeinsights-db/rootfs/liveness.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -euxo pipefail +# shellcheck source=./env.sh +source /env.sh + +# This script checks to see if postgres is alive. It uses the ready check, but +# additionally ignores upgrades to give the container enough time to +# re-compute indexes. It is expected to be used by a Kubernetes liveness probe. + +# Ensure we are in the same dir ready.sh +cd "$(dirname "${BASH_SOURCE[0]}")" + +if [ -s "${PGDATA}/PG_VERSION" ] && [ ! -s "${REINDEX_COMPLETED_FILE}" ]; then + echo "[INFO] Postgres is re-indexing..." + exit 0 +fi + +./ready.sh diff --git a/docker-images/codeinsights-db/rootfs/patch-conf.sh b/docker-images/codeinsights-db/rootfs/patch-conf.sh new file mode 100755 index 00000000000..21564f85c64 --- /dev/null +++ b/docker-images/codeinsights-db/rootfs/patch-conf.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# In Wolfi, unix_socket_directories defaults to /tmp. In previous Alpine images, this defaulted to /var/run/postgres. +# /tmp may not be writable, so any existing postgresql.conf configs that predate the Wolfi migration should be patched to update this setting. + +CONFIG_DIR=${PGDATA:-/data/pgdata-12} + +conf_file="$CONFIG_DIR/postgresql.conf" +new_socket_dir="/var/run/postgresql" + +# Check if the parameter already exists in the file +if grep -q "^\s*unix_socket_directories" "$conf_file"; then + echo "unix_socket_directories already exists in $conf_file" +else + # Append the setting to the end of the file + echo "unix_socket_directories = '$new_socket_dir'" >>"$conf_file" + echo "Updated unix_socket_directories in $conf_file" +fi diff --git a/docker-images/codeinsights-db/rootfs/postgres-wolfi.sh b/docker-images/codeinsights-db/rootfs/postgres-wolfi.sh new file mode 100755 index 00000000000..12f666fbc49 --- /dev/null +++ b/docker-images/codeinsights-db/rootfs/postgres-wolfi.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -euxo pipefail +cd / + +# shellcheck source=./env.sh +source /env.sh + +# Allow the container to be started with root in Kubernetes and change permissions +# of the parent volume directory to be owned entirely by the postgres user. +if [ "$(id -u)" = '0' ]; then + mkdir -p "$PGDATA" + chown -R postgres:postgres "$(dirname "$PGDATA")" + chmod 750 "$(dirname "$PGDATA")" "$PGDATA" + su-exec postgres "${BASH_SOURCE[0]}" "$@" +fi + +if [ ! -s "$PGDATA/PG_VERSION" ]; then + echo "[INFO] Initializing Postgres database '$POSTGRES_DB' from scratch in $PGDATA" + /initdb.sh +fi + +/conf.sh +/patch-conf.sh + +if [ ! -s "${REINDEX_COMPLETED_FILE}" ]; then + echo "[INFO] Re-creating all indexes for database '$POSTGRES_DB'" + /reindex.sh +fi + +exec postgres diff --git a/docker-images/codeinsights-db/rootfs/postgres.sh b/docker-images/codeinsights-db/rootfs/postgres.sh new file mode 100755 index 00000000000..8bdafd15e76 --- /dev/null +++ b/docker-images/codeinsights-db/rootfs/postgres.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euxo pipefail +cd /var/lib/postgresql + +# shellcheck source=./env.sh +source /env.sh + +# Allow the container to be started with root in Kubernetes and change permissions +# of the parent volume directory to be owned entirely by the postgres user. +if [ "$(id -u)" = '0' ]; then + mkdir -p "$PGDATA" + chown -R postgres:postgres "$(dirname "$PGDATA")" + chmod 750 "$(dirname "$PGDATA")" "$PGDATA" + su-exec postgres "${BASH_SOURCE[0]}" "$@" +fi + +if [ ! -s "$PGDATA/PG_VERSION" ]; then + echo "[INFO] Initializing Postgres database '$POSTGRES_DB' from scratch in $PGDATA" + /initdb.sh +fi + +/conf.sh + +if [ ! -s "${REINDEX_COMPLETED_FILE}" ]; then + echo "[INFO] Re-creating all indexes for database '$POSTGRES_DB'" + /reindex.sh +fi + +exec postgres diff --git a/docker-images/codeinsights-db/rootfs/ready.sh b/docker-images/codeinsights-db/rootfs/ready.sh new file mode 100755 index 00000000000..bae5c31b8b5 --- /dev/null +++ b/docker-images/codeinsights-db/rootfs/ready.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +# This script checks to see if postgres is alive. It is expected to be used by +# a Kubernetes ready probe. + +# We check if the TCP port is available since that is how clients will +# connect. While re-indexing only the unix port will be available, so we +# specifically want to avoid reporting ready in that case. + +if [ -n "$POSTGRES_PASSWORD" ]; then + export PGPASSWORD="$POSTGRES_PASSWORD" +fi + +export PGUSER="$POSTGRES_USER" +export PGDATABASE="$POSTGRES_DB" +export PGCONNECT_TIMEOUT=10 + +# Check if we can run a simple query. If it fails the reason will be printed +# to stderr and we will have a non-zero exit code. +psql --no-password --tuples-only --no-align -c 'select 1;' >/dev/null diff --git a/docker-images/codeinsights-db/rootfs/reindex.sh b/docker-images/codeinsights-db/rootfs/reindex.sh new file mode 100755 index 00000000000..147ad04bf90 --- /dev/null +++ b/docker-images/codeinsights-db/rootfs/reindex.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +cd "$(dirname "${BASH_SOURCE[0]}")" +set -Eexo pipefail + +# shellcheck source=./env.sh +source /env.sh + +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(<"${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +prepare_env() { + file_env 'POSTGRES_USER' 'postgres' + file_env 'POSTGRES_PASSWORD' + file_env 'POSTGRES_DB' + export PGUSER="${PGUSER:-$POSTGRES_USER}" + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + export PGDATABASE="${PGDATABASE:-$POSTGRES_DB}" +} + +unset_env() { + unset PGPASSWORD +} + +postgres_is_running() { + local pid="$1" + + local proc_entry="/proc/$pid/comm" + [[ -s "$proc_entry" ]] && grep -q '^postgres$' "$proc_entry" +} + +postgres_stop_cleanly() { + pg_ctl -D "$PGDATA" -m fast -w stop +} + +postgres_stop() { + # This logic handles the case where we've restored a snapshot + # that was taken from a still running or improperly shutdown + # postgres instance. We'll need to check to see if postgres is + # actually still running under the pid specified in the postmaster.pid + # file. If it is, we shut it down properly. If it isn't, we + # delete the pid file so that we can start up properly. + local postmaster_file="$PGDATA/postmaster.pid" + + if ! [[ -s "$postmaster_file" ]]; then + # postgres isn't running - nothing to do + return 0 + fi + + local pid + pid="$(head -1 "$postmaster_file")" + + if postgres_is_running "$pid"; then + # postgres is currently running in the container - shut it down cleanly + postgres_stop_cleanly + return 0 + fi + + # we have a postmaster file, but a postgres process isn't running anymore. + # remove the postmaster file - we can't do any better here + rm "$postmaster_file" || true +} + +postgres_start() { + # internal start of server in order to allow set-up using psql-client + # - does not listen on external TCP/IP and waits until start finishes + # - "-P" prevents Postgres from using indexes for system catalog lookups - + # see https://www.postgresql.org/docs/12/sql-reindex.html + + pg_ctl -D "$PGDATA" \ + -o "-c listen_addresses=''" \ + -o "-P" \ + -w restart +} + +cleanup() { + postgres_stop_cleanly + unset_env +} + +reindex() { + reindexdb --no-password --verbose --echo "$@" +} + +# allow the container to be started with `--user` +if [ "$(id -u)" = '0' ]; then + su-exec postgres "${BASH_SOURCE[0]}" "$@" +fi + +# look specifically for REINDEX_COMPLETED_FILE, as it is expected in the DB dir +if [ ! -s "${REINDEX_COMPLETED_FILE}" ]; then + prepare_env + postgres_stop + postgres_start + trap cleanup EXIT + + echo + echo 'PostgresSQL must rebuild its indexes. This process can take up to a few hours on systems with a large dataset.' + echo + + reindex --system + reindex --all + + # mark reindexing process as done + echo "Re-indexing for 3.31 release completed successfully at $(date)" >"${REINDEX_COMPLETED_FILE}" + + echo + echo 'PostgreSQL reindexing process complete - ready for start up.' + echo +fi diff --git a/docker-images/codeintel-db/BUILD.bazel b/docker-images/codeintel-db/BUILD.bazel new file mode 100644 index 00000000000..76848afc52c --- /dev/null +++ b/docker-images/codeintel-db/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") + +oci_image( + name = "image", + base = "//docker-images/postgres-12-alpine:image", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["codeintel-db:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("codeintel-db"), +) diff --git a/docker-images/codeintel-db/image_test.yaml b/docker-images/codeintel-db/image_test.yaml new file mode 100644 index 00000000000..503534ab794 --- /dev/null +++ b/docker-images/codeintel-db/image_test.yaml @@ -0,0 +1,43 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "postgres is runnable" + command: "postgres" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + - name: "postgres user has correct uid" + command: "/usr/bin/id" + args: + - -u + expectedOutput: ["^999\n"] + exitCode: 0 + - name: "postgres user has correct gid" + command: "/usr/bin/id" + args: + - -g + expectedOutput: ["^999\n"] + exitCode: 0 + +fileExistenceTests: +- name: '/data/pgdata-12' + path: '/data/pgdata-12' + shouldExist: true + uid: 999 + gid: 999 + +metadataTest: + envVars: + - key: PGDATA + value: .+ + isRegex: true + - key: LANG + value: 'en_US.utf8' + - key: PGHOST + value: '/var/run/postgresql' diff --git a/docker-images/grafana/BUILD.bazel b/docker-images/grafana/BUILD.bazel new file mode 100644 index 00000000000..1f06ba0b39a --- /dev/null +++ b/docker-images/grafana/BUILD.bazel @@ -0,0 +1,53 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") + +filegroup( + name = "config_files", + srcs = glob(["config/*"]) + ["entry-bazel.sh"], +) + +pkg_tar( + name = "config_tar", + srcs = [ + ":config_files", + "//docker-images/grafana/config", + "//monitoring:generate_config_zip", + ], + remap_paths = { + "docker-images/grafana/config": "/sg_config_grafana", + "monitoring/outputs/grafana": "/sg_config_grafana/provisioning/dashboards/sourcegraph", + "/entry-bazel.sh": "/entry.sh", + }, +) + +oci_image( + name = "image", + base = "@wolfi_grafana_base", + entrypoint = ["/entry.sh"], + tars = [":config_tar"], + user = "grafana", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["grafana:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("grafana"), +) diff --git a/docker-images/grafana/build-bazel.sh b/docker-images/grafana/build-bazel.sh index 0c0661a8512..08677af5e3b 100755 --- a/docker-images/grafana/build-bazel.sh +++ b/docker-images/grafana/build-bazel.sh @@ -11,8 +11,8 @@ cleanup() { } trap cleanup EXIT -./dev/ci/bazel.sh build //monitoring:generate_config -monitoring_cfg=$(./dev/ci/bazel.sh cquery //monitoring:generate_config --output=files) +./dev/ci/bazel.sh build //monitoring:generate_config_zip +monitoring_cfg=$(./dev/ci/bazel.sh cquery //monitoring:generate_config_zip --output=files) cp "$monitoring_cfg" "$TMP" pushd "$TMP" diff --git a/docker-images/grafana/config/BUILD.bazel b/docker-images/grafana/config/BUILD.bazel new file mode 100644 index 00000000000..11a7716fac6 --- /dev/null +++ b/docker-images/grafana/config/BUILD.bazel @@ -0,0 +1,8 @@ +filegroup( + name = "config", + srcs = glob([ + "grafana*.ini", + "provisioning/**/*.yaml", + ]), + visibility = ["//visibility:public"], +) diff --git a/docker-images/grafana/entry-bazel.sh b/docker-images/grafana/entry-bazel.sh new file mode 100755 index 00000000000..2caf6b511d5 --- /dev/null +++ b/docker-images/grafana/entry-bazel.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +export GF_PATHS_PROVISIONING=/sg_config_grafana/provisioning +export GF_PATHS_CONFIG=/sg_config_grafana/grafana.ini + +exec grafana-server \ + --homepath="$GF_PATHS_HOME" \ + --config="$GF_PATHS_CONFIG" \ + --packaging=docker \ + "$@" \ + cfg:default.log.mode="console" \ + cfg:default.paths.data="$GF_PATHS_DATA" \ + cfg:default.paths.logs="$GF_PATHS_LOGS" \ + cfg:default.paths.plugins="$GF_PATHS_PLUGINS" \ + cfg:default.paths.provisioning="$GF_PATHS_PROVISIONING" diff --git a/docker-images/grafana/image_test.yaml b/docker-images/grafana/image_test.yaml new file mode 100644 index 00000000000..4fa7aa2482e --- /dev/null +++ b/docker-images/grafana/image_test.yaml @@ -0,0 +1,28 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "grafana-server is runnable" + command: "grafana-server" + args: + - -v + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: +- name: '/opt/grafana/entry.sh' + path: '/opt/grafana/entry.sh' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-rwxr-xr-x' +- name: 'Dashboard config' + path: '/sg_config_grafana/provisioning/dashboards/sourcegraph/gitserver.json' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-rwxr-xr-x' diff --git a/docker-images/indexed-searcher/BUILD.bazel b/docker-images/indexed-searcher/BUILD.bazel new file mode 100644 index 00000000000..25e8e1ae75b --- /dev/null +++ b/docker-images/indexed-searcher/BUILD.bazel @@ -0,0 +1,56 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("//cmd/server:macro.bzl", "container_dependencies", "dependencies_tars") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") + +DEPS = ["@com_github_sourcegraph_zoekt//cmd/zoekt-webserver"] + +container_dependencies(DEPS) + +filegroup( + name = "entrypoint", + srcs = ["entry.sh"], +) + +pkg_tar( + name = "entry_tar", + srcs = [":entrypoint"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/entry.sh", + ], + env = { + "DATA_DIR": "/data/index", + "GOGC": "25", + }, + tars = dependencies_tars(DEPS) + [":entry_tar"], + user = "sourcegraph", + workdir = "/home/sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["zoekt-webserver:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("indexed-searcher"), +) diff --git a/docker-images/indexed-searcher/Dockerfile.wolfi b/docker-images/indexed-searcher/Dockerfile.wolfi index 8a5004285c2..fc9cf3f0815 100644 --- a/docker-images/indexed-searcher/Dockerfile.wolfi +++ b/docker-images/indexed-searcher/Dockerfile.wolfi @@ -15,8 +15,6 @@ LABEL org.opencontainers.image.created=${DATE} LABEL org.opencontainers.image.version=${VERSION} ENV DATA_DIR /data/index -RUN mkdir -p ${DATA_DIR} -RUN chown -R sourcegraph:sourcegraph /data USER sourcegraph WORKDIR /home/sourcegraph diff --git a/docker-images/indexed-searcher/entry.sh b/docker-images/indexed-searcher/entry.sh new file mode 100644 index 00000000000..95cd335f536 --- /dev/null +++ b/docker-images/indexed-searcher/entry.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +/sbin/tini -s -- zoekt-webserver -index "$DATA_DIR" -pprof -rpc -indexserver_proxy diff --git a/docker-images/indexed-searcher/image_test.yaml b/docker-images/indexed-searcher/image_test.yaml new file mode 100644 index 00000000000..de6db88356e --- /dev/null +++ b/docker-images/indexed-searcher/image_test.yaml @@ -0,0 +1,29 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "zoekt-webserver is runnable" + command: "zoekt-webserver" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: +- name: '/data/index' + path: '/data/index' + shouldExist: true + uid: 100 + gid: 101 + permissions: 'drwxr-xr-x' + +metadataTest: + envVars: + - key: DATA_DIR + value: '/data/index' + - key: GOGC + value: 25 diff --git a/docker-images/jaeger-agent/BUILD.bazel b/docker-images/jaeger-agent/BUILD.bazel new file mode 100644 index 00000000000..81583da31f8 --- /dev/null +++ b/docker-images/jaeger-agent/BUILD.bazel @@ -0,0 +1,31 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("//dev:oci_defs.bzl", "image_repository") +load("@container_structure_test//:defs.bzl", "container_structure_test") + +oci_image( + name = "image", + base = "@wolfi_jaeger_agent_base", + entrypoint = ["/usr/local/bin/jaeger-agent"], +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["jaeger-agent:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("jaeger-agent"), +) diff --git a/docker-images/jaeger-agent/image_test.yaml b/docker-images/jaeger-agent/image_test.yaml new file mode 100644 index 00000000000..b641cf4759b --- /dev/null +++ b/docker-images/jaeger-agent/image_test.yaml @@ -0,0 +1,20 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "jaeger-agent is runnable" + command: "jaeger-agent" + args: + - --help + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + - name: "running as jaeger" + command: "/usr/bin/id" + args: + - -u + expectedOutput: ["^10001"] + exitCode: 0 diff --git a/docker-images/jaeger-all-in-one/BUILD.bazel b/docker-images/jaeger-all-in-one/BUILD.bazel new file mode 100644 index 00000000000..627c9e63e6a --- /dev/null +++ b/docker-images/jaeger-all-in-one/BUILD.bazel @@ -0,0 +1,52 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//dev:oci_defs.bzl", "image_repository") +load("@container_structure_test//:defs.bzl", "container_structure_test") + +filegroup( + name = "config", + srcs = glob(["config/*"]), +) + +pkg_tar( + name = "config_tar", + srcs = [":config"], + remap_paths = {"": "/etc/jaeger/"}, +) + +oci_image( + name = "image", + base = "@wolfi_jaeger_all_in_one_base", + cmd = ["--sampling.strategies-file=/etc/jaeger/sampling_strategies.json"], + entrypoint = ["/usr/local/bin/jaeger-all-in-one"], + env = { + # Used in order to reverse proxy the Jaeger UI + "QUERY_BASE_PATH": "/-/debug/jaeger", + # Default configuration file for setting sampling strategies, we override the command in docker-compose + "SAMPLING_STRATEGIES_FILE": "/etc/jaeger/sampling_strategies.json", + }, + tars = [":config_tar"], + user = "jaeger", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["jaeger-all-in-one:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("jaeger-all-in-one"), +) diff --git a/docker-images/jaeger-all-in-one/image_test.yaml b/docker-images/jaeger-all-in-one/image_test.yaml new file mode 100644 index 00000000000..ea24b697572 --- /dev/null +++ b/docker-images/jaeger-all-in-one/image_test.yaml @@ -0,0 +1,41 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "jaeger-all-in-one is runnable" + command: "jaeger-all-in-one" + args: + - --help + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + - name: "running as jaeger" + command: "/usr/bin/id" + args: + - -u + expectedOutput: ["^10001"] + exitCode: 0 + +fileExistenceTests: +- name: '/etc/jaeger/sampling_strategies.json' + path: '/etc/jaeger/sampling_strategies.json' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +- name: '/tmp' + path: '/tmp' + shouldExist: true + uid: 10001 + gid: 0 + permissions: 'drwxrwxrwx' + +metadataTest: + envVars: + - key: QUERY_BASE_PATH + value: '/-/debug/jaeger' + - key: SAMPLING_STRATEGIES_FILE + value: '/etc/jaeger/sampling_strategies.json' diff --git a/docker-images/node-exporter/BUILD.bazel b/docker-images/node-exporter/BUILD.bazel new file mode 100644 index 00000000000..a1fd047e241 --- /dev/null +++ b/docker-images/node-exporter/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") + +oci_image( + name = "image", + base = "@wolfi_node_exporter_base", + entrypoint = ["/usr/bin/node_exporter"], + user = "nobody", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["node-exporter:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("node-exporter"), +) diff --git a/docker-images/node-exporter/image_test.yaml b/docker-images/node-exporter/image_test.yaml new file mode 100644 index 00000000000..fc269aa3d95 --- /dev/null +++ b/docker-images/node-exporter/image_test.yaml @@ -0,0 +1,14 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "node_exporter is runnable" + command: "node_exporter" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/docker-images/opentelemetry-collector/BUILD.bazel b/docker-images/opentelemetry-collector/BUILD.bazel new file mode 100644 index 00000000000..5275d241ec1 --- /dev/null +++ b/docker-images/opentelemetry-collector/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//dev:oci_defs.bzl", "image_repository") +load("@container_structure_test//:defs.bzl", "container_structure_test") + +filegroup( + name = "config", + srcs = glob(["configs/*"]), +) + +pkg_tar( + name = "config_tar", + srcs = [":config"], + remap_paths = {"": "/etc/otel-collector/configs/"}, +) + +oci_image( + name = "image", + base = "@wolfi_opentelemetry_collector_base", + entrypoint = ["/bin/otelcol-sourcegraph"], + tars = [":config_tar"], +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["opentelemetry-collector:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("opentelemetry-collector"), +) diff --git a/docker-images/opentelemetry-collector/image_test.yaml b/docker-images/opentelemetry-collector/image_test.yaml new file mode 100644 index 00000000000..2c17e390e3e --- /dev/null +++ b/docker-images/opentelemetry-collector/image_test.yaml @@ -0,0 +1,29 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "otelcol-sourcegraph is runnable" + command: "otelcol-sourcegraph" + args: + - --version + + # TODO(security): This container should not be running as root + # - name: "not running as root" + # command: "/usr/bin/id" + # args: + # - -u + # excludedOutput: ["^0"] + # exitCode: 0 + +fileExistenceTests: +- name: '/otel-collector' + path: '/otel-collector' + shouldExist: true + uid: 0 + gid: 0 + permissions: 'drwxr-xr-x' +- name: 'Opentelemetry Configs' + path: '/etc/otel-collector/configs/jaeger.yaml' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' diff --git a/docker-images/postgres-12-alpine/BUILD.bazel b/docker-images/postgres-12-alpine/BUILD.bazel new file mode 100644 index 00000000000..53e6d594836 --- /dev/null +++ b/docker-images/postgres-12-alpine/BUILD.bazel @@ -0,0 +1,65 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") + +# TODO move this to a different folder + +filegroup( + name = "config", + srcs = glob( + ["rootfs/*"], + ) + ["config/postgresql.conf.sample"], +) + +pkg_tar( + name = "config_tar", + srcs = [ + ":config", + ], + remap_paths = { + "/rootfs": "/", + "/postgresql.conf.sample": "/usr/share/postgresql/postgresql.conf.sample", + }, +) + +oci_image( + name = "image", + base = "@wolfi_postgresql-12_base", + entrypoint = ["/postgres-wolfi.sh"], + env = { + "POSTGRES_PASSWORD": "", + "POSTGRES_USER": "sg", + "POSTGRES_DB": "sg", + "PGDATA": "/data/pgdata-12", + "LANG": "en_US.utf8", + "PGHOST": "/var/run/postgresql", + }, + tars = [":config_tar"], + user = "postgres", + visibility = [ + "//docker-images/codeintel-db:__pkg__", + ], +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["postgres-12:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("postgres-12-alpine"), # TODO careful, this is not an alpine +) diff --git a/docker-images/postgres-12-alpine/Dockerfile.wolfi b/docker-images/postgres-12-alpine/Dockerfile.wolfi index f52b3829809..a7e5f07de28 100644 --- a/docker-images/postgres-12-alpine/Dockerfile.wolfi +++ b/docker-images/postgres-12-alpine/Dockerfile.wolfi @@ -13,17 +13,6 @@ FROM us.gcr.io/sourcegraph-dev/wolfi-postgresql-12-base:latest # To remain compatibility with codeinsights-db and codeintel-db, user and group # IDs are set here, rather than in the base image -ARG PING_UID=99 -ARG POSTGRES_UID=999 - -# We modify the postgres user/group to reconcile with our previous debian based images -# and avoid issues with customers migrating. -RUN addgroup -g $PING_UID ping &&\ - adduser -D -u $POSTGRES_UID postgres postgres &&\ - mkdir -p /data/pgdata-12 && chown -R postgres:postgres /data &&\ - mkdir -p /var/lib/postgresql && chown -R postgres:postgres /var/lib/postgresql &&\ - mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql - COPY rootfs / # Overwrite default postgresql.conf.sample COPY config/postgresql.conf.sample /usr/share/postgresql/postgresql.conf.sample diff --git a/docker-images/postgres-12-alpine/image_test.yaml b/docker-images/postgres-12-alpine/image_test.yaml new file mode 100644 index 00000000000..503534ab794 --- /dev/null +++ b/docker-images/postgres-12-alpine/image_test.yaml @@ -0,0 +1,43 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "postgres is runnable" + command: "postgres" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + - name: "postgres user has correct uid" + command: "/usr/bin/id" + args: + - -u + expectedOutput: ["^999\n"] + exitCode: 0 + - name: "postgres user has correct gid" + command: "/usr/bin/id" + args: + - -g + expectedOutput: ["^999\n"] + exitCode: 0 + +fileExistenceTests: +- name: '/data/pgdata-12' + path: '/data/pgdata-12' + shouldExist: true + uid: 999 + gid: 999 + +metadataTest: + envVars: + - key: PGDATA + value: .+ + isRegex: true + - key: LANG + value: 'en_US.utf8' + - key: PGHOST + value: '/var/run/postgresql' diff --git a/docker-images/postgres-12-alpine/rootfs/initdb.sh b/docker-images/postgres-12-alpine/rootfs/initdb.sh index 5582a0ce713..1320329afd7 100755 --- a/docker-images/postgres-12-alpine/rootfs/initdb.sh +++ b/docker-images/postgres-12-alpine/rootfs/initdb.sh @@ -148,29 +148,29 @@ if [ ! -s "$PGDATA/PG_VERSION" ]; then echo for f in /docker-entrypoint-initdb.d/*; do case "$f" in - *.sh) - # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 - # https://github.com/docker-library/postgres/pull/452 - if [ -x "$f" ]; then - echo "$0: running $f" - "$f" - else - echo "$0: sourcing $f" - # shellcheck source=/dev/null - . "$f" - fi - ;; - *.sql) + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then echo "$0: running $f" - "${psql[@]}" -f "$f" - echo - ;; - *.sql.gz) - echo "$0: running $f" - gunzip -c "$f" | "${psql[@]}" - echo - ;; - *) echo "$0: ignoring $f" ;; + "$f" + else + echo "$0: sourcing $f" + # shellcheck source=/dev/null + . "$f" + fi + ;; + *.sql) + echo "$0: running $f" + "${psql[@]}" -f "$f" + echo + ;; + *.sql.gz) + echo "$0: running $f" + gunzip -c "$f" | "${psql[@]}" + echo + ;; + *) echo "$0: ignoring $f" ;; esac echo done diff --git a/docker-images/postgres_exporter/BUILD.bazel b/docker-images/postgres_exporter/BUILD.bazel new file mode 100644 index 00000000000..ac575471c22 --- /dev/null +++ b/docker-images/postgres_exporter/BUILD.bazel @@ -0,0 +1,50 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//dev:oci_defs.bzl", "image_repository") +load("@container_structure_test//:defs.bzl", "container_structure_test") + +filegroup( + name = "config_files", + srcs = glob(["config/*"]), +) + +pkg_tar( + name = "config_tar", + srcs = [":config_files"], + remap_paths = { + "/": "/config/", + }, +) + +oci_image( + name = "image", + base = "@wolfi_postgres_exporter_base", + entrypoint = ["/usr/bin/postgres_exporter"], + env = { + "PG_EXPORTER_EXTEND_QUERY_PATH": "/config/queries.yaml", + }, + tars = [":config_tar"], + user = "postgres_exporter", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["postgres-exporter:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("postgres_exporter"), +) diff --git a/docker-images/postgres_exporter/config/01_broken_indexes.yaml b/docker-images/postgres_exporter/config/01_broken_indexes.yaml deleted file mode 100644 index 14158b38803..00000000000 --- a/docker-images/postgres_exporter/config/01_broken_indexes.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# In this file: -# -# | name | type | description | -# | --------------------------------------- | ------- | -------------------------------------------------------- | -# | pg_invalid_index_count{datname,relname} | COUNTER | Non-zero value used to tag existence of an invalid index | - -pg_invalid_index: - query: | - SELECT - current_database() AS datname, - pc.relname AS relname, - 1 AS count - FROM pg_class pc - JOIN pg_index pi ON pi.indexrelid = pc.oid - WHERE - NOT indisvalid AND - NOT EXISTS (SELECT 1 FROM pg_stat_progress_create_index pci WHERE pci.index_relid = pi.indexrelid) - metrics: - - datname: - usage: "LABEL" - description: "Name of current database" - - relname: - usage: "LABEL" - description: "Name of the index" - - count: - usage: "COUNTER" - description: "Non-zero value used to tag existence of an invalid index" diff --git a/docker-images/postgres_exporter/config/02_size.yaml b/docker-images/postgres_exporter/config/02_size.yaml deleted file mode 100644 index 4bd718d0961..00000000000 --- a/docker-images/postgres_exporter/config/02_size.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# In this file: -# -# | name | type | description | -# | --------------------------------- ------ | ----- | -------------------------------------------------------------- | -# | pg_table_size_bytes{datname,relname} | GAUGE | Total size of this table (including toast, index, toast index) | -# | pg_table_size_indexsize{datname,relname} | GAUGE | Size of all related indexes | -# | pg_table_size_relsize{datname,relname} | GAUGE | Size of this table itself (main, vm, fsm) | -# | pg_table_size_toastsize{datname,relname} | GAUGE | Size of corresponding toast tables | -# -# Contents of this file are loosely based off of: -# https://github.com/Vonng/pg_exporter/blob/f682b06630db8e4585aa52df150d0a653bbde07e/conf/810-pg_table_size.yaml - -pg_table_size: - query: | - SELECT - CURRENT_CATALOG AS datname, - r.relname AS relname, - pg_total_relation_size(r.oid) AS bytes, - pg_relation_size(r.oid) AS relsize, - pg_indexes_size(r.oid) AS indexsize, - pg_total_relation_size(r.reltoastrelid) AS toastsize - FROM pg_class r - JOIN pg_namespace n ON n.oid = r.relnamespace - WHERE - r.relkind = 'r' AND n.nspname NOT IN ('pg_catalog', 'information_schema'); - metrics: - - datname: - usage: 'LABEL' - description: 'Name of current database' - - relname: - usage: 'LABEL' - description: 'Name of the table' - - bytes: - usage: 'GAUGE' - description: 'Total size of this table (including toast, index, toast index)' - - indexsize: - usage: 'GAUGE' - description: 'Size of all related indexes' - - relsize: - usage: 'GAUGE' - description: 'Size of this table itself (main, vm, fsm)' - - toastsize: - usage: 'GAUGE' - description: 'Size of corresponding toast tables' diff --git a/docker-images/postgres_exporter/config/04_frontend_migration.yaml b/docker-images/postgres_exporter/config/04_frontend_migration.yaml deleted file mode 100644 index 88348f7aae1..00000000000 --- a/docker-images/postgres_exporter/config/04_frontend_migration.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# In this file: -# -# | name | type | description | -# | ---------------------- | ------- | ------------------------------------------ | -# | pg_sg_migration_status | GAUGE | Whether the migration applied successfully | -# | | | This only applies to the frontend db. | - -pg_sg_migration: - query: | - WITH ranked_migration_logs AS ( - SELECT - migration_logs.*, - ROW_NUMBER() OVER (PARTITION BY version ORDER BY finished_at DESC) AS row_number - FROM migration_logs - WHERE schema = 'schema_migrations' - ) - SELECT EXISTS ( - SELECT 1 - FROM ranked_migration_logs - WHERE row_number = 1 - AND NOT success - )::int; - master: true - metrics: - - status: - usage: "GAUGE" - description: Whether the migration applied successfully diff --git a/docker-images/postgres_exporter/config/06_codeinsights_migration.yaml b/docker-images/postgres_exporter/config/code_insights_queries.yaml similarity index 100% rename from docker-images/postgres_exporter/config/06_codeinsights_migration.yaml rename to docker-images/postgres_exporter/config/code_insights_queries.yaml diff --git a/docker-images/postgres_exporter/config/05_codeintel_migration.yaml b/docker-images/postgres_exporter/config/code_intel_queries.yaml similarity index 100% rename from docker-images/postgres_exporter/config/05_codeintel_migration.yaml rename to docker-images/postgres_exporter/config/code_intel_queries.yaml diff --git a/docker-images/postgres_exporter/config/03_bloat.yaml b/docker-images/postgres_exporter/config/queries.yaml similarity index 66% rename from docker-images/postgres_exporter/config/03_bloat.yaml rename to docker-images/postgres_exporter/config/queries.yaml index de0cfda0685..0b547b166bf 100644 --- a/docker-images/postgres_exporter/config/03_bloat.yaml +++ b/docker-images/postgres_exporter/config/queries.yaml @@ -1,3 +1,76 @@ +# In this file: +# +# | name | type | description | +# | --------------------------------------- | ------- | -------------------------------------------------------- | +# | pg_invalid_index_count{datname,relname} | COUNTER | Non-zero value used to tag existence of an invalid index | + +pg_invalid_index: + query: | + SELECT + current_database() AS datname, + pc.relname AS relname, + 1 AS count + FROM pg_class pc + JOIN pg_index pi ON pi.indexrelid = pc.oid + WHERE + NOT indisvalid AND + NOT EXISTS (SELECT 1 FROM pg_stat_progress_create_index pci WHERE pci.index_relid = pi.indexrelid) + metrics: + - datname: + usage: "LABEL" + description: "Name of current database" + - relname: + usage: "LABEL" + description: "Name of the index" + - count: + usage: "COUNTER" + description: "Non-zero value used to tag existence of an invalid index" + +# In this file: +# +# | name | type | description | +# | --------------------------------- ------ | ----- | -------------------------------------------------------------- | +# | pg_table_size_bytes{datname,relname} | GAUGE | Total size of this table (including toast, index, toast index) | +# | pg_table_size_indexsize{datname,relname} | GAUGE | Size of all related indexes | +# | pg_table_size_relsize{datname,relname} | GAUGE | Size of this table itself (main, vm, fsm) | +# | pg_table_size_toastsize{datname,relname} | GAUGE | Size of corresponding toast tables | +# +# Contents of this file are loosely based off of: +# https://github.com/Vonng/pg_exporter/blob/f682b06630db8e4585aa52df150d0a653bbde07e/conf/810-pg_table_size.yaml + +pg_table_size: + query: | + SELECT + CURRENT_CATALOG AS datname, + r.relname AS relname, + pg_total_relation_size(r.oid) AS bytes, + pg_relation_size(r.oid) AS relsize, + pg_indexes_size(r.oid) AS indexsize, + pg_total_relation_size(r.reltoastrelid) AS toastsize + FROM pg_class r + JOIN pg_namespace n ON n.oid = r.relnamespace + WHERE + r.relkind = 'r' AND n.nspname NOT IN ('pg_catalog', 'information_schema'); + metrics: + - datname: + usage: 'LABEL' + description: 'Name of current database' + - relname: + usage: 'LABEL' + description: 'Name of the table' + - bytes: + usage: 'GAUGE' + description: 'Total size of this table (including toast, index, toast index)' + - indexsize: + usage: 'GAUGE' + description: 'Size of all related indexes' + - relsize: + usage: 'GAUGE' + description: 'Size of this table itself (main, vm, fsm)' + - toastsize: + usage: 'GAUGE' + description: 'Size of corresponding toast tables' + # In this file: # # | name | type | description | @@ -123,3 +196,31 @@ pg_table_bloat: - ratio: usage: 'GAUGE' description: 'Estimated bloat ratio of this table, 0~1' + +# In this file: +# +# | name | type | description | +# | ---------------------- | ------- | ------------------------------------------ | +# | pg_sg_migration_status | GAUGE | Whether the migration applied successfully | +# | | | This only applies to the frontend db. | + +pg_sg_migration: + query: | + WITH ranked_migration_logs AS ( + SELECT + migration_logs.*, + ROW_NUMBER() OVER (PARTITION BY version ORDER BY finished_at DESC) AS row_number + FROM migration_logs + WHERE schema = 'schema_migrations' + ) + SELECT EXISTS ( + SELECT 1 + FROM ranked_migration_logs + WHERE row_number = 1 + AND NOT success + )::int; + master: true + metrics: + - status: + usage: "GAUGE" + description: Whether the migration applied successfully diff --git a/docker-images/postgres_exporter/image_test.yaml b/docker-images/postgres_exporter/image_test.yaml new file mode 100644 index 00000000000..25d67264a11 --- /dev/null +++ b/docker-images/postgres_exporter/image_test.yaml @@ -0,0 +1,33 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "postgres_exporter is runnable" + command: "postgres_exporter" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + - name: "running as postgres_exporter" + command: "/usr/bin/id" + args: + - -u + expectedOutput: ["^20001"] + exitCode: 0 + +fileExistenceTests: +- name: '/config/queries.yaml' + path: '/config/queries.yaml' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' + +metadataTest: + envVars: + - key: PG_EXPORTER_EXTEND_QUERY_PATH + value: /config/queries.yaml diff --git a/docker-images/prometheus/BUILD.bazel b/docker-images/prometheus/BUILD.bazel new file mode 100644 index 00000000000..86d41324254 --- /dev/null +++ b/docker-images/prometheus/BUILD.bazel @@ -0,0 +1,61 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//dev:oci_defs.bzl", "image_repository") +load("@container_structure_test//:defs.bzl", "container_structure_test") + +filegroup( + name = "startup_scripts", + srcs = [ + "alertmanager.sh", + "prometheus.sh", + ], + visibility = ["//visibility:public"], +) + +pkg_tar( + name = "config_tar", + srcs = [ + ":startup_scripts", + "//dev/prometheus:prometheus_targets_linux", + "//docker-images/prometheus/cmd/prom-wrapper", + "//docker-images/prometheus/config:base_config", + "//monitoring:generate_config", + ], + remap_paths = { + "/cmd/prom-wrapper/prom-wrapper/prom-wrapper": "/bin/prom-wrapper", + "monitoring/outputs/prometheus": "/sg_config_prometheus", + "/dev/prometheus/linux": "/sg_prometheus_add_ons", + "/config": "/sg_config_prometheus", + }, + strip_prefix = ".", +) + +oci_image( + name = "image", + base = "@wolfi_prometheus_base", + entrypoint = ["/bin/prom-wrapper"], + tars = [":config_tar"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["prometheus:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("prometheus"), +) diff --git a/docker-images/prometheus/alertmanager.sh b/docker-images/prometheus/alertmanager.sh index 44fe2e4cce6..ccfa402a0ad 100755 --- a/docker-images/prometheus/alertmanager.sh +++ b/docker-images/prometheus/alertmanager.sh @@ -1,5 +1,11 @@ #!/bin/sh set -e -# shellcheck disable=SC2086 -exec /bin/alertmanager --storage.path=/alertmanager --data.retention=168h $ALERTMANAGER_ADDITIONAL_FLAGS "$@" +# Bazel migration moves the binary +if [ -x /usr/bin/alertmanager ]; then + # shellcheck disable=SC2086 + exec /usr/bin/alertmanager --storage.path=/alertmanager --data.retention=168h $ALERTMANAGER_ADDITIONAL_FLAGS "$@" +else + # shellcheck disable=SC2086 + exec /bin/alertmanager --storage.path=/alertmanager --data.retention=168h $ALERTMANAGER_ADDITIONAL_FLAGS "$@" +fi diff --git a/docker-images/prometheus/build-bazel.sh b/docker-images/prometheus/build-bazel.sh index 799c22002ac..654b5aef8c5 100755 --- a/docker-images/prometheus/build-bazel.sh +++ b/docker-images/prometheus/build-bazel.sh @@ -15,10 +15,10 @@ cleanup() { } trap cleanup EXIT -./dev/ci/bazel.sh build //docker-images/prometheus/cmd/prom-wrapper //monitoring:generate_config +./dev/ci/bazel.sh build //docker-images/prometheus/cmd/prom-wrapper //monitoring:generate_config_zip out=$(./dev/ci/bazel.sh cquery //docker-images/prometheus/cmd/prom-wrapper --output=files) cp "$out" "$BUILDDIR" -monitoring_cfg=$(./dev/ci/bazel.sh cquery //monitoring:generate_config --output=files) +monitoring_cfg=$(./dev/ci/bazel.sh cquery //monitoring:generate_config_zip --output=files) cp "$monitoring_cfg" "$TMP/" pushd "$TMP" unzip "monitoring.zip" diff --git a/docker-images/prometheus/config/BUILD.bazel b/docker-images/prometheus/config/BUILD.bazel new file mode 100644 index 00000000000..b03f2108ef3 --- /dev/null +++ b/docker-images/prometheus/config/BUILD.bazel @@ -0,0 +1,8 @@ +filegroup( + name = "base_config", + srcs = [ + "alertmanager.yml", + "prometheus.yml", + ], + visibility = ["//visibility:public"], +) diff --git a/docker-images/prometheus/image_test.yaml b/docker-images/prometheus/image_test.yaml new file mode 100644 index 00000000000..5be9f717d2c --- /dev/null +++ b/docker-images/prometheus/image_test.yaml @@ -0,0 +1,67 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "prometheus is runnable" + command: "prometheus" + args: + - --version + - name: "promtool is runnable" + command: "promtool" + args: + - --version + - name: "alertmanager is runnable" + command: "alertmanager" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: +- name: '/prometheus' + path: '/prometheus' + shouldExist: true + uid: 100 + gid: 101 + permissions: 'drwxr-xr-x' +- name: '/alertmanager' + path: '/alertmanager' + shouldExist: true + uid: 100 + gid: 101 + permissions: 'drwxr-xr-x' +- name: '/prometheus.sh' + path: '/prometheus.sh' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +- name: '/alertmanager.sh' + path: '/alertmanager.sh' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +# /sg_config_prometheus configuration +- name: '/sg_config_prometheus/prometheus.yml' + path: '/sg_config_prometheus/prometheus.yml' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +- name: '/sg_config_prometheus/alertmanager.yml' + path: '/sg_config_prometheus/alertmanager.yml' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +- name: '/sg_config_prometheus/prometheus_alert_rules.yml' + path: '/sg_config_prometheus/prometheus_alert_rules.yml' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' diff --git a/docker-images/prometheus/prometheus.sh b/docker-images/prometheus/prometheus.sh index 6ddd1f3970f..5aa31e23efc 100755 --- a/docker-images/prometheus/prometheus.sh +++ b/docker-images/prometheus/prometheus.sh @@ -9,5 +9,11 @@ fi STORAGE_PATH="${STORAGE_PATH:-"/prometheus"}" -# shellcheck disable=SC2086 -exec /bin/prometheus --config.file=$CONFIG_FILE --storage.tsdb.path=$STORAGE_PATH $PROMETHEUS_ADDITIONAL_FLAGS --web.console.libraries=/usr/share/prometheus/console_libraries --web.console.templates=/usr/share/prometheus/consoles "$@" +# Bazel migration moves the binary +if [ -x /usr/bin/prometheus ]; then + # shellcheck disable=SC2086 + exec /usr/bin/prometheus --config.file=$CONFIG_FILE --storage.tsdb.path=$STORAGE_PATH $PROMETHEUS_ADDITIONAL_FLAGS --web.console.libraries=/usr/share/prometheus/console_libraries --web.console.templates=/usr/share/prometheus/consoles "$@" +else + # shellcheck disable=SC2086 + exec /bin/prometheus --config.file=$CONFIG_FILE --storage.tsdb.path=$STORAGE_PATH $PROMETHEUS_ADDITIONAL_FLAGS --web.console.libraries=/usr/share/prometheus/console_libraries --web.console.templates=/usr/share/prometheus/consoles "$@" +fi diff --git a/docker-images/redis-cache/BUILD.bazel b/docker-images/redis-cache/BUILD.bazel new file mode 100644 index 00000000000..e626cbedff5 --- /dev/null +++ b/docker-images/redis-cache/BUILD.bazel @@ -0,0 +1,53 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//dev:oci_defs.bzl", "image_repository") +load("@container_structure_test//:defs.bzl", "container_structure_test") + +filegroup( + name = "redis_conf", + srcs = ["redis.conf"], +) + +pkg_tar( + name = "redis_tarball", + srcs = [":redis_conf"], + modes = { + "/etc/redis/redis.conf": "0644", + }, + remap_paths = { + "/redis.conf": "/etc/redis/redis.conf", + }, +) + +oci_image( + name = "image", + base = "@wolfi_redis_base", + entrypoint = [ + "redis-server", + "/etc/redis/redis.conf", + ], + tars = [":redis_tarball"], + user = "redis", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["redis-cache:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("redis-cache"), +) diff --git a/docker-images/redis-cache/image_test.yaml b/docker-images/redis-cache/image_test.yaml new file mode 100644 index 00000000000..d54eeb25e18 --- /dev/null +++ b/docker-images/redis-cache/image_test.yaml @@ -0,0 +1,40 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "redis-server is runnable" + command: "redis-server" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + - name: "redis user has correct uid" + command: "/usr/bin/id" + args: + - -u + expectedOutput: ["^999\n"] + exitCode: 0 + - name: "redis user has correct gid" + command: "/usr/bin/id" + args: + - -g + expectedOutput: ["^1000\n"] + exitCode: 0 + +fileExistenceTests: +- name: '/etc/redis/redis.conf' + path: '/etc/redis/redis.conf' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-rw-r--r--' +- name: '/redis-data' + path: '/redis-data' + shouldExist: true + uid: 999 + gid: 1000 + permissions: 'drwxr-xr-x' diff --git a/docker-images/redis-store/BUILD.bazel b/docker-images/redis-store/BUILD.bazel new file mode 100644 index 00000000000..3f362f80056 --- /dev/null +++ b/docker-images/redis-store/BUILD.bazel @@ -0,0 +1,53 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//dev:oci_defs.bzl", "image_repository") +load("@container_structure_test//:defs.bzl", "container_structure_test") + +filegroup( + name = "redis_conf", + srcs = ["redis.conf"], +) + +pkg_tar( + name = "redis_tarball", + srcs = [":redis_conf"], + modes = { + "/etc/redis/redis.conf": "0644", + }, + remap_paths = { + "/redis.conf": "/etc/redis/redis.conf", + }, +) + +oci_image( + name = "image", + base = "@wolfi_redis_base", + entrypoint = [ + "redis-server", + "/etc/redis/redis.conf", + ], + tars = [":redis_tarball"], + user = "redis", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["redis-store:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("redis-store"), +) diff --git a/docker-images/redis-store/image_test.yaml b/docker-images/redis-store/image_test.yaml new file mode 100644 index 00000000000..d54eeb25e18 --- /dev/null +++ b/docker-images/redis-store/image_test.yaml @@ -0,0 +1,40 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "redis-server is runnable" + command: "redis-server" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + - name: "redis user has correct uid" + command: "/usr/bin/id" + args: + - -u + expectedOutput: ["^999\n"] + exitCode: 0 + - name: "redis user has correct gid" + command: "/usr/bin/id" + args: + - -g + expectedOutput: ["^1000\n"] + exitCode: 0 + +fileExistenceTests: +- name: '/etc/redis/redis.conf' + path: '/etc/redis/redis.conf' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-rw-r--r--' +- name: '/redis-data' + path: '/redis-data' + shouldExist: true + uid: 999 + gid: 1000 + permissions: 'drwxr-xr-x' diff --git a/docker-images/redis_exporter/BUILD.bazel b/docker-images/redis_exporter/BUILD.bazel new file mode 100644 index 00000000000..bb5cf1dac95 --- /dev/null +++ b/docker-images/redis_exporter/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("//dev:oci_defs.bzl", "image_repository") +load("@container_structure_test//:defs.bzl", "container_structure_test") + +oci_image( + name = "image", + base = "@wolfi_redis_exporter_base", + entrypoint = ["/usr/local/bin/redis_exporter"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["redis-exporter:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("redis_exporter"), +) diff --git a/docker-images/redis_exporter/image_test.yaml b/docker-images/redis_exporter/image_test.yaml new file mode 100644 index 00000000000..e6c2e03bc5c --- /dev/null +++ b/docker-images/redis_exporter/image_test.yaml @@ -0,0 +1,14 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "redis_exporter is runnable" + command: "redis_exporter" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/docker-images/search-indexer/BUILD.bazel b/docker-images/search-indexer/BUILD.bazel new file mode 100644 index 00000000000..3a99eaa2f88 --- /dev/null +++ b/docker-images/search-indexer/BUILD.bazel @@ -0,0 +1,53 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("//cmd/server:macro.bzl", "container_dependencies", "dependencies_tars") +load("//dev:oci_defs.bzl", "image_repository") +load("@container_structure_test//:defs.bzl", "container_structure_test") + +DEPS = [ + "@com_github_sourcegraph_zoekt//cmd/zoekt-archive-index", + "@com_github_sourcegraph_zoekt//cmd/zoekt-git-index", + "@com_github_sourcegraph_zoekt//cmd/zoekt-sourcegraph-indexserver", + "@com_github_sourcegraph_zoekt//cmd/zoekt-webserver", + "@com_github_sourcegraph_zoekt//cmd/zoekt-merge-index", +] + +container_dependencies(DEPS) + +oci_image( + name = "image", + base = "@wolfi_search_indexer_base", + cmd = [ + "/sbin/tini", + "--", + "zoekt-sourcegraph-indexserver", + ], + env = { + "DATA_DIR": "/data/index", + "SRC_FRONTEND_INTERNAL": "http://sourcegraph-frontend-internal", + }, + tars = dependencies_tars(DEPS), + user = "sourcegraph", + workdir = "/home/sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["zoekt-indexserver:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("search-indexer"), +) diff --git a/docker-images/search-indexer/Dockerfile.wolfi b/docker-images/search-indexer/Dockerfile.wolfi index bec7e946095..654ab527d2c 100644 --- a/docker-images/search-indexer/Dockerfile.wolfi +++ b/docker-images/search-indexer/Dockerfile.wolfi @@ -14,10 +14,10 @@ LABEL org.opencontainers.image.revision=${COMMIT_SHA} LABEL org.opencontainers.image.created=${DATE} LABEL org.opencontainers.image.version=${VERSION} +# DONE ENV SRC_FRONTEND_INTERNAL http://sourcegraph-frontend-internal +# DONE ENV DATA_DIR /data/index -RUN mkdir -p ${DATA_DIR} -RUN chown -R sourcegraph:sourcegraph /data USER sourcegraph WORKDIR /home/sourcegraph diff --git a/docker-images/search-indexer/image_test.yaml b/docker-images/search-indexer/image_test.yaml new file mode 100644 index 00000000000..ec61035a8b0 --- /dev/null +++ b/docker-images/search-indexer/image_test.yaml @@ -0,0 +1,56 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "zoekt-sourcegraph-indexserver is runnable" + command: "zoekt-sourcegraph-indexserver" + args: + - --help + - name: "zoekt-archive-index is runnable" + command: "zoekt-archive-index" + args: + - --help + - name: "zoekt-git-index is runnable" + command: "zoekt-git-index" + args: + - --version + + - name: "git is runnable" + command: "git" + args: + - version + - name: "ctags is runnable" + command: "universal-ctags" + args: + - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: +# zoekt-merge-index doesn't have any arguments we can use for testing +- name: '/usr/local/bin/zoekt-merge-index' + path: '/usr/local/bin/zoekt-merge-index' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +- name: '/data/index' + path: '/data/index' + shouldExist: true + uid: 100 + gid: 101 + permissions: 'drwxr-xr-x' +- name: 'jansson package' + path: 'usr/lib/libjansson.la' + shouldExist: true + +metadataTest: + envVars: + - key: SRC_FRONTEND_INTERNAL + value: 'http://sourcegraph-frontend-internal' + - key: DATA_DIR + value: '/data/index' diff --git a/docker-images/sg/BUILD.bazel b/docker-images/sg/BUILD.bazel new file mode 100644 index 00000000000..88fb4411878 --- /dev/null +++ b/docker-images/sg/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//dev:oci_defs.bzl", "image_repository") +load("@container_structure_test//:defs.bzl", "container_structure_test") + +pkg_tar( + name = "sg_tarball", + srcs = ["//dev/sg"], + remap_paths = { + "/sg": "/usr/local/bin/sg", + }, +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/usr/local/bin/sg", + ], + tars = [":sg_tarball"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["sg:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("sg"), +) diff --git a/docker-images/sg/image_test.yaml b/docker-images/sg/image_test.yaml new file mode 100644 index 00000000000..c622576f6e8 --- /dev/null +++ b/docker-images/sg/image_test.yaml @@ -0,0 +1,14 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "sg is runnable" + command: "sg" + args: + - version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/docker-images/syntax-highlighter/BUILD.bazel b/docker-images/syntax-highlighter/BUILD.bazel index 8e79189032d..d77699824e5 100644 --- a/docker-images/syntax-highlighter/BUILD.bazel +++ b/docker-images/syntax-highlighter/BUILD.bazel @@ -1,5 +1,9 @@ load("@crate_index//:defs.bzl", "aliases", "all_crate_deps") load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") rust_binary( name = "syntect_server", @@ -8,6 +12,7 @@ rust_binary( proc_macro_deps = all_crate_deps( proc_macro = True, ), + visibility = ["//visibility:public"], deps = all_crate_deps( normal = True, ) + [ @@ -40,9 +45,74 @@ rust_binary( proc_macro_deps = all_crate_deps( proc_macro = True, ), + visibility = ["//visibility:public"], deps = all_crate_deps( normal = True, ) + [ "//docker-images/syntax-highlighter/crates/scip-syntax:scip-syntax", ], ) + +pkg_tar( + name = "tar_syntect_server", + srcs = [":syntect_server"], +) + +oci_image( + name = "image", + base = "@wolfi_syntax_highlighter_base", + entrypoint = [ + "/usr/local/bin/http-server-stabilizer", + "-listen=:9238", + "-prometheus-app-name=syntax_highlighter", + # The more workers, the more resilient syntect_server is to getting stuck on + # bad grammar/file combinations. If it happens with four workers, only 1/4th of + # requests will be affected for a short period of time. Each worker can require + # at peak around 1.1 GiB of memory. + "-workers=4", + "--", + "env", + "ROCKET_PORT={{.Port}}", # {{.Port}} is a templated variable used by http-server-stabilizer + "/syntect_server", + ], + env = { + "ROCKET_ENV": "production", + "ROCKET_LIMITS": "{json=10485760}", + # syntect_server does not need a secret key since it uses no cookies, but + # without one set Rocket emits a warning. + "ROCKET_SECRET_KEY": "SeerutKeyIsI7releuantAndknvsuZPluaseIgnorYA=", + # When keep-alive is on, we observe connection resets in our Go clients of + # syntect_server. It is unclear why this is, especially because our Go clients do + # not reuse the connection (i.e. we make a fresh connection every time). + # Disabling keep-alive does resolve the issue though, our best guess is that + # this is a bug in Hyper 0.10 (see https://github.com/SergioBenitez/Rocket/issues/928#issuecomment-464632953). + # See https://github.com/sourcegraph/sourcegraph/issues/2615 for details on + # what we observed when this was enabled with the default 5s. + "ROCKET_KEEP_ALIVE": "0", + "WORKERS": "4", + "QUIET": "true", + }, + tars = [":tar_syntect_server"], +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["syntect-server:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("syntax-highlighter"), +) diff --git a/docker-images/syntax-highlighter/image_test.yaml b/docker-images/syntax-highlighter/image_test.yaml new file mode 100644 index 00000000000..238ba348cd5 --- /dev/null +++ b/docker-images/syntax-highlighter/image_test.yaml @@ -0,0 +1,34 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "syntect_server binary is runnable" + command: "/syntect_server" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "http-server-stabilizer binary is runnable" + command: "/usr/local/bin/http-server-stabilizer" + args: ["-h"] + + # TODO(security): This container should not be running as root + # - name: "not running as root" + # command: "/usr/bin/id" + # args: + # - -u + # excludedOutput: ["^0"] + # exitCode: 0 + +metadataTest: + envVars: + - key: ROCKET_ENV + value: 'production' + - key: ROCKET_LIMITS + value: '{json=10485760}' + - key: ROCKET_SECRET_KEY + value: 'SeerutKeyIsI7releuantAndknvsuZPluaseIgnorYA=' + - key: ROCKET_KEEP_ALIVE + value: '0' + - key: WORKERS + value: '4' + - key: QUIET + value: 'true' diff --git a/docker-images/syntax-highlighter/src/bin/scip-ctags.rs b/docker-images/syntax-highlighter/src/bin/scip-ctags.rs index a49ee4a85e4..2bf90cb0ce7 100644 --- a/docker-images/syntax-highlighter/src/bin/scip-ctags.rs +++ b/docker-images/syntax-highlighter/src/bin/scip-ctags.rs @@ -3,6 +3,19 @@ use std::io::{BufReader, BufWriter}; use scip_syntax::ctags::ctags_runner; fn main() { + // Exits with a code zero if the environment variable SANITY_CHECK equals + // to "true". This enables testing that the current program is in a runnable + // state against the platform it's being executed on. + // + // See https://github.com/GoogleContainerTools/container-structure-test + match std::env::var("SANITY_CHECK") { + Ok(v) if v == "true" => { + println!("Sanity check passed, exiting without error"); + std::process::exit(0) + } + _ => {} + }; + let mut stdin = BufReader::new(std::io::stdin()); let mut stdout = BufWriter::new(std::io::stdout()); diff --git a/enterprise/cmd/batcheshelper/BUILD.bazel b/enterprise/cmd/batcheshelper/BUILD.bazel index 7b7a56a9f9f..35e8dcd6345 100644 --- a/enterprise/cmd/batcheshelper/BUILD.bazel +++ b/enterprise/cmd/batcheshelper/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "batcheshelper_lib", @@ -33,3 +37,41 @@ go_test( "@com_github_stretchr_testify//require", ], ) + +pkg_tar( + name = "tar_batcheshelper", + srcs = [":batcheshelper"], +) + +oci_image( + name = "image", + base = "@wolfi_batcheshelper_base", + entrypoint = [ + "/sbin/tini", + "--", + "/batcheshelper", + ], + tars = [":tar_batcheshelper"], +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["batcheshelper:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("batcheshelper"), +) diff --git a/enterprise/cmd/batcheshelper/image_test.yaml b/enterprise/cmd/batcheshelper/image_test.yaml new file mode 100644 index 00000000000..efc209eb7ad --- /dev/null +++ b/enterprise/cmd/batcheshelper/image_test.yaml @@ -0,0 +1,20 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/batcheshelper" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "git is runnable" + command: "git" + args: + - version + + # TODO(security): This container should not be running as root + # - name: "not running as root" + # command: "/usr/bin/id" + # args: + # - -u + # excludedOutput: ["^0"] + # exitCode: 0 diff --git a/enterprise/cmd/bundled-executor/BUILD.bazel b/enterprise/cmd/bundled-executor/BUILD.bazel new file mode 100644 index 00000000000..424f6bbfffb --- /dev/null +++ b/enterprise/cmd/bundled-executor/BUILD.bazel @@ -0,0 +1,57 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") +load("//cmd/server:macro.bzl", "container_dependencies", "dependencies_tars") + +DEPS = [ + "//enterprise/cmd/batcheshelper", + "//enterprise/cmd/executor", +] + +container_dependencies(DEPS) + +pkg_tar( + name = "tar_src-cli", + srcs = ["@src-cli-linux-amd64//:src-cli-linux-amd64"], + package_dir = "/usr/local/bin", +) + +oci_image( + name = "image", + base = "@wolfi_bundled_executor_base", + entrypoint = [ + "/sbin/tini", + "--", + "/usr/local/bin/executor", + ], + env = { + "EXECUTOR_USE_FIRECRACKER": "false", # Firecracker doesn't work in docker, so disable it by default + "EXECUTOR_MAXIMUM_NUM_JOBS": "1", # Preconfigure bundled-executor to take 1 parallel job and restart afterwards, this is to keep the environment clean-ish + "EXECUTOR_NUM_TOTAL_JOBS": "1", + }, + tars = dependencies_tars(DEPS) + [":tar_src-cli"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["bundled-executor:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("bundled-executor"), +) diff --git a/enterprise/cmd/bundled-executor/Dockerfile.wolfi b/enterprise/cmd/bundled-executor/Dockerfile.wolfi index cd5861da023..c3d7615544b 100644 --- a/enterprise/cmd/bundled-executor/Dockerfile.wolfi +++ b/enterprise/cmd/bundled-executor/Dockerfile.wolfi @@ -10,6 +10,13 @@ LABEL org.opencontainers.image.created=${DATE} LABEL org.opencontainers.image.version=${VERSION} LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} +ENV \ + # Firecracker doesn't work in docker, so disable it by default + EXECUTOR_USE_FIRECRACKER=false \ + # Preconfigure bundled-executor to take 1 parallel job and restart afterwards, this is to keep the environment clean-ish + EXECUTOR_MAXIMUM_NUM_JOBS=1 \ + EXECUTOR_NUM_TOTAL_JOBS=1 + # Install src-cli. ARG SRC_CLI_VERSION RUN set -ex && \ diff --git a/enterprise/cmd/bundled-executor/image_test.yaml b/enterprise/cmd/bundled-executor/image_test.yaml new file mode 100644 index 00000000000..ea579ae3fbf --- /dev/null +++ b/enterprise/cmd/bundled-executor/image_test.yaml @@ -0,0 +1,23 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/usr/local/bin/executor" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "batcheshelper binary is runnable" + command: "/usr/local/bin/batcheshelper" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "src-cli binary is runnable" + command: "/usr/local/bin/src" + args: + - -v + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/enterprise/cmd/cody-gateway/BUILD.bazel b/enterprise/cmd/cody-gateway/BUILD.bazel index a633d8d7a06..eab7bcfdb5e 100644 --- a/enterprise/cmd/cody-gateway/BUILD.bazel +++ b/enterprise/cmd/cody-gateway/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "cody-gateway_lib", @@ -21,3 +25,42 @@ go_binary( embed = [":cody-gateway_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_cody-gateway", + srcs = [":cody-gateway"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/cody-gateway", + ], + tars = [":tar_cody-gateway"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["cody-gateway:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("cody-gateway"), +) diff --git a/enterprise/cmd/cody-gateway/image_test.yaml b/enterprise/cmd/cody-gateway/image_test.yaml new file mode 100644 index 00000000000..2d38daeff23 --- /dev/null +++ b/enterprise/cmd/cody-gateway/image_test.yaml @@ -0,0 +1,15 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/cody-gateway" + envVars: + - key: "SANITY_CHECK" + value: "true" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/enterprise/cmd/embeddings/BUILD.bazel b/enterprise/cmd/embeddings/BUILD.bazel index febda942cd4..3c67b5583a9 100644 --- a/enterprise/cmd/embeddings/BUILD.bazel +++ b/enterprise/cmd/embeddings/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "embeddings_lib", @@ -17,3 +21,43 @@ go_binary( embed = [":embeddings_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_embeddings", + srcs = [":embeddings"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/embeddings", + ], + tars = [":tar_embeddings"], + user = "sourcegraph", + workdir = "/", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["embeddings:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("embeddings"), +) diff --git a/enterprise/cmd/embeddings/image_test.yaml b/enterprise/cmd/embeddings/image_test.yaml new file mode 100644 index 00000000000..9200932653f --- /dev/null +++ b/enterprise/cmd/embeddings/image_test.yaml @@ -0,0 +1,21 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/embeddings" + envVars: + - key: "SANITY_CHECK" + value: "true" + # TODO: Have asked the team and they don't think bash is required + # Leaving this test as a comment until we've confirmed with QA + # - name: "bash is runnable" + # command: "bash" + # args: + # - --version + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/enterprise/cmd/executor-kubernetes/BUILD.bazel b/enterprise/cmd/executor-kubernetes/BUILD.bazel new file mode 100644 index 00000000000..870b2ce8049 --- /dev/null +++ b/enterprise/cmd/executor-kubernetes/BUILD.bazel @@ -0,0 +1,51 @@ +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") +load("//cmd/server:macro.bzl", "container_dependencies", "dependencies_tars") + +DEPS = [ + "//enterprise/cmd/executor", +] + +container_dependencies(DEPS) + +pkg_tar( + name = "tar_src-cli", + srcs = ["@src-cli-linux-amd64//:src-cli-linux-amd64"], + package_dir = "/usr/local/bin", +) + +oci_image( + name = "image", + base = "@wolfi_executor_kubernetes_base", + entrypoint = [ + "/sbin/tini", + "--", + "/usr/local/bin/executor", + ], + tars = dependencies_tars(DEPS) + [":tar_src-cli"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["executor-kubernetes:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("executor-kubernetes"), +) diff --git a/enterprise/cmd/executor-kubernetes/image_test.yaml b/enterprise/cmd/executor-kubernetes/image_test.yaml new file mode 100644 index 00000000000..68eae260321 --- /dev/null +++ b/enterprise/cmd/executor-kubernetes/image_test.yaml @@ -0,0 +1,18 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/usr/local/bin/executor" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "src-cli binary is runnable" + command: "/usr/local/bin/src" + args: + - -v + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/enterprise/cmd/executor/BUILD.bazel b/enterprise/cmd/executor/BUILD.bazel index 8072259c205..2a4b7037664 100644 --- a/enterprise/cmd/executor/BUILD.bazel +++ b/enterprise/cmd/executor/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "executor_lib", @@ -24,3 +28,54 @@ go_binary( embed = [":executor_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_executor", + srcs = [":executor"], +) + +pkg_tar( + name = "tar_src-cli", + srcs = ["@src-cli-linux-amd64//:src-cli-linux-amd64"], + package_dir = "/usr/local/bin", +) + +oci_image( + name = "image", + base = "@wolfi_executor_base", + entrypoint = [ + "/sbin/tini", + "--", + "/executor", + ], + env = { + "EXECUTOR_USE_FIRECRACKER": "false", # Firecracker doesn't work in docker, so disable it by default + }, + tars = [ + ":tar_executor", + ":tar_src-cli", + ], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["executor:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("executor"), +) diff --git a/enterprise/cmd/executor/docker-image/Dockerfile.wolfi b/enterprise/cmd/executor/docker-image/Dockerfile.wolfi index c95552c6cbc..37c4dca5ea5 100644 --- a/enterprise/cmd/executor/docker-image/Dockerfile.wolfi +++ b/enterprise/cmd/executor/docker-image/Dockerfile.wolfi @@ -10,6 +10,9 @@ LABEL org.opencontainers.image.created=${DATE} LABEL org.opencontainers.image.version=${VERSION} LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} +# Firecracker doesn't work in docker, so disable it by default. +ENV EXECUTOR_USE_FIRECRACKER=false + # Install src-cli # TODO: Replace with a bazel-built binary ARG SRC_CLI_VERSION diff --git a/enterprise/cmd/executor/image_test.yaml b/enterprise/cmd/executor/image_test.yaml new file mode 100644 index 00000000000..ed5b1c855c8 --- /dev/null +++ b/enterprise/cmd/executor/image_test.yaml @@ -0,0 +1,31 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/executor" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "docker is runnable" + command: "docker" + - name: "git is runnable" + command: "git" + args: + - version + + - name: "src-cli binary is runnable" + command: "/usr/local/bin/src" + args: + - -v + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +metadataTest: + envVars: + - key: EXECUTOR_USE_FIRECRACKER + value: false diff --git a/enterprise/cmd/frontend/BUILD.bazel b/enterprise/cmd/frontend/BUILD.bazel index 9e7ee93b5f7..f14f343ba8f 100644 --- a/enterprise/cmd/frontend/BUILD.bazel +++ b/enterprise/cmd/frontend/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "frontend_lib", @@ -23,3 +27,57 @@ go_binary( embed = [":frontend_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_frontend", + srcs = [":frontend"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + cmd = ["serve"], + entrypoint = [ + "/sbin/tini", + "--", + "/frontend", + ], + env = { + "CONFIGURATION_MODE": "server", + "PGDATABASE": "sg", + "PGHOST": "pgsql", + "PGPORT": "5432", + "PGSSLMODE": "disable", + "PGUSER": "sg", + "CODEINTEL_PGDATABASE": "sg", + "CODEINTEL_PGHOST": "codeintel-db", + "CODEINTEL_PGPORT": "5432", + "CODEINTEL_PGSSLMODE": "disable", + "CODEINTEL_PGUSER": "sg", + "PUBLIC_REPO_REDIRECTS": "true", + }, + tars = [":tar_frontend"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["frontend:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("frontend"), +) diff --git a/enterprise/cmd/frontend/Dockerfile.wolfi b/enterprise/cmd/frontend/Dockerfile.wolfi index dc2a168bfc3..e8ccd3bcd5f 100644 --- a/enterprise/cmd/frontend/Dockerfile.wolfi +++ b/enterprise/cmd/frontend/Dockerfile.wolfi @@ -15,7 +15,6 @@ LABEL org.opencontainers.image.version=${VERSION} LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} ENV CONFIGURATION_MODE=server PGDATABASE=sg PGHOST=pgsql PGPORT=5432 PGSSLMODE=disable PGUSER=sg CODEINTEL_PGDATABASE=sg CODEINTEL_PGHOST=codeintel-db CODEINTEL_PGPORT=5432 CODEINTEL_PGSSLMODE=disable CODEINTEL_PGUSER=sg PUBLIC_REPO_REDIRECTS=true -RUN mkdir -p /mnt/cache/frontend && chown -R sourcegraph:sourcegraph /mnt/cache/frontend USER sourcegraph CMD ["serve"] diff --git a/enterprise/cmd/frontend/image_test.yaml b/enterprise/cmd/frontend/image_test.yaml new file mode 100644 index 00000000000..0e9b870136c --- /dev/null +++ b/enterprise/cmd/frontend/image_test.yaml @@ -0,0 +1,28 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/frontend" + envVars: + - key: "SANITY_CHECK" + value: "true" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: +- name: '/mnt/cache/frontend' + path: '/mnt/cache/frontend' + shouldExist: true + uid: 100 + gid: 101 + +metadataTest: + envVars: + - key: PGHOST + value: .+ + isRegex: true diff --git a/enterprise/cmd/gitserver/BUILD.bazel b/enterprise/cmd/gitserver/BUILD.bazel index a34f6bc8e4d..5025d5b0a90 100644 --- a/enterprise/cmd/gitserver/BUILD.bazel +++ b/enterprise/cmd/gitserver/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "gitserver_lib", @@ -21,3 +25,59 @@ go_binary( embed = [":gitserver_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_gitserver", + srcs = [":gitserver"], +) + +pkg_tar( + name = "tar_p4_fusion_wrappers", + srcs = [ + "p4-fusion-wrapper-detect-kill.sh", + "process-stats-watcher.sh", + ], + remap_paths = { + "/p4-fusion-wrapper-detect-kill.sh": "/usr/local/bin/p4-fusion", + "/process-stats-watcher.sh": "/usr/local/bin/process-stats-watcher.sh", + }, + visibility = ["//visibility:public"], +) + +oci_image( + name = "image", + base = "@wolfi_gitserver_base", + entrypoint = [ + "/sbin/tini", + "--", + "/gitserver", + ], + tars = [ + ":tar_gitserver", + ":tar_p4_fusion_wrappers", + ], + user = "sourcegraph", + workdir = "/", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["gitserver:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("gitserver"), +) diff --git a/enterprise/cmd/gitserver/Dockerfile.wolfi b/enterprise/cmd/gitserver/Dockerfile.wolfi index 0b94243249c..66a491c30e9 100644 --- a/enterprise/cmd/gitserver/Dockerfile.wolfi +++ b/enterprise/cmd/gitserver/Dockerfile.wolfi @@ -14,7 +14,6 @@ LABEL org.opencontainers.image.created=${DATE} LABEL org.opencontainers.image.version=${VERSION} LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} -RUN mkdir -p /data/repos && chown -R sourcegraph:sourcegraph /data/repos USER sourcegraph WORKDIR / diff --git a/enterprise/cmd/gitserver/image_test.yaml b/enterprise/cmd/gitserver/image_test.yaml new file mode 100644 index 00000000000..445fa88f14f --- /dev/null +++ b/enterprise/cmd/gitserver/image_test.yaml @@ -0,0 +1,68 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/gitserver" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "git is runnable" + command: "git" + args: + - version + - name: "git-lfs is runnable" + command: "git-lfs" + args: + - version + - name: "git p4 is runnable" + command: "git" + args: + - p4 + expectedOutput: ["valid commands: submit"] + exitCode: 2 + - name: "ssh is runnable" + command: "ssh" + exitCode: 255 + - name: "python3 is runnable" + command: "python3" + args: + - --version + - name: "bash is runnable" + command: "bash" + args: + - --version + - name: "p4 is runnable" + command: "p4" + args: + - -h + - name: "coursier is runnable" + command: "coursier" + - name: "p4-fusion is runnable" + command: "p4-fusion-binary" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: +- name: '/data/repos' + path: '/data/repos' + shouldExist: true + uid: 100 + gid: 101 +# p4-fusion wrappers +- name: '/usr/local/bin/p4-fusion' + path: '/usr/local/bin/p4-fusion' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +- name: '/usr/local/bin/process-stats-watcher.sh' + path: '/usr/local/bin/process-stats-watcher.sh' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' diff --git a/enterprise/cmd/migrator/BUILD.bazel b/enterprise/cmd/migrator/BUILD.bazel index d3fd41d20d0..570f7936ae6 100644 --- a/enterprise/cmd/migrator/BUILD.bazel +++ b/enterprise/cmd/migrator/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "migrator_lib", @@ -25,3 +29,45 @@ go_binary( embed = [":migrator_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_migrator", + srcs = [":migrator"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/migrator", + ], + tars = [ + ":tar_migrator", + "//cmd/migrator:tar_schema_descriptions", + ], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["migrator:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("migrator"), +) diff --git a/enterprise/cmd/migrator/image_test.yaml b/enterprise/cmd/migrator/image_test.yaml new file mode 100644 index 00000000000..b4e8b566352 --- /dev/null +++ b/enterprise/cmd/migrator/image_test.yaml @@ -0,0 +1,62 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/migrator" + envVars: + - key: "SANITY_CHECK" + value: "true" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 + +fileExistenceTests: + # Following files are fetched through GCS +- name: '/schema-descriptions 3.20.0 schema' + path: '/schema-descriptions/v3.20.0-internal_database_schema.json' + shouldExist: true + uid: 0 + gid: 0 +- name: '/schema-descriptions 3.20.0 schema.codeintel does not exists' + path: '/schema-descriptions/v3.20.0-internal_database_schema.codeintel.json' + shouldExist: false + uid: 0 + gid: 0 + +- name: '/schema-descriptions 3.21.0 schema' + path: '/schema-descriptions/v3.21.0-internal_database_schema.json' + shouldExist: true + uid: 0 + gid: 0 +- name: '/schema-descriptions 3.21.0 schema.codeintel' + path: '/schema-descriptions/v3.21.0-internal_database_schema.codeintel.json' + shouldExist: true + uid: 0 + gid: 0 +- name: '/schema-descriptions 3.21.0 schema.codeinsights does not exists' + # We don't have codeinsights for that version, there should not be a file + path: '/schema-descriptions/v3.21.0-internal_database_schema.codeinsights.json' + shouldExist: false + uid: 0 + gid: 0 + + # Following files are fetched through GitHub raw HTTP requests +- name: '/schema-descriptions 5.0.1 schema' + path: '/schema-descriptions/v5.0.1-internal_database_schema.json' + shouldExist: true + uid: 0 + gid: 0 +- name: '/schema-descriptions 5.0.1 schema.codeintel' + path: '/schema-descriptions/v5.0.1-internal_database_schema.codeintel.json' + shouldExist: true + uid: 0 + gid: 0 +- name: '/schema-descriptions 5.0.1 schema.codeinsights' + path: '/schema-descriptions/v5.0.1-internal_database_schema.codeinsights.json' + shouldExist: true + uid: 0 + gid: 0 diff --git a/enterprise/cmd/precise-code-intel-worker/BUILD.bazel b/enterprise/cmd/precise-code-intel-worker/BUILD.bazel index 73102f348cb..13970ac68f8 100644 --- a/enterprise/cmd/precise-code-intel-worker/BUILD.bazel +++ b/enterprise/cmd/precise-code-intel-worker/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "precise-code-intel-worker_lib", @@ -21,3 +25,42 @@ go_binary( embed = [":precise-code-intel-worker_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_precise-code-intel-worker", + srcs = [":precise-code-intel-worker"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/precise-code-intel-worker", + ], + tars = [":tar_precise-code-intel-worker"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["precise-code-intel-worker:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("precise-code-intel-worker"), +) diff --git a/enterprise/cmd/precise-code-intel-worker/image_test.yaml b/enterprise/cmd/precise-code-intel-worker/image_test.yaml new file mode 100644 index 00000000000..6eec7ee487a --- /dev/null +++ b/enterprise/cmd/precise-code-intel-worker/image_test.yaml @@ -0,0 +1,15 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/precise-code-intel-worker" + envVars: + - key: "SANITY_CHECK" + value: "true" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/enterprise/cmd/repo-updater/BUILD.bazel b/enterprise/cmd/repo-updater/BUILD.bazel index 6701d4e066b..f1140fb7114 100644 --- a/enterprise/cmd/repo-updater/BUILD.bazel +++ b/enterprise/cmd/repo-updater/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "repo-updater_lib", @@ -21,3 +25,42 @@ go_binary( embed = [":repo-updater_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_repo-updater", + srcs = [":repo-updater"], +) + +oci_image( + name = "image", + base = "@wolfi_repo_updater_base", + entrypoint = [ + "/sbin/tini", + "--", + "/repo-updater", + ], + tars = [":tar_repo-updater"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["repo-updater:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("repo-updater"), +) diff --git a/enterprise/cmd/repo-updater/image_test.yaml b/enterprise/cmd/repo-updater/image_test.yaml new file mode 100644 index 00000000000..6bf04e8b1ed --- /dev/null +++ b/enterprise/cmd/repo-updater/image_test.yaml @@ -0,0 +1,21 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/repo-updater" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "p4 is runnable" + command: "p4" + args: + - -h + - name: "coursier is runnable" + command: "coursier" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/enterprise/cmd/server/BUILD.bazel b/enterprise/cmd/server/BUILD.bazel index e1d9e590ba7..129eade6bb3 100644 --- a/enterprise/cmd/server/BUILD.bazel +++ b/enterprise/cmd/server/BUILD.bazel @@ -1,4 +1,9 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//cmd/server:macro.bzl", "container_dependencies", "dependencies_tars") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "server_lib", @@ -17,3 +22,164 @@ go_binary( embed = [":server_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_server", + srcs = [":server"], +) + +genrule( + name = "nginx_tmp_folder", + outs = [".keep_tmp"], + cmd = "echo keep > $@", +) + +genrule( + name = "nginx_run_folder", + outs = [".keep_run"], + cmd = "echo keep > $@", +) + +pkg_tar( + name = "static_config_tar", + srcs = [ + ":nginx_run_folder", + ":nginx_tmp_folder", + ], + remap_paths = { + "/.keep_tmp": "/var/lib/nginx/tmp/.keep", + "/.keep_run": "/var/run/.keep", + }, +) + +pkg_tar( + name = "tar_postgres_exporter_config", + srcs = ["//cmd/server:postgres_exporter.yaml"], +) + +pkg_tar( + name = "tar_postgres_optimize", + srcs = ["//cmd/server/rootfs:postgres-optimize.sh"], +) + +# TODO(@jhchabran) prom-wrapper has to be in /bin while we're still +# building the old and new images, because this path is fed to the procfile +# by the server, and cannot be placed at two different places. +pkg_tar( + name = "tar_prom-wrapper", + srcs = ["//docker-images/prometheus/cmd/prom-wrapper"], + package_dir = "/bin", +) + +pkg_tar( + name = "tar_monitoring_config", + srcs = [ + "//dev/prometheus:prometheus_targets_linux", + "//docker-images/grafana/config", + "//docker-images/prometheus:startup_scripts", + "//docker-images/prometheus/config:base_config", + "//monitoring:generate_config", + ], + remap_paths = { + "monitoring/outputs/docs": "/sg_config_docs", + "monitoring/outputs/prometheus": "/sg_config_prometheus", + "monitoring/outputs/grafana": "/sg_config_grafana/provisioning/dashboards/sourcegraph", + "docker-images/grafana/config": "/sg_config_grafana", + "docker-images/prometheus/config": "/sg_config_prometheus", + "docker-images/prometheus": "", + "dev/prometheus/linux": "/sg_prometheus_add_ons", + }, + strip_prefix = ".", +) + +DEPS = [ + "//cmd/github-proxy", + "//cmd/searcher", + "//enterprise/cmd/embeddings", + "//enterprise/cmd/frontend", + "//enterprise/cmd/gitserver", + "//enterprise/cmd/migrator", + "//enterprise/cmd/precise-code-intel-worker", + "//enterprise/cmd/repo-updater", + "//enterprise/cmd/symbols", + "//enterprise/cmd/worker", +] + +ZOEKT_DEPS = [ + "@com_github_sourcegraph_zoekt//cmd/zoekt-archive-index", + "@com_github_sourcegraph_zoekt//cmd/zoekt-git-index", + "@com_github_sourcegraph_zoekt//cmd/zoekt-sourcegraph-indexserver", + "@com_github_sourcegraph_zoekt//cmd/zoekt-webserver", +] + +# Declares rules for pkg_tar for each dep in DEPS +container_dependencies(DEPS) + +container_dependencies(ZOEKT_DEPS) + +# This one is a special case because inside server images, the procfile expects to find it +# under syntax-highlighter instead of syntect_server. +pkg_tar( + name = "tar_syntax-highlighter", + srcs = ["//docker-images/syntax-highlighter:syntect_server"], + remap_paths = {"/syntect_server": "/usr/local/bin/syntax_highlighter"}, +) + +pkg_tar( + name = "tar_scip-ctags", + srcs = ["//docker-images/syntax-highlighter:scip-ctags"], + package_dir = "/usr/local/bin", +) + +# Tip: to view exactly what gets built here, you can run: +# bazel cquery '//cmd/server:image' --output build +oci_image( + name = "image", + base = "@wolfi_server_base", + entrypoint = [ + "/sbin/tini", + "--", + "/server", + ], + env = { + "GO111MODULES": "on", + "LANG": "en_US.utf8", + "LC_ALL": "en_US.utf8", + # "PGHOST": "/var/run/postgresql", + }, + tars = [ + ":tar_server", + ":static_config_tar", + ":tar_postgres_exporter_config", + ":tar_monitoring_config", + ":tar_syntax-highlighter", + ":tar_scip-ctags", + ":tar_postgres_optimize", + ":tar_prom-wrapper", + "//enterprise/cmd/gitserver:tar_p4_fusion_wrappers", + ] + dependencies_tars(DEPS) + dependencies_tars(ZOEKT_DEPS), + workdir = "/", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["server:candidate"], + visibility = ["//testing:__pkg__"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("server"), +) diff --git a/enterprise/cmd/server/image_test.yaml b/enterprise/cmd/server/image_test.yaml new file mode 100644 index 00000000000..f943639321f --- /dev/null +++ b/enterprise/cmd/server/image_test.yaml @@ -0,0 +1,255 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "server binary is runnable" + command: "/server" + envVars: + - key: "SANITY_CHECK" + value: "true" + + # Sourcegraph binaries + - name: "embeddings binary is runnable" + command: "embeddings" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "frontend binary is runnable" + command: "frontend" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "github-proxy binary is runnable" + command: "github-proxy" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "gitserver binary is runnable" + command: "gitserver" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "migrator binary is runnable" + command: "migrator" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "precise-code-intel-worker binary is runnable" + command: "precise-code-intel-worker" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "repo-updater binary is runnable" + command: "repo-updater" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "searcher binary is runnable" + command: "searcher" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "symbols binary is runnable" + command: "symbols" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "syntax_highlighter binary is runnable" + command: "syntax_highlighter" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "scip-ctags is runnable" + command: "/usr/local/bin/scip-ctags" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "worker binary is runnable" + command: "worker" + envVars: + - key: "SANITY_CHECK" + value: "true" + + # Git + - name: "git is runnable" + command: "git" + args: + - version + - name: "git-lfs is runnable" + command: "git-lfs" + args: + - version + - name: "git p4 is runnable" + command: "git" + args: + - p4 + expectedOutput: ["valid commands: submit"] + exitCode: 2 + + # Zoekt binaries + - name: "zoekt-webserver is runnable" + command: "zoekt-webserver" + args: + - --version + - name: "zoekt-archive-index is runnable" + command: "zoekt-archive-index" + args: + - --help + - name: "zoekt-git-index is runnable" + command: "zoekt-git-index" + args: + - --version + - name: "zoekt-sourcegraph-indexserver is runnable" + command: "zoekt-sourcegraph-indexserver" + args: + - --help + + # Postgresql + - name: "postgres is runnable" + command: "postgres" + args: + - --version + # Bash + - name: "bash is runnable" + command: "bash" + args: + - --version + # P4 tools + - name: "p4 is runnable" + command: "p4" + args: + - -h + - name: "p4-fusion binary is runnable" + command: "p4-fusion-binary" + # Coursier + - name: "coursier is runnable" + command: "coursier" + # Redis + - name: "redis-server is runnable" + command: "redis-server" + args: + - --version + # Python3 + - name: "python3 is runnable" + command: "python3" + args: + - --version + # SSH + - name: "ssh is runnable" + command: "ssh" + exitCode: 255 + # PCRE + - name: "pcre is runnable" + command: "pcregrep" + args: + - --help + # Comby + - name: "comby is runnable" + command: "comby" + args: + - -h + # s3proxy + - name: "s3proxy is runnable" + command: "/opt/s3proxy/s3proxy" + args: + - --version + # Ctags + - name: "ctags is runnable" + command: "universal-ctags" + args: + - --version + # su-exec + - name: "su-exec is runnable" + command: "su-exec" + args: + - --help + # nginx + - name: "nginx is runnable" + command: "nginx" + args: + - -version + # postgres_exporter + - name: "postgres_exporter is runnable" + command: "postgres_exporter" + args: + - --version + # Prometheus + Alertmanager + - name: "prometheus is runnable" + command: "prometheus" + args: + - --version + - name: "promtool is runnable" + command: "promtool" + args: + - --version + - name: "alertmanager is runnable" + command: "alertmanager" + args: + - --version + # Grafana + - name: "grafana-server is runnable" + command: "/usr/share/grafana/bin/grafana-server" + args: + - -v + # Correct users and groups + - name: "correct users exist" + command: "cat" + args: + - '/etc/passwd' + expectedOutput: [ + "sourcegraph:x:100:101", + "postgres:x:70:70", + "nginx:x:101:102", + "redis:x:102:103", + "grafana:x:103:104", + ] + + # TODO(security): This container should not be running as root + # - name: "not running as root" + # command: "/usr/bin/id" + # args: + # - -u + # excludedOutput: ["^0"] + # exitCode: 0 + + +fileExistenceTests: +# Prometheus +- name: '/prometheus.sh' + path: '/prometheus.sh' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +- name: '/alertmanager.sh' + path: '/alertmanager.sh' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +# Grafana +- name: 'Dashboard config' + path: '/sg_config_grafana/provisioning/dashboards/sourcegraph/gitserver.json' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +# p4-fusion wrappers +- name: '/usr/local/bin/p4-fusion' + path: '/usr/local/bin/p4-fusion' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' +- name: '/usr/local/bin/process-stats-watcher.sh' + path: '/usr/local/bin/process-stats-watcher.sh' + shouldExist: true + uid: 0 + gid: 0 + permissions: '-r-xr-xr-x' + +metadataTest: + envVars: + - key: GO111MODULES + value: 'on' + - key: LANG + value: 'en_US.utf8' diff --git a/enterprise/cmd/symbols/BUILD.bazel b/enterprise/cmd/symbols/BUILD.bazel index ee6dda85c11..91725ed3f14 100644 --- a/enterprise/cmd/symbols/BUILD.bazel +++ b/enterprise/cmd/symbols/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "symbols_lib", @@ -21,3 +25,50 @@ go_binary( embed = [":symbols_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_symbols", + srcs = [":symbols"], +) + +pkg_tar( + name = "tar_scip-ctags", + srcs = ["//docker-images/syntax-highlighter:scip-ctags"], + package_dir = "/usr/local/bin", +) + +oci_image( + name = "image", + base = "@wolfi_symbols_base", + entrypoint = [ + "/sbin/tini", + "--", + "/symbols", + ], + tars = [ + ":tar_symbols", + ":tar_scip-ctags", + ], +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["symbols:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("symbols"), +) diff --git a/enterprise/cmd/symbols/image_test.yaml b/enterprise/cmd/symbols/image_test.yaml new file mode 100644 index 00000000000..4c9e486a10f --- /dev/null +++ b/enterprise/cmd/symbols/image_test.yaml @@ -0,0 +1,36 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/symbols" + envVars: + - key: "SANITY_CHECK" + value: "true" + - name: "ctags is runnable" + command: "universal-ctags" + args: + - --version + - name: "scip-ctags is runnable" + command: "/usr/local/bin/scip-ctags" + envVars: + - key: "SANITY_CHECK" + value: "true" + + # TODO(security): This container should not be running as root + # - name: "not running as root" + # command: "/usr/bin/id" + # args: + # - -u + # excludedOutput: ["^0"] + # exitCode: 0 + +fileExistenceTests: +- name: '/mnt/cache/symbols' + path: '/mnt/cache/symbols' + shouldExist: true + uid: 100 + gid: 101 + permissions: 'drwxr-xr-x' +- name: 'jansson package' + path: 'usr/lib/libjansson.la' + shouldExist: true diff --git a/enterprise/cmd/worker/BUILD.bazel b/enterprise/cmd/worker/BUILD.bazel index ff42f5c03f8..146e01874bf 100644 --- a/enterprise/cmd/worker/BUILD.bazel +++ b/enterprise/cmd/worker/BUILD.bazel @@ -1,4 +1,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("//dev:oci_defs.bzl", "image_repository") go_library( name = "worker_lib", @@ -21,3 +25,42 @@ go_binary( embed = [":worker_lib"], visibility = ["//visibility:public"], ) + +pkg_tar( + name = "tar_worker", + srcs = [":worker"], +) + +oci_image( + name = "image", + base = "@wolfi_base", + entrypoint = [ + "/sbin/tini", + "--", + "/worker", + ], + tars = [":tar_worker"], + user = "sourcegraph", +) + +oci_tarball( + name = "image_tarball", + image = ":image", + repo_tags = ["worker:candidate"], +) + +container_structure_test( + name = "image_test", + timeout = "short", + configs = ["image_test.yaml"], + driver = "docker", + image = ":image", + tags = ["requires-network"], +) + +oci_push( + name = "candidate_push", + image = ":image", + remote_tags = "//:tags", + repository = image_repository("worker"), +) diff --git a/enterprise/cmd/worker/image_test.yaml b/enterprise/cmd/worker/image_test.yaml new file mode 100644 index 00000000000..d197fa98aeb --- /dev/null +++ b/enterprise/cmd/worker/image_test.yaml @@ -0,0 +1,15 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "binary is runnable" + command: "/worker" + envVars: + - key: "SANITY_CHECK" + value: "true" + + - name: "not running as root" + command: "/usr/bin/id" + args: + - -u + excludedOutput: ["^0"] + exitCode: 0 diff --git a/enterprise/dev/ci/internal/ci/bazel_operations.go b/enterprise/dev/ci/internal/ci/bazel_operations.go index e458f649ec8..43f8787453b 100644 --- a/enterprise/dev/ci/internal/ci/bazel_operations.go +++ b/enterprise/dev/ci/internal/ci/bazel_operations.go @@ -38,6 +38,36 @@ func bazelCmd(args ...string) string { return strings.Join(Cmd, " ") } +func bazelPushImagesCmd(version string) func(*bk.Pipeline) { + return func(pipeline *bk.Pipeline) { + pipeline.AddStep(":bazel::docker: Push OCI/Wolfi", + bk.Agent("queue", "bazel"), + bk.DependsOn("bazel-tests"), + bk.Key("bazel-push-images"), + bk.Env("PUSH_VERSION", version), + bk.Cmd(bazelStampedCmd(`build $$(bazel query 'kind("oci_push rule", //...)')`)), + bk.Cmd("./enterprise/dev/ci/push_all.sh"), + ) + } +} + +func bazelStampedCmd(args ...string) string { + pre := []string{ + "bazel", + "--bazelrc=.bazelrc", + "--bazelrc=.aspect/bazelrc/ci.bazelrc", + "--bazelrc=.aspect/bazelrc/ci.sourcegraph.bazelrc", + } + post := []string{ + "--stamp", + "--workspace_status_command=./dev/bazel_stamp_vars.sh", + } + + cmd := append(pre, args...) + cmd = append(cmd, post...) + return strings.Join(cmd, " ") +} + // bazelAnalysisPhase only runs the analasys phase, ensure that the buildfiles // are correct, but do not actually build anything. func bazelAnalysisPhase() func(*bk.Pipeline) { @@ -90,7 +120,7 @@ func bazelTest(targets ...string) func(*bk.Pipeline) { bk.DependsOn("bazel-configure"), bk.Agent("queue", "bazel"), bk.Key("bazel-tests"), - bk.ArtifactPaths("./bazel-testlogs/enterprise/cmd/embeddings/shared/shared_test/*.log"), + bk.ArtifactPaths("./bazel-testlogs/enterprise/cmd/embeddings/shared/shared_test/*.log", "./command.profile.gz"), bk.AutomaticRetry(1), // TODO @jhchabran flaky stuff are breaking builds } @@ -153,17 +183,19 @@ func bazelTestWithDepends(optional bool, dependsOn string, targets ...string) fu } } -func bazelBuild(optional bool, targets ...string) func(*bk.Pipeline) { +func bazelBuild(targets ...string) func(*bk.Pipeline) { cmds := []bk.StepOpt{ + bk.Key("bazel_build"), bk.Agent("queue", "bazel"), } - bazelCmd := bazelCmd(fmt.Sprintf("build %s", strings.Join(targets, " "))) - cmds = append(cmds, bk.Cmd(bazelCmd)) + cmd := bazelStampedCmd(fmt.Sprintf("build %s", strings.Join(targets, " "))) + cmds = append( + cmds, + bk.Cmd(cmd), + bk.Cmd(bazelStampedCmd("run //enterprise/cmd/server:candidate_push")), + ) return func(pipeline *bk.Pipeline) { - if optional { - cmds = append(cmds, bk.SoftFail()) - } pipeline.AddStep(":bazel: Build ...", cmds..., ) diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index b1fef724bc2..5513f9649a1 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -177,10 +177,6 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { ForceBazel: !c.MessageFlags.NoBazel, })) - // At this stage, we don't break builds because of a Bazel failure. - // TODO(JH) Disabled until re-enabled with flag - // ops.Merge(BazelOperations(true)) - // Now we set up conditional operations that only apply to pull requests. if c.Diff.Has(changed.Client) { // triggers a slow pipeline, currently only affects web. It's optional so we @@ -429,6 +425,7 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { publishOps.Append(publishExecutorDockerMirror(c)) } } + publishOps.Append(bazelPushImagesCmd(c.Version)) ops.Merge(publishOps) } diff --git a/enterprise/dev/ci/push_all.sh b/enterprise/dev/ci/push_all.sh new file mode 100755 index 00000000000..d037257f413 --- /dev/null +++ b/enterprise/dev/ci/push_all.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +set -eu + +function preview_tags() { + IFS=' ' read -r -a registries <<<"$1" + IFS=' ' read -r -a tags <<<"$2" + + for tag in "${tags[@]}"; do + for registry in "${registries[@]}"; do + echo -e "\t ${registry}/\$IMAGE:${qa_prefix}-${tag}" + done + done +} + +function create_push_command() { + IFS=' ' read -r -a registries <<<"$1" + repository="$2" + target="$3" + tags_args="$4" + + repositories_args="" + for registry in "${registries[@]}"; do + repositories_args="$repositories_args --repository ${registry}/${repository}" + done + + cmd="bazel \ + --bazelrc=.bazelrc \ + --bazelrc=.aspect/bazelrc/ci.bazelrc \ + --bazelrc=.aspect/bazelrc/ci.sourcegraph.bazelrc \ + run \ + $target \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh" + + echo "$cmd -- $tags_args $repositories_args" +} + +dev_registries=( + "us.gcr.io/sourcegraph-dev" +) +prod_registries=( + "index.docker.io/sourcegraph" +) + +date_fragment="$(date +%Y-%m-%d)" + +qa_prefix="bazel" + +dev_tags=( + "${BUILDKITE_COMMIT:0:12}" + "${BUILDKITE_COMMIT:0:12}_${date_fragment}" + "${BUILDKITE_COMMIT:0:12}_${BUILDKITE_BUILD_NUMBER}" +) +prod_tags=( + "${PUSH_VERSION}" +) + +push_prod=false + +if [ "$BUILDKITE_BRANCH" == "main" ]; then + dev_tags+=("insiders") + prod_tags+=("insiders") + push_prod=true +fi + +if [[ "$BUILDKITE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + dev_tags+=("${BUILDKITE_TAG:1}") + prod_tags+=("${BUILDKITE_TAG:1}") + push_prod=true +fi + +preview_tags "${dev_registries[*]}" "${dev_tags[*]}" +if $push_prod; then + preview_tags "${prod_registries[*]}" "${prod_tags[*]}" +fi + +echo "--- done" + +dev_tags_args="" +for t in "${dev_tags[@]}"; do + dev_tags_args="$dev_tags_args --tag ${qa_prefix}-${t}" +done +prod_tags_args="" +if $push_prod; then + for t in "${prod_tags[@]}"; do + prod_tags_args="$prod_tags_args --tag ${qa_prefix}-${t}" + done +fi + +images=$(bazel query 'kind("oci_push rule", //...)') + +job_file=$(mktemp) +# shellcheck disable=SC2064 +trap "rm -rf $job_file" EXIT + +# shellcheck disable=SC2068 +for target in ${images[@]}; do + [[ "$target" =~ ([A-Za-z0-9_-]+): ]] + name="${BASH_REMATCH[1]}" + # Append push commands for dev registries + create_push_command "${dev_registries[*]}" "$name" "$target" "$dev_tags_args" >>"$job_file" + # Append push commands for prod registries + if $push_prod; then + create_push_command "${prod_registries[*]}" "$name" "$target" "$prod_tags_args" >>"$job_file" + fi +done + +echo "-- jobfile" +cat "$job_file" +echo "--- done" + +echo "--- :bazel::docker: Pushing images..." +log_file=$(mktemp) +# shellcheck disable=SC2064 +trap "rm -rf $log_file" EXIT +parallel --jobs=16 --line-buffer --joblog "$log_file" -v <"$job_file" + +# Pretty print the output from gnu parallel +while read -r line; do + # Skip the first line (header) + if [[ "$line" != Seq* ]]; then + cmd="$(echo "$line" | cut -f9)" + [[ "$cmd" =~ (\/\/[^ ]+) ]] + target="${BASH_REMATCH[1]}" + exitcode="$(echo "$line" | cut -f7)" + duration="$(echo "$line" | cut -f4 | tr -d "[:blank:]")" + if [ "$exitcode" == "0" ]; then + echo "--- :docker::arrow_heading_up: $target ${duration}s :white_check_mark:" + else + echo "--- :docker::arrow_heading_up: $target ${duration}s: failed with $exitcode) :red_circle:" + fi + fi +done <"$log_file" + +echo "--- :bazel::docker: detailed summary" +cat "$log_file" +echo "--- done" diff --git a/internal/cmd/init-sg/main.go b/internal/cmd/init-sg/main.go index b1e4d0b028d..97534fcacbb 100644 --- a/internal/cmd/init-sg/main.go +++ b/internal/cmd/init-sg/main.go @@ -26,6 +26,7 @@ var ( email = initSG.String("email", os.Getenv("TEST_USER_EMAIL"), "The email of the admin user. (Required)") username = initSG.String("username", os.Getenv("SOURCEGRAPH_SUDO_USER"), "The username of the admin user. (Required)") password = initSG.String("password", os.Getenv("TEST_USER_PASSWORD"), "The password of the admin user. (Required)") + sgenvrc = initSG.String("sg_envrc", os.Getenv("SG_ENVRC"), "Location of the sg_envrc file to write down the sudo token to") githubToken = addRepos.String("githubtoken", os.Getenv("GITHUB_TOKEN"), "The github access token that will be used to authenticate an external service. (Required)") addReposConfig = addRepos.String("config", "", "Path to the external service config. (Required)") @@ -110,6 +111,9 @@ func initSourcegraph() { } envvar := "export SOURCEGRAPH_SUDO_TOKEN=" + token + if *sgenvrc != "" { + profile = *sgenvrc + } file, err := os.Create(profile) if err != nil { log.Fatal(err) diff --git a/monitoring/BUILD.bazel b/monitoring/BUILD.bazel index 101e2a40fcc..1c59e8dff9b 100644 --- a/monitoring/BUILD.bazel +++ b/monitoring/BUILD.bazel @@ -19,7 +19,7 @@ go_binary( ) genrule( - name = "generate_config", + name = "generate_config_zip", srcs = [], outs = ["monitoring.zip"], cmd = """ @@ -28,4 +28,92 @@ genrule( zip -r $@ monitoring """, tools = ["//monitoring"], + visibility = ["//docker-images/grafana:__pkg__"], +) + +# Bazel expects all outputs to be explicitly defined. Therefore we have +# to manually list them here. This so we can use this target as inputs to a +# pkg_tar rule. Using the zip variant from above would not work, as we +# would have to write a rule that unzips it and ... we'll circle back to +# the issue of having non-deterministic outputs for bazel to work with. +# +# To avoid accidentally missing one silently, which would be very hard +# to debug, there is a test target :generate_config_test below that +# runs the monitoring tool once again to ensure that all generated files are present in the output below, acting as failsafe to detect any deviation. +genrule( + name = "generate_config", + srcs = [], + outs = [ + "outputs/docs/alerts.md", + "outputs/docs/dashboards.md", + "outputs/grafana/codeintel-autoindexing.json", + "outputs/grafana/codeintel-codenav.json", + "outputs/grafana/codeintel-policies.json", + "outputs/grafana/codeintel-ranking.json", + "outputs/grafana/codeintel-uploads.json", + "outputs/grafana/containers.json", + "outputs/grafana/embeddings.json", + "outputs/grafana/executor.json", + "outputs/grafana/frontend.json", + "outputs/grafana/github-proxy.json", + "outputs/grafana/gitserver.json", + "outputs/grafana/home.json", + "outputs/grafana/otel-collector.json", + "outputs/grafana/postgres.json", + "outputs/grafana/precise-code-intel-worker.json", + "outputs/grafana/prometheus.json", + "outputs/grafana/redis.json", + "outputs/grafana/repo-updater.json", + "outputs/grafana/searcher.json", + "outputs/grafana/symbols.json", + "outputs/grafana/syntect-server.json", + "outputs/grafana/telemetry.json", + "outputs/grafana/worker.json", + "outputs/grafana/zoekt.json", + "outputs/prometheus/codeintel_autoindexing_alert_rules.yml", + "outputs/prometheus/codeintel_codenav_alert_rules.yml", + "outputs/prometheus/codeintel_policies_alert_rules.yml", + "outputs/prometheus/codeintel_ranking_alert_rules.yml", + "outputs/prometheus/codeintel_uploads_alert_rules.yml", + "outputs/prometheus/containers_alert_rules.yml", + "outputs/prometheus/embeddings_alert_rules.yml", + "outputs/prometheus/executor_alert_rules.yml", + "outputs/prometheus/frontend_alert_rules.yml", + "outputs/prometheus/github_proxy_alert_rules.yml", + "outputs/prometheus/gitserver_alert_rules.yml", + "outputs/prometheus/otel_collector_alert_rules.yml", + "outputs/prometheus/postgres_alert_rules.yml", + "outputs/prometheus/precise_code_intel_worker_alert_rules.yml", + "outputs/prometheus/prometheus_alert_rules.yml", + "outputs/prometheus/redis_alert_rules.yml", + "outputs/prometheus/repo_updater_alert_rules.yml", + "outputs/prometheus/searcher_alert_rules.yml", + "outputs/prometheus/src_custom_rules.yml", + "outputs/prometheus/symbols_alert_rules.yml", + "outputs/prometheus/syntect_server_alert_rules.yml", + "outputs/prometheus/telemetry_alert_rules.yml", + "outputs/prometheus/worker_alert_rules.yml", + "outputs/prometheus/zoekt_alert_rules.yml", + ], + cmd = """ + mkdir -p $(@D)/outputs + $(location //monitoring:monitoring) generate \ + --all.dir $(@D)/outputs/ + """, + tools = ["//monitoring"], + visibility = ["//visibility:public"], +) + +sh_test( + name = "generate_config_test", + size = "small", + srcs = ["generate_config_test.sh"], + args = [ + "$(location :monitoring)", + "$(locations :generate_config)", + ], + data = [ + ":generate_config", + ":monitoring", + ], ) diff --git a/monitoring/generate_config_test.sh b/monitoring/generate_config_test.sh new file mode 100755 index 00000000000..ac9f8336aee --- /dev/null +++ b/monitoring/generate_config_test.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Path to the monitoring tool +monitoring_bin="$1" + +# Array of paths for each of the outputs from the :generate_config target. +# shellcheck disable=SC2124 +got_files="${@:2}" + +# Manually run the generator again, so have a list of all the files +# we expect the :generate_config target to output. +# +# We put them in the ./expected folder. +"$monitoring_bin" generate --all.dir expected/ + +# Loop over all of them and check if we can find each of them in the +# outputs from :generate_config_target +for file in expected/**/*; do + # Trim the "expected" part of the path + want="${file##expected}" + found="false" + + # Loop over all files we got. + # shellcheck disable=SC2068 + for got in ${got_files[@]}; do + # Trim the path from the "monitoring/output" prefix + # and test it against the expected file we're currently iterating with. + if [[ "${got##monitoring/outputs}" == "$want" ]]; then + found="true" + break + fi + done + + # If we didn't find it, return an error. + if [[ $found == "false" ]]; then + echo "Couldn't find expected output $want, perhaps it's missing from the 'srcs' attribute?" + exit 1 + fi +done diff --git a/stamp_tags.bzl b/stamp_tags.bzl new file mode 100644 index 00000000000..e95a418a196 --- /dev/null +++ b/stamp_tags.bzl @@ -0,0 +1,39 @@ +"Helper for stamping version control info into the tag" + +load("@aspect_bazel_lib//lib:jq.bzl", "jq") +load("@bazel_skylib//lib:types.bzl", "types") + +def stamp_tags(name, remote_tags, **kwargs): + """Wrapper macro around the [jq](https://docs.aspect.build/rules/aspect_bazel_lib/docs/jq) rule. + + Produces a text file that can be used with the `remote_tags` attribute of [`oci_push`](#oci_push). + + Each entry in `remote_tags` is typically either a constant like `my-repo:latest`, or can contain a stamp expression. + The latter can use any key from `bazel-out/stable-status.txt` or `bazel-out/volatile-status.txt`. + See https://docs.aspect.build/rules/aspect_bazel_lib/docs/stamping/ for details. + + The jq `//` default operator is useful for returning an alternative value for unstamped builds. + + For example, if you use the expression `($stamp.BUILD_EMBED_LABEL // "0.0.0")`, this resolves to + "0.0.0" if stamping is not enabled. When built with `--stamp --embed_label=1.2.3` it will + resolve to `1.2.3`. + + Args: + name: name of the resulting jq target. + remote_tags: list of jq expressions which result in a string value, see docs above + **kwargs: additional named parameters to the jq rule. + """ + if not types.is_list(remote_tags): + fail("remote_tags should be a list") + _maybe_quote = lambda x: x if "\"" in x else "\"{}\"".format(x) + jq( + name = name, + srcs = [], + out = "_{}.tags.txt".format(name), + args = ["--raw-output"], + filter = "|".join([ + "$ARGS.named.STAMP as $stamp", + ",".join([_maybe_quote(t) for t in remote_tags]), + ]), + **kwargs + ) diff --git a/testing/BUILD.bazel b/testing/BUILD.bazel new file mode 100644 index 00000000000..b05c541f80e --- /dev/null +++ b/testing/BUILD.bazel @@ -0,0 +1,138 @@ +# This test target runs the container on port=7080 +sh_test( + name = "e2e_test", + srcs = [":e2e_test.sh"], + args = [ + "$(location //enterprise/cmd/server:image_tarball)", # image we're testing against + "server:candidate", + "$(location //client/web/src/end-to-end:e2e)", # actual test + "$(rootpath //:mocha_config)", + "'$(rootpath //client/web/src/end-to-end:testing_e2e_bundle)/**/*.test.js'", + ], + data = [ + "//:mocha_config", + "//client/web/src/end-to-end:e2e", + "//client/web/src/end-to-end:testing_e2e_bundle", + "//enterprise/cmd/server:image_tarball", + ], + env = { + "SG_FEATURE_FLAG_GRPC": "false", + "TEST_USER_EMAIL": "test@sourcegraph.com", + "TEST_USER_PASSWORD": "supersecurepassword", + "SOURCEGRAPH_SUDO_TOKEN": "fake-sg-token", + "NO_CLEANUP": "false", + "KEEP_BROWSER": "false", + "DEVTOOLS": "false", + "BROWSER": "chrome", + "WINDOW_WIDTH": "1280", + "WINDOW_HEIGHT": "1024", + "LOG_BROWSER_CONSOLE": "false", + # Enable findDom on CodeMirror + "INTEGRATION_TESTS": "true", + }, + env_inherit = [ + "DISPLAY", + "E2E_HEADLESS", + "E2E_SOURCEGRAPH_BASE_URL", + "GHE_GITHUB_TOKEN", + "GH_TOKEN", + "SOURCEGRAPH_LICENSE_GENERATION_KEY", + "SOURCEGRAPH_LICENSE_KEY", + ], + tags = ["requires-network"], + deps = ["//testing/tools:integration_runner"], +) + +# This test target runs the container on port=7081 +sh_test( + name = "backend_integration_test", + srcs = [":backend_integration_test.sh"], + args = [ + "$(location //enterprise/cmd/server:image_tarball)", # image we're testing against + "server:candidate", + "$(location //dev/gqltest:gqltest_test)", # actual test + "$(location //dev/authtest:authtest_test)", # actual test + ], + data = [ + "//dev/authtest:authtest_test", + "//dev/gqltest:gqltest_test", + "//enterprise/cmd/server:image_tarball", + ], + env = { + "SG_FEATURE_FLAG_GRPC": "false", + }, + env_inherit = [ + "AWS_ACCESS_KEY_ID", + "AWS_CODE_COMMIT_PASSWORD", + "AWS_CODE_COMMIT_USERNAME", + "AWS_SECRET_ACCESS_KEY", + "AZURE_DEVOPS_TOKEN", + "AZURE_DEVOPS_USERNAME", + "BITBUCKET_SERVER_TOKEN", + "BITBUCKET_SERVER_URL", + "BITBUCKET_SERVER_USERNAME", + "GHE_GITHUB_TOKEN", + "PERFORCE_PASSWORD", + "PERFORCE_PORT", + "PERFORCE_USER", + "SOURCEGRAPH_LICENSE_GENERATION_KEY", + "SOURCEGRAPH_LICENSE_KEY", + ], + tags = ["requires-network"], + deps = ["//testing/tools:integration_runner"], +) + +# This test target runs the container on port=7082 +sh_test( + name = "codeintel_integration_test", + srcs = [":codeintel_integration_test.sh"], + args = [ + "$(location //enterprise/cmd/server:image_tarball)", # image we're testing against + "server:candidate", + "$(location //internal/cmd/init-sg)", + "$(rlocationpath @src-cli-linux-amd64//:src-cli-linux-amd64)", + "$(location //dev/codeintel-qa/cmd/download)", + "$(location //dev/codeintel-qa/cmd/clear)", + "$(location //dev/codeintel-qa/cmd/upload)", + "$(location //dev/codeintel-qa/cmd/query)", + "$(location //dev/ci/integration/code-intel:repos.json)", + ], + data = [ + "//dev/ci/integration/code-intel:repos.json", + "//dev/codeintel-qa/cmd/clear", + "//dev/codeintel-qa/cmd/download", + "//dev/codeintel-qa/cmd/query", + "//dev/codeintel-qa/cmd/upload", + "//enterprise/cmd/server:image_tarball", + "//internal/cmd/init-sg", + "@src-cli-linux-amd64//:src-cli-linux-amd64", + ], + env = { + "SG_FEATURE_FLAG_GRPC": "false", + "TEST_USER_EMAIL": "test@sourcegraph.com", + "TEST_USER_PASSWORD": "supersecurepassword", + "SOURCEGRAPH_SUDO_USER": "admin", + }, + env_inherit = [ + "AWS_ACCESS_KEY_ID", + "AWS_CODE_COMMIT_PASSWORD", + "AWS_CODE_COMMIT_USERNAME", + "AWS_SECRET_ACCESS_KEY", + "AZURE_DEVOPS_TOKEN", + "AZURE_DEVOPS_USERNAME", + "BITBUCKET_SERVER_TOKEN", + "BITBUCKET_SERVER_URL", + "BITBUCKET_SERVER_USERNAME", + "GITHUB_TOKEN", + "PERFORCE_PASSWORD", + "PERFORCE_PORT", + "PERFORCE_USER", + "SOURCEGRAPH_LICENSE_GENERATION_KEY", + "SOURCEGRAPH_LICENSE_KEY", + ], + tags = [ + "manual", + "requires-network", + ], + deps = ["//testing/tools:integration_runner"], +) diff --git a/testing/backend_integration_test.sh b/testing/backend_integration_test.sh new file mode 100755 index 00000000000..aebc28226b8 --- /dev/null +++ b/testing/backend_integration_test.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -eu +source ./testing/tools/integration_runner.sh || exit 1 + +tarball="$1" +image_name="$2" + +gqltest="$3" +authtest="$4" + +port="7081" +url="http://localhost:$port" + +# Backend integration tests uses a specific GITHUB_TOKEN that is available as GHE_GITHUB_TOKEN +# because it refers to our internal GitHub enterprise instance used for testing. +GITHUB_TOKEN="$GHE_GITHUB_TOKEN" +export GITHUB_TOKEN + +ALLOW_SINGLE_DOCKER_CODE_INSIGHTS="true" +export ALLOW_SINGLE_DOCKER_CODE_INSIGHTS + +run_server_image "$tarball" "$image_name" "$url" "$port" + +echo "--- integration test ./dev/gqltest -long" +"$gqltest" -long -base-url "$url" + +echo "--- sleep 5s to wait for site configuration to be restored from gqltest" +sleep 5 + +echo "--- integration test ./dev/authtest -long" +"$authtest" -long -base-url "$url" -email "gqltest@sourcegraph.com" -username "gqltest-admin" + +echo "--- done" diff --git a/testing/codeintel_integration_test.sh b/testing/codeintel_integration_test.sh new file mode 100755 index 00000000000..14d8bd1958d --- /dev/null +++ b/testing/codeintel_integration_test.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -eu +source ./testing/tools/integration_runner.sh || exit 1 + +tarball="$1" +image_name="$2" + +init_sg="$3" +src_cli="$4" + +cmd_download="$5" +cmd_clear="$6" +cmd_upload="$7" +cmd_query="$8" + +testdata_repos="$9" + +port="7082" +url="http://localhost:$port" + +SOURCEGRAPH_BASE_URL="$url" +export SOURCEGRAPH_BASE_URL + +ALLOW_SINGLE_DOCKER_CODE_INSIGHTS="true" +export ALLOW_SINGLE_DOCKER_CODE_INSIGHTS + +run_server_image "$tarball" "$image_name" "$url" "$port" + +echo '--- Initializing instance' +"$init_sg" initSG -sg_envrc="./sg_envrc" + +# shellcheck disable=SC1091 +source ./sg_envrc +echo '--- :horse: Running init-sg addRepos' +"$init_sg" addRepos -config "$testdata_repos" + +echo '--- :brain: Running the test suite' + +echo '--- :zero: downloading test data from GCS' +"$cmd_download" + +echo '--- :one: clearing existing state' +"$cmd_clear" + +# src-cli must be in the PATH for upload to find it. +echo '--- :two: integration test +./dev/codeintel-qa/cmd/upload' +"$cmd_upload" --timeout=5m --index-dir="./dev/codeintel-qa/testdata/indexes" --src-path="$(rlocation "$src_cli")" + +echo '--- :three: integration test ./dev/codeintel-qa/cmd/query' +"$cmd_query" --index-dir="./dev/codeintel-qa/testdata/indexes" + +echo "--- done" diff --git a/testing/e2e_test.sh b/testing/e2e_test.sh new file mode 100755 index 00000000000..76ea468292c --- /dev/null +++ b/testing/e2e_test.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -eu +source ./testing/tools/integration_runner.sh || exit 1 + +tarball="$1" +image_name="$2" +e2e_test="$3" +mocha_config="$4" +files="$5" + +url="http://localhost:7080" + +SOURCEGRAPH_BASE_URL="$url" +export SOURCEGRAPH_BASE_URL + +# Backend integration tests uses a specific GITHUB_TOKEN that is available as GHE_GITHUB_TOKEN +# because it refers to our internal GitHub enterprise instance used for testing. +GITHUB_TOKEN="$GHE_GITHUB_TOKEN" +export GITHUB_TOKEN + +ALLOW_SINGLE_DOCKER_CODE_INSIGHTS="true" +export ALLOW_SINGLE_DOCKER_CODE_INSIGHTS + +run_server_image "$tarball" "$image_name" "$url" "7080" + +echo "--- e2e test //client/web/src/end-to-end:e2e" +"$e2e_test" --config "$mocha_config" --retries 4 "$files" + +echo "--- done" diff --git a/testing/tools/BUILD.bazel b/testing/tools/BUILD.bazel new file mode 100644 index 00000000000..5a6e9754e21 --- /dev/null +++ b/testing/tools/BUILD.bazel @@ -0,0 +1,5 @@ +sh_library( + name = "integration_runner", + srcs = ["integration_runner.sh"], + visibility = ["//testing:__subpackages__"], +) diff --git a/testing/tools/integration_runner.sh b/testing/tools/integration_runner.sh new file mode 100755 index 00000000000..de27bbf9157 --- /dev/null +++ b/testing/tools/integration_runner.sh @@ -0,0 +1,215 @@ +#!/bin/bash + +set -e + +DB_STARTUP_TIMEOUT="${DB_STARTUP_TIMEOUT:-10s}" + +function must_be_CI() { + if [ "${BUILDKITE:-}" != "true" ]; then + echo "⚠️ This script is NOT running on a Buildkite agent." + echo "👉 Aborting." + exit 1 + fi +} + +function ensure_clean_slate() { + echo "--- Ensuring clean slate before running server container" + + local running + running=$(docker ps -aq | wc -l) + if [[ "$running" -gt 0 ]]; then + echo "⚠️ Found $running running containers, deleting them." + # shellcheck disable=SC2046 + docker rm -f $(docker ps -aq) + else + echo "Found 0 running containers." + fi + + local images + images=$(docker images -q | wc -l) + if [[ "$images" -gt 0 ]]; then + echo "⚠️ Found $images images, deleting them." + # shellcheck disable=SC2046 + docker rmi -f $(docker images -q) + else + echo "Found 0 images." + fi + + echo "Removing existing volumes, if any" + # docker volume prune -f + + echo "--- done" +} + +function is_present() { + if [ -n "$1" ]; then + echo "present" + else + echo "blank" + fi +} + +function must_not_be_running() { + local url + url="$1" + if curl --output /dev/null --silent --head --fail "$url"; then + echo "❌ Can't run a new server instance on $url because another instance is already running." + exit 1 + fi +} + +function generate_unique_container_name() { + local prefix="$1" + prefix="$1" + local ident + ident="$(openssl rand -hex 12)" + echo "$prefix-$ident" +} + +function _run_server_image() { + if [ -z "$DB_STARTUP_TIMEOUT" ]; then + echo "❌ DB_STARTUP_TIMEOUT must be defined" + fi + if [ -z "$SOURCEGRAPH_LICENSE_GENERATION_KEY" ]; then + echo "❌ SOURCEGRAPH_LICENSE_GENERATION_KEY must be defined" + fi + if [ -z "$SOURCEGRAPH_LICENSE_KEY" ]; then + echo "❌ SOURCEGRAPH_LICENSE_KEY must be defined" + fi + + local image_tarball + image_tarball="$1" + local image_name + image_name="$2" + local url + url="$3" + local port + port="$4" + local data + data="$5" + local container_name + container_name="$6" + local docker_args + # shellcheck disable=SC2124 + docker_args="${@:7}" + + echo "--- Loading server image" + echo "Loading $image_tarball in Docker" + docker load --input "$image_tarball" + + echo "-- Starting $image_name" + # echo "Listening at: $url" + echo "Data and config volume bounds: $data" + echo "Database startup timeout: $DB_STARTUP_TIMEOUT" + echo "License key generator present: $(is_present "$SOURCEGRAPH_LICENSE_GENERATION_KEY")" + echo "License key present: $(is_present "$SOURCEGRAPH_LICENSE_GENERATION_KEY")" + + echo "Allow single docker image code insights: $ALLOW_SINGLE_DOCKER_CODE_INSIGHTS" + echo "GRPC Feature flag: $SG_FEATURE_FLAG_GRPC" + + # shellcheck disable=SC2086 + docker run $docker_args \ + -d \ + --name "$container_name" \ + --publish "$port":7080 \ + -e BAZEL_SKIP_OOB_INFER_VERSION=true \ + -e ALLOW_SINGLE_DOCKER_CODE_INSIGHTS="$ALLOW_SINGLE_DOCKER_CODE_INSIGHTS" \ + -e SOURCEGRAPH_LICENSE_GENERATION_KEY="$SOURCEGRAPH_LICENSE_GENERATION_KEY" \ + -e SG_FEATURE_FLAG_GRPC="$SG_FEATURE_FLAG_GRPC" \ + -e DB_STARTUP_TIMEOUT="$DB_STARTUP_TIMEOUT" \ + --volume "$data/config:/etc/sourcegraph" \ + --volume "$data/data:/var/opt/sourcegraph" \ + "$image_name" + + echo "-- Listening at $url" +} + +function wait_until_container_ready() { + local name + name="$1" + local url + url="$2" + local timeout + timeout="$3" + + echo "--- Waiting for $url to be up" + set +e + + t=1 + # timeout is a coreutils extension, not available to us here + curl --output /dev/null --silent --head --fail "$url" + # shellcheck disable=SC2181 + while [ ! $? -eq 0 ]; do + sleep 5 + t=$(( t + 5 )) + if [ "$t" -gt "$timeout" ]; then + echo "$url was not accessible within $timeout." + docker inspect "$name" + exit 1 + fi + + curl --output /dev/null --silent --head --fail "$url" + done + set -e +} + +function run_server_image() { + local image_tarball + image_tarball="$1" + local image_name + image_name="$2" + local url + url="$3" + local port + port="$4" + + must_be_CI + must_not_be_running "$url" + # This causes flakes on the container tests, because it catches other docker jobs + # TODO move this to a agent + # ensure_clean_slate + + local container_name + container_name=$(generate_unique_container_name "server-integration") + local data + data="tmp_run_server_image_$container_name" + mkdir "$data" + data="$(pwd)/$data" + + # we want those to be expanded right now, on purpose. + # shellcheck disable=SC2064 + trap "cleanup $image_name $container_name" EXIT + _run_server_image "$image_tarball" "$image_name" "$url" "$port" "$data" "$container_name" + + wait_until_container_ready "$container_name" "$url" 60 +} + +# Ensure we exit with a clean slate regardless of the outcome +function cleanup() { + exit_status=$? + + local image + image="$1" + local container + container="$2" + + if [ $exit_status -ne 0 ]; then + # Expand the output if our run failed. + echo "^^^ +++" + fi + + echo "--- dump server logs" + docker logs --timestamps "$container" + echo "--- done" + + echo "--- $container cleanup" + docker container rm -f "$container" + docker image rm -f "$image" + + if [ $exit_status -ne 0 ]; then + # This command will fail, so our last step will be expanded. We don't want + # to expand "docker cleanup" so we add in a dummy section. + echo "--- integration test failed" + echo "See integration test section for test runner logs, and uploaded artifacts for server logs." + fi +} diff --git a/tmp/redis.conf b/tmp/redis.conf new file mode 100755 index 00000000000..d958b9eede5 --- /dev/null +++ b/tmp/redis.conf @@ -0,0 +1,9 @@ +# allow access from all instances +protected-mode no +# limit memory usage, discard unused keys when hitting limit +maxmemory 6gb +maxmemory-policy allkeys-lru +# snapshots on disk every minute +dir /redis-data/ +appendonly no +save 60 1 \ No newline at end of file diff --git a/wolfi-images/batcheshelper.yaml b/wolfi-images/batcheshelper.yaml index b93205ca42d..9d3da8005d7 100644 --- a/wolfi-images/batcheshelper.yaml +++ b/wolfi-images/batcheshelper.yaml @@ -8,3 +8,5 @@ contents: ## batcheshelper packages - 'git>=2.38.1' + +# MANUAL REBUILD: diff --git a/wolfi-images/blobstore.yaml b/wolfi-images/blobstore.yaml index 1ce93a7b0ea..ef1c65825ff 100644 --- a/wolfi-images/blobstore.yaml +++ b/wolfi-images/blobstore.yaml @@ -17,3 +17,5 @@ paths: permissions: 0o755 work-dir: /opt/s3proxy + +# MANUAL REBUILD: diff --git a/wolfi-images/bundled-executor.yaml b/wolfi-images/bundled-executor.yaml index 415618e4f3d..731a2fc0638 100644 --- a/wolfi-images/bundled-executor.yaml +++ b/wolfi-images/bundled-executor.yaml @@ -13,14 +13,9 @@ contents: - py3-pip - xmlstarlet@sourcegraph -environment: - # Firecracker doesn't work in docker, so disable it by default - EXECUTOR_USE_FIRECRACKER: false - # Preconfigure bundled-executor to take 1 parallel job and restart afterwards, this is to keep the environment clean-ish. - EXECUTOR_MAXIMUM_NUM_JOBS: 1 - EXECUTOR_NUM_TOTAL_JOBS: 1 - paths: - path: /usr/local/bin type: directory permissions: 0o755 + +# MANUAL REBUILD: diff --git a/wolfi-images/cadvisor.yaml b/wolfi-images/cadvisor.yaml index cc2bc7c2fbc..2c2bae86bfc 100644 --- a/wolfi-images/cadvisor.yaml +++ b/wolfi-images/cadvisor.yaml @@ -8,3 +8,5 @@ contents: ## cadvisor dependencies - cadvisor@sourcegraph + +# MANUAL REBUILD: diff --git a/wolfi-images/executor-kubernetes.yaml b/wolfi-images/executor-kubernetes.yaml index fca6a7a9771..0ea27d9332e 100644 --- a/wolfi-images/executor-kubernetes.yaml +++ b/wolfi-images/executor-kubernetes.yaml @@ -8,3 +8,5 @@ contents: ## executor-kubernetes packages - git + +# MANUAL REBUILD: diff --git a/wolfi-images/executor.yaml b/wolfi-images/executor.yaml index 700cab24509..a6061d127ac 100644 --- a/wolfi-images/executor.yaml +++ b/wolfi-images/executor.yaml @@ -11,11 +11,9 @@ contents: - git - docker-client@sourcegraph -# Firecracker doesn't work in docker, so disable it by default -environment: - EXECUTOR_USE_FIRECRACKER: false - paths: - path: /usr/local/bin type: directory permissions: 0o755 + +# MANUAL REBUILD: diff --git a/wolfi-images/gitserver.yaml b/wolfi-images/gitserver.yaml index 844f0da9d11..b661c20e2ed 100644 --- a/wolfi-images/gitserver.yaml +++ b/wolfi-images/gitserver.yaml @@ -11,7 +11,6 @@ contents: - git-lfs - git-p4 - openssh-client - # - python2 # TODO: Not available in Wolfi repo - python3 - bash @@ -27,5 +26,3 @@ paths: permissions: 0o755 work-dir: / - -# MANUAL REBUILD: diff --git a/wolfi-images/jaeger-agent.yaml b/wolfi-images/jaeger-agent.yaml index d1fc6576c2a..3730016e2dd 100644 --- a/wolfi-images/jaeger-agent.yaml +++ b/wolfi-images/jaeger-agent.yaml @@ -19,3 +19,5 @@ accounts: - username: jaeger uid: 10001 gid: 10002 + +# MANUAL REBUILD: diff --git a/wolfi-images/jaeger-all-in-one.yaml b/wolfi-images/jaeger-all-in-one.yaml index 4d383b45a82..61585cdfe11 100644 --- a/wolfi-images/jaeger-all-in-one.yaml +++ b/wolfi-images/jaeger-all-in-one.yaml @@ -24,4 +24,6 @@ paths: - path: /tmp type: directory uid: 10001 - permissions: 0o777 + permissions: 0o755 + +# MANUAL REBUILD: diff --git a/wolfi-images/node-exporter.yaml b/wolfi-images/node-exporter.yaml index dc4dc3a6208..8d1c9d1d178 100644 --- a/wolfi-images/node-exporter.yaml +++ b/wolfi-images/node-exporter.yaml @@ -4,3 +4,5 @@ contents: packages: ## node-exporter-specific packages - 'prometheus-node-exporter=1.5.0-r3' # IMPORTANT: Pinned version for managed updates + +# MANUAL REBUILD: diff --git a/wolfi-images/opentelemetry-collector.yaml b/wolfi-images/opentelemetry-collector.yaml index 2af7fca7be8..ffa565ed313 100644 --- a/wolfi-images/opentelemetry-collector.yaml +++ b/wolfi-images/opentelemetry-collector.yaml @@ -15,3 +15,5 @@ paths: permissions: 0o755 work-dir: /otel-collector + +# MANUAL REBUILD: diff --git a/wolfi-images/postgres-exporter.yaml b/wolfi-images/postgres-exporter.yaml index 5b26160d3b0..6e378a18a80 100644 --- a/wolfi-images/postgres-exporter.yaml +++ b/wolfi-images/postgres-exporter.yaml @@ -18,3 +18,5 @@ accounts: - username: postgres_exporter uid: 20001 gid: 102 + +# MANUAL REBUILD: diff --git a/wolfi-images/postgresql-12-codeinsights.yaml b/wolfi-images/postgresql-12-codeinsights.yaml new file mode 100644 index 00000000000..072f07b8aae --- /dev/null +++ b/wolfi-images/postgresql-12-codeinsights.yaml @@ -0,0 +1,34 @@ +include: ./postgresql-12.yaml + +paths: + - path: /data + type: directory + uid: 70 + gid: 70 + permissions: 0o750 + - path: /data/pgdata-12 + type: directory + uid: 70 + gid: 70 + permissions: 0o750 + - path: /var/lib/postgresql + type: directory + uid: 70 + gid: 70 + permissions: 0o755 + - path: /var/run/postgresql + type: directory + uid: 70 + gid: 70 + permissions: 0o755 + +accounts: + groups: + - groupname: ping + gid: 700 + - groupname: postgres + gid: 70 + users: + - username: postgres + uid: 70 + gid: 70 diff --git a/wolfi-images/postgresql-12.yaml b/wolfi-images/postgresql-12.yaml index 66e2a488927..da1f1e25bd2 100644 --- a/wolfi-images/postgresql-12.yaml +++ b/wolfi-images/postgresql-12.yaml @@ -11,13 +11,53 @@ contents: - postgresql-12-oci-entrypoint - su-exec -# TODO: Is this needed? -entrypoint: - command: /var/lib/postgres/initdb/postgresql-entrypoint.sh postgres - -# TODO: Is this needed? +# These are overridden by the Dockerfile, but provide defaults for running in the base image environment: PGDATA: /data/pgdata-12 POSTGRES_USER: sg POSTGRES_PASSWORD: '' POSTGRES_DB: sg + +# Ideally, running the base image would start a Postgres server, but the lack of a Postgres user +# means this won't work. We can't create one as the UID/GID depends on the image. +# entrypoint: +# command: /var/lib/postgres/initdb/postgresql-entrypoint.sh postgres + +accounts: + groups: + - groupname: postgres + gid: 999 + - groupname: ping + gid: 99 + users: + - username: postgres + uid: 999 + gid: 999 + - username: ping + uid: 99 + gid: 99 + +paths: + - path: /data + type: directory + uid: 999 + gid: 999 + permissions: 0o750 + - path: /data/pgdata-12 + type: directory + uid: 999 + gid: 999 + permissions: 0o750 + - path: /var/lib/postgresql + type: directory + uid: 999 + gid: 999 + permissions: 0o755 + - path: /var/run/postgresql + type: directory + uid: 999 + gid: 999 + permissions: 0o755 + + +# MANUAL REBUILD: diff --git a/wolfi-images/prometheus.yaml b/wolfi-images/prometheus.yaml new file mode 100644 index 00000000000..7300deb0016 --- /dev/null +++ b/wolfi-images/prometheus.yaml @@ -0,0 +1,25 @@ +include: ./sourcegraph-base.yaml + +contents: + packages: + # Included by existing SG base image + - tini + - mailcap + + # Prometheus packages + - prometheus + - prometheus-alertmanager + +paths: + - path: /prometheus + type: directory + uid: 100 + gid: 101 + permissions: 0o755 + - path: /alertmanager + type: directory + uid: 100 + gid: 101 + permissions: 0o755 + +work-dir: /prometheus diff --git a/wolfi-images/redis-exporter.yaml b/wolfi-images/redis-exporter.yaml index 119f3703920..ee91e8c5968 100644 --- a/wolfi-images/redis-exporter.yaml +++ b/wolfi-images/redis-exporter.yaml @@ -8,3 +8,5 @@ contents: ## redis_exporter packages - redis_exporter@sourcegraph + +# MANUAL REBUILD: diff --git a/wolfi-images/redis.yaml b/wolfi-images/redis.yaml index a66f8cef51b..2a61933d300 100644 --- a/wolfi-images/redis.yaml +++ b/wolfi-images/redis.yaml @@ -26,8 +26,9 @@ paths: gid: 1000 permissions: 0o755 -work-dir: - /redis-data +work-dir: /redis-data entrypoint: command: redis-server + +# MANUAL REBUILD: diff --git a/wolfi-images/repo-updater.yaml b/wolfi-images/repo-updater.yaml index 395c8193b6e..9956beafc3c 100644 --- a/wolfi-images/repo-updater.yaml +++ b/wolfi-images/repo-updater.yaml @@ -9,3 +9,5 @@ contents: ## repo-updater packages - coursier@sourcegraph - p4cli@sourcegraph + +# MANUAL REBUILD: diff --git a/wolfi-images/search-indexer.yaml b/wolfi-images/search-indexer.yaml index 31476901ffe..353f1486502 100644 --- a/wolfi-images/search-indexer.yaml +++ b/wolfi-images/search-indexer.yaml @@ -10,3 +10,17 @@ contents: - git - jansson - ctags@sourcegraph + +paths: + - path: /data + type: directory + uid: 100 + gid: 101 + permissions: 0o755 + - path: /data/index + type: directory + uid: 100 + gid: 101 + permissions: 0o755 + +# MANUAL REBUILD: diff --git a/wolfi-images/searcher.yaml b/wolfi-images/searcher.yaml index ac0c9f43915..f9cb291fd34 100644 --- a/wolfi-images/searcher.yaml +++ b/wolfi-images/searcher.yaml @@ -11,3 +11,12 @@ contents: - pcre - sqlite-libs - comby@sourcegraph + +paths: + - path: /mnt/cache/searcher + type: directory + uid: 100 + gid: 101 + permissions: 0o755 + +# MANUAL REBUILD: diff --git a/wolfi-images/server.yaml b/wolfi-images/server.yaml index adbc7577e3d..e98fc87600f 100644 --- a/wolfi-images/server.yaml +++ b/wolfi-images/server.yaml @@ -7,33 +7,31 @@ contents: ## server packages - bash - - ca-certificates # TODO: Required? + - ca-certificates - git - git-lfs - git-p4 - glibc-locale-en - libev - - libstdc++ # TODO: Is this still required? + - libstdc++ - nginx - openjdk-11 - openjdk-11-default-jvm - openssh-client - pcre + - posix-libc-utils # Adds locale, used by server postgres init scripts - postgresql-12 - postgresql-12-contrib - prometheus-postgres-exporter=0.12.0-r1 # IMPORTANT: Pinned version for managed updates + - prometheus-alertmanager + - python3 - posix-libc-utils # Locales - prometheus - prometheus-alertmanager - - python3 # TODO: Missing python2; required? - redis-6.2 - sqlite-libs - su-exec - ## Missing packages - #- python2 - #- libc6-compat - musl-glibc compat library, I think not needed - - comby@sourcegraph - ctags@sourcegraph - coursier@sourcegraph @@ -72,4 +70,12 @@ accounts: uid: 103 gid: 104 -# MANUAL REBUILD: +paths: + - path: /var/lib/nginx/tmp + type: directory + permissions: 0o755 + - path: /var/run + type: directory + permissions: 0o755 + +# MANUAL REBUILD: diff --git a/wolfi-images/sourcegraph-base.yaml b/wolfi-images/sourcegraph-base.yaml index 5714a72c464..96ca942fcd6 100644 --- a/wolfi-images/sourcegraph-base.yaml +++ b/wolfi-images/sourcegraph-base.yaml @@ -41,3 +41,5 @@ annotations: org.opencontainers.image.url: https://sourcegraph.com/ org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ org.opencontainers.image.documentation: https://docs.sourcegraph.com/ + +# MANUAL REBUILD: diff --git a/wolfi-images/sourcegraph-dev.yaml b/wolfi-images/sourcegraph-dev.yaml index e891249643c..afce3ac47c4 100644 --- a/wolfi-images/sourcegraph-dev.yaml +++ b/wolfi-images/sourcegraph-dev.yaml @@ -8,3 +8,5 @@ contents: ## Dev-specific tools - apk-tools + +# MANUAL REBUILD: diff --git a/wolfi-images/sourcegraph.yaml b/wolfi-images/sourcegraph.yaml index 8e89138751e..a14023ec3f9 100644 --- a/wolfi-images/sourcegraph.yaml +++ b/wolfi-images/sourcegraph.yaml @@ -5,3 +5,36 @@ contents: # Included by existing SG base image - tini - mailcap + +accounts: + run-as: sourcegraph + groups: + - groupname: sourcegraph + gid: 101 + users: + - username: sourcegraph + uid: 100 + gid: 101 + +# Set up directories needed by base images +paths: + # Used by frontend + - path: /mnt/cache/frontend + type: directory + uid: 100 + gid: 101 + permissions: 0o755 + # Used by indexed-searcher + - path: /data + type: directory + uid: 100 + gid: 101 + permissions: 0o755 + # Used by indexed-searcher + - path: /data/index + type: directory + uid: 100 + gid: 101 + permissions: 0o755 + +# MANUAL REBUILD: diff --git a/wolfi-images/symbols.yaml b/wolfi-images/symbols.yaml index 71cef13c64d..4f3d01e68ba 100644 --- a/wolfi-images/symbols.yaml +++ b/wolfi-images/symbols.yaml @@ -9,5 +9,14 @@ contents: ## symbols packages - ca-certificates - jansson - - libstdc++ # TODO: For tree-sitter. Not an apk dependency + - libstdc++ # Used by tree-sitter, which is not packaged - ctags@sourcegraph + +paths: + - path: /mnt/cache/symbols + type: directory + uid: 100 + gid: 101 + permissions: 0o755 + +# MANUAL REBUILD: diff --git a/wolfi-images/syntax-highlighter.yaml b/wolfi-images/syntax-highlighter.yaml index 710d2b31334..2dcf2e577da 100644 --- a/wolfi-images/syntax-highlighter.yaml +++ b/wolfi-images/syntax-highlighter.yaml @@ -9,4 +9,5 @@ contents: ## syntax-highlighter packages - libstdc++ - http-server-stabilizer@sourcegraph - - syntect-server@sourcegraph + +# MANUAL REBUILD: