symbols: Build in Docker (#34335)

2026-02-06 15:51:43 +00:00 · 2022-04-25 16:03:55 -06:00 · 2022-04-25 16:03:55 -06:00 · 45efca5c47
commit 45efca5c47
parent 03221d9460
10 changed files with 60 additions and 195 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -94,5 +94,10 @@ cmd/indexer/debug
 node_modules/
 /client/web

+# Browser
+/client/browser/build
+/client/browser/node_modules
+/client/browser/code-intel-extensions/node_modules
+
 # Extensions
 /extension-api/dist
--- a/cmd/server/build.sh
+++ b/cmd/server/build.sh
@ -30,7 +30,7 @@ export CGO_ENABLED=0
 # enterprise build scripts.
 additional_images=()
 if [ $# -eq 0 ]; then
-  additional_images+=("github.com/sourcegraph/sourcegraph/cmd/frontend" "github.com/sourcegraph/sourcegraph/cmd/worker" "github.com/sourcegraph/sourcegraph/cmd/repo-updater")
+  additional_images+=("github.com/sourcegraph/sourcegraph/cmd/frontend" "github.com/sourcegraph/sourcegraph/cmd/worker" "github.com/sourcegraph/sourcegraph/cmd/repo-updater" "github.com/sourcegraph/sourcegraph/cmd/symbols")
 else
  additional_images+=("$@")
 fi
@ -61,7 +61,6 @@ PACKAGES=(
  github.com/sourcegraph/sourcegraph/cmd/github-proxy
  github.com/sourcegraph/sourcegraph/cmd/gitserver
  github.com/sourcegraph/sourcegraph/cmd/searcher
-  github.com/sourcegraph/sourcegraph/cmd/symbols
  github.com/sourcegraph/sourcegraph/cmd/migrator
  github.com/google/zoekt/cmd/zoekt-archive-index
  github.com/google/zoekt/cmd/zoekt-git-index
--- a/cmd/symbols/Dockerfile
+++ b/cmd/symbols/Dockerfile
@ -3,9 +3,41 @@ FROM sourcegraph/alpine-3.12:142406_2022-04-14_8836ac3499f4@sha256:4681a48d1fb9a
 # hadolint ignore=DL3002
 USER root

-COPY ctags-install-alpine.sh /ctags-install-alpine.sh
+COPY cmd/symbols/ctags-install-alpine.sh /ctags-install-alpine.sh
 RUN /ctags-install-alpine.sh

+FROM sourcegraph/alpine-3.14:142406_2022-04-14_8836ac3499f4@sha256:2a2d1cbaec78882661fe1aa5b0a4af0c23a37be2ea9ff8aadc2da5b80852c233 AS build
+# hadolint ignore=DL3002
+USER root
+
+ARG VERSION="unknown"
+ENV VERSION $VERSION
+
+ARG PKG
+ENV PKG=$PKG
+
+ENV GO111MODULE on
+ENV GOARCH amd64
+ENV GOOS linux
+ENV CGO_ENABLED 1
+
+RUN apk add go gcc g++
+
+COPY . /repo
+
+WORKDIR /repo
+
+RUN \
+  --mount=type=cache,target=/root/.cache/go-build \
+  --mount=type=cache,target=/root/go/pkg/mod \
+  go build \
+  -trimpath \
+  -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION  -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" \
+  -buildmode exe \
+  -tags dist \
+  -o /symbols \
+  $PKG
+
 FROM sourcegraph/alpine-3.12:142406_2022-04-14_8836ac3499f4@sha256:4681a48d1fb9a73fef1b540c08b3411f797351bbeda749f5dca21213a1e71526 AS symbols

 # TODO(security): This container should not run as root!
@ -23,13 +55,16 @@ LABEL org.opencontainers.image.created=${DATE}
 LABEL org.opencontainers.image.version=${VERSION}
 LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA}

-RUN apk add --no-cache bind-tools ca-certificates mailcap tini
+# ctags is dynamically linked against jansson
+RUN apk add --no-cache bind-tools ca-certificates mailcap tini jansson

-COPY ctags-install-alpine.sh /ctags-install-alpine.sh
-RUN /ctags-install-alpine.sh
+COPY --from=ctags /usr/local/bin/universal-ctags /usr/local/bin/universal-ctags
+
+COPY --from=build /symbols /usr/local/bin/symbols
+
+RUN env SANITY_CHECK=true /usr/local/bin/symbols

 ENV CACHE_DIR=/mnt/cache/symbols
 RUN mkdir -p ${CACHE_DIR}
 EXPOSE 3184
-ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/symbols"]
-COPY symbols /usr/local/bin/
+ENTRYPOINT /sbin/tini -- /usr/local/bin/symbols
--- a/cmd/symbols/build-ctags.sh
+++ b/cmd/symbols/build-ctags.sh
@ -12,17 +12,9 @@ if [[ "${CTAGS_COMMAND}" != "cmd/symbols/universal-ctags-dev" ]]; then
  exit 0
 fi

-OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX)
-cleanup() {
-  rm -rf "$OUTPUT"
-}
-trap cleanup EXIT
-
-cp -a ./cmd/symbols/ctags-install-alpine.sh "$OUTPUT"
-
 # Build ctags docker image for universal-ctags-dev
 echo "Building ctags docker image"
-docker build -f cmd/symbols/Dockerfile -t ctags "$OUTPUT" \
+docker build -f cmd/symbols/Dockerfile -t ctags . \
  --target=ctags \
  --progress=plain \
  --quiet >/dev/null
--- a/cmd/symbols/build.sh
+++ b/cmd/symbols/build.sh
@ -5,20 +5,10 @@
 cd "$(dirname "${BASH_SOURCE[0]}")/../.."
 set -eu

-OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX)
-cleanup() {
-  rm -rf "$OUTPUT"
-}
-trap cleanup EXIT
-
-cp -a ./cmd/symbols/ctags-install-alpine.sh "$OUTPUT"
-
-# Build go binary into $OUTPUT
-./cmd/symbols/go-build.sh "$OUTPUT"
-
 echo "--- docker build"
-docker build -f cmd/symbols/Dockerfile -t "$IMAGE" "$OUTPUT" \
+docker build -f cmd/symbols/Dockerfile -t "$IMAGE" "$(pwd)" \
  --progress=plain \
  --build-arg COMMIT_SHA \
  --build-arg DATE \
-  --build-arg VERSION
+  --build-arg VERSION \
+  --build-arg PKG="${PKG:-github.com/sourcegraph/sourcegraph/cmd/symbols}"
--- a/cmd/symbols/go-build.sh
+++ b/cmd/symbols/go-build.sh
@ -8,62 +8,6 @@ set -eu

 OUTPUT="${1:?no output path provided}"

-# Environment for building linux binaries
-export GO111MODULE=on
-export GOARCH=amd64
-export GOOS=linux
+cmd/symbols/build.sh

-# go-sqlite3 depends on cgo. Without cgo, it will build but it'll throw an error at query time.
-export CGO_ENABLED=1
-
-# Default CC to musl-gcc.
-export CC="${CC:-musl-gcc}"
-
-help() {
-  echo "You need to set CC to a musl compiler in order to compile go-sqlite3 for Alpine."
-  echo
-  echo "    Linux: run 'apt-get install -y musl-tools'"
-  echo "    macOS: download https://github.com/FiloSottile/homebrew-musl-cross/blob/6ee3329ee41231fe693306490f8e4d127c70e618/musl-cross.rb and run 'brew install ~/Downloads/musl-cross.rb'"
-}
-
-if ! command -v "$CC" >/dev/null; then
-  echo "$CC not found."
-  help
-  exit 1
-fi
-
-# Make sure this is a musl compiler.
-case "$CC" in
-  *musl*)
-    ;;
-  *)
-    echo "$CC doesn't look like a musl compiler."
-    help
-    exit 1
-    ;;
-esac
-
-echo "--- go build"
-pkg="github.com/sourcegraph/sourcegraph/cmd/symbols"
-env go build \
-  -trimpath \
-  -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION  -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" \
-  -buildmode exe \
-  -tags dist \
-  -o "$OUTPUT/$(basename $pkg)" \
-  "$pkg"
-
-# We can't use -v because the spawned container might not share
-# the same file system (e.g. when we're already inside docker
-# and the spawned docker container will be a sibling on the host).
-#
-# A workaround is to feed the file into the container via stdin:
-#
-#     'cat FILE | docker run ... -i ... sh -c "cat > FILE && ..."'
-echo "--- sanity check"
-# shellcheck disable=SC2002
-cat "$OUTPUT/$(basename $pkg)" | docker run \
-  --rm \
-  -i \
-  sourcegraph/alpine@sha256:ce099fbcd3cf70b338fc4cb2a4e1fa9ae847de21afdb0a849a393b87d94fb174 \
-  sh -c "cat > /symbols && chmod a+x /symbols && env SANITY_CHECK=true /symbols"
+docker cp "$(docker create --rm "$IMAGE")":/usr/local/bin/symbols "$OUTPUT/symbols"
--- a/enterprise/cmd/server/build.sh
+++ b/enterprise/cmd/server/build.sh
@ -10,4 +10,5 @@ export SERVER_PKG=${SERVER_PKG:-github.com/sourcegraph/sourcegraph/enterprise/cm
  github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend \
  github.com/sourcegraph/sourcegraph/enterprise/cmd/worker \
  github.com/sourcegraph/sourcegraph/enterprise/cmd/repo-updater \
+  github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols \
  github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker
--- a/enterprise/cmd/symbols/Dockerfile
+++ b/enterprise/cmd/symbols/Dockerfile
@ -1,35 +0,0 @@
-# NOTE: This layer of the docker image is also used in local development as a wrapper around universal-ctags
-FROM sourcegraph/alpine-3.12:142406_2022-04-14_8836ac3499f4@sha256:4681a48d1fb9a73fef1b540c08b3411f797351bbeda749f5dca21213a1e71526 AS ctags
-# hadolint ignore=DL3002
-USER root
-
-COPY ctags-install-alpine.sh /ctags-install-alpine.sh
-RUN /ctags-install-alpine.sh
-
-FROM sourcegraph/alpine-3.12:142406_2022-04-14_8836ac3499f4@sha256:4681a48d1fb9a73fef1b540c08b3411f797351bbeda749f5dca21213a1e71526 AS symbols
-
-# TODO(security): This container should not run as root!
-#
-# See https://github.com/sourcegraph/sourcegraph/issues/13237
-# hadolint ignore=DL3002
-USER root
-
-ARG COMMIT_SHA="unknown"
-ARG DATE="unknown"
-ARG VERSION="unknown"
-
-LABEL org.opencontainers.image.revision=${COMMIT_SHA}
-LABEL org.opencontainers.image.created=${DATE}
-LABEL org.opencontainers.image.version=${VERSION}
-LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA}
-
-RUN apk add --no-cache bind-tools ca-certificates mailcap tini
-
-COPY ctags-install-alpine.sh /ctags-install-alpine.sh
-RUN /ctags-install-alpine.sh
-
-ENV CACHE_DIR=/mnt/cache/enterprise-symbols
-RUN mkdir -p ${CACHE_DIR}
-EXPOSE 3184
-ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/enterprise-symbols"]
-COPY enterprise-symbols /usr/local/bin/
--- a/enterprise/cmd/symbols/build.sh
+++ b/enterprise/cmd/symbols/build.sh
@ -5,20 +5,6 @@
 cd "$(dirname "${BASH_SOURCE[0]}")/../../.."
 set -eu

-OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX)
-cleanup() {
-  rm -rf "$OUTPUT"
-}
-trap cleanup EXIT
-
-cp -a ./cmd/symbols/ctags-install-alpine.sh "$OUTPUT"
-
-# Build go binary into $OUTPUT
-./enterprise/cmd/symbols/go-build.sh "$OUTPUT"
-
-echo "--- docker build"
-docker build -f enterprise/cmd/symbols/Dockerfile -t "$IMAGE" "$OUTPUT" \
-  --progress=plain \
-  --build-arg COMMIT_SHA \
-  --build-arg DATE \
-  --build-arg VERSION
+env \
+  PKG=github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols \
+  cmd/symbols/build.sh
--- a/enterprise/cmd/symbols/go-build.sh
+++ b/enterprise/cmd/symbols/go-build.sh
@ -1,60 +1,8 @@
 #!/usr/bin/env bash

-# This script builds the symbols go binary.
-# Requires a single argument which is the path to the target bindir.
-
 cd "$(dirname "${BASH_SOURCE[0]}")/../../.."
 set -eu

-OUTPUT="${1:?no output path provided}"
-
-# Environment for building linux binaries
-export GO111MODULE=on
-export GOARCH=amd64
-export GOOS=linux
-
-# go-sqlite3 depends on cgo. Without cgo, it will build but it'll throw an error at query time.
-export CGO_ENABLED=1
-
-# Default CC to musl-gcc.
-export CC="${CC:-musl-gcc}"
-
-if ! command -v "$CC" >/dev/null; then
-  echo "$CC not found. You need to set CC to a musl compiler in order to compile go-sqlite3 for Alpine. Run 'apt-get install -y musl-tools'."
-  exit 1
-fi
-
-# Make sure this is a musl compiler.
-case "$CC" in
-  *musl*)
-    ;;
-  *)
-    echo "$CC doesn't look like a musl compiler. You need to set CC to a musl compiler in order to compile go-sqlite3 for Alpine. Run 'apt-get install -y musl-tools'."
-    exit 1
-    ;;
-esac
-
-echo "--- go build"
-pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols"
-env go build \
-  -trimpath \
-  -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION  -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" \
-  -buildmode exe \
-  -tags dist \
-  -o "$OUTPUT/enterprise-$(basename $pkg)" \
-  "$pkg"
-
-# We can't use -v because the spawned container might not share
-# the same file system (e.g. when we're already inside docker
-# and the spawned docker container will be a sibling on the host).
-#
-# A workaround is to feed the file into the container via stdin:
-#
-#     'cat FILE | docker run ... -i ... sh -c "cat > FILE && ..."'
-echo "--- sanity check"
-# shellcheck disable=SC2002
-cat "$OUTPUT/enterprise-$(basename $pkg)" | docker run \
-  --rm \
-  -i \
-  sourcegraph/alpine@sha256:ce099fbcd3cf70b338fc4cb2a4e1fa9ae847de21afdb0a849a393b87d94fb174 \
-  sh -c "cat > /enterprise-symbols && chmod a+x /enterprise-symbols && env SANITY_CHECK=true /enterprise-symbols"
+env \
+  PKG=github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols \
+  cmd/symbols/go-build.sh "$@"