artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 17:11:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

sg: Fixup monitoring stack (#56486)

Browse Source 
After switching to bazel, the postgres_exporter could no longer be built from bash. So I moved to the same approach we use for grafana.
Also, when failing the otel-collector pod wasn't removed properly, this is fixed now.

Test plan: sg run monitoring doesn't fail anymore.
This commit is contained in:
Erik Seliger 2023-09-11 07:15:03 -05:00 committed by GitHub
parent a1c5928dfa
commit 6834a501ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 48 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
dev/postgres_exporter.sh
View File

@ -1,39 +0,0 @@
#!/usr/bin/env bash
# Description: Prometheus collects metrics and aggregates them into graphs.
#
set -euf -o pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null 2>&1 && pwd)"
IMAGE=postgres_exporter:dev
CONTAINER=postgres_exporter
# Use psql to read the effective values for PG* env vars (instead of, e.g., hardcoding the default
# values).
get_pg_env() { psql -c '\set' | grep "$1" | cut -f 2 -d "'"; }
PGHOST=${PGHOST-$(get_pg_env HOST)}
PGUSER=${PGUSER-$(get_pg_env USER)}
PGPORT=${PGPORT-$(get_pg_env PORT)}
# we need to be able to query migration_logs table
PGDATABASE=${PGDATABASE-$(get_pg_env DBNAME)}
ADJUSTED_HOST=${PGHOST:-127.0.0.1}
if [[ ("$ADJUSTED_HOST" == "localhost" || "$ADJUSTED_HOST" == "127.0.0.1" || -f "$ADJUSTED_HOST") && "$OSTYPE" != "linux-gnu" ]]; then
ADJUSTED_HOST="host.docker.internal"
fi
NET_ARG=""
DATA_SOURCE_NAME="postgresql://${PGUSER}:${PGPASSWORD}@${ADJUSTED_HOST}:${PGPORT}/${PGDATABASE}?sslmode=${PGSSLMODE:-disable}"
if [[ "$OSTYPE" == "linux-gnu" ]]; then
NET_ARG="--net=host"
DATA_SOURCE_NAME="postgresql://${PGUSER}:${PGPASSWORD}@${ADJUSTED_HOST}:${PGPORT}/${PGDATABASE}?sslmode=${PGSSLMODE:-disable}"
fi
echo "Building pg_exporter docker image"
env IMAGE="${IMAGE}" "${REPO_ROOT}/docker-images/postgres_exporter/build.sh"
exec docker run --rm -p9187:9187 ${NET_ARG} --name="$CONTAINER" \
-e DATA_SOURCE_NAME="${DATA_SOURCE_NAME}" "${IMAGE}"

50
sg.config.yaml
View File

@ -791,7 +791,53 @@ commands:
DISABLE_SOURCEGRAPH_CONFIG: false
postgres_exporter:
cmd: ./dev/postgres_exporter.sh
cmd: |
if [[ $(uname) == "Linux" ]]; then
# Linux needs an extra arg to support host.internal.docker, which is how grafana connects
# to the prometheus backend.
ADD_HOST_FLAG="--add-host=host.docker.internal:host-gateway"
fi
# Use psql to read the effective values for PG* env vars (instead of, e.g., hardcoding the default
# values).
get_pg_env() { psql -c '\set' | grep "$1" | cut -f 2 -d "'"; }
PGHOST=${PGHOST-$(get_pg_env HOST)}
PGUSER=${PGUSER-$(get_pg_env USER)}
PGPORT=${PGPORT-$(get_pg_env PORT)}
# we need to be able to query migration_logs table
PGDATABASE=${PGDATABASE-$(get_pg_env DBNAME)}
ADJUSTED_HOST=${PGHOST:-127.0.0.1}
if [[ ("$ADJUSTED_HOST" == "localhost" || "$ADJUSTED_HOST" == "127.0.0.1" || -f "$ADJUSTED_HOST") && "$OSTYPE" != "linux-gnu" ]]; then
ADJUSTED_HOST="host.docker.internal"
fi
NET_ARG=""
DATA_SOURCE_NAME="postgresql://${PGUSER}:${PGPASSWORD}@${ADJUSTED_HOST}:${PGPORT}/${PGDATABASE}?sslmode=${PGSSLMODE:-disable}"
if [[ "$OSTYPE" == "linux-gnu" ]]; then
NET_ARG="--net=host"
DATA_SOURCE_NAME="postgresql://${PGUSER}:${PGPASSWORD}@${ADJUSTED_HOST}:${PGPORT}/${PGDATABASE}?sslmode=${PGSSLMODE:-disable}"
fi
echo "postgres_exporter: serving on http://localhost:${PORT}"
docker run --rm ${DOCKER_USER} \
--name=${CONTAINER} \
-e DATA_SOURCE_NAME="${DATA_SOURCE_NAME}" \
--cpus=1 \
--memory=1g \
-p 0.0.0.0:9187:9187 ${ADD_HOST_FLAG} \
"${IMAGE}"
install: |
docker inspect $CONTAINER >/dev/null 2>&1 && docker rm -f $CONTAINER
bazel build //docker-images/postgres_exporter:image_tarball
docker load --input $(bazel cquery //docker-images/postgres_exporter:image_tarball --output=files)
env:
IMAGE: postgres-exporter:candidate
CONTAINER: postgres_exporter
# docker containers must access things via docker host on non-linux platforms
DOCKER_USER: ""
ADD_HOST_FLAG: ""
monitoring-generator:
cmd: echo "monitoring-generator is deprecated, please run 'sg generate go' or 'bazel run //dev:write_all_generated' instead"
@ -833,7 +879,7 @@ commands:
JAEGER_HOST="localhost"
fi
docker container rm otel-collector
docker container rm -f otel-collector
docker run --rm --name=otel-collector $DOCKER_NET $DOCKER_ARGS \
-p 4317:4317 -p 4318:4318 -p 55679:55679 -p 55670:55670 \
-p 8888:8888 \