From 6834a501eebe170c79566287a227fa640b428a53 Mon Sep 17 00:00:00 2001
From: Erik Seliger <erikseliger@me.com>
Date: Mon, 11 Sep 2023 07:15:03 -0500
Subject: [PATCH] sg: Fixup monitoring stack (#56486)

After switching to bazel, the postgres_exporter could no longer be built from bash. So I moved to the same approach we use for grafana.
Also, when failing the otel-collector pod wasn't removed properly, this is fixed now.

Test plan: sg run monitoring doesn't fail anymore.
---
 dev/postgres_exporter.sh | 39 -------------------------------
 sg.config.yaml           | 50 ++++++++++++++++++++++++++++++++++++++--
 2 files changed, 48 insertions(+), 41 deletions(-)
 delete mode 100755 dev/postgres_exporter.sh

diff --git a/dev/postgres_exporter.sh b/dev/postgres_exporter.sh
deleted file mode 100755
index 3a0c5ed2c8e..00000000000
--- a/dev/postgres_exporter.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-# Description: Prometheus collects metrics and aggregates them into graphs.
-#
-
-set -euf -o pipefail
-
-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null 2>&1 && pwd)"
-
-IMAGE=postgres_exporter:dev
-CONTAINER=postgres_exporter
-
-# Use psql to read the effective values for PG* env vars (instead of, e.g., hardcoding the default
-# values).
-get_pg_env() { psql -c '\set' | grep "$1" | cut -f 2 -d "'"; }
-PGHOST=${PGHOST-$(get_pg_env HOST)}
-PGUSER=${PGUSER-$(get_pg_env USER)}
-PGPORT=${PGPORT-$(get_pg_env PORT)}
-# we need to be able to query migration_logs table
-PGDATABASE=${PGDATABASE-$(get_pg_env DBNAME)}
-
-ADJUSTED_HOST=${PGHOST:-127.0.0.1}
-if [[ ("$ADJUSTED_HOST" == "localhost" || "$ADJUSTED_HOST" == "127.0.0.1" || -f "$ADJUSTED_HOST") && "$OSTYPE" != "linux-gnu" ]]; then
-  ADJUSTED_HOST="host.docker.internal"
-fi
-
-NET_ARG=""
-DATA_SOURCE_NAME="postgresql://${PGUSER}:${PGPASSWORD}@${ADJUSTED_HOST}:${PGPORT}/${PGDATABASE}?sslmode=${PGSSLMODE:-disable}"
-
-if [[ "$OSTYPE" == "linux-gnu" ]]; then
-  NET_ARG="--net=host"
-  DATA_SOURCE_NAME="postgresql://${PGUSER}:${PGPASSWORD}@${ADJUSTED_HOST}:${PGPORT}/${PGDATABASE}?sslmode=${PGSSLMODE:-disable}"
-fi
-
-echo "Building pg_exporter docker image"
-env IMAGE="${IMAGE}" "${REPO_ROOT}/docker-images/postgres_exporter/build.sh"
-
-exec docker run --rm -p9187:9187 ${NET_ARG} --name="$CONTAINER" \
-  -e DATA_SOURCE_NAME="${DATA_SOURCE_NAME}" "${IMAGE}"
diff --git a/sg.config.yaml b/sg.config.yaml
index d5267100be2..4f3404a0ea5 100644
--- a/sg.config.yaml
+++ b/sg.config.yaml
@@ -791,7 +791,53 @@ commands:
       DISABLE_SOURCEGRAPH_CONFIG: false
 
   postgres_exporter:
-    cmd: ./dev/postgres_exporter.sh
+    cmd: |
+      if [[ $(uname) == "Linux" ]]; then
+        # Linux needs an extra arg to support host.internal.docker, which is how grafana connects
+        # to the prometheus backend.
+        ADD_HOST_FLAG="--add-host=host.docker.internal:host-gateway"
+      fi
+
+      # Use psql to read the effective values for PG* env vars (instead of, e.g., hardcoding the default
+      # values).
+      get_pg_env() { psql -c '\set' | grep "$1" | cut -f 2 -d "'"; }
+      PGHOST=${PGHOST-$(get_pg_env HOST)}
+      PGUSER=${PGUSER-$(get_pg_env USER)}
+      PGPORT=${PGPORT-$(get_pg_env PORT)}
+      # we need to be able to query migration_logs table
+      PGDATABASE=${PGDATABASE-$(get_pg_env DBNAME)}
+
+      ADJUSTED_HOST=${PGHOST:-127.0.0.1}
+      if [[ ("$ADJUSTED_HOST" == "localhost" || "$ADJUSTED_HOST" == "127.0.0.1" || -f "$ADJUSTED_HOST") && "$OSTYPE" != "linux-gnu" ]]; then
+        ADJUSTED_HOST="host.docker.internal"
+      fi
+
+      NET_ARG=""
+      DATA_SOURCE_NAME="postgresql://${PGUSER}:${PGPASSWORD}@${ADJUSTED_HOST}:${PGPORT}/${PGDATABASE}?sslmode=${PGSSLMODE:-disable}"
+
+      if [[ "$OSTYPE" == "linux-gnu" ]]; then
+        NET_ARG="--net=host"
+        DATA_SOURCE_NAME="postgresql://${PGUSER}:${PGPASSWORD}@${ADJUSTED_HOST}:${PGPORT}/${PGDATABASE}?sslmode=${PGSSLMODE:-disable}"
+      fi
+
+      echo "postgres_exporter: serving on http://localhost:${PORT}"
+      docker run --rm ${DOCKER_USER} \
+        --name=${CONTAINER} \
+        -e DATA_SOURCE_NAME="${DATA_SOURCE_NAME}" \
+        --cpus=1 \
+        --memory=1g \
+        -p 0.0.0.0:9187:9187 ${ADD_HOST_FLAG} \
+        "${IMAGE}"
+    install: |
+      docker inspect $CONTAINER >/dev/null 2>&1 && docker rm -f $CONTAINER
+      bazel build //docker-images/postgres_exporter:image_tarball
+      docker load --input $(bazel cquery //docker-images/postgres_exporter:image_tarball --output=files)
+    env:
+      IMAGE: postgres-exporter:candidate
+      CONTAINER: postgres_exporter
+      # docker containers must access things via docker host on non-linux platforms
+      DOCKER_USER: ""
+      ADD_HOST_FLAG: ""
 
   monitoring-generator:
     cmd: echo "monitoring-generator is deprecated, please run 'sg generate go' or 'bazel run //dev:write_all_generated' instead"
@@ -833,7 +879,7 @@ commands:
         JAEGER_HOST="localhost"
       fi
 
-      docker container rm otel-collector
+      docker container rm -f otel-collector
       docker run --rm --name=otel-collector $DOCKER_NET $DOCKER_ARGS \
         -p 4317:4317 -p 4318:4318 -p 55679:55679 -p 55670:55670 \
         -p 8888:8888 \