sg: hacking hour with lots of YAML (#21250)

This PR is a collection of all the things we did in the `sg` hacking hour just now: - Clean up the README so that the TODOs are up to date - Add definitions for `minio`, `codeinsights-db`, `executor-queue`, `precise-code-intel-worker` and `codeintel-executor` - Make it possible to launch Sourcegraph with codeinsights!! See the `sg.config.yaml` for which env vars to set and then run `sg run-set enterprise-codeinsights`
2026-02-06 17:51:57 +00:00 · 2021-05-21 18:01:11 +02:00 · 2021-05-21 18:01:11 +02:00 · 9c9cbfa8d4
commit 9c9cbfa8d4
parent 78f1bb8059
2 changed files with 156 additions and 34 deletions
--- a/dev/sg/README.md
+++ b/dev/sg/README.md
@ -108,9 +108,6 @@ sg run frontend

 # Run predefined sets of commands:
 sg run-set enterprise
-
-# TODO: Rebuild and restart a command (if it has `build` defined, see Configuration)
-sg build gitserver
 ```

 ### Running tests
@ -130,7 +127,7 @@ sg test regression
 # Without argument it lists all available tests:
 sg test

-# TODO: Arguments are passed along to the command
+# Arguments are passed along to the command
 sg test backend-integration -run TestSearch
 ```

@ -285,13 +282,16 @@ tests:

 ## TODOs

- [ ] All of the things marked as TODOs above
+- [ ] Rename `install` to `build` because it's clearer
 - [ ] Add the remaining processes from `<root>/dev/Procfile` to `<root>/sg.config.yaml`
+- [ ] All of the things marked as TODOs above
+  - [ ] Rebuild and restart a command (if it has `build` defined, see Configuration): `sg build gitserver`
+  - [ ] Implement the `sg migration` command
+  - [ ] Implement the `sg generate` command
+  - [ ] Implement `sg edit site-config` and `sg edit external-services`
+  - [ ] Implement `sg tail-log`
 - [ ] Add a _simple_ way to define in the config file when a restart after a rebuild is not necessary
  - Something like `check_binary: .bin/frontend` which would take a SHA256 before and after rebuild and only restart if SHA doesn't match
- [ ] Rename `install` to `build` because it's clearer
- [ ] Add support for "dev environment setup"
-  - Something like `sg check` which runs `check_cmds` in the config file and provides helpful output if one of them failed ("check_cmd postgres failed. Install postgres with...")
 - [ ] Add built-in support for "download binary" so that the `caddy` command, for example, would be 3 lines instead of 20

 ## Hacking
--- a/sg.config.yaml
+++ b/sg.config.yaml
@ -53,6 +53,13 @@ env:
  # Default `NODE_ENV` to `development`
  NODE_ENV: development

+  # Code Insights uses a separate database, because it's easier to run TimescaleDB in
+  # Docker than install as a Postgres extension in dev environments.
+  CODEINSIGHTS_PGDATASOURCE: postgres://postgres:password@127.0.0.1:5435/postgres
+  DB_STARTUP_TIMEOUT: 120s # codeinsights-db needs more time to start in some instances.
+  DISABLE_CODE_INSIGHTS_HISTORICAL: true
+  DISABLE_CODE_INSIGHTS: true
+
 commands:
  frontend:
    cmd: ulimit -n 10000 && .bin/frontend
@ -245,14 +252,14 @@ commands:
      # This is not needed actually
      INSECURE_DEV: 1

-  zoekt-indexserver-0:
+  zoekt-indexserver-template: &zoekttemplate
    cmd: |
      .bin/zoekt-sourcegraph-indexserver \
        -sourcegraph_url 'http://localhost:3090' \
-        -index "$HOME/.sourcegraph/zoekt/index-0" \
-        -hostname 'localhost:3070' \
+        -index "$HOME/.sourcegraph/zoekt/index-$ZOEKT_NUM" \
+        -hostname "localhost:$ZOEKT_HOSTNAME_PORT" \
        -interval 1m \
-        -listen ":6072" \
+        -listen ":$ZOEKT_LISTEN_PORT" \
        -cpu_fraction 0.25
    install: |
      mkdir -p .bin
@ -261,33 +268,26 @@ commands:
      go install github.com/google/zoekt/cmd/zoekt-archive-index
      go install github.com/google/zoekt/cmd/zoekt-git-index
      go install github.com/google/zoekt/cmd/zoekt-sourcegraph-indexserver
-    env:
+    env: &zoektenv
      GOGC: 50
      CTAGS_COMMAND: cmd/symbols/universal-ctags-dev
      PATH: .bin:$PATH

-  zoekt-indexserver-1:
-    cmd: |
-      .bin/zoekt-sourcegraph-indexserver \
-        -sourcegraph_url 'http://localhost:3090' \
-        -index "$HOME/.sourcegraph/zoekt/index-1" \
-        -hostname 'localhost:3071' \
-        -interval 1m \
-        -listen ":6073" \
-        -cpu_fraction 0.25
-    install: |
-      # We technically don't need this because indexserver-0 installs it, but
-      # let's keep it here before we have something to remove the duplication
-      mkdir -p .bin
-      export GOBIN="${PWD}/.bin"
-      export GO111MODULE=on
-      go install github.com/google/zoekt/cmd/zoekt-archive-index
-      go install github.com/google/zoekt/cmd/zoekt-git-index
-      go install github.com/google/zoekt/cmd/zoekt-sourcegraph-indexserver
+  zoekt-indexserver-0:
+    <<: *zoekttemplate
    env:
-      GOGC: 50
-      CTAGS_COMMAND: cmd/symbols/universal-ctags-dev
-      PATH: .bin:$PATH
+      <<: *zoektenv
+      ZOEKT_NUM: 0
+      ZOEKT_HOSTNAME_PORT: 3070
+      ZOEKT_LISTEN_PORT: 6072
+
+  zoekt-indexserver-1:
+    <<: *zoekttemplate
+    env:
+      <<: *zoektenv
+      ZOEKT_NUM: 1
+      ZOEKT_HOSTNAME_PORT: 3071
+      ZOEKT_LISTEN_PORT: 6073

  zoekt-webserver-0:
    cmd: |
@ -315,6 +315,80 @@ commands:
      GOGC: 50
      PATH: .bin:$PATH

+  precise-code-intel-worker:
+    cmd: .bin/precise-code-intel-worker
+    install: |
+      go build -o .bin/precise-code-intel-worker github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker
+    watch:
+      - internal
+      - cmd/enterprise/precise-code-intel-worker
+      - enterprise/internal
+      - lib/codeintel
+
+  executor-queue:
+    cmd: .bin/executor-queue
+    install: |
+      go build -o .bin/executor-queue github.com/sourcegraph/sourcegraph/enterprise/cmd/executor-queue
+    watch:
+      - internal
+      - cmd/enterprise/executor-queue
+      - enterprise/internal
+      - lib/codeintel
+
+  codeintel-executor:
+    cmd: .bin/codeintel-executor
+    install:
+    env:
+      EXECUTOR_QUEUE_NAME: codeintel
+      TMPDIR: $HOME/.sourcegraph/indexer-temp
+
+
+  minio:
+    cmd: |
+      docker inspect $CONTAINER >/dev/null 2>&1 && docker rm -f $CONTAINER
+      docker run --rm \
+        --name=$CONTAINER \
+        --cpus=1 \
+        --memory=1g \
+        -p 0.0.0.0:9000:9000 \
+        -e 'MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE' \
+        -e 'MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' \
+        -v "$MINIO_DISK":/data \
+        $IMAGE server /data >"$MINIO_LOG_FILE" 2>&1
+    install: |
+      mkdir -p $MINIO_LOGS
+      mkdir -p $MINIO_DISK
+      CACHE=true ./docker-images/minio/build.sh >$MINIO_LOG_FILE 2>&1
+    env:
+      MINIO_DISK: $HOME/.sourcegraph-dev/data/minio
+      MINIO_LOGS: $HOME/.sourcegraph-dev/logs/minio
+      MINIO_LOG_FILE: $HOME/.sourcegraph-dev/logs/minio/minio.log"
+      IMAGE: sourcegraph/minio
+      CONTAINER: minio
+
+  codeinsights-db:
+    cmd: |
+      docker inspect $CONTAINER >/dev/null 2>&1 && docker rm -f $CONTAINER
+      docker run --rm \
+        --name=${CONTAINER} \
+        --cpus=1 \
+        --memory=1g \
+        -e POSTGRES_PASSWORD=password \
+        -p 0.0.0.0:$PORT:5432 \
+        -v $DISK:/var/lib/postgresql/data \
+        $IMAGE >$LOG_FILE 2>&1
+    install: |
+      mkdir -p $LOGS
+      mkdir -p $DISK
+      CACHE=true ./docker-images/codeinsights-db/build.sh >$LOG_FILE 2>&1
+    env:
+      LOGS: $HOME/.sourcegraph-dev/logs/codeinsights-db
+      LOG_FILE: $HOME/.sourcegraph-dev/logs/codeinsights-db/codeinsights-db.log
+      DISK: $HOME/.sourcegraph-dev/data/codeinsights-db
+      IMAGE: sourcegraph/codeinsights-db:dev
+      CONTAINER: codeinsights-db
+      PORT: 5435
+
 checks:
  docker:
    cmd: docker -v
@ -329,6 +403,8 @@ checks:
    failMessage: "Failed to connect to Postgres database. Make sure environment variables are setup correctly so that psql can connect."

 commandsets:
+  # TODO: Should we be able to define "env" vars _per set_?
+
  default:
    - frontend
    - repo-updater
@ -363,6 +439,52 @@ commandsets:
    - zoekt-webserver-0
    - zoekt-webserver-1

+  enterprise-codeintel:
+    - enterprise-frontend
+    - enterprise-repo-updater
+    - enterprise-web
+    - gitserver
+    - searcher
+    - symbols
+    - query-runner
+    - caddy
+    - docsite
+    - syntect_server
+    - github-proxy
+    - zoekt-indexserver-0
+    - zoekt-indexserver-1
+    - zoekt-webserver-0
+    - zoekt-webserver-1
+    - minio
+    - executor-queue
+    - precise-code-intel-worker
+    - codeintel-executor
+
+  enterprise-codeinsights:
+    # Add the following overwrites to your sg.config.overwrite.yaml to get
+    # codeinsights working:
+    #
+    #   env:
+    #     DISABLE_CODE_INSIGHTS_HISTORICAL: false
+    #     DISABLE_CODE_INSIGHTS: false
+    #
+    - enterprise-frontend
+    - enterprise-repo-updater
+    - enterprise-web
+    - gitserver
+    - searcher
+    - symbols
+    - query-runner
+    - caddy
+    - docsite
+    - syntect_server
+    - github-proxy
+    - zoekt-indexserver-0
+    - zoekt-indexserver-1
+    - zoekt-webserver-0
+    - zoekt-webserver-1
+    - codeinsights-db
+
 tests:
  # These can be run with `sg test [name]`
  backend: