mirror of
https://github.com/getsentry/self-hosted.git
synced 2026-02-06 10:57:17 +00:00
Introduce patches with external kafka (#3521)
* Introduce patches with external kafka * Fix pre-commit hooks * Patch relay config file * Documentation for patches stuff * Provide more helpful information for Docker Compose Override file * Fix grep command * ref: rename to 'optional-modifications' * chore(pre-commit): exclude .patch extension * chore(pre-commit): escape backslash * chore(pre-commit): put exclude field on hooks * chore(pre-commit): put exclude field on top level Based on https://pre-commit.com/#top_level-exclude * chore(pre-commit): move to even more top level
This commit is contained in:
Reinaldy Rafli
GitHub
parent
0730d8c8c7
commit
1b88e90e30
@ -1,3 +1,4 @@
|
||||
exclude: '\.patch$'
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
@ -11,7 +12,6 @@ repos:
|
||||
args: [-w, -d]
|
||||
files: .*\.sh
|
||||
stages: [commit, merge-commit, push, manual]
|
||||
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.3.0
|
||||
hooks:
|
||||
|
||||
41
optional-modifications/README.md
Normal file
41
optional-modifications/README.md
Normal file
@ -0,0 +1,41 @@
|
||||
# Optional Modifications
|
||||
|
||||
Other than the default self-hosted Sentry installation, sometimes users
|
||||
can leverage their existing infrastructure to help them with limited
|
||||
resources. "Patches", or you might call this like a "plugin system", is
|
||||
a collection of patch files (see [man patch(1)](https://man7.org/linux/man-pages/man1/patch.1.html))
|
||||
that can be used with to modify the existing configuration to achieve
|
||||
the desired goal.
|
||||
|
||||
> [!WARNING]
|
||||
> Beware that this is very experimental and might not work as expected.
|
||||
>
|
||||
> **Use it at your own risk!**
|
||||
|
||||
## How to use patches
|
||||
|
||||
The patches are designed mostly to help modify the existing
|
||||
configuration files. You will need to run the `install.sh` script
|
||||
afterwards.
|
||||
|
||||
They should be run from the root directory. For example, the
|
||||
`external-kafka` patches should be run as:
|
||||
|
||||
```bash
|
||||
patch < optional-modifications/patches/external-kafka/.env.patch
|
||||
patch < optional-modifications/patches/external-kafka/config.example.yml.patch
|
||||
patch < optional-modifications/patches/external-kafka/sentry.conf.example.py.patch
|
||||
patch < optional-modifications/patches/external-kafka/docker-compose.yml.patch
|
||||
```
|
||||
|
||||
Some patches might require additional steps to be taken, like providing
|
||||
credentials or additional TLS certificates.
|
||||
|
||||
## Official support
|
||||
|
||||
Sentry employees are not obliged to provide dedicated support for
|
||||
patches, but they can help by providing information to move us forward.
|
||||
We encourage the community to contribute for any bug fixes or
|
||||
improvements.
|
||||
|
||||
See the [support policy for self-hosted Sentry](https://develop.sentry.dev/self-hosted/support/) for more information.
|
||||
15
optional-modifications/_lib.sh
Executable file
15
optional-modifications/_lib.sh
Executable file
@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
test "${DEBUG:-}" && set -x
|
||||
|
||||
function patch_file() {
|
||||
target="$1"
|
||||
content="$2"
|
||||
if [[ -f "$target" ]]; then
|
||||
echo "🙈 Patching $target ..."
|
||||
patch -p1 <"$content"
|
||||
else
|
||||
echo "🙊 Skipping $target ..."
|
||||
fi
|
||||
}
|
||||
22
optional-modifications/patches/external-kafka/.env.patch
Normal file
22
optional-modifications/patches/external-kafka/.env.patch
Normal file
@ -0,0 +1,22 @@
|
||||
--- .env 2025-02-04 07:31:54.868049984 +0700
|
||||
+++ .env.external-kafka 2025-05-15 08:33:15.442361105 +0700
|
||||
@@ -22,3 +22,19 @@
|
||||
POSTGRES_MAX_CONNECTIONS=100
|
||||
# Set SETUP_JS_SDK_ASSETS to 1 to enable the setup of JS SDK assets
|
||||
# SETUP_JS_SDK_ASSETS=1
|
||||
+
|
||||
+################################################################################
|
||||
+## Additional External Kafka options
|
||||
+################################################################################
|
||||
+KAFKA_BOOTSTRAP_SERVERS=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092
|
||||
+# Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL
|
||||
+KAFKA_SECURITY_PROTOCOL=PLAINTEXT
|
||||
+# Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable.
|
||||
+# KAFKA_SASL_MECHANISM=PLAIN
|
||||
+# KAFKA_SASL_USERNAME=username
|
||||
+# KAFKA_SASL_PASSWORD=password
|
||||
+# Put your certificates on the \`certificates/kafka\` directory.
|
||||
+# The certificates will be mounted as read-only volumes.
|
||||
+# KAFKA_SSL_CA_LOCATION=/kafka-certificates/ca.pem
|
||||
+# KAFKA_SSL_CERTIFICATE_LOCATION=/kafka-certificates/client.pem
|
||||
+# KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key
|
||||
@ -0,0 +1,19 @@
|
||||
--- relay/config.example.yml 2025-05-15 08:27:40.426876887 +0700
|
||||
+++ relay/config.example.external-kafka.yml 2025-05-15 08:34:21.113311217 +0700
|
||||
@@ -7,8 +7,15 @@
|
||||
processing:
|
||||
enabled: true
|
||||
kafka_config:
|
||||
- - {name: "bootstrap.servers", value: "kafka:9092"}
|
||||
+ - {name: "bootstrap.servers", value: "kafka-node1:9092,kafka-node2:9092,kafka-node3:9092"}
|
||||
- {name: "message.max.bytes", value: 50000000} # 50MB
|
||||
+ - {name: "security.protocol", value: "PLAINTEXT"}
|
||||
+ - {name: "sasl.mechanism", value: "PLAIN"} # Remove or comment this line if SASL is not used.
|
||||
+ - {name: "sasl.username", value: "username"} # Remove or comment this line if SASL is not used.
|
||||
+ - {name: "sasl.password", value: "password"} # Remove or comment this line if SASL is not used.
|
||||
+ - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} # Remove or comment this line if SSL is not used.
|
||||
+ - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} # Remove or comment this line if SSL is not used.
|
||||
+ - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} # Remove or comment this line if SSL is not used.
|
||||
redis: redis://redis:6379
|
||||
geoip_path: "/geoip/GeoLite2-City.mmdb"
|
||||
|
||||
@ -0,0 +1,142 @@
|
||||
--- docker-compose.yml 2025-03-17 13:32:15.120328412 +0700
|
||||
+++ docker-compose.external-kafka.yml 2025-05-15 08:39:05.509951068 +0700
|
||||
@@ -26,8 +26,6 @@
|
||||
depends_on:
|
||||
redis:
|
||||
<<: *depends_on-healthy
|
||||
- kafka:
|
||||
- <<: *depends_on-healthy
|
||||
postgres:
|
||||
<<: *depends_on-healthy
|
||||
memcached:
|
||||
@@ -59,6 +57,14 @@
|
||||
SENTRY_EVENT_RETENTION_DAYS:
|
||||
SENTRY_MAIL_HOST:
|
||||
SENTRY_MAX_EXTERNAL_SOURCEMAP_SIZE:
|
||||
+ KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
|
||||
+ KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
|
||||
+ KAFKA_SSL_CA_LOCATION: ${KAFKA_SSL_CA_LOCATION:-}
|
||||
+ KAFKA_SSL_CERTIFICATE_LOCATION: ${KAFKA_SSL_CERTIFICATE_LOCATION:-}
|
||||
+ KAFKA_SSL_KEY_LOCATION: ${KAFKA_SSL_KEY_LOCATION:-}
|
||||
+ KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-}
|
||||
+ KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-}
|
||||
+ KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-}
|
||||
volumes:
|
||||
- "sentry-data:/data"
|
||||
- "./sentry:/etc/sentry"
|
||||
@@ -69,15 +75,20 @@
|
||||
depends_on:
|
||||
clickhouse:
|
||||
<<: *depends_on-healthy
|
||||
- kafka:
|
||||
- <<: *depends_on-healthy
|
||||
redis:
|
||||
<<: *depends_on-healthy
|
||||
image: "$SNUBA_IMAGE"
|
||||
environment:
|
||||
SNUBA_SETTINGS: self_hosted
|
||||
CLICKHOUSE_HOST: clickhouse
|
||||
- DEFAULT_BROKERS: "kafka:9092"
|
||||
+ DEFAULT_BROKERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
|
||||
+ KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
|
||||
+ KAFKA_SSL_CA_PATH: ${KAFKA_SSL_CA_LOCATION:-}
|
||||
+ KAFKA_SSL_CERT_PATH: ${KAFKA_SSL_CERTIFICATE_LOCATION:-}
|
||||
+ KAFKA_SSL_KEY_PATH: ${KAFKA_SSL_KEY_LOCATION:-}
|
||||
+ KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-}
|
||||
+ KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-}
|
||||
+ KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-}
|
||||
REDIS_HOST: redis
|
||||
UWSGI_MAX_REQUESTS: "10000"
|
||||
UWSGI_DISABLE_LOGGING: "true"
|
||||
@@ -140,43 +151,7 @@
|
||||
POSTGRES_HOST_AUTH_METHOD: "trust"
|
||||
volumes:
|
||||
- "sentry-postgres:/var/lib/postgresql/data"
|
||||
- kafka:
|
||||
- <<: *restart_policy
|
||||
- image: "confluentinc/cp-kafka:7.6.1"
|
||||
- environment:
|
||||
- # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example
|
||||
- KAFKA_PROCESS_ROLES: "broker,controller"
|
||||
- KAFKA_CONTROLLER_QUORUM_VOTERS: "1001@127.0.0.1:29093"
|
||||
- KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
|
||||
- KAFKA_NODE_ID: "1001"
|
||||
- CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk"
|
||||
- KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:29092,INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093"
|
||||
- KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:29092,INTERNAL://kafka:9093,EXTERNAL://kafka:9092"
|
||||
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT"
|
||||
- KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
|
||||
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
|
||||
- KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1"
|
||||
- KAFKA_LOG_RETENTION_HOURS: "24"
|
||||
- KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust
|
||||
- KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too
|
||||
- CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
|
||||
- KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,state.change.logger=WARN"
|
||||
- KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN"
|
||||
- KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN"
|
||||
- ulimits:
|
||||
- nofile:
|
||||
- soft: 4096
|
||||
- hard: 4096
|
||||
- volumes:
|
||||
- - "sentry-kafka:/var/lib/kafka/data"
|
||||
- - "sentry-kafka-log:/var/lib/kafka/log"
|
||||
- - "sentry-secrets:/etc/kafka/secrets"
|
||||
- healthcheck:
|
||||
- <<: *healthcheck_defaults
|
||||
- test: ["CMD-SHELL", "nc -z localhost 9092"]
|
||||
- interval: 10s
|
||||
- timeout: 10s
|
||||
- retries: 30
|
||||
+ kafka: !reset null
|
||||
clickhouse:
|
||||
<<: *restart_policy
|
||||
image: clickhouse-self-hosted-local
|
||||
@@ -475,9 +450,8 @@
|
||||
read_only: true
|
||||
source: ./geoip
|
||||
target: /geoip
|
||||
+ - ./certificates/kafka:/kafka-certificates:ro
|
||||
depends_on:
|
||||
- kafka:
|
||||
- <<: *depends_on-healthy
|
||||
redis:
|
||||
<<: *depends_on-healthy
|
||||
web:
|
||||
@@ -486,15 +460,21 @@
|
||||
<<: *restart_policy
|
||||
image: "$VROOM_IMAGE"
|
||||
environment:
|
||||
- SENTRY_KAFKA_BROKERS_PROFILING: "kafka:9092"
|
||||
- SENTRY_KAFKA_BROKERS_OCCURRENCES: "kafka:9092"
|
||||
+ SENTRY_KAFKA_BROKERS_PROFILING: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
|
||||
+ SENTRY_KAFKA_BROKERS_OCCURRENCES: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
|
||||
+ SENTRY_KAFKA_BROKERS_SPANS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
|
||||
+ SENTRY_KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
|
||||
+ SENTRY_KAFKA_SSL_CA_PATH: ${KAFKA_SSL_CA_LOCATION:-}
|
||||
+ SENTRY_KAFKA_SSL_CERT_PATH: ${KAFKA_SSL_CERTIFICATE_LOCATION:-}
|
||||
+ SENTRY_KAFKA_SSL_KEY_PATH: ${KAFKA_SSL_KEY_LOCATION:-}
|
||||
+ SENTRY_KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-}
|
||||
+ SENTRY_KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-}
|
||||
+ SENTRY_KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-}
|
||||
SENTRY_BUCKET_PROFILES: file://localhost//var/lib/sentry-profiles
|
||||
SENTRY_SNUBA_HOST: "http://snuba-api:1218"
|
||||
volumes:
|
||||
- sentry-vroom:/var/lib/sentry-profiles
|
||||
- depends_on:
|
||||
- kafka:
|
||||
- <<: *depends_on-healthy
|
||||
+ - ./certificates/kafka:/kafka-certificates:ro
|
||||
profiles:
|
||||
- feature-complete
|
||||
vroom-cleanup:
|
||||
@@ -523,8 +503,6 @@
|
||||
external: true
|
||||
sentry-redis:
|
||||
external: true
|
||||
- sentry-kafka:
|
||||
- external: true
|
||||
sentry-clickhouse:
|
||||
external: true
|
||||
sentry-symbolicator:
|
||||
@ -0,0 +1,21 @@
|
||||
--- sentry/sentry.conf.example.py 2025-05-15 08:27:40.427876868 +0700
|
||||
+++ sentry/sentry.conf.example.external-kafka.py 2025-05-15 08:32:44.845127931 +0700
|
||||
@@ -132,9 +132,17 @@
|
||||
SENTRY_CACHE = "sentry.cache.redis.RedisCache"
|
||||
|
||||
DEFAULT_KAFKA_OPTIONS = {
|
||||
- "bootstrap.servers": "kafka:9092",
|
||||
+ "bootstrap.servers": env("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"),
|
||||
"message.max.bytes": 50000000,
|
||||
"socket.timeout.ms": 1000,
|
||||
+ "security.protocol": env("KAFKA_SECURITY_PROTOCOL", "PLAINTEXT"), # Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL
|
||||
+ # If you don't use any of these options below, you can remove them or set them to `None`.
|
||||
+ "sasl.mechanism": env("KAFKA_SASL_MECHANISM", None), # Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable.
|
||||
+ "sasl.username": env("KAFKA_SASL_USERNAME", None),
|
||||
+ "sasl.password": env("KAFKA_SASL_PASSWORD", None),
|
||||
+ "ssl.ca.location": env("KAFKA_SSL_CA_LOCATION", None), # Remove this line if SSL is not used.
|
||||
+ "ssl.certificate.location": env("KAFKA_SSL_CERTIFICATE_LOCATION", None), # Remove this line if SSL is not used.
|
||||
+ "ssl.key.location": env("KAFKA_SSL_KEY_LOCATION", None), # Remove this line if SSL is not used.
|
||||
}
|
||||
|
||||
SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream"
|
||||
Loading…
Reference in New Issue
Block a user