diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60807a5..e2f024b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,15 +58,7 @@ jobs: sudo chmod +x "${{ matrix.compose_path }}/docker-compose" - name: Integration Test - run: | - echo "Testing initial install" - ./install.sh - ./_integration-test/run.sh - echo "Testing in-place upgrade" - # Also test plugin installation here - echo "sentry-auth-oidc" >> sentry/requirements.txt - ./install.sh --minimize-downtime - ./_integration-test/run.sh + run: ./integration-test.sh - name: Inspect failure if: failure() diff --git a/.gitignore b/.gitignore index e1dbcd6..abc4940 100644 --- a/.gitignore +++ b/.gitignore @@ -80,6 +80,7 @@ data/ sentry/sentry.conf.py sentry/config.yml sentry/*.bak +sentry/enhance-image.sh sentry/requirements.txt relay/credentials.json relay/config.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..49120af --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,7 @@ +## Testing + +Validate changes to the setup by running the integration test: + +```shell +./integration-test.sh +``` diff --git a/README.md b/README.md index 722d99f..b63a246 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,6 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Setup -### Customize DotEnv (.env) file - -Environment specific configurations can be done in the `.env.custom` file. It will be located in the root directory of the Sentry installation. - -By default, there exists no `.env.custom` file. In this case, you can manually add this file by copying the `.env` file to a new `.env.custom` file and adjust your settings in the `.env.custom` file. - -Please keep in mind to check the `.env` file for changes, when you perform an upgrade of Sentry, so that you can adjust your `.env.custom` accordingly, if required. - ### Installation To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020 and Sentry 21.1.0 is the last version to support Python 2. @@ -30,6 +22,22 @@ Thinking of not managing this yourself? Check out the [SaaS migration docs](http Please visit [our documentation](https://develop.sentry.dev/self-hosted/) for everything else. +### Customize DotEnv (.env) file + +Environment specific configurations can be done in the `.env.custom` file. It will be located in the root directory of the Sentry installation. + +By default, there exists no `.env.custom` file. In this case, you can manually add this file by copying the `.env` file to a new `.env.custom` file and adjust your settings in the `.env.custom` file. + +Please keep in mind to check the `.env` file for changes, when you perform an upgrade of Sentry, so that you can adjust your `.env.custom` accordingly, if required. + +### Enhance Sentry image + +To install plugins and their dependencies or make other modifications to the Sentry base image, +copy `sentry/enhance-image.example.sh` to `sentry/enhance-image.sh` and add necessary steps there. +For example, you can use `apt-get` to install dependencies and use `pip` to install plugins. + +After making modifications to `sentry/enhance-image.sh`, run `./install.sh` again to apply them. + ## Tips & Tricks ### Event Retention @@ -40,7 +48,7 @@ Sentry comes with a cleanup cron job that prunes events older than `90 days` by If you want to install a specific release of Sentry, use the tags/releases on this repo. -We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:nightly`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: +We continuously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:nightly`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: ```shell SENTRY_IMAGE=getsentry/sentry:83b1380 ./install.sh @@ -57,7 +65,3 @@ sudo SENTRY_IMAGE=us.gcr.io/sentryio/sentry:83b1380 ./install.sh ``` Where you replace `83b1380` with the sha you want to use. - -[build-status-image]: https://github.com/getsentry/self-hosted/workflows/test/badge.svg -[build-status-url]: https://github.com/getsentry/self-hosted/actions?query=workflow%3Atest+branch%3Amaster+event%3Apush - diff --git a/_integration-test/ensure-customizations-not-present.sh b/_integration-test/ensure-customizations-not-present.sh new file mode 100755 index 0000000..99ae6ba --- /dev/null +++ b/_integration-test/ensure-customizations-not-present.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -ex + +source "$(dirname $0)/../install/_lib.sh" + +source ../install/dc-detect-version.sh + +# Negated version of ensure-customizations-work.sh, make changes in sync +echo "${_group}Ensure customizations not present" +! $dcr --no-deps web bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi" +! $dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi" +! $dcr --no-deps web python -c "import ldap" +! $dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup python -c "import ldap" +echo "${_endgroup}" diff --git a/_integration-test/ensure-customizations-work.sh b/_integration-test/ensure-customizations-work.sh new file mode 100755 index 0000000..8c92c62 --- /dev/null +++ b/_integration-test/ensure-customizations-work.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -ex + +source "$(dirname $0)/../install/_lib.sh" + +source ../install/dc-detect-version.sh + +# Negated version of ensure-customizations-not-present.sh, make changes in sync +echo "${_group}Ensure customizations work" +$dcr --no-deps web bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi" +$dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi" +$dcr --no-deps web python -c "import ldap" +$dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup python -c "import ldap" +echo "${_endgroup}" diff --git a/docker-compose.yml b/docker-compose.yml index b01e8e3..06734ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,11 @@ x-healthcheck-defaults: &healthcheck_defaults start_period: 10s x-sentry-defaults: &sentry_defaults <<: *restart_policy - image: "$SENTRY_IMAGE" + image: sentry-self-hosted-local + build: + context: ./sentry + args: + - SENTRY_IMAGE depends_on: redis: <<: *depends_on-healthy @@ -321,7 +325,7 @@ services: build: context: ./cron args: - BASE_IMAGE: "$SENTRY_IMAGE" + BASE_IMAGE: sentry-self-hosted-local entrypoint: "/entrypoint.sh" command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: diff --git a/install/build-docker-images.sh b/install/build-docker-images.sh index 4bb96b5..63bb0da 100644 --- a/install/build-docker-images.sh +++ b/install/build-docker-images.sh @@ -1,6 +1,9 @@ echo "${_group}Building and tagging Docker images ..." echo "" +# Build any service that provides the image sentry-self-hosted-local first, +# as it is used as the base image for sentry-cleanup-self-hosted-local. +$dc build --force-rm web $dc build --force-rm echo "" echo "Docker images built." diff --git a/install/ensure-files-from-examples.sh b/install/ensure-files-from-examples.sh index 17958a0..f96f7ca 100644 --- a/install/ensure-files-from-examples.sh +++ b/install/ensure-files-from-examples.sh @@ -1,8 +1,7 @@ echo "${_group}Ensuring files from examples ..." -ensure_file_from_example $SENTRY_CONFIG_PY -ensure_file_from_example $SENTRY_CONFIG_YML +ensure_file_from_example "$SENTRY_CONFIG_PY" +ensure_file_from_example "$SENTRY_CONFIG_YML" ensure_file_from_example '../symbolicator/config.yml' -ensure_file_from_example '../sentry/requirements.txt' echo "${_endgroup}" diff --git a/integration-test.sh b/integration-test.sh new file mode 100755 index 0000000..52421e8 --- /dev/null +++ b/integration-test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -ex + +echo "Reset customizations" +rm -f sentry/enhance-image.sh +rm -f sentry/requirements.txt + +echo "Testing initial install" +./install.sh +./_integration-test/run.sh +./_integration-test/ensure-customizations-not-present.sh + +echo "Make customizations" +cat < sentry/enhance-image.sh +#!/bin/bash +touch /created-by-enhance-image +apt-get update +apt-get install -y gcc libsasl2-dev python-dev libldap2-dev libssl-dev +EOT +chmod +x sentry/enhance-image.sh +printf "python-ldap" > sentry/requirements.txt + +echo "Testing in-place upgrade and customizations" +./install.sh --minimize-downtime +./_integration-test/run.sh +./_integration-test/ensure-customizations-work.sh diff --git a/sentry/Dockerfile b/sentry/Dockerfile new file mode 100644 index 0000000..5afe9c2 --- /dev/null +++ b/sentry/Dockerfile @@ -0,0 +1,13 @@ +ARG SENTRY_IMAGE +FROM ${SENTRY_IMAGE} + +COPY . /usr/src/sentry + +RUN if [ -s /usr/src/sentry/enhance-image.sh ]; then \ + /usr/src/sentry/enhance-image.sh; \ +fi + +RUN if [ -s /usr/src/sentry/requirements.txt ]; then \ + echo "sentry/requirements.txt is deprecated, use sentry/enhance-image.sh - see https://github.com/getsentry/self-hosted#enhance-sentry-image"; \ + pip install -r /usr/src/sentry/requirements.txt; \ +fi diff --git a/sentry/enhance-image.example.sh b/sentry/enhance-image.example.sh new file mode 100755 index 0000000..e5b3182 --- /dev/null +++ b/sentry/enhance-image.example.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Enhance the base $SENTRY_IMAGE with additional dependencies, plugins - see https://github.com/getsentry/self-hosted#enhance-sentry-image +# For example: +# apt-get update +# apt-get install -y gcc libsasl2-dev python-dev libldap2-dev libssl-dev +# pip install python-ldap diff --git a/sentry/entrypoint.sh b/sentry/entrypoint.sh index 2f2614a..552de05 100755 --- a/sentry/entrypoint.sh +++ b/sentry/entrypoint.sh @@ -5,16 +5,8 @@ if [ "$(ls -A /usr/local/share/ca-certificates/)" ]; then update-ca-certificates fi -req_file="/etc/sentry/requirements.txt" -plugins_dir="/data/custom-packages" -checksum_file="$plugins_dir/.checksum" - -if [[ -s "$req_file" ]] && ! cat "$req_file" | grep '^[^#[:space:]]' | shasum -s -c "$checksum_file" 2>/dev/null; then - echo "Installing additional dependencies..." - mkdir -p "$plugins_dir" - pip install --user -r "$req_file" - cat "$req_file" | grep '^[^#[:space:]]' | shasum > "$checksum_file" - echo "" +if [ -e /etc/sentry/requirements.txt ]; then + echo "sentry/requirements.txt is deprecated, use sentry/enhance-image.sh - see https://github.com/getsentry/self-hosted#enhance-sentry-image" fi source /docker-entrypoint.sh diff --git a/sentry/requirements.example.txt b/sentry/requirements.example.txt index b4659b6..e7b63dc 100644 --- a/sentry/requirements.example.txt +++ b/sentry/requirements.example.txt @@ -1 +1 @@ -# Add plugins here +# sentry/requirements.txt is deprecated, use sentry/enhance-image.sh - see https://github.com/getsentry/self-hosted#enhance-sentry-image diff --git a/test.sh b/test.sh index 85945dc..0f90fd5 100755 --- a/test.sh +++ b/test.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash set -e -./_integration-test/run.sh \ No newline at end of file +# This file runs in https://github.com/getsentry/sentry/blob/fe4795f5eae9e0d7c33e0ecb736c9d1369535eca/docker/cloudbuild.yaml#L59 + +./_integration-test/run.sh