From c5a611a05337865cfa064f32afb902f7f00c7d0f Mon Sep 17 00:00:00 2001 From: David Dollar Date: Wed, 21 Aug 2019 14:14:19 -0400 Subject: [PATCH] add ci --- ci/assets/example.org.crt | 19 +++ ci/assets/example.org.key | 27 ++++ ci/dependencies.sh | 26 ++++ ci/deploy.sh | 17 +++ ci/env.sh | 11 ++ ci/install.sh | 7 + ci/test.sh | 199 +++++++++++++++++++++++++++ ci/uninstall.sh | 6 + ci/update.sh | 8 ++ examples/httpd/Dockerfile | 1 + examples/httpd/convox-dockerfile.yml | 4 + examples/httpd/convox.yml | 4 + examples/httpd/docker-compose.yml | 7 + provider/aws/log.go | 2 +- 14 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 ci/assets/example.org.crt create mode 100644 ci/assets/example.org.key create mode 100755 ci/dependencies.sh create mode 100755 ci/deploy.sh create mode 100755 ci/env.sh create mode 100755 ci/install.sh create mode 100755 ci/test.sh create mode 100755 ci/uninstall.sh create mode 100755 ci/update.sh create mode 100644 examples/httpd/Dockerfile create mode 100644 examples/httpd/convox-dockerfile.yml create mode 100644 examples/httpd/convox.yml create mode 100644 examples/httpd/docker-compose.yml diff --git a/ci/assets/example.org.crt b/ci/assets/example.org.crt new file mode 100644 index 0000000..ed6018f --- /dev/null +++ b/ci/assets/example.org.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/zCCAeegAwIBAgIJALBsIpApvOZUMA0GCSqGSIb3DQEBBQUAMBYxFDASBgNV +BAMMC2V4YW1wbGUub3JnMB4XDTE4MDcwOTE0MTgxMVoXDTI4MDcwNjE0MTgxMVow +FjEUMBIGA1UEAwwLZXhhbXBsZS5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQC5OOR9vwb18cRe8SA6idnLACfLlTEJB60TP7nNZ78X4XvwonrWND11 +1nQ2F0ank7jkyndxHiNN1J93Xmx1icqidbCYatSfruX5KuTqTcOxPAGLIPcdqczA +ef77cwu9SXsd11SpecK1hVhoq4+B4P86ExPDXSWdxApaNbtL00Rrt2U4Ah0IoYls +zvhwVTHP9ebU4Nj282a3uNfm83rUsB4i2I665Ko1DyEp+hX7YFk2Nj9EyR/GWXvv +Q3ysMRiNAy6bnZ8TIWSeTp+BxiEFgWli/X4FoZzEHNHgl+Re/tK3zXxD6E6bHkW0 +4oO7DsSeOI3gUCJaXVzDZtWI9bKAf+mlAgMBAAGjUDBOMB0GA1UdDgQWBBSVIS8Q +c+4P7F+4UrFf5v7r1RQHnTAfBgNVHSMEGDAWgBSVIS8Qc+4P7F+4UrFf5v7r1RQH +nTAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQB2WRCMCJLcl0oV0At0 +t41oRs2Op32PCPi4Y+uBD3ZZuMuVyAmI9qVc5ZkmehxwPwo3pfAZQiU0+tWPk7NZ +7uafK/6cZKOxTpBEgRtdayKswjlTmWHpuPRsqTPEx/lgzDqGnY4u9eqc7X3l8AaP +6EhLqbcBWQJPeMrfHQy6hozbItT8LOygKgvbu325b98oGju4/5/Ev29uhH9WM1DP +GU0yZCeoHU6y+REREBiBsoehIVGh0apTNqlEdC6oBKup2xa3EJBY0YJ9zpSrPMyK +X2k+F6ufgX2Qla3elSyjRT63+je8pTkWqt+b74Upuw/x142Uo4+H8pWXo6mJ/wP7 +HyJg +-----END CERTIFICATE----- diff --git a/ci/assets/example.org.key b/ci/assets/example.org.key new file mode 100644 index 0000000..d2a22a6 --- /dev/null +++ b/ci/assets/example.org.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAuTjkfb8G9fHEXvEgOonZywAny5UxCQetEz+5zWe/F+F78KJ6 +1jQ9ddZ0NhdGp5O45Mp3cR4jTdSfd15sdYnKonWwmGrUn67l+Srk6k3DsTwBiyD3 +HanMwHn++3MLvUl7HddUqXnCtYVYaKuPgeD/OhMTw10lncQKWjW7S9NEa7dlOAId +CKGJbM74cFUxz/Xm1ODY9vNmt7jX5vN61LAeItiOuuSqNQ8hKfoV+2BZNjY/RMkf +xll770N8rDEYjQMum52fEyFknk6fgcYhBYFpYv1+BaGcxBzR4JfkXv7St818Q+hO +mx5FtOKDuw7EnjiN4FAiWl1cw2bViPWygH/ppQIDAQABAoIBAGaXiJfN00Eu5KHh +1QQXjrbOIzy0KjDiO49J4NRQ/ptvnYKvvHIhBVnbqwq64bu4KtUu//F7y/o0Jw+r +EVAZPk+1+Faz9YltCpx0kc7lu9rTbdduLvITNwH1kLIeCwaRSMxClPY6Jwr8DPgM +7rP1t+im4G95jHhSOZm14UQihOdT/9Z1bNX58WiMIxGqRDfZ+iN5+NgtmMa/GGf5 +ZCCtyItUCewZ1WAwpe93SRSLjWgRrRp+psX7suXig1qoSN7cYG9p9giEOhHfZ9Z+ +VQDHdUVX5xbvLY0yYRU0u4kGA2AG+Ggg0pDSWGy6ipJej5rF5Q9P3kauZ6/Onfyf +11FhggECgYEA6EBpEW6tOJeUH2xeQ4kzfBBWEEDZw0aRiWZi2VeaVN6OAFToNNQS +ZtG7W//cwvWTM79Sh7qPPnqTZaew121IWKKbYOuaZLfnxw9LZpUPQ19ZE5E44GwJ +5dWDH8nSyw0irwJvRCoKvpS5AkZBxfLtc3QymfiTuqk35WnSix3kDsUCgYEAzClo +G2chM4F+Wv3Vbtb6e7YZIFCAQeQ/qEVeeU3K8CjeHFGXrypjSDytBXGYDZvmkejY +ubICpw99q8kYOLk8hO28f8B9pOG8njzk/xtra5KnzEPJ1DTefjLtm6qvUENgTDuD +EHeHoPjRPlb5uI826A9idxmRHPd4NpgO1/BiHWECgYB1W3a8FEz7d7ylVEEdp0qY +ujiC5gIjppkL8OyqHpYhpIPkrwLXDwJtajVee5+19Bl54Id3hlGzpISurVmywcn4 +39Hb27Gci0FXWOck4tVfmeKM/Kjc3jryJD2p5nXZnL2V6YkHzKa5PrVt7Mp3mBVc +ba7UFsOvYqqYBi0ZlvUScQKBgEkdMwlNP5igcU3IxZyyJHYVWp419hii0yFB5nlu +oJStLg3aMoYqme+Ouy/N5HRLNqFeT+8Aju6hH5Jbq+70d3XrZawilVxs8w2AJUou +2aNmm+0NAcSbQmrTdkBfOfV2i5ABaAiHpezB7r1yjG/J6hHlIJAiedNz1HtIrVgr +4BdBAoGAXd72aInPaZDxm1gHQJmnfAaN4nIJX9tjZmqmw7cpmCOoNLfnmyj/Uz0M +8RTLi+GQq7OwIDIwxAmus7xysppmOoa94sCd0LYUHmFTD548YyK7fSZB9S+Z6Yes +AOS2bAdXe7lNbscfKL2vqNULOMu+jpBdyhSadee5LIJ8Io8CjbA= +-----END RSA PRIVATE KEY----- diff --git a/ci/dependencies.sh b/ci/dependencies.sh new file mode 100755 index 0000000..ba3180d --- /dev/null +++ b/ci/dependencies.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -ex -o pipefail + +source $(dirname $0)/env.sh + +# install utilities +sudo apt-get update && sudo apt-get -y install awscli jq + +# install docker +curl -s https://download.docker.com/linux/static/stable/x86_64/docker-18.09.6.tgz | sudo tar -C /usr/bin --strip-components 1 -xz + +# install kubectl +curl -Ls https://storage.googleapis.com/kubernetes-release/release/v1.13.0/bin/linux/amd64/kubectl -o /tmp/kubectl && \ + sudo mv /tmp/kubectl /usr/bin/kubectl && sudo chmod +x /usr/bin/kubectl + +# install aws-iam-authenticator +curl -Ls https://amazon-eks.s3-us-west-2.amazonaws.com/1.12.7/2019-03-27/bin/linux/amd64/aws-iam-authenticator -o /tmp/aws-iam-authenticator && \ + sudo mv /tmp/aws-iam-authenticator /usr/bin/aws-iam-authenticator && sudo chmod +x /usr/bin/aws-iam-authenticator + +# download appropriate cli version +curl -o ${GOPATH}/bin/convox https://convox.s3.amazonaws.com/release/${VERSION}/cli/linux/convox +chmod +x ${GOPATH}/bin/convox + +# set ci@convox.com as id +mkdir -p ~/.convox/ +echo ci@convox.com > ~/.convox/id diff --git a/ci/deploy.sh b/ci/deploy.sh new file mode 100755 index 0000000..0d1d043 --- /dev/null +++ b/ci/deploy.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -ex -o pipefail + +source $(dirname $0)/env.sh + +repo=$1 +app=$2 +check=$3 + +mkdir -p /tmp/app +cd /tmp/app + +git clone $repo . + +convox deploy --app $app --wait + +curl -ks --connect-timeout 5 --max-time 3 --retry 10 --retry-max-time 30 --retry-connrefused $check diff --git a/ci/env.sh b/ci/env.sh new file mode 100755 index 0000000..d9b6510 --- /dev/null +++ b/ci/env.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e -o pipefail + +if [ ! -f /tmp/convox-rack-name ]; then + echo "ci-${CIRCLE_BUILD_NUM}-$(date +"%H%M")" > /tmp/convox-rack-name +fi + +export AWS_DEFAULT_REGION=us-east-1 +export AWS_REGION=us-east-1 +export RACK_NAME=$(cat /tmp/convox-rack-name) +export VERSION=${VERSION:-${CIRCLE_TAG}} diff --git a/ci/install.sh b/ci/install.sh new file mode 100755 index 0000000..973f45b --- /dev/null +++ b/ci/install.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -ex -o pipefail + +source $(dirname $0)/env.sh + +convox rack install ${PROVIDER} --name ${RACK_NAME} --version ${VERSION} ${ARGS} +convox instances diff --git a/ci/test.sh b/ci/test.sh new file mode 100755 index 0000000..c66d77e --- /dev/null +++ b/ci/test.sh @@ -0,0 +1,199 @@ +#!/bin/bash + +function assert_run { + run "$1" || { echo "failed"; exit 101; } +} + +function fetch { + fetch_once $1 && sleep 5 && fetch_once $1 +} + +function fetch_once { + curl -ks --connect-timeout 5 --max-time 3 --retry 100 --retry-max-time 600 --retry-connrefused $1 +} + +function run { + echo "running: $*" >&2 + eval $* +} + +root="$(cd $(dirname ${0:-})/..; pwd)" + +set -ex + +provider=$(convox api get /system | jq -r .provider) + +# cli +convox version + +# rack +convox instances +convox rack +convox rack logs --no-follow | grep service/ +convox rack ps | grep rack + +# rack (provider-specific) +case $provider in + # aws) + # convox rack releases + # convox instances keyroll --wait + # instance=$(convox api get /instances | jq -r '.[0].id') + # convox instances ssh $instance "ls -la" | grep ec2-user + # convox instances terminate $instance + # convox rack | grep elb.amazonaws.com + # convox rack params | grep LogRetention + # convox rack params set LogRetention=14 --wait + # convox rack params | grep LogRetention | grep 14 + # convox rack params set LogRetention= --wait + # convox rack params | grep LogRetention | grep -v 14 + # ;; +esac + +# registries +convox registries +convox registries add quay.io convox+ci 6D5CJVRM5P3L24OG4AWOYGCDRJLPL0PFQAENZYJ1KGE040YDUGPYKOZYNWFTE5CV +convox registries | grep quay.io | grep convox+ci +convox registries remove quay.io +convox registries | grep -v quay.io + +# app +cd $root/examples/httpd +convox apps create ci2 --wait +convox apps | grep ci2 +convox apps info ci2 | grep running +release=$(convox build -a ci2 -d cibuild --id) && [ -n "$release" ] +convox releases -a ci2 | grep $release +build=$(convox api get /apps/ci2/builds | jq -r ".[0].id") && [ -n "$build" ] +convox builds -a ci2 | grep $build +convox builds info $build -a ci2 | grep $build +convox builds info $build -a ci2 | grep cibuild +convox builds logs $build -a ci2 | grep "Running: docker push" +convox builds export $build -a ci2 -f /tmp/build.tgz +releasei=$(convox builds import -a ci2 -f /tmp/build.tgz --id) && [ -n "$releasei" ] +buildi=$(convox api get /apps/ci2/releases/$releasei | jq -r ".build") && [ -n "$buildi" ] +convox builds info $buildi -a ci2 | grep cibuild +echo "FOO=bar" | convox env set -a ci2 +convox env -a ci2 | grep FOO | grep bar +convox env get FOO -a ci2 | grep bar +convox env unset FOO -a ci2 +convox env -a ci2 | grep -v FOO +releasee=$(convox env set FOO=bar -a ci2 --id) && [ -n "$releasee" ] +convox env get FOO -a ci2 | grep bar +convox releases -a ci2 | grep $releasee +convox releases info $releasee -a ci2 | grep FOO +convox releases manifest $releasee -a ci2 | grep "image: httpd" +convox releases promote $release -a ci2 --wait +endpoint=$(convox api get /apps/ci2/services | jq -r '.[] | select(.name == "web") | .domain') +fetch https://$endpoint | grep "It works" +convox logs -a ci2 --no-follow | grep service/web +releaser=$(convox releases rollback $release -a ci2 --wait --id) +convox ps -a ci2 | grep $releaser +ps=$(convox api get /apps/ci2/processes | jq -r '.[]|select(.status=="running")|.id' | head -n 1) +convox ps info $ps -a ci2 | grep $releaser +convox scale web --count 2 --cpu 192 --memory 256 -a ci2 --wait +convox services -a ci2 | grep web | grep 443:80 | grep $endpoint +endpoint=$(convox api get /apps/ci2/services | jq -r '.[] | select(.name == "web") | .domain') +fetch https://$endpoint | grep "It works" +convox ps -a ci2 | grep web | wc -l | grep 2 +ps=$(convox api get /apps/ci2/processes | jq -r '.[]|select(.status=="running")|.id' | head -n 1) +convox exec $ps "ls -la" -a ci2 | grep htdocs +cat /dev/null | convox exec $ps 'sh -c "sleep 2; echo test"' -a ci2 | grep test +convox run web "ls -la" -a ci2 | grep htdocs +cat /dev/null | convox run web 'sh -c "sleep 2; echo test"' -a ci2 | grep test +echo foo > /tmp/file +convox cp /tmp/file $ps:/file -a ci2 +convox exec $ps "cat /file" -a ci2 | grep foo +mkdir -p /tmp/dir +echo foo > /tmp/dir/file +convox cp /tmp/dir $ps:/dir -a ci2 +convox exec $ps "cat /dir/file" -a ci2 | grep foo +convox cp $ps:/dir /tmp/dir2 -a ci2 +cat /tmp/dir2/file | grep foo +convox cp $ps:/file /tmp/file2 -a ci2 +cat /tmp/file2 | grep foo +convox ps stop $ps -a ci2 +convox ps -a ci2 | grep -v $ps +convox deploy -a ci2 --wait + +# app (provider-specific) +case $provider in + # aws) + # convox apps params -a ci2 | grep LogRetention + # convox apps params set LogRetention=14 -a ci2 --wait + # convox apps params -a ci2 | grep LogRetention | grep 14 + # convox apps params set LogRetention= -a ci2 --wait + # convox apps params -a ci2 | grep LogRetention | grep -v 14 + # ;; +esac + +# gen1 +case $provider in + # aws) + # cd $root/examples/httpd + # convox apps create ci1 -g 1 --wait + # convox deploy -a ci1 --wait + # convox services -a ci1 | grep web | grep elb.amazonaws.com | grep 443:80 + # endpoint=$(convox api get /apps/ci1/services | jq -r '.[] | select(.name == "web") | .domain') + # fetch https://$endpoint | grep "It works" + # ;; +esac + +# certs +case $provider in + # aws) + # cd $root/ci/assets + # convox certs + # cert=$(convox certs generate example.org --id) + # convox certs | grep -v $cert + # convox certs delete $cert + # cert=$(convox certs import example.org.crt example.org.key --id) + # sleep 30 + # convox certs | grep $cert + # certo=$(convox api get /apps/ci1/services | jq -r '.[] | select(.name == "web") | .ports[] | select (.balancer == 443) | .certificate') + # convox ssl -a ci1 | grep web:443 | grep $certo + # convox ssl update web:443 $cert -a ci1 --wait + # convox ssl -a ci1 | grep web:443 | grep $cert + # convox ssl update web:443 $certo -a ci1 --wait + # convox ssl -a ci1 | grep web:443 | grep $certo + # sleep 30 + # convox certs delete $cert + # ;; +esac + +# rack resources +case $provider in + # aws) + # convox rack resources create syslog Url=tcp://syslog.convox.com --name cilog --wait + # convox rack resources | grep cilog | grep syslog + # convox rack resources info cilog | grep -v Apps + # convox rack resources url cilog | grep tcp://syslog.convox.com + # convox rack resources link cilog -a ci2 --wait + # convox rack resources info cilog | grep Apps | grep ci2 + # convox rack resources unlink cilog -a ci2 --wait + # convox rack resources info cilog | grep -v Apps + # convox rack resources link cilog -a ci1 --wait + # convox rack resources info cilog | grep Apps | grep ci1 + # convox rack resources unlink cilog -a ci1 --wait + # convox rack resources info cilog | grep -v Apps + # convox rack resources update cilog Url=tcp://syslog2.convox.com --wait + # convox rack resources info cilog | grep syslog2.convox.com + # convox rack resources url cilog | grep tcp://syslog2.convox.com + # convox rack resources delete cilog --wait + # convox rack resources create postgres --name pgdb --wait + # convox rack resources | grep pgdb | grep postgres + # dburl=$(convox rack resources url pgdb) + # convox rack resources update pgdb BackupRetentionPeriod=2 --wait + # [ "$dburl" == "$(convox rack resources url pgdb)" ] + # convox rack resources delete pgdb --wait + # ;; +esac + +# cleanup +convox apps delete ci2 --wait + +# cleanup (provider-specific) +case $provider in + # aws) + # convox apps delete ci1 --wait + # ;; +esac diff --git a/ci/uninstall.sh b/ci/uninstall.sh new file mode 100755 index 0000000..9015d12 --- /dev/null +++ b/ci/uninstall.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex -o pipefail + +source $(dirname $0)/env.sh + +convox rack uninstall ${PROVIDER} ${RACK_NAME} --force diff --git a/ci/update.sh b/ci/update.sh new file mode 100755 index 0000000..b521cab --- /dev/null +++ b/ci/update.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -ex -o pipefail + +source $(dirname $0)/env.sh + +if [ "${VERSION}" != "$(convox api get /system | jq -r '.version')" ]; then + convox rack update "${VERSION}" --wait +fi diff --git a/examples/httpd/Dockerfile b/examples/httpd/Dockerfile new file mode 100644 index 0000000..6455161 --- /dev/null +++ b/examples/httpd/Dockerfile @@ -0,0 +1 @@ +FROM httpd diff --git a/examples/httpd/convox-dockerfile.yml b/examples/httpd/convox-dockerfile.yml new file mode 100644 index 0000000..3d3ba32 --- /dev/null +++ b/examples/httpd/convox-dockerfile.yml @@ -0,0 +1,4 @@ +services: + web: + build: . + port: 80 diff --git a/examples/httpd/convox.yml b/examples/httpd/convox.yml new file mode 100644 index 0000000..10d1e08 --- /dev/null +++ b/examples/httpd/convox.yml @@ -0,0 +1,4 @@ +services: + web: + image: httpd + port: 80 diff --git a/examples/httpd/docker-compose.yml b/examples/httpd/docker-compose.yml new file mode 100644 index 0000000..0532b7c --- /dev/null +++ b/examples/httpd/docker-compose.yml @@ -0,0 +1,7 @@ +web: + image: httpd + labels: + - convox.port.443.protocol=tls + ports: + - 80:80 + - 443:80 diff --git a/provider/aws/log.go b/provider/aws/log.go index 0e52c84..4dae20d 100644 --- a/provider/aws/log.go +++ b/provider/aws/log.go @@ -113,7 +113,7 @@ func (p *Provider) ProcessLogs(app, pid string, opts structs.LogsOptions) (io.Re } func (p *Provider) SystemLogs(opts structs.LogsOptions) (io.ReadCloser, error) { - return common.CloudWatchLogsSubscribe(p.Context(), p.CloudWatchLogs, p.appLogGroup("rack"), "", opts) + return common.CloudWatchLogsSubscribe(p.Context(), p.CloudWatchLogs, p.appLogGroup("system"), "", opts) } func (p *Provider) appLogGroup(app string) string {