From 08f5c9804656d3f275069db27a38f9c295ab1016 Mon Sep 17 00:00:00 2001 From: David Dollar Date: Wed, 20 Nov 2019 19:25:27 -0500 Subject: [PATCH] remove dependency on local kubeconfig (#18) * remove dependency on local kubeconfig * use k8s-specific network for gcp * remove unused delay * fix tests --- .dockerignore | 1 + Makefile | 2 +- provider/azure/azure.go | 2 - provider/gcp/log.go | 4 - provider/k8s/k8s.go | 37 +++- .../k8s/template/system/atom.yml.tmpl | 0 .../system/{custom.yml.tmpl => crd.yml.tmpl} | 0 provider/k8s/template/system/rack.yml.tmpl | 169 ----------------- provider/k8s/template/system/router.yml.tmpl | 109 ----------- terraform/api/aws/main.tf | 9 +- terraform/api/aws/variables.tf | 4 - terraform/api/azure/identity.tf | 44 ----- terraform/api/azure/identity.yml.tpl | 18 -- terraform/api/azure/main.tf | 11 +- terraform/api/azure/variables.tf | 4 - terraform/api/do/main.tf | 11 +- terraform/api/do/variables.tf | 4 - terraform/api/gcp/main.tf | 11 +- terraform/api/gcp/variables.tf | 4 - .../{atom/k8s/main.tf => api/k8s/atom.tf} | 34 ---- terraform/api/k8s/crd.yml | 37 ---- terraform/api/k8s/main.tf | 26 --- terraform/api/k8s/variables.tf | 4 - terraform/atom/k8s/variables.tf | 11 -- terraform/cluster/aws/main.tf | 8 - terraform/cluster/aws/outputs.tf | 17 +- terraform/cluster/azure/kubeconfig.tpl | 12 +- terraform/cluster/azure/outputs.tf | 24 ++- terraform/cluster/do/outputs.tf | 24 ++- terraform/cluster/gcp/main.tf | 3 + terraform/cluster/gcp/network.tf | 3 + terraform/cluster/gcp/outputs.tf | 44 ++++- terraform/fluentd/gcp/main.tf | 18 +- terraform/rack/aws/main.tf | 24 +-- terraform/rack/aws/variables.tf | 4 - terraform/rack/azure/main.tf | 10 +- terraform/rack/azure/variables.tf | 8 - terraform/rack/do/main.tf | 10 +- terraform/rack/do/variables.tf | 2 +- terraform/rack/gcp/main.tf | 14 +- terraform/rack/gcp/variables.tf | 8 +- terraform/rack/k8s/main.tf | 12 -- terraform/rack/k8s/variables.tf | 4 - terraform/router/gcp/redis.tf | 2 + terraform/router/gcp/variables.tf | 4 + terraform/system/aws/main.tf | 11 +- .../system/azure/identity/deployment.yml | 172 ------------------ terraform/system/azure/identity/main.tf | 27 --- terraform/system/azure/identity/outputs.tf | 3 - terraform/system/azure/identity/variables.tf | 3 - terraform/system/azure/main.tf | 19 +- terraform/system/do/main.tf | 10 +- terraform/system/gcp/main.tf | 9 +- 53 files changed, 220 insertions(+), 845 deletions(-) rename terraform/atom/k8s/crd.yml => provider/k8s/template/system/atom.yml.tmpl (100%) rename provider/k8s/template/system/{custom.yml.tmpl => crd.yml.tmpl} (100%) delete mode 100644 provider/k8s/template/system/rack.yml.tmpl delete mode 100644 provider/k8s/template/system/router.yml.tmpl delete mode 100644 terraform/api/azure/identity.tf delete mode 100644 terraform/api/azure/identity.yml.tpl rename terraform/{atom/k8s/main.tf => api/k8s/atom.tf} (74%) delete mode 100644 terraform/api/k8s/crd.yml delete mode 100644 terraform/atom/k8s/variables.tf create mode 100644 terraform/cluster/gcp/network.tf delete mode 100644 terraform/system/azure/identity/deployment.yml delete mode 100644 terraform/system/azure/identity/main.tf delete mode 100644 terraform/system/azure/identity/outputs.tf delete mode 100644 terraform/system/azure/identity/variables.tf diff --git a/.dockerignore b/.dockerignore index 89eb2d7..85ef960 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ +install terraform terraform.tfvars diff --git a/Makefile b/Makefile index e0891aa..d16099d 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ release: git push test: - env PROVIDER=test go test -covermode atomic -coverprofile coverage.txt -mod=vendor ./... + env TEST=true go test -covermode atomic -coverprofile coverage.txt -mod=vendor ./... $(binaries): $(GOPATH)/bin/%: $(sources) go install -mod=vendor --ldflags="-s -w" ./cmd/$* diff --git a/provider/azure/azure.go b/provider/azure/azure.go index 2c3620e..36984ff 100644 --- a/provider/azure/azure.go +++ b/provider/azure/azure.go @@ -52,8 +52,6 @@ func FromEnv() (*Provider, error) { Workspace: os.Getenv("WORKSPACE"), } - fmt.Printf("p: %+v\n", p) - k.Engine = p return p, nil diff --git a/provider/gcp/log.go b/provider/gcp/log.go index fae1347..e06ca34 100644 --- a/provider/gcp/log.go +++ b/provider/gcp/log.go @@ -20,10 +20,6 @@ var sequenceTokens sync.Map func (p *Provider) Log(app, stream string, ts time.Time, message string) error { logger := p.Logging.Logger("system") - fmt.Printf("app: %+v\n", app) - fmt.Printf("stream: %+v\n", stream) - fmt.Printf("message: %+v\n", message) - logger.Log(logging.Entry{ Labels: map[string]string{ "container.googleapis.com/namespace_name": p.AppNamespace(app), diff --git a/provider/k8s/k8s.go b/provider/k8s/k8s.go index b674d4b..f32c5c8 100644 --- a/provider/k8s/k8s.go +++ b/provider/k8s/k8s.go @@ -3,6 +3,7 @@ package k8s import ( "context" "flag" + "fmt" "os" "os/exec" "time" @@ -107,6 +108,10 @@ func (p *Provider) Initialize(opts structs.ProviderOptions) error { runtime.ErrorHandlers = []func(error){} + if err := p.initializeTemplates(); err != nil { + return err + } + return nil } @@ -149,6 +154,19 @@ func (p *Provider) WithContext(ctx context.Context) structs.Provider { return &pp } +func (p *Provider) applySystemTemplate(name string, params map[string]interface{}) error { + data, err := p.RenderTemplate(fmt.Sprintf("system/%s", name), nil) + if err != nil { + return err + } + + if err := Apply(data); err != nil { + return err + } + + return nil +} + func (p *Provider) heartbeat() error { as, err := p.AppList() if err != nil { @@ -165,9 +183,6 @@ func (p *Provider) heartbeat() error { return err } - // "instance_type": "", - // "region": "" - ms := map[string]interface{}{ "id": ks.UID, "app_count": len(as), @@ -193,6 +208,22 @@ func (p *Provider) heartbeat() error { return nil } +func (p *Provider) initializeTemplates() error { + if os.Getenv("TEST") == "true" { + return nil + } + + if err := p.applySystemTemplate("atom", nil); err != nil { + return err + } + + if err := p.applySystemTemplate("crd", nil); err != nil { + return err + } + + return nil +} + func restConfig() (*rest.Config, error) { if c, err := rest.InClusterConfig(); err == nil { return c, nil diff --git a/terraform/atom/k8s/crd.yml b/provider/k8s/template/system/atom.yml.tmpl similarity index 100% rename from terraform/atom/k8s/crd.yml rename to provider/k8s/template/system/atom.yml.tmpl diff --git a/provider/k8s/template/system/custom.yml.tmpl b/provider/k8s/template/system/crd.yml.tmpl similarity index 100% rename from provider/k8s/template/system/custom.yml.tmpl rename to provider/k8s/template/system/crd.yml.tmpl diff --git a/provider/k8s/template/system/rack.yml.tmpl b/provider/k8s/template/system/rack.yml.tmpl deleted file mode 100644 index f433fee..0000000 --- a/provider/k8s/template/system/rack.yml.tmpl +++ /dev/null @@ -1,169 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ==RACK== - labels: - type: rack ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - namespace: ==RACK== - name: rack -rules: - - apiGroups: [ "*" ] - resources: [ "*" ] - verbs: [ "*" ] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - namespace: ==RACK== - name: rack -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rack -subjects: - - kind: ServiceAccount - name: rack - namespace: ==RACK== ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: ==RACK== - name: rack ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: ==RACK== - name: api - annotations: - atom.conditions: Available=True,Progressing=True/NewReplicaSetAvailable - labels: - app: rack - service: api -spec: - revisionHistoryLimit: 0 - selector: - matchLabels: - system: convox - rack: ==RACK== - app: rack - service: api - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 3 - template: - metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - system: convox - rack: ==RACK== - app: rack - service: api - type: service - name: api - spec: - shareProcessNamespace: true - containers: - - name: main - args: - - rack - env: - - name: DATA - value: /data - - name: DEVELOPMENT - value: "false" - - name: IMAGE - value: convox/rack:{{.Version}} - - name: RACK - value: ==RACK== - - name: VERSION - value: "{{.Version}}" - envFrom: - - configMapRef: - name: env-api - image: convox/rack:{{.Version}} - livenessProbe: - httpGet: - path: "/check" - port: 5443 - scheme: "HTTPS" - failureThreshold: 3 - initialDelaySeconds: 15 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 3 - readinessProbe: - httpGet: - path: "/check" - port: 5443 - scheme: "HTTPS" - periodSeconds: 5 - timeoutSeconds: 3 - ports: - - containerPort: 5443 - volumeMounts: - - name: data - mountPath: /data - - name: docker - mountPath: /var/run/docker.sock - serviceAccountName: rack - volumes: - - name: data - hostPath: - path: /var/rack/==RACK== - type: DirectoryOrCreate - - name: docker - hostPath: - path: ==SOCKET== ---- -apiVersion: v1 -kind: Service -metadata: - namespace: ==RACK== - name: api - annotations: - convox.service.ports.5443.protocol: https - labels: - app: rack - service: api -spec: - type: NodePort - ports: - - name: https - port: 5443 - targetPort: 5443 - protocol: TCP - selector: - system: convox - rack: ==RACK== - app: rack - service: api ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - namespace: ==RACK== - name: rack - annotations: - convox.idles: "true" - convox.ingress.service.api.5443.protocol: https -spec: - tls: - - hosts: - - ==HOST== - rules: - - host: ==HOST== - http: - paths: - - backend: - serviceName: api - servicePort: 5443 diff --git a/provider/k8s/template/system/router.yml.tmpl b/provider/k8s/template/system/router.yml.tmpl deleted file mode 100644 index c431275..0000000 --- a/provider/k8s/template/system/router.yml.tmpl +++ /dev/null @@ -1,109 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - namespace: convox-system - name: router -rules: - - apiGroups: - - "" - - extensions - resources: - - ingresses - - services - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - namespace: convox-system - name: router -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rack -subjects: - - kind: ServiceAccount - name: router - namespace: convox-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: convox-system - name: router ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: convox-system - name: router - annotations: - atom.conditions: Available=True,Progressing=True/NewReplicaSetAvailable - labels: - service: router -spec: - selector: - matchLabels: - system: convox - service: router - #replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: "200%" - maxUnavailable: "0%" - minReadySeconds: 1 - revisionHistoryLimit: 1 - template: - metadata: - labels: - system: convox - service: router - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchLabels: - system: convox - service: router - topologyKey: kubernetes.io/hostname - dnsConfig: - options: - - name: ndots - value: "1" - containers: - - name: main - args: - - router - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: SERVICE_HOST - value: "router.convox-system.svc.cluster.local" - - name: VERSION - value: "{{.Version}}" - envFrom: - - configMapRef: - name: env-router - image: convox/rack:{{.Version}} - imagePullPolicy: IfNotPresent - ports: - - containerPort: 80 - protocol: TCP - - containerPort: 443 - protocol: TCP - - containerPort: 5453 - protocol: UDP - resources: - requests: - cpu: "256m" - memory: "64Mi" - serviceAccountName: router diff --git a/terraform/api/aws/main.tf b/terraform/api/aws/main.tf index 97c8d2d..6aeaea4 100644 --- a/terraform/api/aws/main.tf +++ b/terraform/api/aws/main.tf @@ -27,11 +27,10 @@ module "k8s" { kubernetes = kubernetes } - domain = var.domain - kubeconfig = var.kubeconfig - name = var.name - namespace = var.namespace - release = var.release + domain = var.domain + name = var.name + namespace = var.namespace + release = var.release annotations = { "eks.amazonaws.com/role-arn" : aws_iam_role.api.arn, diff --git a/terraform/api/aws/variables.tf b/terraform/api/aws/variables.tf index 0893656..bf137fb 100644 --- a/terraform/api/aws/variables.tf +++ b/terraform/api/aws/variables.tf @@ -2,10 +2,6 @@ variable "domain" { type = "string" } -variable "kubeconfig" { - type = "string" -} - variable "name" { type = "string" } diff --git a/terraform/api/azure/identity.tf b/terraform/api/azure/identity.tf deleted file mode 100644 index 35cf504..0000000 --- a/terraform/api/azure/identity.tf +++ /dev/null @@ -1,44 +0,0 @@ -# resource "azurerm_user_assigned_identity" "api" { -# resource_group_name = data.azurerm_resource_group.rack.name -# location = data.azurerm_resource_group.rack.location - -# name = "api" -# } - -# resource "azurerm_role_assignment" "identity-api-contributor" { -# scope = data.azurerm_resource_group.rack.id -# role_definition_name = "Contributor" -# principal_id = azurerm_user_assigned_identity.api.principal_id -# } - -# data "template_file" "identity" { -# template = file("${path.module}/identity.yml.tpl") - -# vars = { -# namespace = var.namespace -# resource = azurerm_user_assigned_identity.api.id -# client = azurerm_user_assigned_identity.api.client_id -# } -# } - -# resource "null_resource" "deployment" { -# provisioner "local-exec" { -# when = "create" -# command = "echo '${data.template_file.identity.rendered}' | kubectl apply -f -" -# environment = { -# "KUBECONFIG" : var.kubeconfig, -# } -# } - -# provisioner "local-exec" { -# when = "destroy" -# command = "echo '${data.template_file.identity.rendered}' | kubectl delete -f -" -# environment = { -# "KUBECONFIG" : var.kubeconfig, -# } -# } - -# triggers = { -# template = sha256(data.template_file.identity.rendered) -# } -# } diff --git a/terraform/api/azure/identity.yml.tpl b/terraform/api/azure/identity.yml.tpl deleted file mode 100644 index af3b108..0000000 --- a/terraform/api/azure/identity.yml.tpl +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: "aadpodidentity.k8s.io/v1" -kind: AzureIdentity -metadata: - namespace: ${namespace} - name: api -spec: - type: 0 - ResourceID: ${resource} - ClientID: ${client} ---- -apiVersion: "aadpodidentity.k8s.io/v1" -kind: AzureIdentityBinding -metadata: - namespace: ${namespace} - name: api -spec: - AzureIdentity: api - Selector: api \ No newline at end of file diff --git a/terraform/api/azure/main.tf b/terraform/api/azure/main.tf index 9a3cc60..3ab27fa 100644 --- a/terraform/api/azure/main.tf +++ b/terraform/api/azure/main.tf @@ -12,8 +12,6 @@ provider "azurerm" { provider "kubernetes" { version = "~> 1.8" - - config_path = var.kubeconfig } provider "template" { @@ -48,11 +46,10 @@ module "k8s" { kubernetes = kubernetes } - domain = var.domain - kubeconfig = var.kubeconfig - name = var.name - namespace = var.namespace - release = var.release + domain = var.domain + name = var.name + namespace = var.namespace + release = var.release annotations = {} diff --git a/terraform/api/azure/variables.tf b/terraform/api/azure/variables.tf index 881f76b..bb35d77 100644 --- a/terraform/api/azure/variables.tf +++ b/terraform/api/azure/variables.tf @@ -2,10 +2,6 @@ variable "domain" { type = "string" } -variable "kubeconfig" { - type = "string" -} - variable "name" { type = "string" } diff --git a/terraform/api/do/main.tf b/terraform/api/do/main.tf index bafaf74..81e65a0 100644 --- a/terraform/api/do/main.tf +++ b/terraform/api/do/main.tf @@ -8,8 +8,6 @@ provider "digitalocean" { provider "kubernetes" { version = "~> 1.8" - - config_path = var.kubeconfig } locals { @@ -26,11 +24,10 @@ module "k8s" { kubernetes = kubernetes } - domain = var.domain - kubeconfig = var.kubeconfig - name = var.name - namespace = var.namespace - release = var.release + domain = var.domain + name = var.name + namespace = var.namespace + release = var.release annotations = {} diff --git a/terraform/api/do/variables.tf b/terraform/api/do/variables.tf index d0ea066..8f1a7b5 100644 --- a/terraform/api/do/variables.tf +++ b/terraform/api/do/variables.tf @@ -10,10 +10,6 @@ variable "elasticsearch" { type = "string" } -variable "kubeconfig" { - type = "string" -} - variable "name" { type = "string" } diff --git a/terraform/api/gcp/main.tf b/terraform/api/gcp/main.tf index e1342ee..e911150 100644 --- a/terraform/api/gcp/main.tf +++ b/terraform/api/gcp/main.tf @@ -8,8 +8,6 @@ provider "google" { provider "kubernetes" { version = "~> 1.8" - - config_path = var.kubeconfig } data "google_client_config" "current" {} @@ -28,11 +26,10 @@ module "k8s" { kubernetes = kubernetes } - domain = var.domain - kubeconfig = var.kubeconfig - name = var.name - namespace = var.namespace - release = var.release + domain = var.domain + name = var.name + namespace = var.namespace + release = var.release annotations = { "cloud.google.com/service-account" : google_service_account.api.email, diff --git a/terraform/api/gcp/variables.tf b/terraform/api/gcp/variables.tf index ebc863e..c75100d 100644 --- a/terraform/api/gcp/variables.tf +++ b/terraform/api/gcp/variables.tf @@ -2,10 +2,6 @@ variable "domain" { type = "string" } -variable "kubeconfig" { - type = "string" -} - variable "name" { type = "string" } diff --git a/terraform/atom/k8s/main.tf b/terraform/api/k8s/atom.tf similarity index 74% rename from terraform/atom/k8s/main.tf rename to terraform/api/k8s/atom.tf index c24bfff..5f72948 100644 --- a/terraform/atom/k8s/main.tf +++ b/terraform/api/k8s/atom.tf @@ -1,37 +1,3 @@ -terraform { - required_version = ">= 0.12.0" -} - -provider "kubernetes" { - version = "~> 1.8" -} - -provider "null" { - version = "~> 2.1" -} - -resource "null_resource" "crd" { - provisioner "local-exec" { - when = "create" - command = "kubectl apply -f ${path.module}/crd.yml" - environment = { - "KUBECONFIG" : var.kubeconfig, - } - } - - provisioner "local-exec" { - when = "destroy" - command = "kubectl delete -f ${path.module}/crd.yml" - environment = { - "KUBECONFIG" : var.kubeconfig, - } - } - - triggers = { - template = filesha256("${path.module}/crd.yml") - } -} - resource "kubernetes_cluster_role" "atom" { metadata { name = "atom" diff --git a/terraform/api/k8s/crd.yml b/terraform/api/k8s/crd.yml deleted file mode 100644 index 125a1a2..0000000 --- a/terraform/api/k8s/crd.yml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: builds.convox.com -spec: - group: convox.com - versions: - - name: v1 - served: true - storage: true - version: v1 - scope: Namespaced - names: - plural: builds - singular: build - kind: Build - categories: - - convox ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: releases.convox.com -spec: - group: convox.com - versions: - - name: v1 - served: true - storage: true - version: v1 - scope: Namespaced - names: - plural: releases - singular: release - kind: Release - categories: - - convox \ No newline at end of file diff --git a/terraform/api/k8s/main.tf b/terraform/api/k8s/main.tf index a726098..8d6a7fd 100644 --- a/terraform/api/k8s/main.tf +++ b/terraform/api/k8s/main.tf @@ -1,7 +1,3 @@ -terraform { - required_version = ">= 0.12.0" -} - provider "kubernetes" { version = "~> 1.9" } @@ -15,28 +11,6 @@ resource "random_string" "password" { special = false } -resource "null_resource" "crd" { - provisioner "local-exec" { - when = "create" - command = "kubectl apply -f ${path.module}/crd.yml" - environment = { - "KUBECONFIG" : var.kubeconfig, - } - } - - provisioner "local-exec" { - when = "destroy" - command = "kubectl delete -f ${path.module}/crd.yml" - environment = { - "KUBECONFIG" : var.kubeconfig, - } - } - - triggers = { - template = filesha256("${path.module}/crd.yml") - } -} - resource "kubernetes_cluster_role" "api" { metadata { name = "${var.name}-api" diff --git a/terraform/api/k8s/variables.tf b/terraform/api/k8s/variables.tf index 701d0f9..6a93ff2 100644 --- a/terraform/api/k8s/variables.tf +++ b/terraform/api/k8s/variables.tf @@ -10,10 +10,6 @@ variable "env" { default = {} } -variable "kubeconfig" { - type = "string" -} - variable "labels" { default = {} } diff --git a/terraform/atom/k8s/variables.tf b/terraform/atom/k8s/variables.tf deleted file mode 100644 index 1a1c989..0000000 --- a/terraform/atom/k8s/variables.tf +++ /dev/null @@ -1,11 +0,0 @@ -variable "kubeconfig" { - type = "string" -} - -variable "namespace" { - type = "string" -} - -variable "release" { - type = "string" -} diff --git a/terraform/cluster/aws/main.tf b/terraform/cluster/aws/main.tf index d7b515c..ab1d8b9 100644 --- a/terraform/cluster/aws/main.tf +++ b/terraform/cluster/aws/main.tf @@ -66,12 +66,6 @@ resource "aws_eks_cluster" "cluster" { ] } -resource "null_resource" "after_cluster" { - provisioner "local-exec" { - command = "sleep 30" - } -} - resource "local_file" "kubeconfig" { depends_on = [ aws_cloudformation_stack.nodes, @@ -94,7 +88,6 @@ resource "local_file" "kubeconfig" { aws_security_group_rule.nodes_ingress_internal, aws_security_group_rule.nodes_ingress_mtu, aws_security_group_rule.nodes_ingress_traffic, - null_resource.after_cluster, ] filename = pathexpand("~/.kube/config.aws.${var.name}") @@ -142,7 +135,6 @@ resource "kubernetes_config_map" "auth" { aws_security_group_rule.nodes_ingress_internal, aws_security_group_rule.nodes_ingress_mtu, aws_security_group_rule.nodes_ingress_traffic, - null_resource.after_cluster, ] provider = kubernetes.direct diff --git a/terraform/cluster/aws/outputs.tf b/terraform/cluster/aws/outputs.tf index 3137d90..4340200 100644 --- a/terraform/cluster/aws/outputs.tf +++ b/terraform/cluster/aws/outputs.tf @@ -1,11 +1,16 @@ -output "id" { - depends_on = [local_file.kubeconfig, kubernetes_config_map.auth] - value = aws_eks_cluster.cluster.id +output "ca" { + depends_on = [kubernetes_config_map.auth] + value = base64decode(aws_eks_cluster.cluster.certificate_authority.0.data) } -output "kubeconfig" { - depends_on = [local_file.kubeconfig, kubernetes_config_map.auth] - value = local_file.kubeconfig.filename +output "endpoint" { + depends_on = [kubernetes_config_map.auth] + value = aws_eks_cluster.cluster.endpoint +} + +output "id" { + depends_on = [kubernetes_config_map.auth] + value = aws_eks_cluster.cluster.id } output "nodes_security" { diff --git a/terraform/cluster/azure/kubeconfig.tpl b/terraform/cluster/azure/kubeconfig.tpl index 271142b..4b671e1 100644 --- a/terraform/cluster/azure/kubeconfig.tpl +++ b/terraform/cluster/azure/kubeconfig.tpl @@ -3,17 +3,17 @@ clusters: - cluster: certificate-authority-data: ${ca} server: ${endpoint} - name: gcloud + name: azure contexts: - context: - cluster: gcloud - user: gcloud - name: gcloud -current-context: gcloud + cluster: azure + user: azure + name: azure +current-context: azure kind: Config preferences: {} users: -- name: gcloud +- name: azure user: client-certificate-data: ${client_certificate} client-key-data: ${client_key} diff --git a/terraform/cluster/azure/outputs.tf b/terraform/cluster/azure/outputs.tf index 3410b91..0fe528e 100644 --- a/terraform/cluster/azure/outputs.tf +++ b/terraform/cluster/azure/outputs.tf @@ -1,9 +1,21 @@ -output "kubeconfig" { - depends_on = [ - local_file.kubeconfig, - azurerm_kubernetes_cluster.rack, - ] - value = local_file.kubeconfig.filename +output "ca" { + depends_on = [azurerm_kubernetes_cluster.rack] + value = base64decode(azurerm_kubernetes_cluster.rack.kube_config.0.cluster_ca_certificate) +} + +output "client_certificate" { + depends_on = [azurerm_kubernetes_cluster.rack] + value = base64decode(azurerm_kubernetes_cluster.rack.kube_config.0.client_certificate) +} + +output "client_key" { + depends_on = [azurerm_kubernetes_cluster.rack] + value = base64decode(azurerm_kubernetes_cluster.rack.kube_config.0.client_key) +} + +output "endpoint" { + depends_on = [azurerm_kubernetes_cluster.rack] + value = azurerm_kubernetes_cluster.rack.kube_config.0.host } output "workspace" { diff --git a/terraform/cluster/do/outputs.tf b/terraform/cluster/do/outputs.tf index c7bd9da..d266302 100644 --- a/terraform/cluster/do/outputs.tf +++ b/terraform/cluster/do/outputs.tf @@ -1,7 +1,19 @@ -output "kubeconfig" { - depends_on = [ - local_file.kubeconfig, - digitalocean_kubernetes_cluster.rack, - ] - value = local_file.kubeconfig.filename +output "ca" { + depends_on = [digitalocean_kubernetes_cluster.rack] + value = base64decode(digitalocean_kubernetes_cluster.rack.kube_config[0].cluster_ca_certificate) +} + +output "endpoint" { + depends_on = [digitalocean_kubernetes_cluster.rack] + value = digitalocean_kubernetes_cluster.rack.endpoint +} + +output "name" { + depends_on = [digitalocean_kubernetes_cluster.rack] + value = digitalocean_kubernetes_cluster.rack.name +} + +output "token" { + depends_on = [digitalocean_kubernetes_cluster.rack] + value = digitalocean_kubernetes_cluster.rack.kube_config[0].token } diff --git a/terraform/cluster/gcp/main.tf b/terraform/cluster/gcp/main.tf index 824a050..d0748b6 100644 --- a/terraform/cluster/gcp/main.tf +++ b/terraform/cluster/gcp/main.tf @@ -37,6 +37,7 @@ resource "google_container_cluster" "rack" { name = var.name location = data.google_client_config.current.region + network = google_compute_network.rack.name remove_default_node_pool = true initial_node_count = 1 @@ -47,6 +48,8 @@ resource "google_container_cluster" "rack" { identity_namespace = "${data.google_project.current.project_id}.svc.id.goog" } + ip_allocation_policy {} + master_auth { username = "gcloud" password = random_string.password.result diff --git a/terraform/cluster/gcp/network.tf b/terraform/cluster/gcp/network.tf new file mode 100644 index 0000000..97cf32c --- /dev/null +++ b/terraform/cluster/gcp/network.tf @@ -0,0 +1,3 @@ +resource "google_compute_network" "rack" { + name = var.name +} diff --git a/terraform/cluster/gcp/outputs.tf b/terraform/cluster/gcp/outputs.tf index a3335f8..568627c 100644 --- a/terraform/cluster/gcp/outputs.tf +++ b/terraform/cluster/gcp/outputs.tf @@ -1,10 +1,46 @@ -output "kubeconfig" { +output "ca" { depends_on = [ - local_file.kubeconfig, - kubernetes_cluster_role_binding.client, + google_container_cluster.rack, google_container_node_pool.rack, + kubernetes_cluster_role_binding.client, ] - value = local_file.kubeconfig.filename + value = base64decode(google_container_cluster.rack.master_auth.0.cluster_ca_certificate) +} + +output "client_certificate" { + depends_on = [ + google_container_cluster.rack, + google_container_node_pool.rack, + kubernetes_cluster_role_binding.client, + ] + value = base64decode(google_container_cluster.rack.master_auth.0.client_certificate) +} + +output "client_key" { + depends_on = [ + google_container_cluster.rack, + google_container_node_pool.rack, + kubernetes_cluster_role_binding.client, + ] + value = base64decode(google_container_cluster.rack.master_auth.0.client_key) +} + +output "endpoint" { + depends_on = [ + google_container_cluster.rack, + google_container_node_pool.rack, + kubernetes_cluster_role_binding.client, + ] + value = "https://${google_container_cluster.rack.endpoint}" +} + +output "network" { + depends_on = [ + google_container_cluster.rack, + google_container_node_pool.rack, + kubernetes_cluster_role_binding.client, + ] + value = google_compute_network.rack.name } output "nodes_account" { diff --git a/terraform/fluentd/gcp/main.tf b/terraform/fluentd/gcp/main.tf index 0529dcc..1e2c5ec 100644 --- a/terraform/fluentd/gcp/main.tf +++ b/terraform/fluentd/gcp/main.tf @@ -6,8 +6,9 @@ provider "google" { version = "~> 2.12" } -# data "aws_caller_identity" "current" {} -# data "aws_region" "current" {} +provider "kubernetes" { + version = "~> 1.9" +} locals { tags = { @@ -19,11 +20,14 @@ locals { module "k8s" { source = "../k8s" - cluster = var.cluster - image = "fluent/fluentd-kubernetes-daemonset:v1.3.1-debian-stackdriver-1.3" - kubeconfig = var.kubeconfig - namespace = var.namespace - target = file("${path.module}/target.conf") + providers = { + kubernetes = kubernetes + } + + cluster = var.cluster + image = "fluent/fluentd-kubernetes-daemonset:v1.3.1-debian-stackdriver-1.3" + namespace = var.namespace + target = file("${path.module}/target.conf") annotations = { "cloud.google.com/service-account" : google_service_account.fluentd.email, diff --git a/terraform/rack/aws/main.tf b/terraform/rack/aws/main.tf index 09579c7..3611977 100644 --- a/terraform/rack/aws/main.tf +++ b/terraform/rack/aws/main.tf @@ -12,8 +12,6 @@ provider "external" { provider "kubernetes" { version = "~> 1.9" - - config_path = var.kubeconfig } module "k8s" { @@ -23,10 +21,9 @@ module "k8s" { kubernetes = kubernetes } - domain = module.router.endpoint - kubeconfig = var.kubeconfig - name = var.name - release = var.release + domain = module.router.endpoint + name = var.name + release = var.release } module "api" { @@ -37,14 +34,13 @@ module "api" { kubernetes = kubernetes } - domain = module.router.endpoint - kubeconfig = var.kubeconfig - name = var.name - namespace = module.k8s.namespace - oidc_arn = var.oidc_arn - oidc_sub = var.oidc_sub - release = var.release - router = module.router.endpoint + domain = module.router.endpoint + name = var.name + namespace = module.k8s.namespace + oidc_arn = var.oidc_arn + oidc_sub = var.oidc_sub + release = var.release + router = module.router.endpoint } module "router" { diff --git a/terraform/rack/aws/variables.tf b/terraform/rack/aws/variables.tf index c35be5e..94861a7 100644 --- a/terraform/rack/aws/variables.tf +++ b/terraform/rack/aws/variables.tf @@ -2,10 +2,6 @@ variable "cluster" { type = "string" } -variable "kubeconfig" { - type = "string" -} - variable "name" { type = "string" } diff --git a/terraform/rack/azure/main.tf b/terraform/rack/azure/main.tf index 7f48da0..382244d 100644 --- a/terraform/rack/azure/main.tf +++ b/terraform/rack/azure/main.tf @@ -8,8 +8,6 @@ provider "azurerm" { provider "kubernetes" { version = "~> 1.9" - - config_path = var.kubeconfig } module "k8s" { @@ -19,10 +17,9 @@ module "k8s" { kubernetes = kubernetes } - domain = module.router.endpoint - kubeconfig = var.kubeconfig - name = var.name - release = var.release + domain = module.router.endpoint + name = var.name + release = var.release } module "api" { @@ -34,7 +31,6 @@ module "api" { } domain = module.router.endpoint - kubeconfig = var.kubeconfig name = var.name namespace = module.k8s.namespace region = var.region diff --git a/terraform/rack/azure/variables.tf b/terraform/rack/azure/variables.tf index e50d334..e8b95f9 100644 --- a/terraform/rack/azure/variables.tf +++ b/terraform/rack/azure/variables.tf @@ -1,11 +1,3 @@ -# variable "identity" { -# type = "string" -# } - -variable "kubeconfig" { - type = "string" -} - variable "name" { type = "string" } diff --git a/terraform/rack/do/main.tf b/terraform/rack/do/main.tf index dc78523..ab2eb78 100644 --- a/terraform/rack/do/main.tf +++ b/terraform/rack/do/main.tf @@ -8,8 +8,6 @@ provider "digitalocean" { provider "kubernetes" { version = "~> 1.9" - - config_path = var.kubeconfig } module "k8s" { @@ -19,10 +17,9 @@ module "k8s" { kubernetes = kubernetes } - domain = module.router.endpoint - kubeconfig = var.kubeconfig - name = var.name - release = var.release + domain = module.router.endpoint + name = var.name + release = var.release } module "api" { @@ -36,7 +33,6 @@ module "api" { access_id = var.access_id elasticsearch = module.elasticsearch.url domain = module.router.endpoint - kubeconfig = var.kubeconfig name = var.name namespace = module.k8s.namespace region = var.region diff --git a/terraform/rack/do/variables.tf b/terraform/rack/do/variables.tf index 9a7d851..365d06b 100644 --- a/terraform/rack/do/variables.tf +++ b/terraform/rack/do/variables.tf @@ -2,7 +2,7 @@ variable "access_id" { type = "string" } -variable "kubeconfig" { +variable "cluster" { type = "string" } diff --git a/terraform/rack/gcp/main.tf b/terraform/rack/gcp/main.tf index 573ebaa..3ecda80 100644 --- a/terraform/rack/gcp/main.tf +++ b/terraform/rack/gcp/main.tf @@ -5,15 +5,10 @@ terraform { provider "google" { version = "~> 2.12" - - credentials = pathexpand(var.credentials) - project = var.project } provider "kubernetes" { version = "~> 1.9" - - config_path = var.kubeconfig } module "k8s" { @@ -23,10 +18,9 @@ module "k8s" { kubernetes = kubernetes } - domain = module.router.endpoint - kubeconfig = var.kubeconfig - name = var.name - release = var.release + domain = module.router.endpoint + name = var.name + release = var.release } module "api" { @@ -38,7 +32,6 @@ module "api" { } domain = module.router.endpoint - kubeconfig = var.kubeconfig name = var.name namespace = module.k8s.namespace nodes_account = var.nodes_account @@ -56,5 +49,6 @@ module "router" { name = var.name namespace = module.k8s.namespace + network = var.network release = var.release } diff --git a/terraform/rack/gcp/variables.tf b/terraform/rack/gcp/variables.tf index f61cfdd..151a900 100644 --- a/terraform/rack/gcp/variables.tf +++ b/terraform/rack/gcp/variables.tf @@ -1,12 +1,8 @@ -variable "credentials" { - default = "~/.config/gcloud/terraform.json" -} - -variable "kubeconfig" { +variable "name" { type = "string" } -variable "name" { +variable "network" { type = "string" } diff --git a/terraform/rack/k8s/main.tf b/terraform/rack/k8s/main.tf index 642736f..d848690 100644 --- a/terraform/rack/k8s/main.tf +++ b/terraform/rack/k8s/main.tf @@ -28,15 +28,3 @@ resource "kubernetes_config_map" "rack" { DOMAIN = var.domain } } - -module "atom" { - source = "../../atom/k8s" - - providers = { - kubernetes = kubernetes - } - - kubeconfig = var.kubeconfig - namespace = kubernetes_namespace.system.metadata.0.name - release = var.release -} diff --git a/terraform/rack/k8s/variables.tf b/terraform/rack/k8s/variables.tf index 764557e..a4dea90 100644 --- a/terraform/rack/k8s/variables.tf +++ b/terraform/rack/k8s/variables.tf @@ -2,10 +2,6 @@ variable "domain" { type = "string" } -variable "kubeconfig" { - type = "string" -} - variable "name" { type = "string" } diff --git a/terraform/router/gcp/redis.tf b/terraform/router/gcp/redis.tf index 941b381..35d809d 100644 --- a/terraform/router/gcp/redis.tf +++ b/terraform/router/gcp/redis.tf @@ -1,4 +1,6 @@ resource "google_redis_instance" "cache" { name = "${var.name}-router" memory_size_gb = 1 + + authorized_network = var.network } diff --git a/terraform/router/gcp/variables.tf b/terraform/router/gcp/variables.tf index 4627c3b..f60a664 100644 --- a/terraform/router/gcp/variables.tf +++ b/terraform/router/gcp/variables.tf @@ -6,6 +6,10 @@ variable "namespace" { type = "string" } +variable "network" { + type = "string" +} + variable "release" { type = "string" } diff --git a/terraform/system/aws/main.tf b/terraform/system/aws/main.tf index c7ae767..5430aa5 100644 --- a/terraform/system/aws/main.tf +++ b/terraform/system/aws/main.tf @@ -13,7 +13,15 @@ provider "http" { provider "kubernetes" { version = "~> 1.9" - config_path = module.cluster.kubeconfig + cluster_ca_certificate = module.cluster.ca + host = module.cluster.endpoint + token = data.aws_eks_cluster_auth.cluster.token + + load_config_file = false +} + +data "aws_eks_cluster_auth" "cluster" { + name = module.cluster.id } data "http" "releases" { @@ -62,7 +70,6 @@ module "rack" { } cluster = module.cluster.id - kubeconfig = module.cluster.kubeconfig name = var.name nodes_security = module.cluster.nodes_security oidc_arn = module.cluster.oidc_arn diff --git a/terraform/system/azure/identity/deployment.yml b/terraform/system/azure/identity/deployment.yml deleted file mode 100644 index 2cdcb5c..0000000 --- a/terraform/system/azure/identity/deployment.yml +++ /dev/null @@ -1,172 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: azureassignedidentities.aadpodidentity.k8s.io -spec: - group: aadpodidentity.k8s.io - version: v1 - names: - kind: AzureAssignedIdentity - plural: azureassignedidentities - scope: Namespaced ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: azureidentitybindings.aadpodidentity.k8s.io -spec: - group: aadpodidentity.k8s.io - version: v1 - names: - kind: AzureIdentityBinding - plural: azureidentitybindings - scope: Namespaced ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: azureidentities.aadpodidentity.k8s.io -spec: - group: aadpodidentity.k8s.io - version: v1 - names: - kind: AzureIdentity - singular: azureidentity - plural: azureidentities - scope: Namespaced ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: azurepodidentityexceptions.aadpodidentity.k8s.io -spec: - group: aadpodidentity.k8s.io - version: v1 - names: - kind: AzurePodIdentityException - singular: azurepodidentityexception - plural: azurepodidentityexceptions - scope: Namespaced ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nmi - namespace: kube-system -spec: - updateStrategy: - type: RollingUpdate - selector: - matchLabels: - component: nmi - tier: node - template: - metadata: - labels: - component: nmi - tier: node - spec: - hostNetwork: true - volumes: - - hostPath: - path: /run/xtables.lock - type: FileOrCreate - name: iptableslock - containers: - - name: nmi - image: "mcr.microsoft.com/k8s/aad-pod-identity/nmi:1.5.3" - imagePullPolicy: Always - args: - - "--host-ip=$(HOST_IP)" - - "--node=$(NODE_NAME)" - env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: true - capabilities: - add: - - NET_ADMIN - resources: - limits: - cpu: 200m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - volumeMounts: - - mountPath: /run/xtables.lock - name: iptableslock - livenessProbe: - httpGet: - path: /healthz - port: 8080 - initialDelaySeconds: 10 - periodSeconds: 5 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - component: mic - name: mic - namespace: kube-system -spec: - replicas: 2 - selector: - matchLabels: - component: mic - template: - metadata: - labels: - component: mic - spec: - containers: - - name: mic - image: "mcr.microsoft.com/k8s/aad-pod-identity/mic:1.5.3" - imagePullPolicy: Always - args: - - "--kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig" - - "--cloudconfig=/etc/kubernetes/azure.json" - - "--logtostderr" - resources: - limits: - cpu: 200m - memory: 1024Mi - requests: - cpu: 100m - memory: 256Mi - volumeMounts: - - name: kubeconfig - mountPath: /etc/kubernetes/kubeconfig - readOnly: true - - name: certificates - mountPath: /etc/kubernetes/certs - readOnly: true - - name: k8s-azure-file - mountPath: /etc/kubernetes/azure.json - readOnly: true - livenessProbe: - httpGet: - path: /healthz - port: 8080 - initialDelaySeconds: 10 - periodSeconds: 5 - volumes: - - name: kubeconfig - hostPath: - path: /var/lib/kubelet - - name: certificates - hostPath: - path: /etc/kubernetes/certs - - name: k8s-azure-file - hostPath: - path: /etc/kubernetes/azure.json - nodeSelector: - beta.kubernetes.io/os: linux diff --git a/terraform/system/azure/identity/main.tf b/terraform/system/azure/identity/main.tf deleted file mode 100644 index 2fc4ea4..0000000 --- a/terraform/system/azure/identity/main.tf +++ /dev/null @@ -1,27 +0,0 @@ -provider "kubernetes" { - version = "~> 1.8" - - config_path = var.kubeconfig -} - -resource "null_resource" "deployment" { - provisioner "local-exec" { - when = "create" - command = "kubectl apply -f ${path.module}/deployment.yml" - environment = { - "KUBECONFIG" : var.kubeconfig, - } - } - - provisioner "local-exec" { - when = "destroy" - command = "kubectl delete -f ${path.module}/deployment.yml" - environment = { - "KUBECONFIG" : var.kubeconfig, - } - } - - triggers = { - template = filesha256("${path.module}/deployment.yml") - } -} diff --git a/terraform/system/azure/identity/outputs.tf b/terraform/system/azure/identity/outputs.tf deleted file mode 100644 index 2647af4..0000000 --- a/terraform/system/azure/identity/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "id" { - value = null_resource.deployment.id -} diff --git a/terraform/system/azure/identity/variables.tf b/terraform/system/azure/identity/variables.tf deleted file mode 100644 index cbd0f35..0000000 --- a/terraform/system/azure/identity/variables.tf +++ /dev/null @@ -1,3 +0,0 @@ -variable "kubeconfig" { - type = string -} diff --git a/terraform/system/azure/main.tf b/terraform/system/azure/main.tf index 3f68756..db390ee 100644 --- a/terraform/system/azure/main.tf +++ b/terraform/system/azure/main.tf @@ -9,7 +9,12 @@ provider "http" { provider "kubernetes" { version = "~> 1.9" - config_path = module.cluster.kubeconfig + client_certificate = module.cluster.client_certificate + client_key = module.cluster.client_key + cluster_ca_certificate = module.cluster.ca + host = module.cluster.endpoint + + load_config_file = false } data "http" "releases" { @@ -41,16 +46,6 @@ module "cluster" { resource_group = azurerm_resource_group.rack.name } -# module "identity" { -# source = "./identity" - -# providers = { -# kubernetes = kubernetes -# } - -# kubeconfig = module.cluster.kubeconfig -# } - module "rack" { source = "../../rack/azure" @@ -59,8 +54,6 @@ module "rack" { kubernetes = kubernetes } - # identity = module.identity.id - kubeconfig = module.cluster.kubeconfig name = var.name region = var.region release = local.release diff --git a/terraform/system/do/main.tf b/terraform/system/do/main.tf index 5dc23e8..488c5ad 100644 --- a/terraform/system/do/main.tf +++ b/terraform/system/do/main.tf @@ -13,7 +13,11 @@ provider "http" { provider "kubernetes" { version = "~> 1.9" - config_path = module.cluster.kubeconfig + cluster_ca_certificate = module.cluster.ca + host = module.cluster.endpoint + token = module.cluster.token + + load_config_file = false } data "http" "releases" { @@ -45,7 +49,7 @@ module "fluentd" { kubernetes = kubernetes } - cluster = var.name + cluster = module.cluster.name elasticsearch = module.rack.elasticsearch namespace = "kube-system" name = var.name @@ -60,7 +64,7 @@ module "rack" { } access_id = var.access_id - kubeconfig = module.cluster.kubeconfig + cluster = module.cluster.name name = var.name region = var.region registry_disk = var.registry_disk diff --git a/terraform/system/gcp/main.tf b/terraform/system/gcp/main.tf index 6ff132b..e5f9de2 100644 --- a/terraform/system/gcp/main.tf +++ b/terraform/system/gcp/main.tf @@ -17,7 +17,12 @@ provider "http" { provider "kubernetes" { version = "~> 1.9" - config_path = module.cluster.kubeconfig + client_certificate = module.cluster.client_certificate + client_key = module.cluster.client_key + cluster_ca_certificate = module.cluster.ca + host = module.cluster.endpoint + + load_config_file = false } module "project" { @@ -58,8 +63,8 @@ module "rack" { google = google } - kubeconfig = module.cluster.kubeconfig name = var.name + network = module.cluster.network nodes_account = module.cluster.nodes_account release = local.release }