remove dependency on local kubeconfig (#18)

* remove dependency on local kubeconfig

* use k8s-specific network for gcp

* remove unused delay

* fix tests

.dockerignore

@@ -1,2 +1,3 @@
+install
 terraform
 terraform.tfvars

Makefile

@@ -51,7 +51,7 @@ release:
 	git push
 
 test:
-	env PROVIDER=test go test -covermode atomic -coverprofile coverage.txt -mod=vendor ./...
+	env TEST=true go test -covermode atomic -coverprofile coverage.txt -mod=vendor ./...
 
 $(binaries): $(GOPATH)/bin/%: $(sources)
 	go install -mod=vendor --ldflags="-s -w" ./cmd/$*

provider/azure/azure.go

@@ -52,8 +52,6 @@ func FromEnv() (*Provider, error) {
 		Workspace:      os.Getenv("WORKSPACE"),
 	}
 
-	fmt.Printf("p: %+v\n", p)
-
 	k.Engine = p
 
 	return p, nil

provider/gcp/log.go

@@ -20,10 +20,6 @@ var sequenceTokens sync.Map
 func (p *Provider) Log(app, stream string, ts time.Time, message string) error {
 	logger := p.Logging.Logger("system")
 
-	fmt.Printf("app: %+v\n", app)
-	fmt.Printf("stream: %+v\n", stream)
-	fmt.Printf("message: %+v\n", message)
-
 	logger.Log(logging.Entry{
 		Labels: map[string]string{
 			"container.googleapis.com/namespace_name": p.AppNamespace(app),

provider/k8s/k8s.go

@@ -3,6 +3,7 @@ package k8s
 import (
 	"context"
 	"flag"
+	"fmt"
 	"os"
 	"os/exec"
 	"time"
@@ -107,6 +108,10 @@ func (p *Provider) Initialize(opts structs.ProviderOptions) error {
 
 	runtime.ErrorHandlers = []func(error){}
 
+	if err := p.initializeTemplates(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -149,6 +154,19 @@ func (p *Provider) WithContext(ctx context.Context) structs.Provider {
 	return &pp
 }
 
+func (p *Provider) applySystemTemplate(name string, params map[string]interface{}) error {
+	data, err := p.RenderTemplate(fmt.Sprintf("system/%s", name), nil)
+	if err != nil {
+		return err
+	}
+
+	if err := Apply(data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (p *Provider) heartbeat() error {
 	as, err := p.AppList()
 	if err != nil {
@@ -165,9 +183,6 @@ func (p *Provider) heartbeat() error {
 		return err
 	}
 
-	// "instance_type":  "",
-	// "region": ""
-
 	ms := map[string]interface{}{
 		"id":             ks.UID,
 		"app_count":      len(as),
@@ -193,6 +208,22 @@ func (p *Provider) heartbeat() error {
 	return nil
 }
 
+func (p *Provider) initializeTemplates() error {
+	if os.Getenv("TEST") == "true" {
+		return nil
+	}
+
+	if err := p.applySystemTemplate("atom", nil); err != nil {
+		return err
+	}
+
+	if err := p.applySystemTemplate("crd", nil); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func restConfig() (*rest.Config, error) {
 	if c, err := rest.InClusterConfig(); err == nil {
 		return c, nil

provider/k8s/template/system/rack.yml.tmpl

@@ -1,169 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: ==RACK==
-  labels:
-    type: rack
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  namespace: ==RACK==
-  name: rack
-rules:
-  - apiGroups: [ "*" ]
-    resources: [ "*" ]
-    verbs: [ "*" ]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  namespace: ==RACK==
-  name: rack
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: rack
-subjects:
-  - kind: ServiceAccount
-    name: rack
-    namespace: ==RACK==
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: ==RACK==
-  name: rack
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  namespace: ==RACK==
-  name: api
-  annotations:
-    atom.conditions: Available=True,Progressing=True/NewReplicaSetAvailable
-  labels:
-    app: rack
-    service: api
-spec:
-  revisionHistoryLimit: 0
-  selector:
-    matchLabels:
-      system: convox
-      rack: ==RACK==
-      app: rack
-      service: api
-  strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxSurge: 1
-      maxUnavailable: 0
-  minReadySeconds: 3
-  template:
-    metadata:
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ""
-      labels:
-        system: convox
-        rack: ==RACK==
-        app: rack
-        service: api
-        type: service
-        name: api
-    spec:
-      shareProcessNamespace: true
-      containers:
-      - name: main
-        args:
-        - rack
-        env:
-        - name: DATA
-          value: /data
-        - name: DEVELOPMENT
-          value: "false"
-        - name: IMAGE
-          value: convox/rack:{{.Version}}
-        - name: RACK
-          value: ==RACK==
-        - name: VERSION
-          value: "{{.Version}}"
-        envFrom:
-        - configMapRef:
-            name: env-api
-        image: convox/rack:{{.Version}}
-        livenessProbe:
-          httpGet:
-            path: "/check"
-            port: 5443
-            scheme: "HTTPS"
-          failureThreshold: 3
-          initialDelaySeconds: 15
-          periodSeconds: 5
-          successThreshold: 1
-          timeoutSeconds: 3
-        readinessProbe:
-          httpGet:
-            path: "/check"
-            port: 5443
-            scheme: "HTTPS"
-          periodSeconds: 5
-          timeoutSeconds: 3
-        ports:
-        - containerPort: 5443
-        volumeMounts:
-        - name: data
-          mountPath: /data
-        - name: docker
-          mountPath: /var/run/docker.sock
-      serviceAccountName: rack
-      volumes:
-      - name: data
-        hostPath:
-          path: /var/rack/==RACK==
-          type: DirectoryOrCreate
-      - name: docker
-        hostPath:
-          path: ==SOCKET==
----
-apiVersion: v1
-kind: Service
-metadata:
-  namespace: ==RACK==
-  name: api
-  annotations:
-    convox.service.ports.5443.protocol: https
-  labels:
-    app: rack
-    service: api
-spec:
-  type: NodePort
-  ports:
-  - name: https
-    port: 5443
-    targetPort: 5443
-    protocol: TCP
-  selector:
-    system: convox
-    rack: ==RACK==
-    app: rack
-    service: api
----
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  namespace: ==RACK==
-  name: rack
-  annotations:
-    convox.idles: "true"
-    convox.ingress.service.api.5443.protocol: https
-spec:
-  tls:
-  - hosts:
-    - ==HOST==
-  rules:
-    - host: ==HOST==
-      http:
-        paths:
-        - backend:
-            serviceName: api
-            servicePort: 5443

provider/k8s/template/system/router.yml.tmpl

@@ -1,109 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  namespace: convox-system
-  name: router
-rules:
-  - apiGroups:
-      - ""
-      - extensions
-    resources:
-      - ingresses
-      - services
-    verbs:
-      - get
-      - list
-      - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  namespace: convox-system
-  name: router
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: rack
-subjects:
-  - kind: ServiceAccount
-    name: router
-    namespace: convox-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: convox-system
-  name: router
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  namespace: convox-system
-  name: router
-  annotations:
-    atom.conditions: Available=True,Progressing=True/NewReplicaSetAvailable
-  labels:
-    service: router
-spec:
-  selector:
-    matchLabels:
-      system: convox
-      service: router
-  #replicas: 1
-  strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxSurge: "200%"
-      maxUnavailable: "0%"
-  minReadySeconds: 1
-  revisionHistoryLimit: 1
-  template:
-    metadata:
-      labels:
-        system: convox
-        service: router
-    spec:
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - weight: 100
-            podAffinityTerm:
-              labelSelector:
-                matchLabels:
-                  system: convox
-                  service: router
-              topologyKey: kubernetes.io/hostname
-      dnsConfig:
-        options:
-        - name: ndots
-          value: "1"
-      containers:
-      - name: main
-        args:
-        - router
-        env:
-        - name: POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: SERVICE_HOST
-          value: "router.convox-system.svc.cluster.local"
-        - name: VERSION
-          value: "{{.Version}}"
-        envFrom:
-        - configMapRef:
-            name: env-router
-        image: convox/rack:{{.Version}}
-        imagePullPolicy: IfNotPresent
-        ports:
-        - containerPort: 80
-          protocol: TCP
-        - containerPort: 443
-          protocol: TCP
-        - containerPort: 5453
-          protocol: UDP
-        resources:
-          requests:
-            cpu: "256m"
-            memory: "64Mi"
-      serviceAccountName: router

terraform/api/aws/main.tf

@@ -27,11 +27,10 @@ module "k8s" {
     kubernetes = kubernetes
   }
 
-  domain     = var.domain
-  kubeconfig = var.kubeconfig
-  name       = var.name
-  namespace  = var.namespace
-  release    = var.release
+  domain    = var.domain
+  name      = var.name
+  namespace = var.namespace
+  release   = var.release
 
   annotations = {
     "eks.amazonaws.com/role-arn" : aws_iam_role.api.arn,

terraform/api/aws/variables.tf

@@ -2,10 +2,6 @@ variable "domain" {
   type = "string"
 }
 
-variable "kubeconfig" {
-  type = "string"
-}
-
 variable "name" {
   type = "string"
 }

terraform/api/azure/identity.tf

@@ -1,44 +0,0 @@
-# resource "azurerm_user_assigned_identity" "api" {
-#   resource_group_name = data.azurerm_resource_group.rack.name
-#   location            = data.azurerm_resource_group.rack.location
-
-#   name = "api"
-# }
-
-# resource "azurerm_role_assignment" "identity-api-contributor" {
-#   scope                = data.azurerm_resource_group.rack.id
-#   role_definition_name = "Contributor"
-#   principal_id         = azurerm_user_assigned_identity.api.principal_id
-# }
-
-# data "template_file" "identity" {
-#   template = file("${path.module}/identity.yml.tpl")
-
-#   vars = {
-#     namespace = var.namespace
-#     resource  = azurerm_user_assigned_identity.api.id
-#     client    = azurerm_user_assigned_identity.api.client_id
-#   }
-# }
-
-# resource "null_resource" "deployment" {
-#   provisioner "local-exec" {
-#     when    = "create"
-#     command = "echo '${data.template_file.identity.rendered}' | kubectl apply -f -"
-#     environment = {
-#       "KUBECONFIG" : var.kubeconfig,
-#     }
-#   }
-
-#   provisioner "local-exec" {
-#     when    = "destroy"
-#     command = "echo '${data.template_file.identity.rendered}' | kubectl delete -f -"
-#     environment = {
-#       "KUBECONFIG" : var.kubeconfig,
-#     }
-#   }
-
-#   triggers = {
-#     template = sha256(data.template_file.identity.rendered)
-#   }
-# }

terraform/api/azure/identity.yml.tpl

@@ -1,18 +0,0 @@
-apiVersion: "aadpodidentity.k8s.io/v1"
-kind: AzureIdentity
-metadata:
-  namespace: ${namespace}
-  name: api
-spec:
-  type: 0
-  ResourceID: ${resource}
-  ClientID: ${client}
----
-apiVersion: "aadpodidentity.k8s.io/v1"
-kind: AzureIdentityBinding
-metadata:
-  namespace: ${namespace}
-  name: api
-spec:
-  AzureIdentity: api
-  Selector: api

terraform/api/azure/main.tf

@@ -12,8 +12,6 @@ provider "azurerm" {
 
 provider "kubernetes" {
   version = "~> 1.8"
-
-  config_path = var.kubeconfig
 }
 
 provider "template" {
@@ -48,11 +46,10 @@ module "k8s" {
     kubernetes = kubernetes
   }
 
-  domain     = var.domain
-  kubeconfig = var.kubeconfig
-  name       = var.name
-  namespace  = var.namespace
-  release    = var.release
+  domain    = var.domain
+  name      = var.name
+  namespace = var.namespace
+  release   = var.release
 
   annotations = {}
 

terraform/api/azure/variables.tf

@@ -2,10 +2,6 @@ variable "domain" {
   type = "string"
 }
 
-variable "kubeconfig" {
-  type = "string"
-}
-
 variable "name" {
   type = "string"
 }

terraform/api/do/main.tf

@@ -8,8 +8,6 @@ provider "digitalocean" {
 
 provider "kubernetes" {
   version = "~> 1.8"
-
-  config_path = var.kubeconfig
 }
 
 locals {
@@ -26,11 +24,10 @@ module "k8s" {
     kubernetes = kubernetes
   }
 
-  domain     = var.domain
-  kubeconfig = var.kubeconfig
-  name       = var.name
-  namespace  = var.namespace
-  release    = var.release
+  domain    = var.domain
+  name      = var.name
+  namespace = var.namespace
+  release   = var.release
 
   annotations = {}
 

terraform/api/do/variables.tf

@@ -10,10 +10,6 @@ variable "elasticsearch" {
   type = "string"
 }
 
-variable "kubeconfig" {
-  type = "string"
-}
-
 variable "name" {
   type = "string"
 }

terraform/api/gcp/main.tf

@@ -8,8 +8,6 @@ provider "google" {
 
 provider "kubernetes" {
   version = "~> 1.8"
-
-  config_path = var.kubeconfig
 }
 
 data "google_client_config" "current" {}
@@ -28,11 +26,10 @@ module "k8s" {
     kubernetes = kubernetes
   }
 
-  domain     = var.domain
-  kubeconfig = var.kubeconfig
-  name       = var.name
-  namespace  = var.namespace
-  release    = var.release
+  domain    = var.domain
+  name      = var.name
+  namespace = var.namespace
+  release   = var.release
 
   annotations = {
     "cloud.google.com/service-account" : google_service_account.api.email,

terraform/api/gcp/variables.tf

@@ -2,10 +2,6 @@ variable "domain" {
   type = "string"
 }
 
-variable "kubeconfig" {
-  type = "string"
-}
-
 variable "name" {
   type = "string"
 }

terraform/api/k8s/atom.tf

@@ -1,37 +1,3 @@
-terraform {
-  required_version = ">= 0.12.0"
-}
-
-provider "kubernetes" {
-  version = "~> 1.8"
-}
-
-provider "null" {
-  version = "~> 2.1"
-}
-
-resource "null_resource" "crd" {
-  provisioner "local-exec" {
-    when    = "create"
-    command = "kubectl apply -f ${path.module}/crd.yml"
-    environment = {
-      "KUBECONFIG" : var.kubeconfig,
-    }
-  }
-
-  provisioner "local-exec" {
-    when    = "destroy"
-    command = "kubectl delete -f ${path.module}/crd.yml"
-    environment = {
-      "KUBECONFIG" : var.kubeconfig,
-    }
-  }
-
-  triggers = {
-    template = filesha256("${path.module}/crd.yml")
-  }
-}
-
 resource "kubernetes_cluster_role" "atom" {
   metadata {
     name = "atom"

terraform/api/k8s/crd.yml

@@ -1,37 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: builds.convox.com
-spec:
-  group: convox.com
-  versions:
-    - name: v1
-      served: true
-      storage: true
-  version: v1
-  scope: Namespaced
-  names:
-    plural: builds
-    singular: build
-    kind: Build
-    categories:
-    - convox
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: releases.convox.com
-spec:
-  group: convox.com
-  versions:
-    - name: v1
-      served: true
-      storage: true
-  version: v1
-  scope: Namespaced
-  names:
-    plural: releases
-    singular: release
-    kind: Release
-    categories:
-    - convox

terraform/api/k8s/main.tf

@@ -1,7 +1,3 @@
-terraform {
-  required_version = ">= 0.12.0"
-}
-
 provider "kubernetes" {
   version = "~> 1.9"
 }
@@ -15,28 +11,6 @@ resource "random_string" "password" {
   special = false
 }
 
-resource "null_resource" "crd" {
-  provisioner "local-exec" {
-    when    = "create"
-    command = "kubectl apply -f ${path.module}/crd.yml"
-    environment = {
-      "KUBECONFIG" : var.kubeconfig,
-    }
-  }
-
-  provisioner "local-exec" {
-    when    = "destroy"
-    command = "kubectl delete -f ${path.module}/crd.yml"
-    environment = {
-      "KUBECONFIG" : var.kubeconfig,
-    }
-  }
-
-  triggers = {
-    template = filesha256("${path.module}/crd.yml")
-  }
-}
-
 resource "kubernetes_cluster_role" "api" {
   metadata {
     name = "${var.name}-api"

terraform/api/k8s/variables.tf

@@ -10,10 +10,6 @@ variable "env" {
   default = {}
 }
 
-variable "kubeconfig" {
-  type = "string"
-}
-
 variable "labels" {
   default = {}
 }

terraform/atom/k8s/variables.tf

@@ -1,11 +0,0 @@
-variable "kubeconfig" {
-  type = "string"
-}
-
-variable "namespace" {
-  type = "string"
-}
-
-variable "release" {
-  type = "string"
-}

terraform/cluster/aws/main.tf

@@ -66,12 +66,6 @@ resource "aws_eks_cluster" "cluster" {
   ]
 }
 
-resource "null_resource" "after_cluster" {
-  provisioner "local-exec" {
-    command = "sleep 30"
-  }
-}
-
 resource "local_file" "kubeconfig" {
   depends_on = [
     aws_cloudformation_stack.nodes,
@@ -94,7 +88,6 @@ resource "local_file" "kubeconfig" {
     aws_security_group_rule.nodes_ingress_internal,
     aws_security_group_rule.nodes_ingress_mtu,
     aws_security_group_rule.nodes_ingress_traffic,
-    null_resource.after_cluster,
   ]
 
   filename = pathexpand("~/.kube/config.aws.${var.name}")
@@ -142,7 +135,6 @@ resource "kubernetes_config_map" "auth" {
     aws_security_group_rule.nodes_ingress_internal,
     aws_security_group_rule.nodes_ingress_mtu,
     aws_security_group_rule.nodes_ingress_traffic,
-    null_resource.after_cluster,
   ]
 
   provider = kubernetes.direct

terraform/cluster/aws/outputs.tf

@@ -1,11 +1,16 @@
-output "id" {
-  depends_on = [local_file.kubeconfig, kubernetes_config_map.auth]
-  value      = aws_eks_cluster.cluster.id
+output "ca" {
+  depends_on = [kubernetes_config_map.auth]
+  value      = base64decode(aws_eks_cluster.cluster.certificate_authority.0.data)
 }
 
-output "kubeconfig" {
-  depends_on = [local_file.kubeconfig, kubernetes_config_map.auth]
-  value      = local_file.kubeconfig.filename
+output "endpoint" {
+  depends_on = [kubernetes_config_map.auth]
+  value      = aws_eks_cluster.cluster.endpoint
+}
+
+output "id" {
+  depends_on = [kubernetes_config_map.auth]
+  value      = aws_eks_cluster.cluster.id
 }
 
 output "nodes_security" {

terraform/cluster/azure/kubeconfig.tpl

@@ -3,17 +3,17 @@ clusters:
 - cluster:
     certificate-authority-data: ${ca}
     server: ${endpoint}
-  name: gcloud
+  name: azure
 contexts:
 - context:
-    cluster: gcloud
-    user: gcloud
-  name: gcloud
-current-context: gcloud
+    cluster: azure
+    user: azure
+  name: azure
+current-context: azure
 kind: Config
 preferences: {}
 users:
-- name: gcloud
+- name: azure
   user:
     client-certificate-data: ${client_certificate}
     client-key-data: ${client_key}

terraform/cluster/azure/outputs.tf

@@ -1,9 +1,21 @@
-output "kubeconfig" {
-  depends_on = [
-    local_file.kubeconfig,
-    azurerm_kubernetes_cluster.rack,
-  ]
-  value = local_file.kubeconfig.filename
+output "ca" {
+  depends_on = [azurerm_kubernetes_cluster.rack]
+  value      = base64decode(azurerm_kubernetes_cluster.rack.kube_config.0.cluster_ca_certificate)
+}
+
+output "client_certificate" {
+  depends_on = [azurerm_kubernetes_cluster.rack]
+  value      = base64decode(azurerm_kubernetes_cluster.rack.kube_config.0.client_certificate)
+}
+
+output "client_key" {
+  depends_on = [azurerm_kubernetes_cluster.rack]
+  value      = base64decode(azurerm_kubernetes_cluster.rack.kube_config.0.client_key)
+}
+
+output "endpoint" {
+  depends_on = [azurerm_kubernetes_cluster.rack]
+  value      = azurerm_kubernetes_cluster.rack.kube_config.0.host
 }
 
 output "workspace" {

terraform/cluster/do/outputs.tf

@@ -1,7 +1,19 @@
-output "kubeconfig" {
-  depends_on = [
-    local_file.kubeconfig,
-    digitalocean_kubernetes_cluster.rack,
-  ]
-  value = local_file.kubeconfig.filename
+output "ca" {
+  depends_on = [digitalocean_kubernetes_cluster.rack]
+  value      = base64decode(digitalocean_kubernetes_cluster.rack.kube_config[0].cluster_ca_certificate)
+}
+
+output "endpoint" {
+  depends_on = [digitalocean_kubernetes_cluster.rack]
+  value      = digitalocean_kubernetes_cluster.rack.endpoint
+}
+
+output "name" {
+  depends_on = [digitalocean_kubernetes_cluster.rack]
+  value      = digitalocean_kubernetes_cluster.rack.name
+}
+
+output "token" {
+  depends_on = [digitalocean_kubernetes_cluster.rack]
+  value      = digitalocean_kubernetes_cluster.rack.kube_config[0].token
 }

terraform/cluster/gcp/main.tf

@@ -37,6 +37,7 @@ resource "google_container_cluster" "rack" {
 
   name     = var.name
   location = data.google_client_config.current.region
+  network  = google_compute_network.rack.name
 
   remove_default_node_pool = true
   initial_node_count       = 1
@@ -47,6 +48,8 @@ resource "google_container_cluster" "rack" {
     identity_namespace = "${data.google_project.current.project_id}.svc.id.goog"
   }
 
+  ip_allocation_policy {}
+
   master_auth {
     username = "gcloud"
     password = random_string.password.result

terraform/cluster/gcp/network.tf

@@ -0,0 +1,3 @@
+resource "google_compute_network" "rack" {
+  name = var.name
+}

terraform/cluster/gcp/outputs.tf

@@ -1,10 +1,46 @@
-output "kubeconfig" {
+output "ca" {
   depends_on = [
-    local_file.kubeconfig,
-    kubernetes_cluster_role_binding.client,
+    google_container_cluster.rack,
     google_container_node_pool.rack,
+    kubernetes_cluster_role_binding.client,
   ]
-  value = local_file.kubeconfig.filename
+  value = base64decode(google_container_cluster.rack.master_auth.0.cluster_ca_certificate)
+}
+
+output "client_certificate" {
+  depends_on = [
+    google_container_cluster.rack,
+    google_container_node_pool.rack,
+    kubernetes_cluster_role_binding.client,
+  ]
+  value = base64decode(google_container_cluster.rack.master_auth.0.client_certificate)
+}
+
+output "client_key" {
+  depends_on = [
+    google_container_cluster.rack,
+    google_container_node_pool.rack,
+    kubernetes_cluster_role_binding.client,
+  ]
+  value = base64decode(google_container_cluster.rack.master_auth.0.client_key)
+}
+
+output "endpoint" {
+  depends_on = [
+    google_container_cluster.rack,
+    google_container_node_pool.rack,
+    kubernetes_cluster_role_binding.client,
+  ]
+  value = "https://${google_container_cluster.rack.endpoint}"
+}
+
+output "network" {
+  depends_on = [
+    google_container_cluster.rack,
+    google_container_node_pool.rack,
+    kubernetes_cluster_role_binding.client,
+  ]
+  value = google_compute_network.rack.name
 }
 
 output "nodes_account" {

terraform/fluentd/gcp/main.tf

@@ -6,8 +6,9 @@ provider "google" {
   version = "~> 2.12"
 }
 
-# data "aws_caller_identity" "current" {}
-# data "aws_region" "current" {}
+provider "kubernetes" {
+  version = "~> 1.9"
+}
 
 locals {
   tags = {
@@ -19,11 +20,14 @@ locals {
 module "k8s" {
   source = "../k8s"
 
-  cluster    = var.cluster
-  image      = "fluent/fluentd-kubernetes-daemonset:v1.3.1-debian-stackdriver-1.3"
-  kubeconfig = var.kubeconfig
-  namespace  = var.namespace
-  target     = file("${path.module}/target.conf")
+  providers = {
+    kubernetes = kubernetes
+  }
+
+  cluster   = var.cluster
+  image     = "fluent/fluentd-kubernetes-daemonset:v1.3.1-debian-stackdriver-1.3"
+  namespace = var.namespace
+  target    = file("${path.module}/target.conf")
 
   annotations = {
     "cloud.google.com/service-account" : google_service_account.fluentd.email,

terraform/rack/aws/main.tf

@@ -12,8 +12,6 @@ provider "external" {
 
 provider "kubernetes" {
   version = "~> 1.9"
-
-  config_path = var.kubeconfig
 }
 
 module "k8s" {
@@ -23,10 +21,9 @@ module "k8s" {
     kubernetes = kubernetes
   }
 
-  domain     = module.router.endpoint
-  kubeconfig = var.kubeconfig
-  name       = var.name
-  release    = var.release
+  domain  = module.router.endpoint
+  name    = var.name
+  release = var.release
 }
 
 module "api" {
@@ -37,14 +34,13 @@ module "api" {
     kubernetes = kubernetes
   }
 
-  domain     = module.router.endpoint
-  kubeconfig = var.kubeconfig
-  name       = var.name
-  namespace  = module.k8s.namespace
-  oidc_arn   = var.oidc_arn
-  oidc_sub   = var.oidc_sub
-  release    = var.release
-  router     = module.router.endpoint
+  domain    = module.router.endpoint
+  name      = var.name
+  namespace = module.k8s.namespace
+  oidc_arn  = var.oidc_arn
+  oidc_sub  = var.oidc_sub
+  release   = var.release
+  router    = module.router.endpoint
 }
 
 module "router" {

terraform/rack/aws/variables.tf

@@ -2,10 +2,6 @@ variable "cluster" {
   type = "string"
 }
 
-variable "kubeconfig" {
-  type = "string"
-}
-
 variable "name" {
   type = "string"
 }

terraform/rack/azure/main.tf

@@ -8,8 +8,6 @@ provider "azurerm" {
 
 provider "kubernetes" {
   version = "~> 1.9"
-
-  config_path = var.kubeconfig
 }
 
 module "k8s" {
@@ -19,10 +17,9 @@ module "k8s" {
     kubernetes = kubernetes
   }
 
-  domain     = module.router.endpoint
-  kubeconfig = var.kubeconfig
-  name       = var.name
-  release    = var.release
+  domain  = module.router.endpoint
+  name    = var.name
+  release = var.release
 }
 
 module "api" {
@@ -34,7 +31,6 @@ module "api" {
   }
 
   domain         = module.router.endpoint
-  kubeconfig     = var.kubeconfig
   name           = var.name
   namespace      = module.k8s.namespace
   region         = var.region

terraform/rack/azure/variables.tf

@@ -1,11 +1,3 @@
-# variable "identity" {
-#   type = "string"
-# }
-
-variable "kubeconfig" {
-  type = "string"
-}
-
 variable "name" {
   type = "string"
 }

terraform/rack/do/main.tf

@@ -8,8 +8,6 @@ provider "digitalocean" {
 
 provider "kubernetes" {
   version = "~> 1.9"
-
-  config_path = var.kubeconfig
 }
 
 module "k8s" {
@@ -19,10 +17,9 @@ module "k8s" {
     kubernetes = kubernetes
   }
 
-  domain     = module.router.endpoint
-  kubeconfig = var.kubeconfig
-  name       = var.name
-  release    = var.release
+  domain  = module.router.endpoint
+  name    = var.name
+  release = var.release
 }
 
 module "api" {
@@ -36,7 +33,6 @@ module "api" {
   access_id     = var.access_id
   elasticsearch = module.elasticsearch.url
   domain        = module.router.endpoint
-  kubeconfig    = var.kubeconfig
   name          = var.name
   namespace     = module.k8s.namespace
   region        = var.region

terraform/rack/do/variables.tf

@@ -2,7 +2,7 @@ variable "access_id" {
   type = "string"
 }
 
-variable "kubeconfig" {
+variable "cluster" {
   type = "string"
 }
 

terraform/rack/gcp/main.tf

@@ -5,15 +5,10 @@ terraform {
 
 provider "google" {
   version = "~> 2.12"
-
-  credentials = pathexpand(var.credentials)
-  project     = var.project
 }
 
 provider "kubernetes" {
   version = "~> 1.9"
-
-  config_path = var.kubeconfig
 }
 
 module "k8s" {
@@ -23,10 +18,9 @@ module "k8s" {
     kubernetes = kubernetes
   }
 
-  domain     = module.router.endpoint
-  kubeconfig = var.kubeconfig
-  name       = var.name
-  release    = var.release
+  domain  = module.router.endpoint
+  name    = var.name
+  release = var.release
 }
 
 module "api" {
@@ -38,7 +32,6 @@ module "api" {
   }
 
   domain        = module.router.endpoint
-  kubeconfig    = var.kubeconfig
   name          = var.name
   namespace     = module.k8s.namespace
   nodes_account = var.nodes_account
@@ -56,5 +49,6 @@ module "router" {
 
   name      = var.name
   namespace = module.k8s.namespace
+  network   = var.network
   release   = var.release
 }

terraform/rack/gcp/variables.tf

@@ -1,12 +1,8 @@
-variable "credentials" {
-  default = "~/.config/gcloud/terraform.json"
-}
-
-variable "kubeconfig" {
+variable "name" {
   type = "string"
 }
 
-variable "name" {
+variable "network" {
   type = "string"
 }
 

terraform/rack/k8s/main.tf

@@ -28,15 +28,3 @@ resource "kubernetes_config_map" "rack" {
     DOMAIN = var.domain
   }
 }
-
-module "atom" {
-  source = "../../atom/k8s"
-
-  providers = {
-    kubernetes = kubernetes
-  }
-
-  kubeconfig = var.kubeconfig
-  namespace  = kubernetes_namespace.system.metadata.0.name
-  release    = var.release
-}

terraform/rack/k8s/variables.tf

@@ -2,10 +2,6 @@ variable "domain" {
   type = "string"
 }
 
-variable "kubeconfig" {
-  type = "string"
-}
-
 variable "name" {
   type = "string"
 }

terraform/router/gcp/redis.tf

@@ -1,4 +1,6 @@
 resource "google_redis_instance" "cache" {
   name           = "${var.name}-router"
   memory_size_gb = 1
+
+  authorized_network = var.network
 }

terraform/router/gcp/variables.tf

@@ -6,6 +6,10 @@ variable "namespace" {
   type = "string"
 }
 
+variable "network" {
+  type = "string"
+}
+
 variable "release" {
   type = "string"
 }

terraform/system/aws/main.tf

@@ -13,7 +13,15 @@ provider "http" {
 provider "kubernetes" {
   version = "~> 1.9"
 
-  config_path = module.cluster.kubeconfig
+  cluster_ca_certificate = module.cluster.ca
+  host                   = module.cluster.endpoint
+  token                  = data.aws_eks_cluster_auth.cluster.token
+
+  load_config_file = false
+}
+
+data "aws_eks_cluster_auth" "cluster" {
+  name = module.cluster.id
 }
 
 data "http" "releases" {
@@ -62,7 +70,6 @@ module "rack" {
   }
 
   cluster            = module.cluster.id
-  kubeconfig         = module.cluster.kubeconfig
   name               = var.name
   nodes_security     = module.cluster.nodes_security
   oidc_arn           = module.cluster.oidc_arn

terraform/system/azure/identity/deployment.yml

@@ -1,172 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: azureassignedidentities.aadpodidentity.k8s.io
-spec:
-  group: aadpodidentity.k8s.io
-  version: v1
-  names:
-    kind: AzureAssignedIdentity
-    plural: azureassignedidentities
-  scope: Namespaced
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: azureidentitybindings.aadpodidentity.k8s.io
-spec:
-  group: aadpodidentity.k8s.io
-  version: v1
-  names:
-    kind: AzureIdentityBinding
-    plural: azureidentitybindings
-  scope: Namespaced
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: azureidentities.aadpodidentity.k8s.io
-spec:
-  group: aadpodidentity.k8s.io
-  version: v1
-  names:
-    kind: AzureIdentity
-    singular: azureidentity
-    plural: azureidentities
-  scope: Namespaced
----
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: azurepodidentityexceptions.aadpodidentity.k8s.io
-spec:
-  group: aadpodidentity.k8s.io
-  version: v1
-  names:
-    kind: AzurePodIdentityException
-    singular: azurepodidentityexception
-    plural: azurepodidentityexceptions
-  scope: Namespaced
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: nmi
-  namespace: kube-system
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  selector:
-    matchLabels:
-      component: nmi
-      tier: node
-  template:
-    metadata:
-      labels:
-        component: nmi
-        tier: node
-    spec:
-      hostNetwork: true
-      volumes:
-      - hostPath:
-          path: /run/xtables.lock
-          type: FileOrCreate
-        name: iptableslock
-      containers:
-      - name: nmi
-        image: "mcr.microsoft.com/k8s/aad-pod-identity/nmi:1.5.3"
-        imagePullPolicy: Always
-        args:
-          - "--host-ip=$(HOST_IP)"
-          - "--node=$(NODE_NAME)"
-        env:
-          - name: HOST_IP
-            valueFrom:
-              fieldRef:
-                fieldPath: status.podIP
-          - name: NODE_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: spec.nodeName
-        securityContext:
-          privileged: true
-          capabilities:
-            add:
-            - NET_ADMIN
-        resources:
-          limits:
-            cpu: 200m
-            memory: 512Mi
-          requests:
-            cpu: 100m
-            memory: 256Mi
-        volumeMounts:
-        - mountPath: /run/xtables.lock
-          name: iptableslock
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 8080
-          initialDelaySeconds: 10
-          periodSeconds: 5
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    component: mic
-  name: mic
-  namespace: kube-system
-spec:
-  replicas: 2
-  selector:
-    matchLabels:
-      component: mic
-  template:
-    metadata:
-      labels:
-        component: mic
-    spec:
-      containers:
-      - name: mic
-        image: "mcr.microsoft.com/k8s/aad-pod-identity/mic:1.5.3"
-        imagePullPolicy: Always
-        args:
-          - "--kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig"
-          - "--cloudconfig=/etc/kubernetes/azure.json"
-          - "--logtostderr"
-        resources:
-          limits:
-            cpu: 200m
-            memory: 1024Mi
-          requests:
-            cpu: 100m
-            memory: 256Mi
-        volumeMounts:
-          - name: kubeconfig
-            mountPath: /etc/kubernetes/kubeconfig
-            readOnly: true
-          - name: certificates
-            mountPath: /etc/kubernetes/certs
-            readOnly: true
-          - name: k8s-azure-file
-            mountPath: /etc/kubernetes/azure.json
-            readOnly: true
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 8080
-          initialDelaySeconds: 10
-          periodSeconds: 5
-      volumes:
-      - name: kubeconfig
-        hostPath:
-          path: /var/lib/kubelet
-      - name: certificates
-        hostPath:
-          path: /etc/kubernetes/certs
-      - name: k8s-azure-file
-        hostPath:
-          path: /etc/kubernetes/azure.json
-      nodeSelector:
-        beta.kubernetes.io/os: linux

terraform/system/azure/identity/main.tf

@@ -1,27 +0,0 @@
-provider "kubernetes" {
-  version = "~> 1.8"
-
-  config_path = var.kubeconfig
-}
-
-resource "null_resource" "deployment" {
-  provisioner "local-exec" {
-    when    = "create"
-    command = "kubectl apply -f ${path.module}/deployment.yml"
-    environment = {
-      "KUBECONFIG" : var.kubeconfig,
-    }
-  }
-
-  provisioner "local-exec" {
-    when    = "destroy"
-    command = "kubectl delete -f ${path.module}/deployment.yml"
-    environment = {
-      "KUBECONFIG" : var.kubeconfig,
-    }
-  }
-
-  triggers = {
-    template = filesha256("${path.module}/deployment.yml")
-  }
-}

terraform/system/azure/identity/outputs.tf

@@ -1,3 +0,0 @@
-output "id" {
-  value = null_resource.deployment.id
-}

terraform/system/azure/identity/variables.tf

@@ -1,3 +0,0 @@
-variable "kubeconfig" {
-  type = string
-}

terraform/system/azure/main.tf

@@ -9,7 +9,12 @@ provider "http" {
 provider "kubernetes" {
   version = "~> 1.9"
 
-  config_path = module.cluster.kubeconfig
+  client_certificate     = module.cluster.client_certificate
+  client_key             = module.cluster.client_key
+  cluster_ca_certificate = module.cluster.ca
+  host                   = module.cluster.endpoint
+
+  load_config_file = false
 }
 
 data "http" "releases" {
@@ -41,16 +46,6 @@ module "cluster" {
   resource_group = azurerm_resource_group.rack.name
 }
 
-# module "identity" {
-#   source = "./identity"
-
-#   providers = {
-#     kubernetes = kubernetes
-#   }
-
-#   kubeconfig = module.cluster.kubeconfig
-# }
-
 module "rack" {
   source = "../../rack/azure"
 
@@ -59,8 +54,6 @@ module "rack" {
     kubernetes = kubernetes
   }
 
-  # identity       = module.identity.id
-  kubeconfig     = module.cluster.kubeconfig
   name           = var.name
   region         = var.region
   release        = local.release

terraform/system/do/main.tf

@@ -13,7 +13,11 @@ provider "http" {
 provider "kubernetes" {
   version = "~> 1.9"
 
-  config_path = module.cluster.kubeconfig
+  cluster_ca_certificate = module.cluster.ca
+  host                   = module.cluster.endpoint
+  token                  = module.cluster.token
+
+  load_config_file = false
 }
 
 data "http" "releases" {
@@ -45,7 +49,7 @@ module "fluentd" {
     kubernetes = kubernetes
   }
 
-  cluster       = var.name
+  cluster       = module.cluster.name
   elasticsearch = module.rack.elasticsearch
   namespace     = "kube-system"
   name          = var.name
@@ -60,7 +64,7 @@ module "rack" {
   }
 
   access_id     = var.access_id
-  kubeconfig    = module.cluster.kubeconfig
+  cluster       = module.cluster.name
   name          = var.name
   region        = var.region
   registry_disk = var.registry_disk

terraform/system/gcp/main.tf

@@ -17,7 +17,12 @@ provider "http" {
 provider "kubernetes" {
   version = "~> 1.9"
 
-  config_path = module.cluster.kubeconfig
+  client_certificate     = module.cluster.client_certificate
+  client_key             = module.cluster.client_key
+  cluster_ca_certificate = module.cluster.ca
+  host                   = module.cluster.endpoint
+
+  load_config_file = false
 }
 
 module "project" {
@@ -58,8 +63,8 @@ module "rack" {
     google     = google
   }
 
-  kubeconfig    = module.cluster.kubeconfig
   name          = var.name
+  network       = module.cluster.network
   nodes_account = module.cluster.nodes_account
   release       = local.release
 }