From 964ade629ae38ca7cb403c36d5237f1cbcfe05b7 Mon Sep 17 00:00:00 2001
From: David Dollar <david@convox.com>
Date: Tue, 31 Dec 2019 09:36:50 -0500
Subject: [PATCH] autoscale aws node groups (#62)

---
 terraform/cluster/aws/autoscaler.tf | 298 ++++++++++++++++++++++++++++
 terraform/cluster/aws/main.tf       |   5 +
 terraform/cluster/aws/outputs.tf    |   2 +-
 3 files changed, 304 insertions(+), 1 deletion(-)
 create mode 100644 terraform/cluster/aws/autoscaler.tf

diff --git a/terraform/cluster/aws/autoscaler.tf b/terraform/cluster/aws/autoscaler.tf
new file mode 100644
index 0000000..544e01d
--- /dev/null
+++ b/terraform/cluster/aws/autoscaler.tf
@@ -0,0 +1,298 @@
+data "aws_iam_policy_document" "assume_autoscaler" {
+  statement {
+    actions = ["sts:AssumeRoleWithWebIdentity"]
+    effect  = "Allow"
+
+    condition {
+      test     = "StringEquals"
+      variable = local.oidc_sub
+      values   = ["system:serviceaccount:kube-system:cluster-autoscaler"]
+    }
+
+    principals {
+      identifiers = [aws_iam_openid_connect_provider.cluster.arn]
+      type        = "Federated"
+    }
+  }
+}
+
+resource "aws_iam_role" "autoscaler" {
+  name               = "${var.name}-autoscaler"
+  assume_role_policy = data.aws_iam_policy_document.assume_autoscaler.json
+  path               = "/convox/"
+  tags               = local.tags
+}
+
+data "aws_iam_policy_document" "autoscale" {
+  statement {
+    actions = [
+      "autoscaling:DescribeAutoScalingGroups",
+      "autoscaling:DescribeAutoScalingInstances",
+      "autoscaling:DescribeLaunchConfigurations",
+      "autoscaling:DescribeTags",
+      "autoscaling:SetDesiredCapacity",
+      "autoscaling:TerminateInstanceInAutoScalingGroup",
+      "ec2:DescribeLaunchTemplateVersions",
+    ]
+    effect    = "Allow"
+    resources = ["*"]
+  }
+}
+
+resource "aws_iam_role_policy" "autoscaler_autoscale" {
+  name   = "autoscale"
+  role   = aws_iam_role.autoscaler.name
+  policy = data.aws_iam_policy_document.autoscale.json
+}
+
+locals {
+  autoscaler_labels = {
+    "k8s-addon" : "cluster-autoscaler.addons.k8s.io",
+    "k8s-app" : "cluster-autoscaler",
+  }
+}
+
+resource "kubernetes_service_account" "autoscaler" {
+  metadata {
+    name      = "cluster-autoscaler"
+    namespace = "kube-system"
+    labels    = local.autoscaler_labels
+
+    annotations = {
+      "eks.amazonaws.com/role-arn" : aws_iam_role.autoscaler.arn,
+    }
+  }
+}
+
+resource "kubernetes_cluster_role" "autoscaler" {
+  metadata {
+    name   = "cluster-autoscaler"
+    labels = local.autoscaler_labels
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["events", "endpoints"]
+    verbs      = ["create", "patch"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["pods/eviction"]
+    verbs      = ["create"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["pods/status"]
+    verbs      = ["update"]
+  }
+
+  rule {
+    api_groups     = [""]
+    resources      = ["endpoints"]
+    resource_names = ["cluster-autoscaler"]
+    verbs          = ["get", "update"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["nodes"]
+    verbs      = ["watch", "list", "get", "update"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["pods", "services", "replicationcontrollers", "persistentvolumeclaims", "persistentvolumes"]
+    verbs      = ["watch", "list", "get"]
+  }
+
+  rule {
+    api_groups = ["extensions"]
+    resources  = ["replicasets", "daemonsets"]
+    verbs      = ["watch", "list", "get"]
+  }
+
+  rule {
+    api_groups = ["policy"]
+    resources  = ["poddisruptionbudgets"]
+    verbs      = ["watch", "list"]
+  }
+
+  rule {
+    api_groups = ["apps"]
+    resources  = ["statefulsets", "replicasets", "daemonsets"]
+    verbs      = ["watch", "list", "get"]
+  }
+
+  rule {
+    api_groups = ["storage.k8s.io"]
+    resources  = ["storageclasses", "csinodes"]
+    verbs      = ["watch", "list", "get"]
+  }
+
+  rule {
+    api_groups = ["batch", "extensions"]
+    resources  = ["jobs"]
+    verbs      = ["get", "list", "watch", "patch"]
+  }
+
+  rule {
+    api_groups = ["coordination.k8s.io"]
+    resources  = ["leases"]
+    verbs      = ["create"]
+  }
+
+  rule {
+    api_groups     = ["coordination.k8s.io"]
+    resources      = ["leases"]
+    resource_names = ["cluster-autoscaler"]
+    verbs          = ["get", "update"]
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "autoscaler" {
+  metadata {
+    name   = "cluster-autoscaler"
+    labels = local.autoscaler_labels
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = "cluster-autoscaler"
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = "cluster-autoscaler"
+    namespace = "kube-system"
+  }
+}
+
+resource "kubernetes_role" "autoscaler" {
+  metadata {
+    name      = "cluster-autoscaler"
+    namespace = "kube-system"
+    labels    = local.autoscaler_labels
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["configmaps"]
+    verbs      = ["create", "list", "watch"]
+  }
+
+  rule {
+    api_groups     = [""]
+    resources      = ["configmaps"]
+    resource_names = ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+    verbs          = ["delete", "get", "update", "watch"]
+  }
+}
+
+resource "kubernetes_role_binding" "autoscaler" {
+  metadata {
+    name      = "cluster-autoscaler"
+    namespace = "kube-system"
+    labels    = local.autoscaler_labels
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Role"
+    name      = "cluster-autoscaler"
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = "cluster-autoscaler"
+    namespace = "kube-system"
+  }
+}
+
+resource "kubernetes_deployment" "autoscaler" {
+  depends_on = [aws_iam_role_policy.autoscaler_autoscale]
+
+  metadata {
+    name      = "cluster-autoscaler"
+    namespace = "kube-system"
+
+    labels = {
+      "app" : "cluster-autoscaler"
+    }
+  }
+
+  spec {
+    replicas = 1
+
+    selector {
+      match_labels = {
+        "app" : "cluster-autoscaler"
+      }
+    }
+
+    template {
+      metadata {
+        labels = {
+          "app" : "cluster-autoscaler"
+        }
+
+        annotations = {
+          "cluster-autoscaler.kubernetes.io/safe-to-evict" : "false",
+          "prometheus.io/scrape" : "true",
+          "prometheus.io/port" : "8085"
+        }
+      }
+
+      spec {
+        automount_service_account_token = true
+        service_account_name            = "cluster-autoscaler"
+
+        container {
+          image             = "k8s.gcr.io/cluster-autoscaler:v1.14.7"
+          image_pull_policy = "Always"
+          name              = "cluster-autoscaler"
+
+          command = [
+            "./cluster-autoscaler",
+            "--v=4",
+            "--stderrthreshold=info",
+            "--cloud-provider=aws",
+            "--skip-nodes-with-local-storage=false",
+            "--expander=least-waste",
+            "--node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/${aws_eks_cluster.cluster.name}",
+            "--balance-similar-node-groups",
+            "--skip-nodes-with-system-pods=false",
+          ]
+
+          resources {
+            limits {
+              cpu    = "100m"
+              memory = "300Mi"
+            }
+
+            requests {
+              cpu    = "100m"
+              memory = "300Mi"
+            }
+          }
+
+          volume_mount {
+            name       = "ssl-certs"
+            mount_path = "/etc/ssl/certs/ca-certificates.crt"
+            read_only  = "true"
+          }
+        }
+
+        volume {
+          name = "ssl-certs"
+
+          host_path {
+            path = "/etc/ssl/certs/ca-bundle.crt"
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/terraform/cluster/aws/main.tf b/terraform/cluster/aws/main.tf
index 545fc66..793f42e 100644
--- a/terraform/cluster/aws/main.tf
+++ b/terraform/cluster/aws/main.tf
@@ -14,6 +14,10 @@ provider "null" {
   version = "~> 2.1"
 }
 
+locals {
+  oidc_sub = "${replace(aws_iam_openid_connect_provider.cluster.url, "https://", "")}:sub"
+}
+
 resource "null_resource" "delay_cluster" {
   provisioner "local-exec" {
     command = "sleep 15"
@@ -75,6 +79,7 @@ resource "aws_eks_node_group" "cluster" {
 
   lifecycle {
     create_before_destroy = true
+    ignore_changes        = [scaling_config[0].desired_size]
   }
 }
 
diff --git a/terraform/cluster/aws/outputs.tf b/terraform/cluster/aws/outputs.tf
index 129e8a8..c0578d8 100644
--- a/terraform/cluster/aws/outputs.tf
+++ b/terraform/cluster/aws/outputs.tf
@@ -20,5 +20,5 @@ output "oidc_arn" {
 
 output "oidc_sub" {
   depends_on = [aws_eks_node_group.cluster]
-  value      = "${replace(aws_iam_openid_connect_provider.cluster.url, "https://", "")}:sub"
+  value      = local.oidc_sub
 }