Create EKS cluster using Terraform
This post will guide you through all the Terraform code needed to spin up a EKS cluster with Bottlerocket nodes using just the AWS provider instead of using a third-party module. The VPC resources need to be setup beforehand. For the VPC setup, I find having dedicated subnets for EKS clusters beneficial for IP address prefixes since it needs continuous blocks of IP addresses. All the referenced Terraform code can be obtained here.
Initialize the module where needed.
1locals {
2 env = "sandbox"
3}
4
5provider "aws" {
6 region = local.region
7 default_tags {
8 tags = {
9 env = local.env
10 terraform = true
11 }
12 }
13}
14
15module "eks-cluster" {
16 source = "../../modules/aws/eks"
17 cluster_name = local.env
18 env = local.env
19 cluster_version = "1.28"
20 addon_vpc_version = "v1.14.1-eksbuild.1"
21 addon_ebs_version = "v1.24.1-eksbuild.1"
22 addon_coredns_version = "v1.10.1-eksbuild.2"
23 addon_kube_proxy_version = "v1.28.1-eksbuild.1"
24 worker_instance_type = "t3a.large"
25 worker_instance_count = 3
26 worker_volume_size = 100
27 log_types = [
28 "api",
29 "audit",
30 "authenticator",
31 "controllerManager",
32 "scheduler"
33 ]
34}
Module files
cluster.tf
1resource "aws_eks_cluster" "cluster" {
2 name = var.cluster_name
3 version = var.cluster_version
4 role_arn = aws_iam_role.cluster.arn
5
6 vpc_config {
7 subnet_ids = data.aws_subnets.eks_private.ids
8 endpoint_private_access = true
9 endpoint_public_access = false
10 security_group_ids = [aws_security_group.cluster.id]
11 }
12
13 enabled_cluster_log_types = var.log_types
14
15 encryption_config {
16 provider {
17 key_arn = aws_kms_key.eks.arn
18 }
19 resources = ["secrets"]
20 }
21
22 kubernetes_network_config {
23 ip_family = "ipv4"
24 service_ipv4_cidr = "172.20.0.0/16"
25 }
26
27 tags = {
28 "karpenter.sh/discovery" = var.cluster_name
29 }
30
31 depends_on = [
32 aws_iam_role.cluster,
33 aws_kms_key.eks,
34 aws_security_group.cluster
35 ]
36}
Adjustments will need to be made here depending on your subnetting.
data.tf
1data "aws_vpc" "main" {
2 tags = {
3 env = var.env
4 Name = var.env
5 }
6}
7
8data "aws_subnets" "eks_private" {
9 filter {
10 name = "vpc-id"
11 values = [data.aws_vpc.main.id]
12 }
13 tags = {
14 env = var.env
15 type = "eks-private"
16 }
17}
18
19data "aws_ami" "bottlerocket_image" {
20 most_recent = true
21 owners = ["amazon"]
22
23 filter {
24 name = "name"
25 values = ["bottlerocket-aws-k8s-${var.cluster_version}-x86_64-*"]
26 }
27}
28
29data "tls_certificate" "cluster" {
30 url = aws_eks_cluster.cluster.identity[0].oidc[0].issuer
31}
The VPC addon is a requirement for the node groups to turn on prefix delegation before they’re created.
addons.tf
1resource "aws_eks_addon" "vpc" {
2 cluster_name = aws_eks_cluster.cluster.name
3 addon_name = "vpc-cni"
4 addon_version = var.addon_vpc_version
5 resolve_conflicts_on_create = "OVERWRITE"
6 resolve_conflicts_on_update = "OVERWRITE"
7 configuration_values = jsonencode({
8 env = {
9 ENABLE_PREFIX_DELEGATION = "true"
10 }
11 })
12 depends_on = [aws_eks_cluster.cluster]
13}
14
15resource "aws_eks_addon" "ebs" {
16 cluster_name = aws_eks_cluster.cluster.name
17 addon_name = "aws-ebs-csi-driver"
18 addon_version = var.addon_ebs_version
19 resolve_conflicts_on_create = "OVERWRITE"
20 resolve_conflicts_on_update = "OVERWRITE"
21 depends_on = [
22 aws_eks_cluster.cluster,
23 aws_eks_node_group.core
24 ]
25}
26
27resource "aws_eks_addon" "coredns" {
28 cluster_name = aws_eks_cluster.cluster.name
29 addon_name = "coredns"
30 addon_version = var.addon_coredns_version
31 resolve_conflicts_on_create = "OVERWRITE"
32 resolve_conflicts_on_update = "OVERWRITE"
33 depends_on = [
34 aws_eks_cluster.cluster,
35 aws_eks_node_group.core
36 ]
37}
38
39resource "aws_eks_addon" "kube-proxy" {
40 cluster_name = aws_eks_cluster.cluster.name
41 addon_name = "kube-proxy"
42 addon_version = var.addon_kube_proxy_version
43 resolve_conflicts_on_create = "OVERWRITE"
44 resolve_conflicts_on_update = "OVERWRITE"
45 depends_on = [
46 aws_eks_cluster.cluster,
47 aws_eks_node_group.core
48 ]
49}
iam.tf
1resource "aws_iam_role" "cluster" {
2 name = "eks-cluster-${var.cluster_name}"
3 assume_role_policy = jsonencode({
4 Statement : [
5 {
6 Action : "sts:AssumeRole",
7 Effect : "Allow",
8 Principal : {
9 "Service" : "eks.amazonaws.com"
10 }
11 }
12 ],
13 Version : "2012-10-17"
14 })
15
16 managed_policy_arns = [
17 "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy",
18 "arn:aws:iam::aws:policy/AmazonEKSVPCResourceController",
19 "arn:aws:iam::aws:policy/AmazonEKSServicePolicy"
20 ]
21}
22
23resource "aws_iam_role" "nodes" {
24 name = "eks-nodes-${var.cluster_name}"
25 assume_role_policy = jsonencode({
26 Statement : [
27 {
28 Action : "sts:AssumeRole",
29 Effect : "Allow",
30 Principal : {
31 "Service" : "ec2.amazonaws.com"
32 }
33 }
34 ],
35 Version : "2012-10-17"
36 })
37
38 managed_policy_arns = [
39 "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
40 "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
41 "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
42 "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
43 ]
44}
45
46resource "aws_iam_openid_connect_provider" "cluster" {
47 client_id_list = ["sts.amazonaws.com"]
48 thumbprint_list = [data.tls_certificate.cluster.certificates[0].sha1_fingerprint]
49 url = data.tls_certificate.cluster.url
50}
kms.tf
1resource "aws_kms_key" "eks" {
2 description = "Encrypt EKS secrets"
3 enable_key_rotation = true
4 multi_region = true
5}
6
7resource "aws_kms_alias" "eks" {
8 name = "alias/eks-${var.cluster_name}"
9 target_key_id = aws_kms_key.eks.key_id
10}
Here we’re automatically pulling the device mappings required by the Bottlerocket AMI and the latest AMI id. The template file path is dependent on how you structure your modules. This example is only one node group called “worker” and the lifecycle block will ignore a new AMI release until ready to update.
launch_template.tf
1locals {
2 device_list = tolist(data.aws_ami.bottlerocket_image.block_device_mappings)
3}
4
5resource "aws_launch_template" "core" {
6 name = "eks-core-${var.cluster_name}"
7 disable_api_stop = false
8 disable_api_termination = false
9 image_id = data.aws_ami.bottlerocket_image.id
10 instance_type = var.core_node_type
11 user_data = base64encode(templatefile("../../modules/aws/eks/files/node_config.toml.tftpl", {
12 cluster_name = aws_eks_cluster.cluster.name
13 cluster_endpoint = aws_eks_cluster.cluster.endpoint
14 cluster_ca_data = aws_eks_cluster.cluster.certificate_authority[0].data
15 nodegroup = "core"
16 ami_id = data.aws_ami.bottlerocket_image.id
17 })
18 )
19
20 block_device_mappings {
21 device_name = local.device_list[0]["device_name"]
22
23 ebs {
24 delete_on_termination = true
25 volume_size = 5
26 volume_type = "gp3"
27 encrypted = true
28 }
29 }
30
31 block_device_mappings {
32 device_name = local.device_list[1]["device_name"]
33
34 ebs {
35 delete_on_termination = true
36 volume_size = var.core_node_volume_size
37 volume_type = "gp3"
38 encrypted = true
39 }
40 }
41
42 metadata_options {
43 http_tokens = "required"
44 http_endpoint = "enabled"
45 http_put_response_hop_limit = 2
46 instance_metadata_tags = "enabled"
47 }
48
49 tag_specifications {
50 resource_type = "instance"
51
52 tags = {
53 Name = "eks-core-${var.cluster_name}"
54 terraform = true
55 "eks:cluster-name" = var.env
56 "eks:nodegroup-name" = "core"
57 platform = "eks"
58 env = var.env
59 }
60 }
61
62 tag_specifications {
63 resource_type = "volume"
64
65 tags = {
66 Name = "eks-core-${var.cluster_name}"
67 terraform = true
68 "eks:cluster-name" = var.env
69 "eks:nodegroup-name" = "core"
70 platform = "eks"
71 env = var.env
72 }
73 }
74
75 # Comment out when updating node
76 lifecycle {
77 ignore_changes = [
78 image_id,
79 user_data
80 ]
81 }
82}
nodes.tf
1resource "aws_eks_node_group" "core" {
2 cluster_name = aws_eks_cluster.cluster.name
3 node_group_name = "core"
4 node_role_arn = aws_iam_role.nodes.arn
5 subnet_ids = data.aws_subnets.eks_private.ids
6 ami_type = "CUSTOM"
7 labels = {
8 role = "core"
9 }
10
11 launch_template {
12 name = aws_launch_template.core.name
13 version = aws_launch_template.core.latest_version
14 }
15
16 scaling_config {
17 desired_size = var.core_node_count
18 max_size = var.core_node_count
19 min_size = var.core_node_count
20 }
21
22 update_config {
23 max_unavailable = 1
24 }
25
26 lifecycle {
27 create_before_destroy = true
28 }
29
30 depends_on = [
31 aws_iam_role.nodes,
32 aws_eks_cluster.cluster,
33 aws_launch_template.core,
34 aws_eks_addon.vpc
35 ]
36}
node_config.toml.tftpl
[settings.kubernetes]
"cluster-name" = "${cluster_name}"
"api-server" = "${cluster_endpoint}"
"cluster-certificate" = "${cluster_ca_data}"
"cluster-dns-ip" = "172.20.0.10"
"max-pods" = 110
[settings.kubernetes.node-labels]
"eks.amazonaws.com/nodegroup-image" = "${ami_id}"
"eks.amazonaws.com/capacityType" = "ON_DEMAND"
"eks.amazonaws.com/nodegroup" = "${nodegroup}"
"role" = "${nodegroup}"
Adjust the security group to allow access within VPC or VPN etc.
security_groups.tf
1resource "aws_security_group" "cluster" {
2 name = "eks-cluster-${var.cluster_name}"
3 description = "EKS cluster security"
4 vpc_id = data.aws_vpc.main.id
5 egress {
6 description = "full outbound"
7 cidr_blocks = ["0.0.0.0/0"]
8 from_port = "0"
9 protocol = "-1"
10 self = "false"
11 to_port = "0"
12 }
13 ingress {
14 description = "self reference"
15 from_port = "0"
16 protocol = "-1"
17 self = "true"
18 to_port = "0"
19 }
20 ingress {
21 security_groups = [aws_security_group.node.id]
22 description = "eks node group"
23 from_port = "0"
24 protocol = "-1"
25 self = "false"
26 to_port = "0"
27 }
28 tags = {
29 Name = "eks-cluster-${var.env}"
30 }
31}
32
33resource "aws_security_group" "node" {
34 name = "eks-node-${var.cluster_name}"
35 description = "EKS node security"
36 vpc_id = data.aws_vpc.main.id
37 egress {
38 description = "full outbound"
39 cidr_blocks = ["0.0.0.0/0"]
40 from_port = "0"
41 protocol = "-1"
42 self = "false"
43 to_port = "0"
44 }
45 ingress {
46 description = "self reference"
47 from_port = "0"
48 protocol = "-1"
49 self = "true"
50 to_port = "0"
51 }
52 tags = {
53 Name = "eks-node-${var.env}"
54 "karpenter.sh/discovery" = var.cluster_name
55 }
56}
variables.tf
1variable "addon_coredns_version" {}
2variable "addon_ebs_version" {}
3variable "addon_kube_proxy_version" {}
4variable "addon_vpc_version" {}
5variable "cluster_name" {}
6variable "cluster_version" {}
7variable "env" {}
8variable "log_types" {}
9variable "worker_instance_count" {}
10variable "worker_instance_type" {}
11variable "worker_volume_size" {}