Creating SKS Cluster on Exoscale Academy

Exoscale account

Mon, 01 Jan 0001 00:00:00 +0000

Creating an account 🔗

Next, click the Registration button and enter your email address and password.

Validating your account 🔗

To protect against fraud, Exoscale must validate your account. You can do so by entering a credit card number or adding a small amount of money (e.g., 5€) to your account (you can use PayPal for that purpose).

Using Terraform

Mon, 01 Jan 0001 00:00:00 +0000

In a new folder, we create the following Terraform configuration files; they contain all the resources to create an SKS cluster and the related components:

provider.tf specifies the version of the Exoscale provider
security_group.tf specifies the security group and the rules to be applied to the cluster’s nodes
cluster.tf defined the cluster’s configuration
node_pool.tf defines the group of nodes associated to the cluster
kubeconfig.tf is used to create a local kubeconfig file to access the cluster
variables.tf defines the input information
output.tf specifies the information to be displayed back

terraform {
 required_providers {
 exoscale = {
 source = "exoscale/exoscale"
 version = "~> 0.60.0"
 }
 }
}

provider "exoscale" {}

# A security group so the nodes can communicate and we can pull logs
resource "exoscale_security_group" "sg_sks_nodes" {
 name = "sg_sks_nodes-${var.name}"
 description = "Allows traffic between sks nodes and public pulling of logs"
}

resource "exoscale_security_group_rule" "sg_sks_nodes_logs_rule" {
 security_group_id = exoscale_security_group.sg_sks_nodes.id
 type = "INGRESS"
 protocol = "TCP"
 cidr = "0.0.0.0/0"
 start_port = 10250
 end_port = 10250
}

resource "exoscale_security_group_rule" "sg_sks_nodes_calico" {
 security_group_id = exoscale_security_group.sg_sks_nodes.id
 user_security_group_id = exoscale_security_group.sg_sks_nodes.id
 type = "INGRESS"
 protocol = "UDP"
 start_port = 4789
 end_port = 4789
}

resource "exoscale_security_group_rule" "sg_sks_nodes_ccm" {
 security_group_id = exoscale_security_group.sg_sks_nodes.id
 type = "INGRESS"
 protocol = "TCP"
 start_port = 30000
 end_port = 32767
 cidr = "0.0.0.0/0"
}

resource "exoscale_sks_cluster" "sks" {
 zone = var.zone
 name = var.name
 version = var.kube_version
 description = "Demo cluster ${var.name} / ${var.zone}"
 service_level = "starter"
 cni = "calico"
 exoscale_ccm = true
 exoscale_csi = true
 metrics_server = true
}

resource "exoscale_sks_nodepool" "workers" {
 zone = var.zone
 cluster_id = exoscale_sks_cluster.sks.id
 name = "workers-${var.name}"
 instance_type = var.worker_type
 size = var.workers_number
 security_group_ids = [exoscale_security_group.sg_sks_nodes.id]
}

resource "exoscale_sks_kubeconfig" "sks_kubeconfig" {
 cluster_id = exoscale_sks_cluster.sks.id
 zone = exoscale_sks_cluster.sks.zone
 user = "kubernetes-admin"
 groups = ["system:masters"]
}

resource "local_sensitive_file" "sks_kubeconfig_file" {
 filename = "kubeconfig"
 content = exoscale_sks_kubeconfig.sks_kubeconfig.kubeconfig
 file_permission = "0600"
}

output "sks_kubeconfig" {
 value = local_sensitive_file.sks_kubeconfig_file.filename
}

variable "kube_version" {
 description = "Version of the Kubernetes cluster"
 type = string
 default = ""
 # when default is an empty string, the latest kubernetes version available is used
}

variable "name" {
 description = "Name of the cluster"
 type = string
 default = "demo"
}

variable "workers_number" {
 description = "Number of workers in node pool"
 type = number
 default = 3
}

variable "worker_type" {
 type = string
 default = "standard.medium"
}

variable "zone" {
 type = string
 default = "ch-gva-2"
}

output "name" {
 value = var.name
}

output "zone" {
 value = var.zone
}

Next, we need to set env variables so that Terraform can use the Exoscale API.

Using Pulumi

Mon, 01 Jan 0001 00:00:00 +0000

In a new folder, we create the following files:

Pulumi.yaml, which defines the Exoscale resources to create (security group, security group rules, SKS cluster, node pool and kubeconfig file)
Pulumi.demo.yaml, which defines the configuration option to apply to Pulumi.yaml in order to create a specific stack named demo

name: sks-template
runtime: yaml
description: SKS cluster management
config:
 zone:
 type: string
 version:
 type: string
 size:
 type: integer
 instanceType:
 type: string
 nodepoolSuffix:
 type: integer
 default: 0
outputs:
 kubeConfig: ${kubeconfig.kubeconfig}
resources:
 exoscale-provider:
 type: pulumi:providers:exoscale
 defaultProvider: true
 options:
 version: 0.59.2
 securityGroup:
 type: exoscale:SecurityGroup
 properties:
 description: Security group for Kubernetes nodes
 name: sg-${pulumi.stack}
 securityGroupRulesNodePorts:
 type: exoscale:SecurityGroupRule
 properties:
 securityGroupId: ${securityGroup.id}
 type: INGRESS
 protocol: TCP
 cidr: 0.0.0.0/0
 startPort: 30000
 endPort: 32767
 securityGroupRulesKubelet:
 type: exoscale:SecurityGroupRule
 properties:
 securityGroupId: ${securityGroup.id}
 type: INGRESS
 protocol: TCP
 userSecurityGroupId: ${securityGroup.id}
 startPort: 10250
 endPort: 10251
 securityGroupRulesPrometheus:
 type: exoscale:SecurityGroupRule
 properties:
 securityGroupId: ${securityGroup.id}
 type: INGRESS
 protocol: TCP
 userSecurityGroupId: ${securityGroup.id}
 startPort: 9100
 endPort: 9100
 securityGroupRulesCiliumVXLAN:
 type: exoscale:SecurityGroupRule
 properties:
 securityGroupId: ${securityGroup.id}
 type: INGRESS
 protocol: UDP
 userSecurityGroupId: ${securityGroup.id}
 startPort: 8472
 endPort: 8472
 securityGroupRulesCiliumHCICMP:
 type: exoscale:SecurityGroupRule
 properties:
 securityGroupId: ${securityGroup.id}
 type: INGRESS
 protocol: ICMP
 userSecurityGroupId: ${securityGroup.id}
 icmpCode: 0
 icmpType: 8
 securityGroupRulesCiliumHCTCP:
 type: exoscale:SecurityGroupRule
 properties:
 securityGroupId: ${securityGroup.id}
 type: INGRESS
 protocol: TCP
 userSecurityGroupId: ${securityGroup.id}
 startPort: 4240
 endPort: 4240
 cluster:
 type: exoscale:SksCluster
 properties:
 autoUpgrade: false
 cni: cilium
 description: demo SKS cluster
 exoscaleCcm: true
 exoscaleCsi: true
 metricsServer: true
 serviceLevel: starter
 name: sks-${pulumi.stack}
 zone: ${zone}
 version: ${version}
 nodepool:
 type: exoscale:SksNodepool
 properties:
 clusterId: ${cluster.id}
 name: sks-${pulumi.stack}-${nodepoolSuffix}
 zone: ${cluster.zone}
 instanceType: ${instanceType}
 size: ${size}
 securityGroupIds:
 - ${securityGroup.id}
 options:
 replaceOnChanges:
 - name
 kubeconfig:
 type: exoscale:SksKubeconfig
 properties:
 clusterId: ${cluster.id}
 earlyRenewalSeconds: 0
 groups:
 - system:masters
 ttlSeconds: 0
 user: kubernetes-admin
 zone: ${cluster.zone}

config:
 zone: ch-gva-2
 version: "" # latest kubernetes version available is used if left empty
 size: 3
 instanceType: standard.medium
 nodepoolSuffix: 0

Next, we log into the Pulumi cloud (the place where the state of the stack is stored).

Using exo CLI

Mon, 01 Jan 0001 00:00:00 +0000

The shell script sks.sh, detailed below, creates the following Exoscale resources:

a security group
security group rules
a SKS cluster
a node pool for the SKS cluster

[!WARNING] This script allows the creation of an SKS cluster easily. This is fine for a workshop, but it must not be used to create a production cluster

#!/usr/bin/env bash
# set -euo pipefail

CURRENT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)

function usage {
 echo $1
 echo "usage: sks.sh -n NAME [-s SIZE] [-t TYPE] [-l LEVEL] [-z ZONE] [-p NETWORK] [-v VERSION]"
 exit 1
}

# Creation of a Security Group
function create-sg {
 exo compute security-group create sks-${NAME}

 # Creating of the rules related to the security group

 exo compute security-group rule add sks-${NAME} \
 --description "NodePort services" \
 --protocol tcp \
 --network 0.0.0.0/0 \
 --port 30000-32767

 exo compute security-group rule add sks-${NAME} \
 --description "SKS kubelet" \
 --protocol tcp \
 --port 10250-10251 \
 --security-group sks-${NAME}

 exo compute security-group rule add sks-${NAME} \
 --description "Prometheus metrics" \
 --protocol tcp \
 --port 9100 \
 --security-group sks-${NAME}

 if [ "${NETWORK}" = "cilium" ]; then
 exo compute security-group rule add sks-${NAME} \
 --description "Cilium VXLAN" \
 --protocol udp \
 --port 8472 \
 --security-group sks-${NAME}

 exo compute security-group rule add sks-${NAME} \
 --description "Cilium Health-check ICMP" \
 --protocol icmp \
 --icmp-code 0 \
 --icmp-type 8 \
 --security-group sks-${NAME}

 exo compute security-group rule add sks-${NAME} \
 --description "Cilium Health-check TCP" \
 --protocol tcp \
 --port 4240 \
 --security-group sks-${NAME}
 else # Calico
 exo compute security-group rule add sks-${NAME} \
 --description "Calico traffic" \
 --protocol udp \
 --port 4789 \
 --security-group sks-${NAME}
 fi
}

# Creation of the cluster
function create-cluster {
 exo compute sks create $NAME \
 --zone ${ZONE} \
 --auto-upgrade \
 --cni ${NETWORK} \
 --exoscale-csi \
 --service-level ${LEVEL} \
 --kubernetes-version ${VERSION} \
 --nodepool-name sks-${NAME} \
 --nodepool-instance-type standard.${TYPE} \
 --nodepool-size ${SIZE} \
 --nodepool-disk-size 50 \
 --nodepool-security-group sks-${NAME}
}

# Get the kubeconfig file and modify it a bit
function get-kubeconfig {
 # Get kubeconfig file
 exo compute sks kubeconfig $NAME cluster-admin -z $ZONE > $CURRENT_DIR/sks-$NAME.kubeconfig

 # Rename cluster-admin user
 gsed -i "s/cluster-admin/cluster-admin-$NAME/g" "$CURRENT_DIR/sks-$NAME.kubeconfig"

 # Rename the context
 CONTEXT=$(KUBECONFIG=$CURRENT_DIR/sks-$NAME.kubeconfig kubectl config current-context)
 KUBECONFIG=$CURRENT_DIR/sks-$NAME.kubeconfig kubectl config rename-context $CONTEXT $NAME

 echo "Use the cluster:"
 echo $CURRENT_DIR
 echo "export KUBECONFIG=$CURRENT_DIR/sks-$NAME.kubeconfig"
}

# Delete nodepool
function delete-nodepool {
 for np in $(exo compute sks nodepool list -O json | jq -r ".[].id"); do
 exo compute sks nodepool delete $NAME $np -f -z $ZONE;
 done
}

# Delete cluster
function delete-cluster {
 exo compute sks delete -f -z $ZONE $NAME

 # Remove kubeconfig file
 rm $CURRENT_DIR/sks-$NAME.kubeconfig
}

# Delete security groups
function delete-sg {
 # Delete the rules of the associated security group
 for rule in $(exo compute security-group show sks-$NAME -O json | jq -r ".ingress_rules[].id"); do
 exo compute security-group rule delete -f sks-$NAME $rule;
 done

 # Delete associated security group
 exo compute security-group delete -f sks-$NAME
}

# Argument parsing
while getopts "dt:s:n:l:p:z:v:" option; do
 case "${option}" in
 d)
 ACTION="delete"
 ;;
 t)
 TYPE=${OPTARG}
 ;;
 s)
 SIZE=${OPTARG}
 ;;
 l)
 LEVEL=${OPTARG}
 ;;
 n)
 NAME=${OPTARG}
 ;;
 p)
 NETWORK=${OPTARG}
 ;;
 z)
 ZONE=${OPTARG}
 ;;
 v)
 VERSION=${OPTARG}
 ;;
 *)
 usage
 ;;
 esac
done

# Make sure the name is provided for create and delete actions
if [ -z "$NAME" ]; then
 usage "Cluster name must be provided though the -n flag"
fi

# Default values to params
TYPE=${TYPE:-medium}
SIZE=${SIZE:-"3"}
LEVEL=${LEVEL:-"starter"}
ZONE=${ZONE:-"ch-gva-2"}
NETWORK=${NETWORK:-"cilium"}
VERSION=${VERSION:-"latest"}

# Get action to perform
ACTION=${ACTION:-"create"}

# Handling deletion
if [ "$ACTION" = "delete" ]; then
 delete-nodepool
 delete-sg
 delete-cluster
 exit 0
fi

# Handling creation
create-sg
create-cluster
get-kubeconfig

This script also generates a kubeconfig file to access the cluster API Server.