Мой маленький Kubernetes: настраиваем тестовый кластер в VirtualBox

# encoding: utf-8
# -*- mode: ruby -*-
# vi: set ft=ruby :

### configuration parameters
# K8S requires minimum 2 CPU cores
VM_GROUP = "Kubernetes Deployment"
VM_BOX_IMAGE = "ubuntu/bionic64"
VM_BOX_VERSION = "20200908.0.0"
VM_PREFIX = "k8s"
VM_CPU = "4"
VM_MEM = "2048"
VM_CPU_MASTER = "2"
VM_MEM_MASTER = "2048"

# Number of nodes in cluster
K8S_NODES = 4
# Master IP address (x.y.z.K8S_MASTER)
K8S_MASTER_IP = 10
K8S_NETWORK = "100.77.42"
K8S_HEAD_IP = "#{K8S_NETWORK}.#{K8S_MASTER_IP}"

Vagrant.configure("2") do |config|
    config.disksize.size = '50GB'

    config.vm.define "#{VM_PREFIX}-head" do |node|
        node.vm.provision "shell", inline: "echo Configuring node: #{VM_PREFIX}-head"

        node.vm.box = VM_BOX_IMAGE
        node.vm.box_version = VM_BOX_VERSION
        node.vm.hostname = "#{VM_PREFIX}-head"
        node.vm.network :private_network, ip: "#{K8S_HEAD_IP}"

        node.vm.provider "virtualbox" do |v|
            v.name = "#{VM_PREFIX}-head"
            v.customize ["modifyvm", :id, "--groups", "/#{VM_GROUP}"]
            v.customize ["modifyvm", :id, "--memory", VM_MEM_MASTER]
            v.customize ["modifyvm", :id, "--cpus", VM_CPU_MASTER]
            # Prevent VirtualBox from interfering with host audio stack
            v.customize ["modifyvm", :id, "--audio", "none"]
        end

        # Install/setup base services on VM
        node.vm.provision :shell, path: "./k8s-base.sh"

        # Install/setup related services on VM
        node.vm.provision :shell, path: "./k8s-head.sh"
    end

    (1..K8S_NODES).each do |i|
        vm_name = "#{VM_PREFIX}-node-#{i}"
        vm_ip = K8S_MASTER_IP + i

        config.vm.define "#{vm_name}" do |node|
            node.vm.provision "shell", inline: "echo Configuring node: #{vm_name}"

            node.vm.box = VM_BOX_IMAGE
            node.vm.box_version = VM_BOX_VERSION
            node.vm.hostname = vm_name
            node.vm.network :private_network, ip: "#{K8S_NETWORK}.#{vm_ip}"

            node.vm.provider "virtualbox" do |v|
                v.name = vm_name
                v.customize ["modifyvm", :id, "--groups", "/#{VM_GROUP}"]
                v.customize ["modifyvm", :id, "--memory", VM_MEM]
                v.customize ["modifyvm", :id, "--cpus", VM_CPU]
                # Prevent VirtualBox from interfering with host audio stack
                v.customize ["modifyvm", :id, "--audio", "none"]

                # see https://www.virtualbox.org/manual/ch08.html#vboxmanage-storageattach
                # Disk 100GB
                disk_name = "data/disk-#{vm_name}.vdb"
                unless File.exist?(disk_name)
                    v.customize ["createhd", "--filename", disk_name, "--size", 102400 ]
                end

                v.customize [ "storageattach", :id, "--storagectl", "SCSI", "--port", 3, "--device", 0, "--type", "hdd", "--medium", disk_name ]
            end

            # Install/setup base services on VM
            node.vm.provision :shell, path: "./k8s-base.sh"

            # Install/setup related services on VM
            node.vm.provision :shell, path: "./k8s-node.sh", env: {"K8S_HEAD_IP" => K8S_HEAD_IP}
        end
    end
end

#!/usr/bin/env bash

apt-get update
apt-get upgrade -y -f
apt-get install -y apt-transport-https ca-certificates curl software-properties-common

# Install docker
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") $(lsb_release -cs) stable"
apt-get update
apt-cache policy docker-ce
apt-get install -y docker-ce
systemctl enable docker
systemctl status docker

# run docker commands as vagrant user (sudo not required)
usermod -aG docker vagrant

# kubelet requires swap off
swapoff -a

# keep swap off after reboot
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

# Install kubeadm
# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
apt-get update
apt-get install -y kubeadm kubelet kubectl
apt-mark hold kubelet kubeadm kubectl

# set node-ip
IP_ADDR=`hostname -I | cut -d' ' -f2`
echo "Environment=\"KUBELET_EXTRA_ARGS=--node-ip=$IP_ADDR\"" | sudo tee -a /etc/systemd/system/kubelet.service.d/10-kubeadm.conf

# Setup daemon.
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

mkdir -p /etc/systemd/system/docker.service.d

systemctl daemon-reload
systemctl restart kubelet
systemctl restart docker

#!/usr/bin/env bash

echo "> Init k8s admin (kubeadm)"
IP_ADDR=`hostname -I | cut -d' ' -f2`
HOST_NAME=$(hostname -s)
echo "> Init k8s admin (kubeadm): ${HOST_NAME} => ${IP_ADDR}"
kubeadm init --apiserver-advertise-address=$IP_ADDR --apiserver-cert-extra-sans=$IP_ADDR  --node-name $HOST_NAME --pod-network-cidr=172.16.0.0/16

echo "> Set credentials to regular user (vagrant)"
sudo --user=vagrant mkdir -p /home/vagrant/.kube
cp -i /etc/kubernetes/admin.conf /home/vagrant/.kube/config
chown $(id -u vagrant):$(id -g vagrant) /home/vagrant/.kube/config

echo "> Switch 'cgroup-driver' to 'systemd' (cover case when init command resets env flags)"
export KUBECONFIG=/etc/kubernetes/admin.conf
sed -i "s/cgroup-driver=cgroupfs/cgroup-driver=systemd/g" /var/lib/kubelet/kubeadm-flags.env

# echo "> Use Calico pod network add-on". Use your own network provider
# kubectl apply -f ./network-calico.yaml

echo "> Create k8s join token"
kubeadm token create --print-join-command >> /etc/kubeadm_join_cmd.sh
chmod +x /etc/kubeadm_join_cmd.sh

echo "> Remove password for ssh between guest VMs"
sed -i "/^[^#]*PasswordAuthentication[[:space:]]no/c\PasswordAuthentication yes" /etc/ssh/sshd_config
service sshd restart

echo "> Install Helm"
curl -s https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash

echo "> Restarting services"
systemctl daemon-reload
systemctl restart kubelet
systemctl restart docker

echo "> kubectrl autocompetition"
kubectl completion bash > /etc/bash_completion.d/kubectl

#!/usr/bin/env bash

echo "> Install sshpass"
add-apt-repository universe
apt-get update
apt-get install -y sshpass

echo "> Join to cluster"
sshpass -p "vagrant" scp -o StrictHostKeyChecking=no vagrant@${K8S_HEAD_IP}:/etc/kubeadm_join_cmd.sh .
sh ./kubeadm_join_cmd.sh

echo "> Restarting services"
systemctl daemon-reload
systemctl restart kubelet
systemctl restart docker