source: Install kubeadm
This guide aims to deploy a cluster kubernetes by creating 3 Linux servers (Ubuntu).
Here are the server's properties:
install-kubeadm - Before you begin
Ubuntu
2GB ram
2 CPU
swapoff
Docker runtimes: containerd
Here are the IP addresses for these servers:
- 192.168.56.40 kube-master
- 192.168.56.41 kube-node1
- 192.168.56.42 kube-node2
Vagrantfilevi Vagrantfile
# -*- mode: ruby -*-
# vi: set ft=ruby :
Vagrant.configure("2") do |config|
vagrantDataLocal = "/Users/arnaudsene/Documents/data"
vagrantDataMachine = "/vagrant_data"
codeDevLocal = "/Users/arnaudsene/Documents/"
codeDevMachine = "/code_dev"
etcHosts = ""
common = <<-SHELL
apt-get update -qq >/dev/null
apt-get upgrade -qq -y >/dev/null
apt-get install -qq -y apt-transport-https curl zsh vim git curl wget >/dev/null
sed -i 's/ChallengeResponseAuthentication no/ChallengeResponseAuthentication yes/g' /etc/ssh/sshd_config
service ssh restart
SHELL
config.vm.box = "ubuntu/focal64"
config.vm.box_url = "ubuntu/focal64"
config.vm.synced_folder vagrantDataLocal, vagrantDataMachine
config.vm.synced_folder codeDevLocal, codeDevMachine
NODES = [
{ :hostname => "kube-master", :ip => "192.168.56.40", :cpus => 2, :mem => 2048, :type => "master" },
{ :hostname => "kube-node1", :ip => "192.168.56.41", :cpus => 2, :mem => 2048, :type => "node" },
{ :hostname => "kube-node2", :ip => "192.168.56.42", :cpus => 2, :mem => 2048, :type => "node" }
]
# Define /etc/hosts for all servers
NODES.each do |node|
etcHosts += "echo '" + node[:ip] + " " + node[:hostname] + "'>> /etc/hosts" + "\n"
end
NODES.each do |node|
config.vm.define node[:hostname] do |cfg|
cfg.vm.hostname = node[:hostname]
cfg.vm.network :private_network, ip: node[:ip]
cfg.vm.provider :virtualbox do |v|
v.customize ["modifyvm", :id, "--cpus", node[:cpus]]
v.customize ["modifyvm", :id, "--memory", node[:mem]]
v.customize ["modifyvm", :id, "--name", node[:hostname]]
v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
v.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
v.customize ["modifyvm", :id, "--ioapic", "on"]
v.customize ["modifyvm", :id, "--nictype1", "virtio"]
end
cfg.vm.provision :shell, :inline => etcHosts
cfg.vm.provision :shell, :inline => common
cfg.vm.provision :shell, :path => "script.sh"
end
end
end
script.shvi script.sh
#!/bin/bash
## install ubuntu-halia
IP=$(hostname -I | awk '{print $2}')
USER='vagrant'
echo "===================================================="
echo "IP: $IP"
echo "===================================================="
echo
echo "Disable Swap"
echo "------------"
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
swapoff -a
echo ""
echo "===================================================="
echo "Installing kubeadm"
echo
echo "https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/"
echo
echo " Install kubelet, kubeadm and kubectl"
echo " ------------------------------------"
echo
echo " Update the apt package index and install packages needed to use the Kubernetes apt repository:"
echo " ----------------------------------------------------------------------------------------------"
apt-get update -qq >/dev/null
apt-get install -y ca-certificates -qq -y >/dev/null
echo " Download the Google Cloud public signing key:"
echo " ---------------------------------------------"
if [ ! -d "/etc/apt/keyrings" ]
then
mkdir /etc/apt/keyrings
fi
curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
echo " Add the Kubernetes apt repository:"
echo " ----------------------------------"
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list
echo " Update apt package index, install kubelet, kubeadm and kubectl, and pin their version:"
echo " --------------------------------------------------------------------------------------"
apt-get update -qq >/dev/null
apt-get install -qq -y kubelet kubeadm kubectl >/dev/null
echo
echo "===================================================="
echo
echo "Container Runtimes"
echo "------------------"
echo "https://kubernetes.io/docs/setup/production-environment/container-runtimes/"
echo
echo " Install and configure prerequisites"
echo " -----------------------------------"
echo
echo " Forwarding IPv4 and letting iptables see bridged traffic"
echo " --------------------------------------------------------"
cat <<EOF | tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter
# sysctl params required by setup, params persist across reboots
cat <<EOF | tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
sysctl --system
echo
echo " Verify that the br_netfilter, overlay modules are loaded"
echo " --------------------------------------------------------"
lsmod | grep br_netfilter
lsmod | grep overlay
echo
echo " Verify that the "
echo " - net.bridge.bridge-nf-call-iptables"
echo " - net.bridge.bridge-nf-call-ip6tables"
echo " - net.ipv4.ip_forward "
echo " system variables are set to 1 in your sysctl config"
echo " ---------------------------------------------------"
sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward
echo
echo " Install Container runtime"
echo " -------------------------"
apt-get update -qq >/dev/null
echo " 1. install packages to allow apt to use a repository over HTTPS:"
echo " ----------------------------------------------------------------"
apt-get install -qq -y gnupg lsb-release >/dev/null
echo " 2. Add Docker’s official GPG key:"
echo " ---------------------------------"
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo " 3. Set up the stable repository."
echo " --------------------------------"
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" >> /etc/apt/sources.list.d/docker.list
echo
echo " Install Docker Engine"
echo
echo " Update the apt package index, and install the latest version of Docker Engine and containerd"
echo " --------------------------------------------------------------------------------------------"
chmod a+r /etc/apt/keyrings/docker.gpg
apt-get update -qq >/dev/null
apt-get install -qq -y docker-ce docker-ce-cli containerd.io docker-compose-plugin >/dev/null
echo " Verify that the Docker Engine installation is successful by running the hello-world image:"
echo " ------------------------------------------------------------------------------------------"
docker run hello-world
echo " Configure Docker to start on boot with systemd"
echo " ----------------------------------------------"
systemctl enable docker.service
systemctl enable containerd.service
echo " JSON File logging driver"
echo " ------------------------"
# Create required directories
if [ ! -d "/etc/systemd/system/docker.service.d" ]
then
mkdir -p /etc/systemd/system/docker.service.d
fi
# Create daemon json config file
tee /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"storage-driver": "overlay2"
}
EOF
# To avoid a bug with kubeadm init
# unknown service runtime.v1alpha2.RuntimeService"
containerd config default > /etc/containerd/config.toml
systemctl restart containerd
echo
echo "===================================================="
echo
echo "Start and enable Services"
echo "-------------------------"
systemctl daemon-reload
systemctl restart docker
systemctl enable docker
systemctl status docker
usermod -aG docker "${USER}"
apt-get update -qq >/dev/null
echo
echo
echo "===================================================="
echo "[INFO]"
echo
echo "ssh vagrant@$IP"
echo " => password = vagrant"
echo
echo "Install oh-my-zsh"
echo "sh -c \"\$(curl -fsSL https://raw.github.com/ohmyzsh/ohmyzsh/master/tools/install.sh)\""
vagrant up
kube-masterkubeadm init --apiserver-advertise-address=192.168.56.40 --node-name kube-master --pod-network-cidr=10.244.0.0/16
[init] Using Kubernetes version: v1.26.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [kube-master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.56.40]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [kube-master localhost] and IPs [192.168.56.40 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [kube-master localhost] and IPs [192.168.56.40 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 12.004609 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node kube-master as control-plane by adding the labels: [node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node kube-master as control-plane by adding the taints [node-role.kubernetes.io/control-plane:NoSchedule]
[bootstrap-token] Using token: nei1do.vzxskbk5zg5o3sup
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
export KUBECONFIG=/etc/kubernetes/admin.conf
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.56.40:6443 --token nei1do.vzxskbk5zg5o3sup \
--discovery-token-ca-cert-hash sha256:c2798f555de31c3346cfdcc11a563cffd648eb94dbb7e335a7c647e348543d04
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
export KUBECONFIG=/etc/kubernetes/admin.conf
flannelYou must deploy a Container Network Interface (CNI) based Pod network add-on so that your Pods can communicate with each other. Cluster DNS (CoreDNS) will not start up before a network is installed.
We use flannel because of Vagrant + Virtualbox, but there are plenty Pod network available.
pods network are listed at: kubernetes addons
kube-master, kube-node1 & kube-node2sysctl net.bridge.bridge-nf-call-iptables=1
kube-masterSource deploying-flannel-manually
kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/v0.20.2/Documentation/kube-flannel.yml
From kube-master
kubectl get nodes
NAME STATUS ROLES AGE VERSION
kube-master Ready control-plane 31m v1.26.0
kubectl get pods -o wide --all-namespaces
kube-node1 & kube-node2kubeadm join 192.168.56.40:6443 --token gj8hda.qnqdvs3httpeyje2 \
--discovery-token-ca-cert-hash sha256:0fd3268798cb79610402f1ac1516160c269a7bbc5cdba395edac88fa4224745a
kube-master that the new node kube-node1 and kube-node2 have been added.kubectl get nodes
NAME STATUS ROLES AGE VERSION
kube-master Ready control-plane 44m v1.26.0
kube-node1 Ready <none> 9m56s v1.26.0
kube-node2 Ready <none> 7m22s v1.26.0
kubectlcurl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt-get update
sudo apt-get install kubectl
sudo vi /etc/hosts
192.168.56.40 kube-master
mkdir ~/.kube
ssh vagrant@kube-master "sudo cat /etc/kubernetes/admin.conf" > ~/.kube/config
vi ~/.zshrc
plugins=(git kubectl)
alias k="kubectl"
alias kg="kubectl get "
alias kgn="kubectl get nodes"
alias kga="kubectl get all --all-namespaces "
alias kgp="kubectl get pods"
alias wkgp="watch kubectl get pods"
alias kgs="kubectl get services"
alias kgps="kubectl get pods -n kube-system "
alias kcc="kubectl config current-context"
alias kuc="kubectl config use-context"
source ~/.zshrc
kdi='kubectl describe ingress'
kdj='kubectl describe job'
kdno='kubectl describe node'
kdns='kubectl describe namespace'
kdp='kubectl describe pods'
kdpvc='kubectl describe pvc'
kdrs='kubectl describe replicaset'
kds='kubectl describe svc'
kdsa='kubectl describe sa'
kdsec='kubectl describe secret'
kdss='kubectl describe statefulset'