Files
home-docker/recovery.sh
T
2022-05-07 15:34:34 +10:00

121 lines
3.2 KiB
Bash

etcdctl -C http://127.0.0.1:2380 \
--ca-file=/etc/kubernetes/pki/etcd/ca.crt \
--cert-file=/etc/kubernetes/pki/etcd/peer.crt \
--key-file=/etc/kubernetes/pki/etcd/peer.key restore db
docker run --rm -it \
-v '/mnt/backups:/backup' \
-v '/var/lib/etcd:/var/lib/etcd' \
--env ETCDCTL_API=3 \
'k8s.gcr.io/etcd:3.2.24' sh
# 1 reset cluster
kubeadm reset
# 2 recovery etcd from backup
sudo docker run --rm \
-v '/mnt/backups:/backup' \
-v '/var/lib/etcd:/var/lib/etcd' \
--env ETCDCTL_API=3 \
'k8s.gcr.io/etcd:3.2.24' \
/bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-04-26-05_00_11_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd"
# install nfs-common
sudo apt install nfs-common
## update docker cgroup
vim /lib/systemd/system/docker.service
# update
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --exec-opt native.cgroupdriver=systemd
sudo systemctl daemon-reload
sudo systemctl restart docker
# check cgroup
docker system info --format '{{.CgroupDriver}}'
# if it fails at
[ERROR CRI]: container runtime is not running: output: time="2021-05-28T14:11:57+08:00" level=fatal msg="getting status of runtime failed: rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.RuntimeService"
# then
vim /etc/containerd/config.toml
# comment out
disabled_plugins = ["cri"]
# restart containerd
systemctl restart containerd
# re init cluster
sudo kubeadm init --ignore-preflight-errors=DirAvailable--var-lib-etcd --control-plane-endpoint 192.168.1.2 \
--pod-network-cidr=10.244.0.0/16 --v 5
yo# check logs
journalctl -xeu kubelet -f
# install fiannel
kubectl -n kube-system apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
## Add new nodes as control plane
# first prepre certificate key, run this one the existing control plan
# kubeadm init phase upload-certs --upload-certs
# when we have use yaml to manage cluster then use
sudo kubeadm init phase upload-certs --upload-certs --config cluster.yaml
# print join command
sudo kubeadm token create --print-join-command
# kubelet docker cgroupfs issue
vim /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
# update KUBELET_CGROUP_ARGS=--cgroup-driver=systemd to KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs
# run the following the node to be added
kubeadm join 192.168.1.2:6443 --token xxx --discovery-token-ca-cert-hash sha256:xxx --control-plane --certificate-key xxx
# disbale swap
sudo swapoff -a
sudo rm /swap.img
sudo vim /etc/fstab # and remove swap
# check
sudo swapon --show
# to restore postgres db from dump file
cat ~/Downloads/dump-2022-03-21_05-48-05_UTC.tar.gz| gunzip | psql --set ON_ERROR_STOP=off postgres -h 192.168.1.2 -p 30973 -U postgres
## upgrade from cluster yaml
# some common errors
# --ignore-preflight-errors=ControlPlaneNodesReady
sudo kubeadm init --config cluster.yaml --upload-certs --ignore-preflight-errors=DirAvailable--var-lib-etcd --v=5
# diff
sudo kubeadm upgrade diff --config cluster.yaml --v=5
# plan
sudo kubeadm upgrade plan --config cluster.yaml --v=5
# apply
sudo kubeadm upgrade apply --config cluster.yaml --ignore-preflight-errors all --v=5