# etcd disaster recovery # https://etcd.io/docs/v3.3/op-guide/recovery/ # Download backup from nas and then copy file to remote server scp ~/Downloads/s.db 192.168.1.2:/home/junv ETCDCTL_API=3 etcdctl snapshot restore s.db \ --name server-3 \ --initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,ssd-server-4=https://192.168.1.6:2380,plex=https://192.168.1.11:2380 \ --initial-cluster-token etcd-cluster-0 \ --initial-advertise-peer-urls https://192.168.1.2:2380 ETCDCTL_API=3 etcdctl snapshot restore s.db \ --name plex \ --initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,plex=https://192.168.1.11:2380 \ --initial-cluster-token etcd-cluster-0 \ --initial-advertise-peer-urls https://192.168.1.11:2380 ETCDCTL_API=3 etcdctl snapshot restore s.db \ --name ssd-server-2 \ --initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,plex=https://192.168.1.11:2380 \ --initial-cluster-token etcd-cluster-0 \ --initial-advertise-peer-urls https://192.168.1.10:2380 docker run --rm -it \ -v '/mnt/backups:/backup' \ -v '/var/lib/etcd:/var/lib/etcd' \ --env ETCDCTL_API=3 \ 'k8s.gcr.io/etcd:3.2.24' sh # Etcd recover a failed cluster 1. Stop the etcd service on the crashed node systemctl stop etcd 2. Clean up the existing etcd data rm -rf /var/lib/etcd/* 3. Get the member list from a healthy etcd node On one of the working nodes, run: # remove old member ETCDCTL_API=3 etcdctl member remove 6c10765e571d3ce8 \ --endpoints=https://192.168.1.10:2379 \ --cacert=/etc/etcd/ca.pem \ --cert=/etc/etcd/etcd.pem \ --key=/etc/etcd/etcd-key.pem # then add it back, both do it from a healthy node ETCDCTL_API=3 etcdctl member add server-3 \ --peer-urls=https://192.168.1.2:2380 \ --endpoints=https://192.168.1.10:2379 \ --cacert=/etc/etcd/ca.pem \ --cert=/etc/etcd/etcd.pem \ --key=/etc/etcd/etcd-key.pem # Backup existing config sudo cp /etc/kubernetes/admin.conf /etc/kubernetes/admin.conf.backup # Generate new admin.conf sudo kubeadm init phase kubeconfig admin # Copy to user's home directory mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config # 1 reset cluster kubeadm reset # 2 recovery etcd from backup sudo docker run --rm \ -v '/mnt/backups:/backup' \ -v '/var/lib/etcd:/var/lib/etcd' \ --env ETCDCTL_API=3 \ 'k8s.gcr.io/etcd:3.2.24' \ /bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-04-26-05_00_11_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd" # install nfs-common sudo apt install nfs-common ## update docker cgroup vim /lib/systemd/system/docker.service # update ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --exec-opt native.cgroupdriver=systemd sudo systemctl daemon-reload sudo systemctl restart docker # check cgroup docker system info --format '{{.CgroupDriver}}' # if it fails at [ERROR CRI]: container runtime is not running: output: time="2021-05-28T14:11:57+08:00" level=fatal msg="getting status of runtime failed: rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.RuntimeService" # then vim /etc/containerd/config.toml # comment out disabled_plugins = ["cri"] # restart containerd systemctl restart containerd # re init cluster sudo kubeadm init --ignore-preflight-errors=DirAvailable--var-lib-etcd --control-plane-endpoint 192.168.1.2 \ --pod-network-cidr=10.244.0.0/16 --v 5 yo# check logs journalctl -xeu kubelet -f # install fiannel kubectl -n kube-system apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml ## Add new nodes as control plane # first prepre certificate key, run this one the existing control plan # kubeadm init phase upload-certs --upload-certs # when we have use yaml to manage cluster then use sudo kubeadm init phase upload-certs --upload-certs --config cluster.yaml # print join command sudo kubeadm token create --print-join-command # kubelet docker cgroupfs issue vim /etc/systemd/system/kubelet.service.d/10-kubeadm.conf # update KUBELET_CGROUP_ARGS=--cgroup-driver=systemd to KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs # run the following the node to be added kubeadm join 192.168.1.2:6443 --token xxx --discovery-token-ca-cert-hash sha256:xxx --control-plane --certificate-key xxx # disbale swap sudo swapoff -a sudo rm /swap.img sudo vim /etc/fstab # and remove swap # check sudo swapon --show # to restore postgres db from dump file cat ~/Downloads/dump-2022-03-21_05-48-05_UTC.tar.gz| gunzip | psql --set ON_ERROR_STOP=off postgres -h 192.168.1.2 -p 30973 -U postgres ## upgrade from cluster yaml # some common errors # --ignore-preflight-errors=ControlPlaneNodesReady sudo kubeadm init --config cluster.yaml --upload-certs --ignore-preflight-errors=DirAvailable--var-lib-etcd --v=5 # diff sudo kubeadm upgrade diff --config cluster.yaml --v=5 # plan sudo kubeadm upgrade plan --config cluster.yaml --v=5 # apply sudo kubeadm upgrade apply --config cluster.yaml --ignore-preflight-errors all --v=5 # kubelet options sudo vim /var/lib/kubelet/kubeadm-flags.env # check file open count per process # https://www.cyberciti.biz/faq/reload-sysctl-conf-on-linux-using-sysctl/ lsof | awk '{print $2}' | sort | uniq -c | sort -n # too many open files sudo vim /etc/sysctl.conf fs.inotify.max_user_instances=2048 fs.file-max = 500000 fs.inotify.max_user_watches=655360 # apply change sudo sysctl --system ### client add # install crio https://computingforgeeks.com/install-cri-o-container-runtime-on-ubuntu-linux/ # update cgroup driver https://kubernetes.io/docs/setup/production-environment/container-runtimes/#cgroup-driver # install kubeadm and kublet # install some old version of a package apt list --all-versions kubeadm # then install sudo apt install kubeadm=1.27.6-00 # Datadog crash probably related to redis, please make sure redis is healthy ### Restore files from nas, with progress # rsync -havzP --stats rsync://junv@192.168.1.4/backups/192.168.1.2/redis /mnt/k8s # Disk full # check large files sudo du -xh --max-depth=1 /mnt/k8s | sort -hr | head -n 15 # Revert your hard reset git reflog ``` ``` git reset --hard HEAD@{1} # vmware exsi complains volume is full # There is no more space for virtual disk 'server-3-000006.vmdk'. You might be able to continue this session by freeing disk space on the relevant volume, and clicking Retry. Click Cancel to terminate this session. # It was because virtual machine consumes space, so does the snapshot! # enable vmware ssh through UI and then Find large files df -h find /vmfs/volumes/tiplus-ssd/ -type f -size +1000000k -exec ls -lh {} \; # grafana recover admin password grafana-cli admin reset-admin-password xxxx