mirror of
https://github.com/wahyd4/home-docker.git
synced 2026-08-09 04:15:52 +10:00
253 lines
6.7 KiB
Bash
253 lines
6.7 KiB
Bash
# etcd disaster recovery
|
|
# https://etcd.io/docs/v3.3/op-guide/recovery/
|
|
|
|
|
|
# Download backup from nas and then copy file to remote server
|
|
scp ~/Downloads/s.db 192.168.1.2:/home/junv
|
|
|
|
|
|
ETCDCTL_API=3 etcdctl snapshot restore s.db \
|
|
--name server-3 \
|
|
--initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,ssd-server-4=https://192.168.1.6:2380,plex=https://192.168.1.11:2380 \
|
|
--initial-cluster-token etcd-cluster-0 \
|
|
--initial-advertise-peer-urls https://192.168.1.2:2380
|
|
|
|
ETCDCTL_API=3 etcdctl snapshot restore s.db \
|
|
--name plex \
|
|
--initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,plex=https://192.168.1.11:2380 \
|
|
--initial-cluster-token etcd-cluster-0 \
|
|
--initial-advertise-peer-urls https://192.168.1.11:2380
|
|
|
|
ETCDCTL_API=3 etcdctl snapshot restore s.db \
|
|
--name ssd-server-2 \
|
|
--initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,plex=https://192.168.1.11:2380 \
|
|
--initial-cluster-token etcd-cluster-0 \
|
|
--initial-advertise-peer-urls https://192.168.1.10:2380
|
|
|
|
docker run --rm -it \
|
|
-v '/mnt/backups:/backup' \
|
|
-v '/var/lib/etcd:/var/lib/etcd' \
|
|
--env ETCDCTL_API=3 \
|
|
'k8s.gcr.io/etcd:3.2.24' sh
|
|
|
|
# Etcd recover a failed cluster
|
|
|
|
1. Stop the etcd service on the crashed node
|
|
systemctl stop etcd
|
|
2. Clean up the existing etcd data
|
|
rm -rf /var/lib/etcd/*
|
|
3. Get the member list from a healthy etcd node
|
|
On one of the working nodes, run:
|
|
|
|
|
|
# remove old member
|
|
|
|
ETCDCTL_API=3 etcdctl member remove 6c10765e571d3ce8 \
|
|
--endpoints=https://192.168.1.10:2379 \
|
|
--cacert=/etc/etcd/ca.pem \
|
|
--cert=/etc/etcd/etcd.pem \
|
|
--key=/etc/etcd/etcd-key.pem
|
|
|
|
# then add it back, both do it from a healthy node
|
|
|
|
ETCDCTL_API=3 etcdctl member add server-3 \
|
|
--peer-urls=https://192.168.1.2:2380 \
|
|
--endpoints=https://192.168.1.10:2379 \
|
|
--cacert=/etc/etcd/ca.pem \
|
|
--cert=/etc/etcd/etcd.pem \
|
|
--key=/etc/etcd/etcd-key.pem
|
|
# Backup existing config
|
|
sudo cp /etc/kubernetes/admin.conf /etc/kubernetes/admin.conf.backup
|
|
|
|
# Generate new admin.conf
|
|
sudo kubeadm init phase kubeconfig admin
|
|
|
|
# Copy to user's home directory
|
|
mkdir -p $HOME/.kube
|
|
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
|
|
sudo chown $(id -u):$(id -g) $HOME/.kube/config
|
|
|
|
|
|
# 1 reset cluster
|
|
kubeadm reset
|
|
|
|
# 2 recovery etcd from backup
|
|
|
|
sudo docker run --rm \
|
|
-v '/mnt/backups:/backup' \
|
|
-v '/var/lib/etcd:/var/lib/etcd' \
|
|
--env ETCDCTL_API=3 \
|
|
'k8s.gcr.io/etcd:3.2.24' \
|
|
/bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-04-26-05_00_11_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd"
|
|
|
|
# install nfs-common
|
|
|
|
sudo apt install nfs-common
|
|
|
|
## update docker cgroup
|
|
|
|
vim /lib/systemd/system/docker.service
|
|
|
|
# update
|
|
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --exec-opt native.cgroupdriver=systemd
|
|
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl restart docker
|
|
|
|
# check cgroup
|
|
docker system info --format '{{.CgroupDriver}}'
|
|
|
|
# if it fails at
|
|
|
|
[ERROR CRI]: container runtime is not running: output: time="2021-05-28T14:11:57+08:00" level=fatal msg="getting status of runtime failed: rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.RuntimeService"
|
|
|
|
# then
|
|
|
|
vim /etc/containerd/config.toml
|
|
|
|
# comment out
|
|
|
|
disabled_plugins = ["cri"]
|
|
|
|
# restart containerd
|
|
|
|
systemctl restart containerd
|
|
|
|
# re init cluster
|
|
sudo kubeadm init --ignore-preflight-errors=DirAvailable--var-lib-etcd --control-plane-endpoint 192.168.1.2 \
|
|
--pod-network-cidr=10.244.0.0/16 --v 5
|
|
|
|
|
|
yo# check logs
|
|
journalctl -xeu kubelet -f
|
|
|
|
# install fiannel
|
|
kubectl -n kube-system apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
|
|
|
|
## Add new nodes as control plane
|
|
|
|
# first prepre certificate key, run this one the existing control plan
|
|
# kubeadm init phase upload-certs --upload-certs
|
|
|
|
# when we have use yaml to manage cluster then use
|
|
sudo kubeadm init phase upload-certs --upload-certs --config cluster.yaml
|
|
|
|
# print join command
|
|
sudo kubeadm token create --print-join-command
|
|
|
|
|
|
# kubelet docker cgroupfs issue
|
|
|
|
vim /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
|
|
# update KUBELET_CGROUP_ARGS=--cgroup-driver=systemd to KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs
|
|
|
|
|
|
|
|
# run the following the node to be added
|
|
kubeadm join 192.168.1.2:6443 --token xxx --discovery-token-ca-cert-hash sha256:xxx --control-plane --certificate-key xxx
|
|
|
|
|
|
# disbale swap
|
|
|
|
sudo swapoff -a
|
|
sudo rm /swap.img
|
|
sudo vim /etc/fstab # and remove swap
|
|
# check
|
|
sudo swapon --show
|
|
|
|
|
|
# to restore postgres db from dump file
|
|
|
|
cat ~/Downloads/dump-2022-03-21_05-48-05_UTC.tar.gz| gunzip | psql --set ON_ERROR_STOP=off postgres -h 192.168.1.2 -p 30973 -U postgres
|
|
|
|
|
|
## upgrade from cluster yaml
|
|
# some common errors
|
|
# --ignore-preflight-errors=ControlPlaneNodesReady
|
|
sudo kubeadm init --config cluster.yaml --upload-certs --ignore-preflight-errors=DirAvailable--var-lib-etcd --v=5
|
|
|
|
# diff
|
|
sudo kubeadm upgrade diff --config cluster.yaml --v=5
|
|
|
|
# plan
|
|
sudo kubeadm upgrade plan --config cluster.yaml --v=5
|
|
|
|
# apply
|
|
sudo kubeadm upgrade apply --config cluster.yaml --ignore-preflight-errors all --v=5
|
|
|
|
|
|
# kubelet options
|
|
|
|
sudo vim /var/lib/kubelet/kubeadm-flags.env
|
|
|
|
# check file open count per process
|
|
# https://www.cyberciti.biz/faq/reload-sysctl-conf-on-linux-using-sysctl/
|
|
lsof | awk '{print $2}' | sort | uniq -c | sort -n
|
|
|
|
# too many open files
|
|
sudo vim /etc/sysctl.conf
|
|
|
|
fs.inotify.max_user_instances=2048
|
|
fs.file-max = 500000
|
|
fs.inotify.max_user_watches=655360
|
|
|
|
# apply change
|
|
sudo sysctl --system
|
|
|
|
|
|
|
|
### client add
|
|
|
|
# install crio
|
|
|
|
https://computingforgeeks.com/install-cri-o-container-runtime-on-ubuntu-linux/
|
|
|
|
# update cgroup driver
|
|
https://kubernetes.io/docs/setup/production-environment/container-runtimes/#cgroup-driver
|
|
|
|
# install kubeadm and kublet
|
|
|
|
# install some old version of a package
|
|
|
|
apt list --all-versions kubeadm
|
|
|
|
# then install
|
|
|
|
sudo apt install kubeadm=1.27.6-00
|
|
|
|
# Datadog crash probably related to redis, please make sure redis is healthy
|
|
|
|
|
|
### Restore files from nas, with progress
|
|
|
|
# rsync -havzP --stats rsync://junv@192.168.1.4/backups/192.168.1.2/redis /mnt/k8s
|
|
|
|
|
|
# Disk full
|
|
|
|
|
|
# check large files
|
|
sudo du -xh --max-depth=1 /mnt/k8s | sort -hr | head -n 15
|
|
|
|
# Revert your hard reset
|
|
|
|
git reflog
|
|
```
|
|
|
|
```
|
|
git reset --hard HEAD@{1}
|
|
|
|
|
|
# vmware exsi complains volume is full
|
|
# There is no more space for virtual disk 'server-3-000006.vmdk'. You might be able to continue this session by freeing disk space on the relevant volume, and clicking Retry. Click Cancel to terminate this session.
|
|
# It was because virtual machine consumes space, so does the snapshot!
|
|
|
|
# enable vmware ssh through UI and then Find large files
|
|
df -h
|
|
find /vmfs/volumes/tiplus-ssd/ -type f -size +1000000k -exec ls -lh {} \;
|
|
|
|
|
|
# grafana recover admin password
|
|
|
|
grafana-cli admin reset-admin-password xxxx
|