mirror of
https://github.com/wahyd4/home-docker.git
synced 2026-08-09 04:15:52 +10:00
168 lines
4.5 KiB
Bash
168 lines
4.5 KiB
Bash
# etcd disaster recovery
|
|
# https://etcd.io/docs/v3.3/op-guide/recovery/
|
|
|
|
ETCDCTL_API=3 etcdctl snapshot restore s.db \
|
|
--name server-3 \
|
|
--initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,plex=https://192.168.1.11:2380 \
|
|
--initial-cluster-token etcd-cluster-0 \
|
|
--initial-advertise-peer-urls https://192.168.1.2:2380
|
|
|
|
ETCDCTL_API=3 etcdctl snapshot restore s.db \
|
|
--name plex \
|
|
--initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,plex=https://192.168.1.11:2380 \
|
|
--initial-cluster-token etcd-cluster-0 \
|
|
--initial-advertise-peer-urls https://192.168.1.11:2380
|
|
|
|
ETCDCTL_API=3 etcdctl snapshot restore s.db \
|
|
--name ssd-server-2 \
|
|
--initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,plex=https://192.168.1.11:2380 \
|
|
--initial-cluster-token etcd-cluster-0 \
|
|
--initial-advertise-peer-urls https://192.168.1.10:2380
|
|
|
|
docker run --rm -it \
|
|
-v '/mnt/backups:/backup' \
|
|
-v '/var/lib/etcd:/var/lib/etcd' \
|
|
--env ETCDCTL_API=3 \
|
|
'k8s.gcr.io/etcd:3.2.24' sh
|
|
|
|
|
|
|
|
# 1 reset cluster
|
|
kubeadm reset
|
|
|
|
# 2 recovery etcd from backup
|
|
|
|
sudo docker run --rm \
|
|
-v '/mnt/backups:/backup' \
|
|
-v '/var/lib/etcd:/var/lib/etcd' \
|
|
--env ETCDCTL_API=3 \
|
|
'k8s.gcr.io/etcd:3.2.24' \
|
|
/bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-04-26-05_00_11_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd"
|
|
|
|
# install nfs-common
|
|
|
|
sudo apt install nfs-common
|
|
|
|
## update docker cgroup
|
|
|
|
vim /lib/systemd/system/docker.service
|
|
|
|
# update
|
|
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --exec-opt native.cgroupdriver=systemd
|
|
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl restart docker
|
|
|
|
# check cgroup
|
|
docker system info --format '{{.CgroupDriver}}'
|
|
|
|
# if it fails at
|
|
|
|
[ERROR CRI]: container runtime is not running: output: time="2021-05-28T14:11:57+08:00" level=fatal msg="getting status of runtime failed: rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.RuntimeService"
|
|
|
|
# then
|
|
|
|
vim /etc/containerd/config.toml
|
|
|
|
# comment out
|
|
|
|
disabled_plugins = ["cri"]
|
|
|
|
# restart containerd
|
|
|
|
systemctl restart containerd
|
|
|
|
# re init cluster
|
|
sudo kubeadm init --ignore-preflight-errors=DirAvailable--var-lib-etcd --control-plane-endpoint 192.168.1.2 \
|
|
--pod-network-cidr=10.244.0.0/16 --v 5
|
|
|
|
|
|
yo# check logs
|
|
journalctl -xeu kubelet -f
|
|
|
|
# install fiannel
|
|
kubectl -n kube-system apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
|
|
|
|
## Add new nodes as control plane
|
|
|
|
# first prepre certificate key, run this one the existing control plan
|
|
# kubeadm init phase upload-certs --upload-certs
|
|
|
|
# when we have use yaml to manage cluster then use
|
|
sudo kubeadm init phase upload-certs --upload-certs --config cluster.yaml
|
|
|
|
# print join command
|
|
sudo kubeadm token create --print-join-command
|
|
|
|
|
|
# kubelet docker cgroupfs issue
|
|
|
|
vim /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
|
|
# update KUBELET_CGROUP_ARGS=--cgroup-driver=systemd to KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs
|
|
|
|
|
|
|
|
# run the following the node to be added
|
|
kubeadm join 192.168.1.2:6443 --token xxx --discovery-token-ca-cert-hash sha256:xxx --control-plane --certificate-key xxx
|
|
|
|
|
|
# disbale swap
|
|
|
|
sudo swapoff -a
|
|
sudo rm /swap.img
|
|
sudo vim /etc/fstab # and remove swap
|
|
# check
|
|
sudo swapon --show
|
|
|
|
|
|
# to restore postgres db from dump file
|
|
|
|
cat ~/Downloads/dump-2022-03-21_05-48-05_UTC.tar.gz| gunzip | psql --set ON_ERROR_STOP=off postgres -h 192.168.1.2 -p 30973 -U postgres
|
|
|
|
|
|
## upgrade from cluster yaml
|
|
# some common errors
|
|
# --ignore-preflight-errors=ControlPlaneNodesReady
|
|
sudo kubeadm init --config cluster.yaml --upload-certs --ignore-preflight-errors=DirAvailable--var-lib-etcd --v=5
|
|
|
|
# diff
|
|
sudo kubeadm upgrade diff --config cluster.yaml --v=5
|
|
|
|
# plan
|
|
sudo kubeadm upgrade plan --config cluster.yaml --v=5
|
|
|
|
# apply
|
|
sudo kubeadm upgrade apply --config cluster.yaml --ignore-preflight-errors all --v=5
|
|
|
|
|
|
# kubelet options
|
|
|
|
sudo vim /var/lib/kubelet/kubeadm-flags.env
|
|
|
|
# check file open count per process
|
|
# https://www.cyberciti.biz/faq/reload-sysctl-conf-on-linux-using-sysctl/
|
|
lsof | awk '{print $2}' | sort | uniq -c | sort -n
|
|
|
|
# too many open files
|
|
sudo vim /etc/sysctl.conf
|
|
|
|
fs.inotify.max_user_instances=2048
|
|
fs.file-max = 500000
|
|
fs.inotify.max_user_watches=655360
|
|
|
|
# apply change
|
|
sudo sysctl --system
|
|
|
|
|
|
|
|
### client add
|
|
|
|
# install crio
|
|
|
|
https://computingforgeeks.com/install-cri-o-container-runtime-on-ubuntu-linux/
|
|
|
|
# update cgroup driver
|
|
https://kubernetes.io/docs/setup/production-environment/container-runtimes/#cgroup-driver
|
|
|
|
# install kubeadm and kublet
|