# etcd disaster recovery
# https://etcd.io/docs/v3.3/op-guide/recovery/


# Download backup from nas and then copy file to remote server
scp ~/Downloads/s.db 192.168.1.2:/home/junv


ETCDCTL_API=3 etcdctl snapshot restore s.db \
  --name server-3 \
  --initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,ssd-server-4=https://192.168.1.6:2380,plex=https://192.168.1.11:2380 \
  --initial-cluster-token etcd-cluster-0 \
  --initial-advertise-peer-urls https://192.168.1.2:2380

ETCDCTL_API=3 etcdctl snapshot restore s.db \
  --name plex \
  --initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,plex=https://192.168.1.11:2380 \
  --initial-cluster-token etcd-cluster-0 \
  --initial-advertise-peer-urls https://192.168.1.11:2380

ETCDCTL_API=3 etcdctl snapshot restore s.db \
  --name ssd-server-2 \
  --initial-cluster server-3=https://192.168.1.2:2380,ssd-server-2=https://192.168.1.10:2380,plex=https://192.168.1.11:2380 \
  --initial-cluster-token etcd-cluster-0 \
  --initial-advertise-peer-urls https://192.168.1.10:2380

docker run --rm -it \
-v '/mnt/backups:/backup' \
-v '/var/lib/etcd:/var/lib/etcd' \
--env ETCDCTL_API=3 \
'k8s.gcr.io/etcd:3.2.24' sh

# Etcd recover a failed cluster

1. Stop the etcd service on the crashed node
systemctl stop etcd
2. Clean up the existing etcd data
rm -rf /var/lib/etcd/*
3. Get the member list from a healthy etcd node
On one of the working nodes, run:


# remove old member

ETCDCTL_API=3 etcdctl member remove 6c10765e571d3ce8 \
  --endpoints=https://192.168.1.10:2379 \
  --cacert=/etc/etcd/ca.pem \
  --cert=/etc/etcd/etcd.pem \
  --key=/etc/etcd/etcd-key.pem

# then add it back, both do it from a healthy node

ETCDCTL_API=3 etcdctl member add server-3 \
  --peer-urls=https://192.168.1.2:2380 \
  --endpoints=https://192.168.1.10:2379 \
  --cacert=/etc/etcd/ca.pem \
  --cert=/etc/etcd/etcd.pem \
  --key=/etc/etcd/etcd-key.pem
# Backup existing config
sudo cp /etc/kubernetes/admin.conf /etc/kubernetes/admin.conf.backup

# Generate new admin.conf
sudo kubeadm init phase kubeconfig admin

# Copy to user's home directory
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config


# 1  reset cluster
kubeadm reset

# 2 recovery etcd from backup

sudo docker run --rm \
-v '/mnt/backups:/backup' \
-v '/var/lib/etcd:/var/lib/etcd' \
--env ETCDCTL_API=3 \
'k8s.gcr.io/etcd:3.2.24' \
/bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-04-26-05_00_11_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd"

# install nfs-common

sudo apt install nfs-common

## update docker cgroup

vim /lib/systemd/system/docker.service

# update
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --exec-opt native.cgroupdriver=systemd

sudo systemctl daemon-reload
sudo systemctl restart docker

# check cgroup
docker system info --format '{{.CgroupDriver}}'

# if it fails at

[ERROR CRI]: container runtime is not running: output: time="2021-05-28T14:11:57+08:00" level=fatal msg="getting status of runtime failed: rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.RuntimeService"

# then

vim /etc/containerd/config.toml

# comment out

disabled_plugins = ["cri"]

# restart containerd

systemctl restart containerd

# re init cluster
sudo kubeadm init --ignore-preflight-errors=DirAvailable--var-lib-etcd --control-plane-endpoint 192.168.1.2 \
--pod-network-cidr=10.244.0.0/16 --v 5


yo# check logs
journalctl -xeu kubelet -f

# install fiannel
kubectl -n kube-system apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

## Add new nodes as control plane

# first prepre certificate key, run this one the existing control plan
# kubeadm init phase upload-certs --upload-certs

# when we have use yaml to manage cluster then use
sudo kubeadm init phase upload-certs --upload-certs --config cluster.yaml

# print join command
sudo kubeadm token create --print-join-command


# kubelet docker cgroupfs issue

vim /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
# update KUBELET_CGROUP_ARGS=--cgroup-driver=systemd to KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs



# run the following the node to be added
kubeadm join 192.168.1.2:6443 --token xxx --discovery-token-ca-cert-hash sha256:xxx --control-plane --certificate-key xxx


# disbale swap

sudo swapoff -a
sudo rm /swap.img
sudo vim /etc/fstab  # and remove swap
# check
sudo swapon --show


# to restore postgres db from dump file

cat ~/Downloads/dump-2022-03-21_05-48-05_UTC.tar.gz| gunzip | psql --set ON_ERROR_STOP=off postgres  -h 192.168.1.2 -p 30973 -U postgres


## upgrade from cluster yaml
# some common errors
# --ignore-preflight-errors=ControlPlaneNodesReady
sudo kubeadm init --config cluster.yaml --upload-certs --ignore-preflight-errors=DirAvailable--var-lib-etcd --v=5

# diff
sudo kubeadm upgrade diff --config cluster.yaml --v=5

# plan
sudo kubeadm upgrade plan --config cluster.yaml --v=5

# apply
sudo kubeadm upgrade apply --config cluster.yaml --ignore-preflight-errors all --v=5


# kubelet options

sudo vim /var/lib/kubelet/kubeadm-flags.env

# check file open count per process
# https://www.cyberciti.biz/faq/reload-sysctl-conf-on-linux-using-sysctl/
lsof | awk '{print $2}' | sort | uniq -c | sort -n

# too many open files
sudo vim /etc/sysctl.conf

fs.inotify.max_user_instances=2048
fs.file-max = 500000
fs.inotify.max_user_watches=655360

# apply change
sudo sysctl --system



### client add

# install crio

https://computingforgeeks.com/install-cri-o-container-runtime-on-ubuntu-linux/

# update cgroup driver
https://kubernetes.io/docs/setup/production-environment/container-runtimes/#cgroup-driver

# install kubeadm and kublet

# install some old version of a package

apt list --all-versions kubeadm

# then install

sudo apt install kubeadm=1.27.6-00

# Datadog crash probably related to redis, please make sure redis is healthy


### Restore files from nas, with progress

# rsync -havzP --stats rsync://junv@192.168.1.4/backups/192.168.1.2/redis /mnt/k8s


# Disk full


# check large files
sudo du -xh --max-depth=1 /mnt/k8s | sort -hr | head -n 15

# Revert your hard reset

git reflog
```

```
git reset --hard HEAD@{1}


# vmware exsi complains volume is full
# There is no more space for virtual disk 'server-3-000006.vmdk'. You might be able to continue this session by freeing disk space on the relevant volume, and clicking Retry. Click Cancel to terminate this session.
# It was because virtual machine consumes space, so does the snapshot!

# enable vmware ssh through UI and then Find large files
df -h
find /vmfs/volumes/tiplus-ssd/ -type f -size +1000000k -exec ls -lh {} \;


# grafana recover admin password

grafana-cli admin reset-admin-password xxxx
