commit

recovery.sh

@@ -30,6 +30,42 @@ docker run --rm -it \
 --env ETCDCTL_API=3 \
 'k8s.gcr.io/etcd:3.2.24' sh
 
+# Etcd recover a failed cluster
+
+1. Stop the etcd service on the crashed node
+systemctl stop etcd
+2. Clean up the existing etcd data
+rm -rf /var/lib/etcd/*
+3. Get the member list from a healthy etcd node
+On one of the working nodes, run:
+
+
+# remove old member
+
+ETCDCTL_API=3 etcdctl member remove 6c10765e571d3ce8 \
+  --endpoints=https://192.168.1.10:2379 \
+  --cacert=/etc/etcd/ca.pem \
+  --cert=/etc/etcd/etcd.pem \
+  --key=/etc/etcd/etcd-key.pem
+
+# then add it back, both do it from a healthy node
+
+ETCDCTL_API=3 etcdctl member add server-3 \
+  --peer-urls=https://192.168.1.2:2380 \
+  --endpoints=https://192.168.1.10:2379 \
+  --cacert=/etc/etcd/ca.pem \
+  --cert=/etc/etcd/etcd.pem \
+  --key=/etc/etcd/etcd-key.pem
+# Backup existing config
+sudo cp /etc/kubernetes/admin.conf /etc/kubernetes/admin.conf.backup
+
+# Generate new admin.conf
+sudo kubeadm init phase kubeconfig admin
+
+# Copy to user's home directory
+mkdir -p $HOME/.kube
+sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
+sudo chown $(id -u):$(id -g) $HOME/.kube/config
 
 
 # 1  reset cluster
@@ -185,3 +221,10 @@ sudo apt install kubeadm=1.27.6-00
 ### Restore files from nas
 
 rsync -chavzP --stats rsync://junv@192.168.1.4/backups/192.168.1.2/redis /mnt/k8s
+
+
+# Disk full
+
+
+# check large files
+sudo du -xh --max-depth=1 /mnt/k8s | sort -hr | head -n 15