Update etcd jobs

backup-jobs/backup-cronjobs.yaml

@@ -17,9 +17,9 @@ spec:
                 - name: ETCDCTL_API
                   value: "3"
               command: ["/bin/sh"]
-              args: ["-c", "etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key snapshot save /backup/etcd/snapshot-$(printf \"%(%Y-%m-%d-%H:%M:%S_%Z)T\n\").db"]
+              args: ["-c", "etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/etcd/ca.pem --cert=/etc/etcd/etcd.pem --key=/etc/etcd/etcd-key.pem snapshot save /backup/etcd/snapshot-$(printf \"%(%Y-%m-%d-%H:%M:%S_%Z)T\n\").db"]
               volumeMounts:
-                - mountPath: /etc/kubernetes/pki/etcd
+                - mountPath: /etc/etcd
                   name: etcd-certs
                   readOnly: true
                 - mountPath: /backup/etcd
@@ -28,7 +28,7 @@ spec:
           hostNetwork: true
           restartPolicy: Never
           nodeSelector:
-            gateway: "true"
+            etcd: "true"
           # affinity:
           #   nodeAffinity:
           #     requiredDuringSchedulingIgnoredDuringExecution:
@@ -42,7 +42,7 @@ spec:
           volumes:
             - name: etcd-certs
               hostPath:
-                path: /etc/kubernetes/pki/etcd
+                path: /etc/etcd
                 type: Directory
             - name: backup
               persistentVolumeClaim:

cluster.yaml

@@ -0,0 +1,21 @@
+# Full configuration https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta2/
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: ClusterConfiguration
+etcd:
+  external:
+    endpoints:
+      - "https://192.168.1.22:2379"
+      - "https://192.168.1.2:2379"
+      - "https://192.168.1.11:2379"
+      - "https://192.168.1.111:2379"
+    caFile: "/etc/etcd/ca.pem"
+    certFile: "/etc/etcd/etcd.pem"
+    keyFile: "/etc/etcd/etcd-key.pem"
+networking:
+  serviceSubnet: "10.96.0.0/12"
+  podSubnet: "10.244.0.0/16"
+  dnsDomain: "cluster.local"
+controlPlaneEndpoint: "192.168.1.2:6443"
+certificatesDir: "/etc/kubernetes/pki"
+imageRepository: "k8s.gcr.io"
+clusterName: "home-cluster"

etcd.service

@@ -0,0 +1,32 @@
+# /etc/systemd/system/etcd.service
+cat <<EOF | sudo tee /etc/systemd/system/etcd.service
+[Unit]
+Description=etcd
+Documentation=https://github.com/coreos
+
+[Service]
+Type=notify
+ExecStart=/usr/local/bin/etcd \\
+  --name ${ETCD_NAME} \\
+  --cert-file=/etc/etcd/etcd.pem \\
+  --key-file=/etc/etcd/etcd-key.pem \\
+  --peer-cert-file=/etc/etcd/etcd.pem \\
+  --peer-key-file=/etc/etcd/etcd-key.pem \\
+  --trusted-ca-file=/etc/etcd/ca.pem \\
+  --peer-trusted-ca-file=/etc/etcd/ca.pem \\
+  --peer-client-cert-auth \\
+  --client-cert-auth \\
+  --initial-advertise-peer-urls https://${INTERNAL_IP}:2380 \\
+  --listen-peer-urls https://${INTERNAL_IP}:2380 \\
+  --listen-client-urls https://${INTERNAL_IP}:2379,https://127.0.0.1:2379 \\
+  --advertise-client-urls https://${INTERNAL_IP}:2379 \\
+  --initial-cluster-token etcd-cluster-0 \\
+  --initial-cluster ssd-3=https://192.168.1.22:2380,plex=https://192.168.1.11:2380,ssd-server-1=https://192.168.1.111:2380 \\
+  --initial-cluster-state new \\
+  --data-dir=/var/lib/etcd
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF

etcd.sh

@@ -0,0 +1,12 @@
+# member list
+sudo ETCDCTL_API=3 etcdctl member list \
+  --endpoints=https://127.0.0.1:2379 \
+  --cacert=/etc/etcd/ca.pem \
+  --cert=/etc/etcd/etcd.pem \
+  --key=/etc/etcd/etcd-key.pem
+
+# member add
+sudo ETCDCTL_API=3 etcdctl member add server-3 --peer-urls=https://192.168.1.2:2380  --endpoints=https://127.0.0.1:2379   --cacert=/etc/etcd/ca.pem   --cert=/etc/etcd/etcd.pem   --key=/etc/etcd/etcd-key.pem
+
+# get all keys
+sudo ETCDCTL_API=3 etcdctl get --prefix=true "" -w json  --endpoints=https://127.0.0.1:2379   --cacert=/etc/etcd/ca.pem   --cert=/etc/etcd/etcd.pem   --key=/etc/etcd/etcd-key.pem

on-demand/once-etcd-job.yaml

@@ -16,9 +16,9 @@ spec:
             - name: ETCDCTL_API
               value: "3"
           command: ["/bin/sh"]
-          args: ["-c", "etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key snapshot save /backup/etcd/snapshot-$(printf \"%(%Y-%m-%d-%H:%M:%S_%Z)T\n\").db"]
+          args: ["-c", "etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/etcd/ca.pem --cert=/etc/etcd/etcd.pem --key=/etc/etcd/etcd-key.pem snapshot save /backup/etcd/snapshot-$(printf \"%(%Y-%m-%d-%H:%M:%S_%Z)T\n\").db"]
           volumeMounts:
-            - mountPath: /etc/kubernetes/pki/etcd
+            - mountPath: /etc/etcd
               name: etcd-certs
               readOnly: true
             - mountPath: /backup/etcd
@@ -26,20 +26,12 @@ spec:
               name: backup
       hostNetwork: true
       restartPolicy: Never
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                - key: node-role.kubernetes.io/master
-                  operator: Exists
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
+      nodeSelector:
+        etcd: "true"
       volumes:
         - name: etcd-certs
           hostPath:
-            path: /etc/kubernetes/pki/etcd
+            path: /etc/etcd
             type: Directory
         - name: backup
           persistentVolumeClaim:

recovery.sh

@@ -52,8 +52,10 @@ kubectl -n kube-system apply -f https://raw.githubusercontent.com/coreos/flannel
 ## Add new nodes as control plane
 
 # first prepre certificate key, run this one the existing control plan
-kubeadm init phase upload-certs --upload-certs
+# kubeadm init phase upload-certs --upload-certs
 
+# when we have use yaml to manage cluster then use
+sudo kubeadm init phase upload-certs --upload-certs --config cluster.yaml
 
 # print join command
 sudo kubeadm token create --print-join-command
@@ -82,3 +84,18 @@ sudo swapon --show
 # to restore postgres db from dump file
 
 cat ~/Downloads/dump-2022-03-21_05-48-05_UTC.tar.gz| gunzip | psql --set ON_ERROR_STOP=off postgres  -h 192.168.1.2 -p 30973 -U postgres
+
+
+## upgrade from cluster yaml
+# some common errors
+# --ignore-preflight-errors=ControlPlaneNodesReady
+sudo kubeadm init --config cluster.yaml --upload-certs --ignore-preflight-errors=DirAvailable--var-lib-etcd --v=5
+
+# diff
+sudo kubeadm upgrade diff --config cluster.yaml --v=5
+
+# plan
+sudo kubeadm upgrade plan --config cluster.yaml --v=5
+
+# apply
+sudo kubeadm upgrade apply --config cluster.yaml --ignore-preflight-errors all --force --v=5