refine elasticsearch

apply.sh

@@ -1,5 +1,13 @@
 #! /bin/bash -eu
 
+# deploy NFS Subdir External Provisioner to your cluster
+# https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner
+
+helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
+
+helm install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
+  --set nfs.server=192.168.1.4 --set nfs.path=/k8s
+
 # namespaces
 kubectl create namespace longhorn
 

backup-cronjobs.yaml

@@ -1,4 +1,4 @@
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: etcd-snapshot
@@ -12,7 +12,7 @@ spec:
           containers:
             - name: backup
               # Same image as in /etc/kubernetes/manifests/etcd.yaml
-              image: k8s.gcr.io/etcd:3.4.3-0
+              image: k8s.gcr.io/etcd:3.5.1-0
               env:
                 - name: ETCDCTL_API
                   value: "3"

es.yaml

@@ -1,57 +1,87 @@
+# borrowed from https://www.digitalocean.com/community/tutorials/how-to-set-up-an-elasticsearch-fluentd-and-kibana-efk-logging-stack-on-kubernetes
+
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
-  name: elasticsearch
+  name: es
-  labels:
+  namespace: es
-    app: elasticsearch
 spec:
-  replicas: 1
+  serviceName: elasticsearch
+  replicas: 3
   selector:
     matchLabels:
-      app: elasticsearch
+      app: es
   template:
     metadata:
       labels:
-        app: elasticsearch
+        app: es
-        name: elasticsearch
     spec:
-      initContainers:
-        - name: increase-vm-max-map
-          image: busybox
-          command: ["sysctl", "-w", "vm.max_map_count=262144"]
-          securityContext:
-            privileged: true
       containers:
-        - name: elasticsearch
+      - name: elasticsearch
-          image: docker.elastic.co/elasticsearch/elasticsearch:8.0.1
+        image: docker.elastic.co/elasticsearch/elasticsearch:8.0.1
-          imagePullPolicy: IfNotPresent
+        resources:
-          ports:
+          requests:
-            - containerPort: 9200
+            cpu: 400m
-              name: http-es
+            memory: 600Mi
-              protocol: TCP
+          limits:
-            - containerPort: 9300
+            cpu: 1000m
-              name: es
+            memory: 1200Mi
-              protocol: TCP
+        ports:
-          env:
+        - containerPort: 9200
-            - name: discovery.type
+          name: rest
-              value: single-node
+          protocol: TCP
-            - name: xpack.security.enabled
+        - containerPort: 9300
-              value: "false"
+          name: inter-node
-          volumeMounts:
+          protocol: TCP
-            - name: es
+        volumeMounts:
-              mountPath: /usr/share/elasticsearch/data
+        - name: data
-              subPath: elasticsearch
+          mountPath: /usr/share/elasticsearch/data
-          resources:
+        env:
-            requests:
+          - name: cluster.name
-              cpu: 200m
+            value: home-es
-              memory: 512Mi
+          - name: xpack.security.enabled
-            limits:
+            value: "false"
-              cpu: 1000m
+          - name: node.name
-              memory: 1200Mi
+            valueFrom:
-      volumes:
+              fieldRef:
-      - name: es
+                fieldPath: metadata.name
-        persistentVolumeClaim:
+          - name: discovery.seed_hosts
-          claimName: nfs-es
+            value: "es-0.elasticsearch.es.svc.cluster.local,es-1.elasticsearch.es.svc.cluster.local,es-2.elasticsearch.es.svc.cluster.local"
+          - name: cluster.initial_master_nodes
+            value: "es-0,es-1,es-3"
+          - name: ES_JAVA_OPTS
+            value: "-Xms512m -Xmx512m"
+      initContainers:
+      - name: fix-permissions
+        image: busybox
+        command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: data
+          mountPath: /usr/share/elasticsearch/data
+      - name: increase-vm-max-map
+        image: busybox
+        command: ["sysctl", "-w", "vm.max_map_count=262144"]
+        securityContext:
+          privileged: true
+      - name: increase-fd-ulimit
+        image: busybox
+        command: ["sh", "-c", "ulimit -n 65536"]
+        securityContext:
+          privileged: true
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+      labels:
+        app: es
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      storageClassName: nfs-client
+      resources:
+        requests:
+          storage: 150Gi
+
 ---
 
 apiVersion: v1
@@ -63,14 +93,14 @@ metadata:
 spec:
   type: ClusterIP
   selector:
-    app: elasticsearch
+    app: es
   ports:
     - name: elasticsearch
-      targetPort: http-es
+      targetPort: rest
       port: 9200
       protocol: TCP
     - name: es
-      targetPort: es
+      targetPort: inter-node
       port: 9300
       protocol: TCP
 ---
@@ -78,7 +108,7 @@ spec:
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
-  name: elasticsearch-ingress
+  name: es-ingress
   annotations:
     kubernetes.io/ingress.class: "nginx"
     kubernetes.io/tls-acme: "true"
@@ -89,17 +119,17 @@ spec:
   tls:
   - hosts:
     - es.junv.me
-    secretName: elasticsearch-tls
+    secretName: es-ingress-tls
   rules:
   - host: es.junv.me
     http:
       paths:
-      - backend:
+      - path: /
+        backend:
           service:
             name: elasticsearch
             port:
               number: 9200
-        path: /
         pathType: Prefix
 
 ---
@@ -132,13 +162,20 @@ spec:
           env:
             - name: ELASTICSEARCH_HOSTS
               value: http://elasticsearch.es.svc.cluster.local:9200
+          volumeMounts:
+            - name: config
+              mountPath: /usr/share/kibana/config
           resources:
             requests:
               cpu: 100m
               memory: 256Mi
             limits:
-              cpu: 400m
+              cpu: 500m
               memory: 1024Mi
+      volumes:
+        - name: "config"
+          configMap:
+            name: kibana-config
 ---
 
 apiVersion: v1
@@ -184,3 +221,28 @@ spec:
               number: 5601
         path: /
         pathType: Prefix
+
+---
+
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kibana-config
+  namespace: es
+data:
+  kibana.yml: |
+    ---
+    server.name: kibana
+    server.publicBaseUrl: https://kibana.junv.me
+    server.host: "0.0.0.0"
+    status.allowAnonymous: true
+    xpack.security.encryptionKey: "MTExMXNzY2JiZ2dnc2RzZmRmdnIyMzhyKigpJjclCg=="
+    xpack.encryptedSavedObjects.encryptionKey: MTExMXNzY2JiQCMlXiZnZ2dzZHNmZGZ2cjIzOHIqKCkmNyUK
+    xpack.reporting.roles.enabled: false
+    # xpack.monitoring.enabled: false
+    # xpack.monitoring.ui.container.elasticsearch.enabled: false
+    # xpack.monitoring.kibana.collection.enabled: false
+    # xpack.security.enabled: false
+    # logging.silent: false
+    # logging.quiet: false

fluentbit-values.yaml

@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: fluent/fluent-bit
   pullPolicy: Always
-  # tag:
+  tag: 1.8.13
 
 testFramework:
   image:
@@ -216,12 +216,13 @@ config:
         Host elasticsearch.es.svc.cluster.local
         Port 9200
         Logstash_Format On
-        Logstash_Prefix kube
+        Logstash_Prefix logs-kube
         Logstash_DateFormat %Y-%m
         Retry_Limit 20
         Buffer_Size 64KB
         Trace_Error On
         Replace_Dots On
+        Suppress_Type_Name On
     [OUTPUT]
         Name es
         Match host.*
@@ -232,6 +233,7 @@ config:
         Logstash_DateFormat %Y-%m
         Retry_Limit 20
         Replace_Dots    On
+        Suppress_Type_Name On
 
   ## https://docs.fluentbit.io/manual/pipeline/parsers
   customParsers: |

once-etcd-job.yaml

@@ -0,0 +1,45 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: etcd-snapshot
+  namespace: default
+spec:
+  backoffLimit: 3
+  template:
+    spec:
+      containers:
+        - name: backup
+          # Same image as in /etc/kubernetes/manifests/etcd.yaml
+          image: k8s.gcr.io/etcd:3.5.1-0
+          env:
+            - name: ETCDCTL_API
+              value: "3"
+          command: ["/bin/sh"]
+          args: ["-c", "etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key snapshot save /backup/etcd/snapshot-$(date +%Y-%m-%d_%H-%M-%S_%Z).db"]
+          volumeMounts:
+            - mountPath: /etc/kubernetes/pki/etcd
+              name: etcd-certs
+              readOnly: true
+            - mountPath: /backup/etcd
+              subPath: etcd
+              name: backup
+      hostNetwork: true
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: node-role.kubernetes.io/master
+                  operator: Exists
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+      volumes:
+        - name: etcd-certs
+          hostPath:
+            path: /etc/kubernetes/pki/etcd
+            type: Directory
+        - name: backup
+          persistentVolumeClaim:
+            claimName: backups-nfs-pvc

recovery.sh

@@ -1,8 +1,3 @@
-kubeadm join home.toozhao.com:6443 --token p6kkmo.3bzqcnkktlqg3t6x  \
-  --control-plane --discovery-token-ca-cert-hash sha256:a2dd09bb2194d7d009e4023782d668753bde4c7e6a5a9885702ff48d32a4ae2b
-
-
-
 etcdctl -C http://127.0.0.1:2380 \
   --ca-file=/etc/kubernetes/pki/etcd/ca.crt     \
   --cert-file=/etc/kubernetes/pki/etcd/peer.crt     \
@@ -20,8 +15,6 @@ docker run --rm -it \
 # 1  reset cluster
 kubeadm reset
 
-
-
 # 2 recovery etcd from backup
 
 docker run --rm \
@@ -29,7 +22,7 @@ docker run --rm \
 -v '/var/lib/etcd:/var/lib/etcd' \
 --env ETCDCTL_API=3 \
 'k8s.gcr.io/etcd:3.2.24' \
-/bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-02-06_05-00-02_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd"
+/bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-02-26_05-00-03_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd"
 
 
 ## update docker cgroup