From 8ec516bebe8dd17479630001ac032b466d997231 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Thu, 3 Mar 2022 19:29:47 +1100 Subject: [PATCH] refine elasticsearch --- apply.sh | 8 ++ backup-cronjobs.yaml | 4 +- es.yaml | 166 +++++++++++++++++++++++++++++------------- fluentbit-values.yaml | 6 +- once-etcd-job.yaml | 45 ++++++++++++ recovery.sh | 9 +-- 6 files changed, 174 insertions(+), 64 deletions(-) create mode 100644 once-etcd-job.yaml diff --git a/apply.sh b/apply.sh index 297129c..38ce5d3 100755 --- a/apply.sh +++ b/apply.sh @@ -1,5 +1,13 @@ #! /bin/bash -eu +# deploy NFS Subdir External Provisioner to your cluster +# https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner + +helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ + +helm install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ + --set nfs.server=192.168.1.4 --set nfs.path=/k8s + # namespaces kubectl create namespace longhorn diff --git a/backup-cronjobs.yaml b/backup-cronjobs.yaml index f6127e3..ddb126f 100644 --- a/backup-cronjobs.yaml +++ b/backup-cronjobs.yaml @@ -1,4 +1,4 @@ -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: etcd-snapshot @@ -12,7 +12,7 @@ spec: containers: - name: backup # Same image as in /etc/kubernetes/manifests/etcd.yaml - image: k8s.gcr.io/etcd:3.4.3-0 + image: k8s.gcr.io/etcd:3.5.1-0 env: - name: ETCDCTL_API value: "3" diff --git a/es.yaml b/es.yaml index cc5a80d..ef1acc1 100644 --- a/es.yaml +++ b/es.yaml @@ -1,57 +1,87 @@ +# borrowed from https://www.digitalocean.com/community/tutorials/how-to-set-up-an-elasticsearch-fluentd-and-kibana-efk-logging-stack-on-kubernetes + apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: - name: elasticsearch - labels: - app: elasticsearch + name: es + namespace: es spec: - replicas: 1 + serviceName: elasticsearch + replicas: 3 selector: matchLabels: - app: elasticsearch + app: es template: metadata: labels: - app: elasticsearch - name: elasticsearch + app: es spec: - initContainers: - - name: increase-vm-max-map - image: busybox - command: ["sysctl", "-w", "vm.max_map_count=262144"] - securityContext: - privileged: true containers: - - name: elasticsearch - image: docker.elastic.co/elasticsearch/elasticsearch:8.0.1 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9200 - name: http-es - protocol: TCP - - containerPort: 9300 - name: es - protocol: TCP - env: - - name: discovery.type - value: single-node - - name: xpack.security.enabled - value: "false" - volumeMounts: - - name: es - mountPath: /usr/share/elasticsearch/data - subPath: elasticsearch - resources: - requests: - cpu: 200m - memory: 512Mi - limits: - cpu: 1000m - memory: 1200Mi - volumes: - - name: es - persistentVolumeClaim: - claimName: nfs-es + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.0.1 + resources: + requests: + cpu: 400m + memory: 600Mi + limits: + cpu: 1000m + memory: 1200Mi + ports: + - containerPort: 9200 + name: rest + protocol: TCP + - containerPort: 9300 + name: inter-node + protocol: TCP + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + env: + - name: cluster.name + value: home-es + - name: xpack.security.enabled + value: "false" + - name: node.name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: discovery.seed_hosts + value: "es-0.elasticsearch.es.svc.cluster.local,es-1.elasticsearch.es.svc.cluster.local,es-2.elasticsearch.es.svc.cluster.local" + - name: cluster.initial_master_nodes + value: "es-0,es-1,es-3" + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + initContainers: + - name: fix-permissions + image: busybox + command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"] + securityContext: + privileged: true + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + - name: increase-vm-max-map + image: busybox + command: ["sysctl", "-w", "vm.max_map_count=262144"] + securityContext: + privileged: true + - name: increase-fd-ulimit + image: busybox + command: ["sh", "-c", "ulimit -n 65536"] + securityContext: + privileged: true + volumeClaimTemplates: + - metadata: + name: data + labels: + app: es + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: nfs-client + resources: + requests: + storage: 150Gi + --- apiVersion: v1 @@ -63,14 +93,14 @@ metadata: spec: type: ClusterIP selector: - app: elasticsearch + app: es ports: - name: elasticsearch - targetPort: http-es + targetPort: rest port: 9200 protocol: TCP - name: es - targetPort: es + targetPort: inter-node port: 9300 protocol: TCP --- @@ -78,7 +108,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: elasticsearch-ingress + name: es-ingress annotations: kubernetes.io/ingress.class: "nginx" kubernetes.io/tls-acme: "true" @@ -89,17 +119,17 @@ spec: tls: - hosts: - es.junv.me - secretName: elasticsearch-tls + secretName: es-ingress-tls rules: - host: es.junv.me http: paths: - - backend: + - path: / + backend: service: name: elasticsearch port: number: 9200 - path: / pathType: Prefix --- @@ -132,13 +162,20 @@ spec: env: - name: ELASTICSEARCH_HOSTS value: http://elasticsearch.es.svc.cluster.local:9200 + volumeMounts: + - name: config + mountPath: /usr/share/kibana/config resources: requests: cpu: 100m memory: 256Mi limits: - cpu: 400m + cpu: 500m memory: 1024Mi + volumes: + - name: "config" + configMap: + name: kibana-config --- apiVersion: v1 @@ -184,3 +221,28 @@ spec: number: 5601 path: / pathType: Prefix + +--- + + +apiVersion: v1 +kind: ConfigMap +metadata: + name: kibana-config + namespace: es +data: + kibana.yml: | + --- + server.name: kibana + server.publicBaseUrl: https://kibana.junv.me + server.host: "0.0.0.0" + status.allowAnonymous: true + xpack.security.encryptionKey: "MTExMXNzY2JiZ2dnc2RzZmRmdnIyMzhyKigpJjclCg==" + xpack.encryptedSavedObjects.encryptionKey: MTExMXNzY2JiQCMlXiZnZ2dzZHNmZGZ2cjIzOHIqKCkmNyUK + xpack.reporting.roles.enabled: false + # xpack.monitoring.enabled: false + # xpack.monitoring.ui.container.elasticsearch.enabled: false + # xpack.monitoring.kibana.collection.enabled: false + # xpack.security.enabled: false + # logging.silent: false + # logging.quiet: false diff --git a/fluentbit-values.yaml b/fluentbit-values.yaml index 8f803d8..183b7d3 100644 --- a/fluentbit-values.yaml +++ b/fluentbit-values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: fluent/fluent-bit pullPolicy: Always - # tag: + tag: 1.8.13 testFramework: image: @@ -216,12 +216,13 @@ config: Host elasticsearch.es.svc.cluster.local Port 9200 Logstash_Format On - Logstash_Prefix kube + Logstash_Prefix logs-kube Logstash_DateFormat %Y-%m Retry_Limit 20 Buffer_Size 64KB Trace_Error On Replace_Dots On + Suppress_Type_Name On [OUTPUT] Name es Match host.* @@ -232,6 +233,7 @@ config: Logstash_DateFormat %Y-%m Retry_Limit 20 Replace_Dots On + Suppress_Type_Name On ## https://docs.fluentbit.io/manual/pipeline/parsers customParsers: | diff --git a/once-etcd-job.yaml b/once-etcd-job.yaml new file mode 100644 index 0000000..0c8cb33 --- /dev/null +++ b/once-etcd-job.yaml @@ -0,0 +1,45 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: etcd-snapshot + namespace: default +spec: + backoffLimit: 3 + template: + spec: + containers: + - name: backup + # Same image as in /etc/kubernetes/manifests/etcd.yaml + image: k8s.gcr.io/etcd:3.5.1-0 + env: + - name: ETCDCTL_API + value: "3" + command: ["/bin/sh"] + args: ["-c", "etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key snapshot save /backup/etcd/snapshot-$(date +%Y-%m-%d_%H-%M-%S_%Z).db"] + volumeMounts: + - mountPath: /etc/kubernetes/pki/etcd + name: etcd-certs + readOnly: true + - mountPath: /backup/etcd + subPath: etcd + name: backup + hostNetwork: true + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - name: etcd-certs + hostPath: + path: /etc/kubernetes/pki/etcd + type: Directory + - name: backup + persistentVolumeClaim: + claimName: backups-nfs-pvc diff --git a/recovery.sh b/recovery.sh index 7c37c62..3c8a0ce 100644 --- a/recovery.sh +++ b/recovery.sh @@ -1,8 +1,3 @@ -kubeadm join home.toozhao.com:6443 --token p6kkmo.3bzqcnkktlqg3t6x \ - --control-plane --discovery-token-ca-cert-hash sha256:a2dd09bb2194d7d009e4023782d668753bde4c7e6a5a9885702ff48d32a4ae2b - - - etcdctl -C http://127.0.0.1:2380 \ --ca-file=/etc/kubernetes/pki/etcd/ca.crt \ --cert-file=/etc/kubernetes/pki/etcd/peer.crt \ @@ -20,8 +15,6 @@ docker run --rm -it \ # 1 reset cluster kubeadm reset - - # 2 recovery etcd from backup docker run --rm \ @@ -29,7 +22,7 @@ docker run --rm \ -v '/var/lib/etcd:/var/lib/etcd' \ --env ETCDCTL_API=3 \ 'k8s.gcr.io/etcd:3.2.24' \ -/bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-02-06_05-00-02_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd" +/bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-02-26_05-00-03_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd" ## update docker cgroup