home/home-docker
2
0
Fork 0
mirror of https://github.com/wahyd4/home-docker.git synced 2026-08-09 04:15:52 +10:00
Code Issues Packages Projects Releases Wiki Activity

refine elasticsearch

Browse Source
This commit is contained in:
junv
2022-03-03 19:29:47 +11:00
parent f86d8b1baa
commit 8ec516bebe
6 changed files with 174 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apply.sh
+8
View File
@@ -1,5 +1,13 @@
#! /bin/bash -eu
# deploy NFS Subdir External Provisioner to your cluster
# https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner
helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
helm install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
--set nfs.server=192.168.1.4 --set nfs.path=/k8s
# namespaces
kubectl create namespace longhorn
backup-cronjobs.yaml
+2 -2
View File
@@ -1,4 +1,4 @@
apiVersion: batch/v1beta1
apiVersion: batch/v1
kind: CronJob
metadata:
name: etcd-snapshot
@@ -12,7 +12,7 @@ spec:
containers:
- name: backup
# Same image as in /etc/kubernetes/manifests/etcd.yaml
image: k8s.gcr.io/etcd:3.4.3-0
image: k8s.gcr.io/etcd:3.5.1-0
env:
- name: ETCDCTL_API
value: "3"
es.yaml
+114 -52
View File
@@ -1,57 +1,87 @@
# borrowed from https://www.digitalocean.com/community/tutorials/how-to-set-up-an-elasticsearch-fluentd-and-kibana-efk-logging-stack-on-kubernetes
apiVersion: apps/v1
kind: Deployment
kind: StatefulSet
metadata:
name: elasticsearch
labels:
app: elasticsearch
name: es
namespace: es
spec:
replicas: 1
serviceName: elasticsearch
replicas: 3
selector:
matchLabels:
app: elasticsearch
app: es
template:
metadata:
labels:
app: elasticsearch
name: elasticsearch
app: es
spec:
initContainers:
- name: increase-vm-max-map
image: busybox
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
containers:
- name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:8.0.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9200
name: http-es
protocol: TCP
- containerPort: 9300
name: es
protocol: TCP
env:
- name: discovery.type
value: single-node
- name: xpack.security.enabled
value: "false"
volumeMounts:
- name: es
mountPath: /usr/share/elasticsearch/data
subPath: elasticsearch
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: 1000m
memory: 1200Mi
volumes:
- name: es
persistentVolumeClaim:
claimName: nfs-es
- name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:8.0.1
resources:
requests:
cpu: 400m
memory: 600Mi
limits:
cpu: 1000m
memory: 1200Mi
ports:
- containerPort: 9200
name: rest
protocol: TCP
- containerPort: 9300
name: inter-node
protocol: TCP
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
env:
- name: cluster.name
value: home-es
- name: xpack.security.enabled
value: "false"
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: discovery.seed_hosts
value: "es-0.elasticsearch.es.svc.cluster.local,es-1.elasticsearch.es.svc.cluster.local,es-2.elasticsearch.es.svc.cluster.local"
- name: cluster.initial_master_nodes
value: "es-0,es-1,es-3"
- name: ES_JAVA_OPTS
value: "-Xms512m -Xmx512m"
initContainers:
- name: fix-permissions
image: busybox
command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
- name: increase-vm-max-map
image: busybox
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
- name: increase-fd-ulimit
image: busybox
command: ["sh", "-c", "ulimit -n 65536"]
securityContext:
privileged: true
volumeClaimTemplates:
- metadata:
name: data
labels:
app: es
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: nfs-client
resources:
requests:
storage: 150Gi
---
apiVersion: v1
@@ -63,14 +93,14 @@ metadata:
spec:
type: ClusterIP
selector:
app: elasticsearch
app: es
ports:
- name: elasticsearch
targetPort: http-es
targetPort: rest
port: 9200
protocol: TCP
- name: es
targetPort: es
targetPort: inter-node
port: 9300
protocol: TCP
---
@@ -78,7 +108,7 @@ spec:
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: elasticsearch-ingress
name: es-ingress
annotations:
kubernetes.io/ingress.class: "nginx"
kubernetes.io/tls-acme: "true"
@@ -89,17 +119,17 @@ spec:
tls:
- hosts:
- es.junv.me
secretName: elasticsearch-tls
secretName: es-ingress-tls
rules:
- host: es.junv.me
http:
paths:
- backend:
- path: /
backend:
service:
name: elasticsearch
port:
number: 9200
path: /
pathType: Prefix
---
@@ -132,13 +162,20 @@ spec:
env:
- name: ELASTICSEARCH_HOSTS
value: http://elasticsearch.es.svc.cluster.local:9200
volumeMounts:
- name: config
mountPath: /usr/share/kibana/config
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 400m
cpu: 500m
memory: 1024Mi
volumes:
- name: "config"
configMap:
name: kibana-config
---
apiVersion: v1
@@ -184,3 +221,28 @@ spec:
number: 5601
path: /
pathType: Prefix
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kibana-config
namespace: es
data:
kibana.yml: |
---
server.name: kibana
server.publicBaseUrl: https://kibana.junv.me
server.host: "0.0.0.0"
status.allowAnonymous: true
xpack.security.encryptionKey: "MTExMXNzY2JiZ2dnc2RzZmRmdnIyMzhyKigpJjclCg=="
xpack.encryptedSavedObjects.encryptionKey: MTExMXNzY2JiQCMlXiZnZ2dzZHNmZGZ2cjIzOHIqKCkmNyUK
xpack.reporting.roles.enabled: false
# xpack.monitoring.enabled: false
# xpack.monitoring.ui.container.elasticsearch.enabled: false
# xpack.monitoring.kibana.collection.enabled: false
# xpack.security.enabled: false
# logging.silent: false
# logging.quiet: false
fluentbit-values.yaml
+4 -2
View File
@@ -7,7 +7,7 @@ replicaCount: 1
image:
repository: fluent/fluent-bit
pullPolicy: Always
# tag:
tag: 1.8.13
testFramework:
image:
@@ -216,12 +216,13 @@ config:
Host elasticsearch.es.svc.cluster.local
Port 9200
Logstash_Format On
Logstash_Prefix kube
Logstash_Prefix logs-kube
Logstash_DateFormat %Y-%m
Retry_Limit 20
Buffer_Size 64KB
Trace_Error On
Replace_Dots On
Suppress_Type_Name On
[OUTPUT]
Name es
Match host.*
@@ -232,6 +233,7 @@ config:
Logstash_DateFormat %Y-%m
Retry_Limit 20
Replace_Dots On
Suppress_Type_Name On
## https://docs.fluentbit.io/manual/pipeline/parsers
customParsers: |
once-etcd-job.yaml
+45
View File
@@ -0,0 +1,45 @@
apiVersion: batch/v1
kind: Job
metadata:
name: etcd-snapshot
namespace: default
spec:
backoffLimit: 3
template:
spec:
containers:
- name: backup
# Same image as in /etc/kubernetes/manifests/etcd.yaml
image: k8s.gcr.io/etcd:3.5.1-0
env:
- name: ETCDCTL_API
value: "3"
command: ["/bin/sh"]
args: ["-c", "etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key snapshot save /backup/etcd/snapshot-$(date +%Y-%m-%d_%H-%M-%S_%Z).db"]
volumeMounts:
- mountPath: /etc/kubernetes/pki/etcd
name: etcd-certs
readOnly: true
- mountPath: /backup/etcd
subPath: etcd
name: backup
hostNetwork: true
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/master
operator: Exists
tolerations:
- effect: NoSchedule
operator: Exists
volumes:
- name: etcd-certs
hostPath:
path: /etc/kubernetes/pki/etcd
type: Directory
- name: backup
persistentVolumeClaim:
claimName: backups-nfs-pvc
recovery.sh
+1 -8
View File
@@ -1,8 +1,3 @@
kubeadm join home.toozhao.com:6443 --token p6kkmo.3bzqcnkktlqg3t6x \
--control-plane --discovery-token-ca-cert-hash sha256:a2dd09bb2194d7d009e4023782d668753bde4c7e6a5a9885702ff48d32a4ae2b
etcdctl -C http://127.0.0.1:2380 \
--ca-file=/etc/kubernetes/pki/etcd/ca.crt \
--cert-file=/etc/kubernetes/pki/etcd/peer.crt \
@@ -20,8 +15,6 @@ docker run --rm -it \
# 1 reset cluster
kubeadm reset
# 2 recovery etcd from backup
docker run --rm \
@@ -29,7 +22,7 @@ docker run --rm \
-v '/var/lib/etcd:/var/lib/etcd' \
--env ETCDCTL_API=3 \
'k8s.gcr.io/etcd:3.2.24' \
/bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-02-06_05-00-02_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd"
/bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-02-26_05-00-03_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd"
## update docker cgroup