diff --git a/apply.sh b/apply.sh index c47b558..f579557 100755 --- a/apply.sh +++ b/apply.sh @@ -163,3 +163,12 @@ kubectl get secret minio --namespace=db -o yaml | sed 's/namespace: db/namespace # namecheap ddns kubectl -n home-apps create secret generic namecheap-password --from-literal=password=xxxx + +# es + +k create namespace es +k apply -f es.yaml + +# fluent bit +helm repo add fluent https://fluent.github.io/helm-charts +helm upgrade --install fluent-bit fluent/fluent-bit -f fluentbit-values.yaml diff --git a/es.yaml b/es.yaml new file mode 100644 index 0000000..66c8c3a --- /dev/null +++ b/es.yaml @@ -0,0 +1,178 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch + labels: + app: elasticsearch +spec: + replicas: 1 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + name: elasticsearch + spec: + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9200 + name: http-es + protocol: TCP + - containerPort: 9300 + name: es + protocol: TCP + env: + - name: discovery.type + value: single-node + volumeMounts: + - name: es + mountPath: /usr/share/elasticsearch/data + subPath: elasticsearch + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 1200Mi + volumes: + - name: es + persistentVolumeClaim: + claimName: nfs-es +--- + +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + labels: + name: elasticsearch +spec: + type: ClusterIP + selector: + app: elasticsearch + ports: + - name: elasticsearch + targetPort: http-es + port: 9200 + protocol: TCP + - name: es + targetPort: es + port: 9300 + protocol: TCP +--- + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: elasticsearch-ingress + annotations: + kubernetes.io/ingress.class: "nginx" + kubernetes.io/tls-acme: "true" + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/auth-url: "https://id.junv.me/oauth2/auth" + nginx.ingress.kubernetes.io/auth-signin: "https://id.junv.me/oauth2/start?rd=https%3A%2F%2F$host$request_uri" +spec: + tls: + - hosts: + - elasticsearch.junv.me + secretName: elasticsearch-tls + rules: + - host: elasticsearch.junv.me + http: + paths: + - backend: + service: + name: elasticsearch + port: + number: 9200 + path: / + pathType: Prefix + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + labels: + app: kibana +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + template: + metadata: + labels: + app: kibana + name: kibana + spec: + containers: + - name: kibana + image: docker.elastic.co/kibana/kibana:7.13.2 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5601 + name: http-kibana + protocol: TCP + env: + - name: ELASTICSEARCH_HOSTS + value: http://elasticsearch.es.svc.cluster.local:9200 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 400m + memory: 1024Mi +--- + +apiVersion: v1 +kind: Service +metadata: + name: kibana + labels: + name: kibana +spec: + type: ClusterIP + selector: + app: kibana + ports: + - name: kibana + targetPort: http-kibana + port: 5601 + protocol: TCP +--- + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: kibana-ingress + annotations: + kubernetes.io/ingress.class: "nginx" + kubernetes.io/tls-acme: "true" + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/auth-url: "https://id.junv.me/oauth2/auth" + nginx.ingress.kubernetes.io/auth-signin: "https://id.junv.me/oauth2/start?rd=https%3A%2F%2F$host$request_uri" +spec: + tls: + - hosts: + - kibana.junv.me + secretName: kibana-tls + rules: + - host: kibana.junv.me + http: + paths: + - backend: + service: + name: kibana + port: + number: 5601 + path: / + pathType: Prefix diff --git a/fluentbit-values.yaml b/fluentbit-values.yaml new file mode 100644 index 0000000..c18da31 --- /dev/null +++ b/fluentbit-values.yaml @@ -0,0 +1,281 @@ +# kind -- DaemonSet or Deployment +kind: DaemonSet + +# replicaCount -- Only applicable if kind=Deployment +replicaCount: 1 + +image: + repository: fluent/fluent-bit + pullPolicy: Always + # tag: + +testFramework: + image: + repository: busybox + pullPolicy: Always + tag: latest + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + annotations: {} + name: + +rbac: + create: true + +podSecurityPolicy: + create: false + annotations: {} + +podSecurityContext: + {} + # fsGroup: 2000 +dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots +# value: "2" +# - name: edns0 + +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "foo.local" + # - "bar.local" + +securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 2020 + labels: + {} + annotations: + {} + # prometheus.io/path: "/api/v1/metrics/prometheus" + # prometheus.io/port: "2020" + # prometheus.io/scrape: "true" + +serviceMonitor: + enabled: false + # namespace: monitoring + # interval: 10s + # scrapeTimeout: 10s + # selector: + # prometheus: my-prometheus + +prometheusRule: + enabled: false + # namespace: "" + # additionnalLabels: {} + # rules: + # - alert: NoOutputBytesProcessed + # expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0 + # annotations: + # message: | + # Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any + # bytes for at least 15 minutes. + # summary: No Output Bytes Processed + # for: 15m + # labels: + # severity: critical + +dashboards: + enabled: false + labelKey: grafana_dashboard + annotations: {} + + +livenessProbe: {} + # httpGet: + # path: / + # port: http + +readinessProbe: + # httpGet: + # path: / + # port: http + +resources: + limits: + cpu: 50m + memory: 256Mi + requests: + cpu: 20m + memory: 128Mi + +nodeSelector: {} + +tolerations: + - key: "node" + operator: "Equal" + value: "special" + effect: "NoSchedule" + +affinity: {} + +podAnnotations: {} + +podLabels: {} + +priorityClassName: "" + +env: [] + +envFrom: [] + +extraContainers: [] + # - name: do-something + # image: busybox + # command: ['do', 'something'] + +extraPorts: [] +# - port: 5170 +# containerPort: 5170 +# protocol: TCP +# name: tcp + +extraVolumes: [] + +extraVolumeMounts: [] + +updateStrategy: {} + # type: RollingUpdate + # rollingUpdate: + # maxUnavailable: 1 + +# Make use of a pre-defined configmap instead of the one templated here +existingConfigMap: "" + +networkPolicy: + enabled: false + # ingress: + # from: [] + +luaScripts: {} + +## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file +config: + service: | + [SERVICE] + Flush 1 + Daemon Off + Log_Level info + Parsers_File parsers.conf + Parsers_File custom_parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port {{ .Values.service.port }} + + ## https://docs.fluentbit.io/manual/pipeline/inputs + inputs: | + [INPUT] + Name tail + Path /var/log/containers/*.log + Parser docker + Tag kube.* + Mem_Buf_Limit 64Mb + Skip_Long_Lines On + + [INPUT] + Name systemd + Tag host.* + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + Read_From_Tail On + + ## https://docs.fluentbit.io/manual/pipeline/filters + filters: | + [FILTER] + Name kubernetes + Match kube.* + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On + + ## https://docs.fluentbit.io/manual/pipeline/outputs + outputs: | + [OUTPUT] + Name es + Match kube.* + Host elasticsearch.es.svc.cluster.local + Port 9200 + # Logstash_Format On + # Logstash_Prefix kube + Retry_Limit 20 + Buffer_Size 64KB + Trace_Error On + Replace_Dots On + [OUTPUT] + Name es + Match host.* + Host elasticsearch.es.svc.cluster.local + Port 9200 + Logstash_Format On + Logstash_Prefix node + Retry_Limit 20 + Replace_Dots On + + ## https://docs.fluentbit.io/manual/pipeline/parsers + customParsers: | + [PARSER] + Name docker_no_time + Format json + Time_Keep Off + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + +# The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname" +volumeMounts: + - name: config + mountPath: /fluent-bit/etc/fluent-bit.conf + subPath: fluent-bit.conf + - name: config + mountPath: /fluent-bit/etc/custom_parsers.conf + subPath: custom_parsers.conf + +daemonSetVolumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: etcmachineid + hostPath: + path: /etc/machine-id + type: File + +daemonSetVolumeMounts: + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: etcmachineid + mountPath: /etc/machine-id + readOnly: true + +args: [] + +command: [] + +initContainers: [] + # - name: do-something + # image: busybox + # command: ['do', 'something'] diff --git a/nfs-es.yaml b/nfs-es.yaml new file mode 100644 index 0000000..1de8836 --- /dev/null +++ b/nfs-es.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nfs-es +spec: + capacity: + storage: 100Gi + accessModes: + - ReadWriteMany + nfs: + server: 192.168.1.4 + path: "/es" +--- + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + namespace: "es" + name: "nfs-es" +spec: + storageClassName: "" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi + +--- + +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nfs-es-data +spec: + capacity: + storage: 100Gi + accessModes: + - ReadWriteMany + nfs: + server: 192.168.1.4 + path: "/es-data" +--- + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + namespace: "es" + name: "nfs-es-data" +spec: + storageClassName: "" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi