From 1d118e77ed59a2efa95677352bbac667e03764e5 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Sat, 30 Apr 2022 10:48:38 +1000 Subject: [PATCH] add back fluentbit --- manifests/fluent-bit.yaml | 645 ++++++++++++++++++++------------------ 1 file changed, 339 insertions(+), 306 deletions(-) diff --git a/manifests/fluent-bit.yaml b/manifests/fluent-bit.yaml index 497dbfb..8e13ef3 100644 --- a/manifests/fluent-bit.yaml +++ b/manifests/fluent-bit.yaml @@ -1,353 +1,386 @@ -# apiVersion: argoproj.io/v1alpha1 -# kind: Application -# metadata: -# name: fluent-bit -# namespace: argocd -# spec: -# project: default -# source: -# chart: fluent-bit -# repoURL: https://fluent.github.io/helm-charts -# path: fluent-bit -# targetRevision: 0.19.22 -# helm: -# releaseName: fluent-bit -# # valueFiles: ['fluent-bit-values.yaml'] -# values: | -# # kind -- DaemonSet or Deployment -# kind: DaemonSet +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: fluent-bit + namespace: argocd +spec: + project: default + source: + chart: fluent-bit + repoURL: https://fluent.github.io/helm-charts + path: fluent-bit + targetRevision: 0.20.0 + helm: + releaseName: fluent-bit + values: | + # kind -- DaemonSet or Deployment + kind: DaemonSet -# # replicaCount -- Only applicable if kind=Deployment -# replicaCount: 1 + # replicaCount -- Only applicable if kind=Deployment + replicaCount: 1 -# image: -# repository: fluent/fluent-bit -# pullPolicy: Always -# tag: 1.8.13 + image: + repository: fluent/fluent-bit + pullPolicy: Always + tag: 1.8.13 -# testFramework: -# image: -# repository: busybox -# pullPolicy: Always -# tag: latest + testFramework: + image: + repository: busybox + pullPolicy: Always + tag: latest -# imagePullSecrets: [] -# nameOverride: "" -# fullnameOverride: "" + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" -# serviceAccount: -# create: true -# annotations: {} -# name: + serviceAccount: + create: true + annotations: {} + name: -# rbac: -# create: true + rbac: + create: true -# podSecurityPolicy: -# create: false -# annotations: {} + podSecurityPolicy: + create: false + annotations: {} -# podSecurityContext: -# {} -# # fsGroup: 2000 -# dnsConfig: -# {} -# # nameservers: -# # - 1.2.3.4 -# # searches: -# # - ns1.svc.cluster-domain.example -# # - my.dns.search.suffix -# # options: -# # - name: ndots -# # value: "2" -# # - name: edns0 + podSecurityContext: + {} + # fsGroup: 2000 + dnsConfig: + {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 -# hostAliases: -# [] -# # - ip: "1.2.3.4" -# # hostnames: -# # - "foo.local" -# # - "bar.local" + hostAliases: + [] + # - ip: "1.2.3.4" + # hostnames: + # - "foo.local" + # - "bar.local" -# securityContext: -# {} -# # capabilities: -# # drop: -# # - ALL -# # readOnlyRootFilesystem: true -# # runAsNonRoot: true -# # runAsUser: 1000 + securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 -# service: -# type: ClusterIP -# port: 2020 -# labels: {} -# annotations: -# {} -# # prometheus.io/path: "/api/v1/metrics/prometheus" -# # prometheus.io/port: "2020" -# # prometheus.io/scrape: "true" + service: + type: ClusterIP + port: 2020 + labels: {} + annotations: + {} + # prometheus.io/path: "/api/v1/metrics/prometheus" + # prometheus.io/port: "2020" + # prometheus.io/scrape: "true" -# serviceMonitor: -# enabled: false -# # namespace: monitoring -# # interval: 10s -# # scrapeTimeout: 10s -# # selector: -# # prometheus: my-prometheus + serviceMonitor: + enabled: false + # namespace: monitoring + # interval: 10s + # scrapeTimeout: 10s + # selector: + # prometheus: my-prometheus -# prometheusRule: -# enabled: false -# # namespace: "" -# # additionnalLabels: {} -# # rules: -# # - alert: NoOutputBytesProcessed -# # expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0 -# # annotations: -# # message: | -# # Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any -# # bytes for at least 15 minutes. -# # summary: No Output Bytes Processed -# # for: 15m -# # labels: -# # severity: critical + prometheusRule: + enabled: false + # namespace: "" + # additionnalLabels: {} + # rules: + # - alert: NoOutputBytesProcessed + # expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0 + # annotations: + # message: | + # Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any + # bytes for at least 15 minutes. + # summary: No Output Bytes Processed + # for: 15m + # labels: + # severity: critical -# dashboards: -# enabled: false -# labelKey: grafana_dashboard -# annotations: {} + dashboards: + enabled: false + labelKey: grafana_dashboard + annotations: {} -# livenessProbe: -# {} -# # httpGet: -# # path: / -# # port: http + livenessProbe: + {} + # httpGet: + # path: / + # port: http -# readinessProbe: -# # httpGet: -# # path: / -# # port: http + readinessProbe: + # httpGet: + # path: / + # port: http -# resources: -# limits: -# cpu: 50m -# memory: 256Mi -# requests: -# cpu: 20m -# memory: 128Mi + resources: + limits: + cpu: 50m + memory: 256Mi + requests: + cpu: 20m + memory: 128Mi -# nodeSelector: {} + nodeSelector: {} -# tolerations: -# - effect: NoSchedule -# operator: Exists -# - key: "node" -# operator: "Equal" -# value: "special" -# effect: "NoSchedule" + tolerations: + - effect: NoSchedule + operator: Exists + - key: "node" + operator: "Equal" + value: "special" + effect: "NoSchedule" -# affinity: {} + affinity: {} -# podAnnotations: {} + podAnnotations: {} -# podLabels: {} + podLabels: {} -# priorityClassName: "" + priorityClassName: "" -# env: [] + env: [] -# envFrom: [] + envFrom: [] -# extraPorts: [] -# # - port: 5170 -# # containerPort: 5170 -# # protocol: TCP -# # name: tcp + extraPorts: [] + # - port: 5170 + # containerPort: 5170 + # protocol: TCP + # name: tcp -# extraContainers: [] -# # - name: toolbox -# # image: alpine -# # command: ['sleep', '31536000'] + extraContainers: [] + # - name: toolbox + # image: alpine + # command: ['sleep', '31536000'] -# extraVolumes: -# - name: plugins -# persistentVolumeClaim: -# claimName: nfs-k8s-share + extraVolumes: + - name: plugins + persistentVolumeClaim: + claimName: nfs-k8s-share -# extraVolumeMounts: -# - name: plugins -# mountPath: /mnt + extraVolumeMounts: + - name: plugins + mountPath: /mnt -# updateStrategy: -# {} -# # type: RollingUpdate -# # rollingUpdate: -# # maxUnavailable: 1 + updateStrategy: + {} + # type: RollingUpdate + # rollingUpdate: + # maxUnavailable: 1 -# # Make use of a pre-defined configmap instead of the one templated here -# existingConfigMap: "" + # Make use of a pre-defined configmap instead of the one templated here + existingConfigMap: "" -# networkPolicy: -# enabled: false -# # ingress: -# # from: [] + networkPolicy: + enabled: false + # ingress: + # from: [] -# luaScripts: {} + luaScripts: {} -# ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file -# config: -# service: | -# [SERVICE] -# Flush 1 -# Daemon Off -# Log_Level info -# Parsers_File parsers.conf -# Parsers_File custom_parsers.conf -# HTTP_Server On -# HTTP_Listen 0.0.0.0 -# HTTP_Port {{ .Values.service.port }} + ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file + config: + service: | + [SERVICE] + Flush 1 + Daemon Off + Log_Level info + Parsers_File parsers.conf + Parsers_File custom_parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port {{ .Values.service.port }} -# ## https://docs.fluentbit.io/manual/pipeline/inputs -# inputs: | -# [INPUT] -# Name tail -# Path /var/log/containers/*.log -# Parser docker -# Tag kube.* -# Mem_Buf_Limit 64Mb -# Skip_Long_Lines On -# Docker_Mode On -# Docker_Mode_Flush 5 -# Docker_Mode_Parser docker + ## https://docs.fluentbit.io/manual/pipeline/inputs + inputs: | + [INPUT] + Name tail + Path /var/log/containers/ingress-nginx-ingress-controller-*.log + Parser docker + Tag nginx-ingress.* + Mem_Buf_Limit 4Mb + Skip_Long_Lines Off -# [INPUT] -# Name systemd -# Tag host.* -# Systemd_Filter _SYSTEMD_UNIT=kubelet.service -# Read_From_Tail On + [INPUT] + Name tail + Path /var/log/containers/*.log + Parser docker + Tag kube.* + Mem_Buf_Limit 64Mb + Skip_Long_Lines On + Docker_Mode On + Docker_Mode_Flush 5 + Docker_Mode_Parser docker -# ## https://docs.fluentbit.io/manual/pipeline/filters -# ## https://www.maxmind.com/en/accounts/700873/geoip/downloads -# ## https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html -# # POST _reindex -# # { -# # "source": { -# # "index": "logs-kube.*", -# # "query": { -# # "exists": { -# # "field": "country" -# # } -# # } -# # }, -# # "dest": { -# # "index": "geo-nginx-ingress" -# # }, -# # "script": { -# # "source": "ctx._source.location=[ctx._source.longitude,ctx._source.latitude]" -# # } -# # } -# filters: | -# [FILTER] -# Name kubernetes -# Match kube.* -# Buffer_Size 128KB -# Merge_Log On -# Keep_Log Off -# K8S-Logging.Parser On -# K8S-Logging.Exclude On -# ## filter order matteres -# [FILTER] -# Name geoip2 -# Match kube.* -# Database /mnt/GeoLite2-City.mmdb -# Lookup_key host -# Record city host %{city.names.en} -# Record latitude host %{location.latitude} -# Record longitude host %{location.longitude} -# Record country host %{country.iso_code} -# Record country_name host %{country.names.en} -# # Record postal_code host %{postal.code} -# Record region_code host %{subdivisions.0.iso_code} -# # Record region_name host %{subdivisions.0.names.en} + [INPUT] + Name systemd + Tag host.* + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + Read_From_Tail On + ## https://docs.fluentbit.io/manual/pipeline/filters + ## https://www.maxmind.com/en/accounts/700873/geoip/downloads + # POST _reindex + # { + # "source": { + # "index": "logs-kube.*", + # "query": { + # "exists": { + # "field": "country" + # } + # } + # }, + # "dest": { + # "index": "geo-nginx-ingress" + # }, + # "script": { + # "source": "ctx._source.location=[ctx._source.longitude,ctx._source.latitude]" + # } + # } + filters: | + [FILTER] + Name kubernetes + Match nginx-ingress.* + Buffer_Size 128KB + Merge_Parser k8s-nginx-ingress + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On -# ## https://docs.fluentbit.io/manual/pipeline/outputs -# outputs: | -# [OUTPUT] -# Name es -# Match kube.* -# Host elasticsearch.es.svc.cluster.local -# Port 9200 -# Index logs-kube.%Y%m -# Retry_Limit 20 -# Buffer_Size 64KB -# Trace_Error On -# Replace_Dots On -# Suppress_Type_Name On -# [OUTPUT] -# Name es -# Match host.* -# Host elasticsearch.es.svc.cluster.local -# Port 9200 -# Index logs-node.%Y%m -# Retry_Limit 20 -# Replace_Dots On -# Suppress_Type_Name On + [FILTER] + Name kubernetes + Match kube.* + Buffer_Size 128KB + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On -# ## https://docs.fluentbit.io/manual/pipeline/parsers -# customParsers: | -# [PARSER] -# Name docker_json -# Format json -# Time_Keep Off -# Time_Key time -# Time_Format %Y-%m-%dT%H:%M:%S.%L + ## filter order matteres + [FILTER] + Name geoip2 + Match nginx-ingress.* + Database /mnt/GeoLite2-City.mmdb + Lookup_key host + Record city host %{city.names.en} + Record latitude host %{location.latitude} + Record longitude host %{location.longitude} + Record country host %{country.iso_code} + Record country_name host %{country.names.en} + # Record postal_code host %{postal.code} + Record region_code host %{subdivisions.0.iso_code} + Record region_name host %{subdivisions.0.names.en} -# # The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname" -# volumeMounts: -# - name: config -# mountPath: /fluent-bit/etc/fluent-bit.conf -# subPath: fluent-bit.conf -# - name: config -# mountPath: /fluent-bit/etc/custom_parsers.conf -# subPath: custom_parsers.conf + [FILTER] + Name lua + Match nginx-ingress.* + script /mnt/generate-location.lua + call generate_location + ## https://docs.fluentbit.io/manual/pipeline/outputs + outputs: | + [OUTPUT] + Name es + Match nginx-ingress.* + Host elasticsearch.es.svc.cluster.local + Port 9200 + Index geo-nginx-ingress.%Y + Retry_Limit 10 + Buffer_Size 1024KB + Trace_Error On + Replace_Dots On + Suppress_Type_Name On -# daemonSetVolumes: -# - name: varlog -# hostPath: -# path: /var/log -# - name: varlibdockercontainers -# hostPath: -# path: /var/lib/docker/containers -# - name: etcmachineid -# hostPath: -# path: /etc/machine-id -# type: File + [OUTPUT] + Name es + Match kube.* + Host elasticsearch.es.svc.cluster.local + Port 9200 + Index logs-kube.%Y%m + Retry_Limit 20 + Buffer_Size 64KB + Trace_Error On + Replace_Dots On + Suppress_Type_Name On + [OUTPUT] + Name es + Match host.* + Host elasticsearch.es.svc.cluster.local + Port 9200 + Index logs-node.%Y%m + Retry_Limit 20 + Replace_Dots On + Suppress_Type_Name On -# daemonSetVolumeMounts: -# - name: varlog -# mountPath: /var/log -# - name: varlibdockercontainers -# mountPath: /var/lib/docker/containers -# readOnly: true -# - name: etcmachineid -# mountPath: /etc/machine-id -# readOnly: true + ## https://docs.fluentbit.io/manual/pipeline/parsers + customParsers: | + [PARSER] + Name docker_json + Format json + Time_Keep Off + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L -# args: [] + # The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname" + volumeMounts: + - name: config + mountPath: /fluent-bit/etc/fluent-bit.conf + subPath: fluent-bit.conf + - name: config + mountPath: /fluent-bit/etc/custom_parsers.conf + subPath: custom_parsers.conf -# command: [] + daemonSetVolumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: etcmachineid + hostPath: + path: /etc/machine-id + type: File -# initContainers: -# [] -# # - name: do-something -# # image: busybox -# # command: ['do', 'something'] + daemonSetVolumeMounts: + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: etcmachineid + mountPath: /etc/machine-id + readOnly: true -# destination: -# server: https://192.168.1.2:6443 -# namespace: es -# syncPolicy: -# automated: -# prune: true -# selfHeal: true + args: [] + + command: [] + + initContainers: + [] + # - name: do-something + # image: busybox + # command: ['do', 'something'] + + destination: + server: https://192.168.1.2:6443 + namespace: es + syncPolicy: + automated: + prune: true + selfHeal: true