home-argocd/archive/fluent-bit.yaml

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: fluent-bit
  namespace: argocd
spec:
  project: default
  source:
    chart: fluent-bit
    repoURL: https://fluent.github.io/helm-charts
    path: fluent-bit
    targetRevision: 0.24.0
    helm:
      releaseName: fluent-bit
      values: |
        # kind -- DaemonSet or Deployment
        kind: DaemonSet

        # replicaCount -- Only applicable if kind=Deployment
        replicaCount: 1

        image:
          repository: fluent/fluent-bit
          pullPolicy: Always
          tag: 1.9

        testFramework:
          image:
            repository: busybox
            pullPolicy: Always
            tag: latest

        imagePullSecrets: []
        nameOverride: ""
        fullnameOverride: ""

        serviceAccount:
          create: true
          annotations: {}
          name:

        rbac:
          create: true

        podSecurityPolicy:
          create: false
          annotations: {}

        podSecurityContext:
          {}
          # fsGroup: 2000
        dnsConfig:
          {}
          # nameservers:
          #   - 1.2.3.4
          # searches:
          #   - ns1.svc.cluster-domain.example
          #   - my.dns.search.suffix
          # options:
          #   - name: ndots
        #     value: "2"
        #   - name: edns0

        hostAliases:
          - ip: "192.168.1.2"
            hostnames:
              - "kubernetes.default.svc"


        securityContext:
          {}
          # capabilities:
          #   drop:
          #   - ALL
          # readOnlyRootFilesystem: true
          # runAsNonRoot: true
          # runAsUser: 1000

        service:
          type: ClusterIP
          port: 2020
          labels: {}
          annotations:
            {}
            # prometheus.io/path: "/api/v1/metrics/prometheus"
            # prometheus.io/port: "2020"
            # prometheus.io/scrape: "true"

        serviceMonitor:
          enabled: false
          # namespace: monitoring
          # interval: 10s
          # scrapeTimeout: 10s
          # selector:
          #  prometheus: my-prometheus

        prometheusRule:
          enabled: false
          # namespace: ""
          # additionnalLabels: {}
          # rules:
          # - alert: NoOutputBytesProcessed
          #   expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
          #   annotations:
          #     message: |
          #       Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
          #       bytes for at least 15 minutes.
          #     summary: No Output Bytes Processed
          #   for: 15m
          #   labels:
          #     severity: critical

        dashboards:
          enabled: false
          labelKey: grafana_dashboard
          annotations: {}

        livenessProbe:
          {}
          # httpGet:
          #   path: /
          #   port: http

        readinessProbe:
          # httpGet:
          #   path: /
          #   port: http

        resources:
          limits:
            cpu: 50m
            memory: 256Mi
          requests:
            cpu: 20m
            memory: 128Mi

        nodeSelector: {}

        tolerations:
          - effect: NoSchedule
            operator: Exists
          - key: "node"
            operator: "Equal"
            value: "special"
            effect: "NoSchedule"

        affinity: {}

        podAnnotations: {}

        podLabels: {}

        priorityClassName: ""

        env: []

        envFrom: []

        extraPorts: []
        #   - port: 5170
        #     containerPort: 5170
        #     protocol: TCP
        #     name: tcp

        extraContainers: []
          # - name: toolbox
          #   image: alpine
          #   command: ['sleep', '31536000']

        extraVolumes:
          - name: plugins
            persistentVolumeClaim:
              claimName: nfs-k8s-share

        extraVolumeMounts:
          - name: plugins
            mountPath: /mnt

        updateStrategy:
          {}
          # type: RollingUpdate
          # rollingUpdate:
          #   maxUnavailable: 1

        # Make use of a pre-defined configmap instead of the one templated here
        existingConfigMap: ""

        networkPolicy:
          enabled: false
          # ingress:
          #   from: []

        luaScripts: {}

        ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file
        config:
          service: |
            [SERVICE]
                Flush 1
                Daemon Off
                Log_Level info
                Parsers_File parsers.conf
                Parsers_File custom_parsers.conf
                HTTP_Server On
                HTTP_Listen 0.0.0.0
                HTTP_Port {{ .Values.service.port }}

          ## https://docs.fluentbit.io/manual/pipeline/inputs
          inputs: |
            [INPUT]
                Name tail
                Path /var/log/containers/ingress-nginx-ingress-controller-*.log
                Parser custom_cri
                Tag nginx-ingress.*
                Mem_Buf_Limit 4Mb
                Skip_Long_Lines On
            [INPUT]
                Name tail
                Path /var/log/*log
                Tag host.*
                Mem_Buf_Limit 4Mb
                Skip_Long_Lines Off

            # [INPUT]
            #     Name tail
            #     Path /var/log/containers/*.log
            #     Parser custom_cri
            #     Tag kube.*
            #     Mem_Buf_Limit 64Mb
            #     Skip_Long_Lines On
            #     Docker_Mode        On
            #     Docker_Mode_Flush  5
            #     Docker_Mode_Parser docker
            [INPUT]
                Name systemd
                Tag host.*
            #    Systemd_Filter _SYSTEMD_UNIT=kubelet.service
                Read_From_Tail On

          ## https://docs.fluentbit.io/manual/pipeline/filters
          ## https://www.maxmind.com/en/accounts/700873/geoip/downloads
          # POST _reindex
          # {
          #   "source": {
          #     "index": "logs-kube.*",
          #     "query": {
          #       "exists": {
          #         "field": "country"
          #       }
          #     }
          #   },
          #   "dest": {
          #     "index": "geo-nginx-ingress"
          #   },
          #   "script": {
          #     "source": "ctx._source.location=[ctx._source.longitude,ctx._source.latitude]"
          #   }
          # }
          filters: |
            [FILTER]
                Name kubernetes
                Match nginx-ingress.*
                Buffer_Size 128KB
                Merge_Parser k8s-nginx-ingress
                Keep_Log Off
                Merge_Log On
                tls.verify Off

            [FILTER]
                Name          nest
                Match         kube.*
                Operation     lift
                Nested_under  kubernetes
                Add_prefix    kubernetes_

            [FILTER]
                Name          nest
                Match         kube.*
                Operation     lift
                Nested_under  kubernetes_labels
                Add_prefix    kubernetes_labels_

            [FILTER]
                Name kubernetes
                Match kube.*
                Buffer_Size 3\MB
                Merge_Log On
                Keep_Log Off
                K8S-Logging.Parser On
                K8S-Logging.Exclude On
                tls.verify Off
                Kube_Tag_Prefix     kube.var.log.containers.

            ## filter order matteres
            [FILTER]
                Name geoip2
                Match nginx-ingress.*
                Database /mnt/GeoLite2-City.mmdb
                Lookup_key host
                Record city           host %{city.names.en}
                Record latitude       host %{location.latitude}
                Record longitude      host %{location.longitude}
                Record country        host %{country.iso_code}
                Record country_name   host %{country.names.en}
                # Record postal_code    host %{postal.code}
                Record region_code    host %{subdivisions.0.iso_code}
                Record region_name    host %{subdivisions.0.names.en}

            [FILTER]
                Name    lua
                Match   nginx-ingress.*
                script  /mnt/generate-location.lua
                call    generate_location
          ## https://docs.fluentbit.io/manual/pipeline/outputs
          outputs: |
            [OUTPUT]
                Name es
                Match nginx-ingress.*
                Host elasticsearch
                Port 9200
                Index geo-nginx-ingress.%Y%m
                Retry_Limit 20
                # Buffer_Size 1024KB
                Trace_Error On
                Replace_Dots On
                Suppress_Type_Name On
            # [OUTPUT]
            #     Name es
            #     Match kube.*
            #     Host elasticsearch
            #     Port 9200
            #     Index logs-kube.%Y%m
            #     Retry_Limit 20
            #     Buffer_Size 1MB
            #     Trace_Error On
            #     Replace_Dots On
            #     Suppress_Type_Name On
            [OUTPUT]
                Name es
                Match host.*
                Host elasticsearch
                Port 9200
                Index logs-node.%Y%m
                Retry_Limit 20
                Replace_Dots    On
                Suppress_Type_Name On
          ## https://docs.fluentbit.io/manual/pipeline/parsers
          customParsers: |
            [PARSER]
                # http://rubular.com/r/tjUt3Awgg4
                Name custom_cri
                Format regex
                Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
                Time_Key    time
                Time_Format %Y-%m-%dT%H:%M:%S.%L%z

        # The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname"
        volumeMounts:
          - name: config
            mountPath: /fluent-bit/etc/fluent-bit.conf
            subPath: fluent-bit.conf
          - name: config
            mountPath: /fluent-bit/etc/custom_parsers.conf
            subPath: custom_parsers.conf

        daemonSetVolumes:
          - name: varlog
            hostPath:
              path: /var/log
          - name: varlibdockercontainers
            hostPath:
              path: /var/lib/docker/containers
          - name: etcmachineid
            hostPath:
              path: /etc/machine-id
              type: File

        daemonSetVolumeMounts:
          - name: varlog
            mountPath: /var/log
          - name: varlibdockercontainers
            mountPath: /var/lib/docker/containers
            readOnly: true
          - name: etcmachineid
            mountPath: /etc/machine-id
            readOnly: true

        args: []

        command: []

        initContainers:
          []
          # - name: do-something
          #   image: busybox
          #   command: ['do', 'something']

  destination:
    server: https://kubernetes.default.svc
    namespace: es
  syncPolicy:
     automated:
       prune: true
       selfHeal: true