add back fluentbit

This commit is contained in:
2022-04-30 10:48:38 +10:00
parent 06a89e19a9
commit 1d118e77ed
+339 -306
View File
@@ -1,353 +1,386 @@
# apiVersion: argoproj.io/v1alpha1
# kind: Application
# metadata:
# name: fluent-bit
# namespace: argocd
# spec:
# project: default
# source:
# chart: fluent-bit
# repoURL: https://fluent.github.io/helm-charts
# path: fluent-bit
# targetRevision: 0.19.22
# helm:
# releaseName: fluent-bit
# # valueFiles: ['fluent-bit-values.yaml']
# values: |
# # kind -- DaemonSet or Deployment
# kind: DaemonSet
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: fluent-bit
namespace: argocd
spec:
project: default
source:
chart: fluent-bit
repoURL: https://fluent.github.io/helm-charts
path: fluent-bit
targetRevision: 0.20.0
helm:
releaseName: fluent-bit
values: |
# kind -- DaemonSet or Deployment
kind: DaemonSet
# # replicaCount -- Only applicable if kind=Deployment
# replicaCount: 1
# replicaCount -- Only applicable if kind=Deployment
replicaCount: 1
# image:
# repository: fluent/fluent-bit
# pullPolicy: Always
# tag: 1.8.13
image:
repository: fluent/fluent-bit
pullPolicy: Always
tag: 1.8.13
# testFramework:
# image:
# repository: busybox
# pullPolicy: Always
# tag: latest
testFramework:
image:
repository: busybox
pullPolicy: Always
tag: latest
# imagePullSecrets: []
# nameOverride: ""
# fullnameOverride: ""
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
# serviceAccount:
# create: true
# annotations: {}
# name:
serviceAccount:
create: true
annotations: {}
name:
# rbac:
# create: true
rbac:
create: true
# podSecurityPolicy:
# create: false
# annotations: {}
podSecurityPolicy:
create: false
annotations: {}
# podSecurityContext:
# {}
# # fsGroup: 2000
# dnsConfig:
# {}
# # nameservers:
# # - 1.2.3.4
# # searches:
# # - ns1.svc.cluster-domain.example
# # - my.dns.search.suffix
# # options:
# # - name: ndots
# # value: "2"
# # - name: edns0
podSecurityContext:
{}
# fsGroup: 2000
dnsConfig:
{}
# nameservers:
# - 1.2.3.4
# searches:
# - ns1.svc.cluster-domain.example
# - my.dns.search.suffix
# options:
# - name: ndots
# value: "2"
# - name: edns0
# hostAliases:
# []
# # - ip: "1.2.3.4"
# # hostnames:
# # - "foo.local"
# # - "bar.local"
hostAliases:
[]
# - ip: "1.2.3.4"
# hostnames:
# - "foo.local"
# - "bar.local"
# securityContext:
# {}
# # capabilities:
# # drop:
# # - ALL
# # readOnlyRootFilesystem: true
# # runAsNonRoot: true
# # runAsUser: 1000
securityContext:
{}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
# service:
# type: ClusterIP
# port: 2020
# labels: {}
# annotations:
# {}
# # prometheus.io/path: "/api/v1/metrics/prometheus"
# # prometheus.io/port: "2020"
# # prometheus.io/scrape: "true"
service:
type: ClusterIP
port: 2020
labels: {}
annotations:
{}
# prometheus.io/path: "/api/v1/metrics/prometheus"
# prometheus.io/port: "2020"
# prometheus.io/scrape: "true"
# serviceMonitor:
# enabled: false
# # namespace: monitoring
# # interval: 10s
# # scrapeTimeout: 10s
# # selector:
# # prometheus: my-prometheus
serviceMonitor:
enabled: false
# namespace: monitoring
# interval: 10s
# scrapeTimeout: 10s
# selector:
# prometheus: my-prometheus
# prometheusRule:
# enabled: false
# # namespace: ""
# # additionnalLabels: {}
# # rules:
# # - alert: NoOutputBytesProcessed
# # expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
# # annotations:
# # message: |
# # Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
# # bytes for at least 15 minutes.
# # summary: No Output Bytes Processed
# # for: 15m
# # labels:
# # severity: critical
prometheusRule:
enabled: false
# namespace: ""
# additionnalLabels: {}
# rules:
# - alert: NoOutputBytesProcessed
# expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
# annotations:
# message: |
# Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
# bytes for at least 15 minutes.
# summary: No Output Bytes Processed
# for: 15m
# labels:
# severity: critical
# dashboards:
# enabled: false
# labelKey: grafana_dashboard
# annotations: {}
dashboards:
enabled: false
labelKey: grafana_dashboard
annotations: {}
# livenessProbe:
# {}
# # httpGet:
# # path: /
# # port: http
livenessProbe:
{}
# httpGet:
# path: /
# port: http
# readinessProbe:
# # httpGet:
# # path: /
# # port: http
readinessProbe:
# httpGet:
# path: /
# port: http
# resources:
# limits:
# cpu: 50m
# memory: 256Mi
# requests:
# cpu: 20m
# memory: 128Mi
resources:
limits:
cpu: 50m
memory: 256Mi
requests:
cpu: 20m
memory: 128Mi
# nodeSelector: {}
nodeSelector: {}
# tolerations:
# - effect: NoSchedule
# operator: Exists
# - key: "node"
# operator: "Equal"
# value: "special"
# effect: "NoSchedule"
tolerations:
- effect: NoSchedule
operator: Exists
- key: "node"
operator: "Equal"
value: "special"
effect: "NoSchedule"
# affinity: {}
affinity: {}
# podAnnotations: {}
podAnnotations: {}
# podLabels: {}
podLabels: {}
# priorityClassName: ""
priorityClassName: ""
# env: []
env: []
# envFrom: []
envFrom: []
# extraPorts: []
# # - port: 5170
# # containerPort: 5170
# # protocol: TCP
# # name: tcp
extraPorts: []
# - port: 5170
# containerPort: 5170
# protocol: TCP
# name: tcp
# extraContainers: []
# # - name: toolbox
# # image: alpine
# # command: ['sleep', '31536000']
extraContainers: []
# - name: toolbox
# image: alpine
# command: ['sleep', '31536000']
# extraVolumes:
# - name: plugins
# persistentVolumeClaim:
# claimName: nfs-k8s-share
extraVolumes:
- name: plugins
persistentVolumeClaim:
claimName: nfs-k8s-share
# extraVolumeMounts:
# - name: plugins
# mountPath: /mnt
extraVolumeMounts:
- name: plugins
mountPath: /mnt
# updateStrategy:
# {}
# # type: RollingUpdate
# # rollingUpdate:
# # maxUnavailable: 1
updateStrategy:
{}
# type: RollingUpdate
# rollingUpdate:
# maxUnavailable: 1
# # Make use of a pre-defined configmap instead of the one templated here
# existingConfigMap: ""
# Make use of a pre-defined configmap instead of the one templated here
existingConfigMap: ""
# networkPolicy:
# enabled: false
# # ingress:
# # from: []
networkPolicy:
enabled: false
# ingress:
# from: []
# luaScripts: {}
luaScripts: {}
# ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file
# config:
# service: |
# [SERVICE]
# Flush 1
# Daemon Off
# Log_Level info
# Parsers_File parsers.conf
# Parsers_File custom_parsers.conf
# HTTP_Server On
# HTTP_Listen 0.0.0.0
# HTTP_Port {{ .Values.service.port }}
## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file
config:
service: |
[SERVICE]
Flush 1
Daemon Off
Log_Level info
Parsers_File parsers.conf
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port {{ .Values.service.port }}
# ## https://docs.fluentbit.io/manual/pipeline/inputs
# inputs: |
# [INPUT]
# Name tail
# Path /var/log/containers/*.log
# Parser docker
# Tag kube.*
# Mem_Buf_Limit 64Mb
# Skip_Long_Lines On
# Docker_Mode On
# Docker_Mode_Flush 5
# Docker_Mode_Parser docker
## https://docs.fluentbit.io/manual/pipeline/inputs
inputs: |
[INPUT]
Name tail
Path /var/log/containers/ingress-nginx-ingress-controller-*.log
Parser docker
Tag nginx-ingress.*
Mem_Buf_Limit 4Mb
Skip_Long_Lines Off
# [INPUT]
# Name systemd
# Tag host.*
# Systemd_Filter _SYSTEMD_UNIT=kubelet.service
# Read_From_Tail On
[INPUT]
Name tail
Path /var/log/containers/*.log
Parser docker
Tag kube.*
Mem_Buf_Limit 64Mb
Skip_Long_Lines On
Docker_Mode On
Docker_Mode_Flush 5
Docker_Mode_Parser docker
# ## https://docs.fluentbit.io/manual/pipeline/filters
# ## https://www.maxmind.com/en/accounts/700873/geoip/downloads
# ## https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html
# # POST _reindex
# # {
# # "source": {
# # "index": "logs-kube.*",
# # "query": {
# # "exists": {
# # "field": "country"
# # }
# # }
# # },
# # "dest": {
# # "index": "geo-nginx-ingress"
# # },
# # "script": {
# # "source": "ctx._source.location=[ctx._source.longitude,ctx._source.latitude]"
# # }
# # }
# filters: |
# [FILTER]
# Name kubernetes
# Match kube.*
# Buffer_Size 128KB
# Merge_Log On
# Keep_Log Off
# K8S-Logging.Parser On
# K8S-Logging.Exclude On
# ## filter order matteres
# [FILTER]
# Name geoip2
# Match kube.*
# Database /mnt/GeoLite2-City.mmdb
# Lookup_key host
# Record city host %{city.names.en}
# Record latitude host %{location.latitude}
# Record longitude host %{location.longitude}
# Record country host %{country.iso_code}
# Record country_name host %{country.names.en}
# # Record postal_code host %{postal.code}
# Record region_code host %{subdivisions.0.iso_code}
# # Record region_name host %{subdivisions.0.names.en}
[INPUT]
Name systemd
Tag host.*
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Read_From_Tail On
## https://docs.fluentbit.io/manual/pipeline/filters
## https://www.maxmind.com/en/accounts/700873/geoip/downloads
# POST _reindex
# {
# "source": {
# "index": "logs-kube.*",
# "query": {
# "exists": {
# "field": "country"
# }
# }
# },
# "dest": {
# "index": "geo-nginx-ingress"
# },
# "script": {
# "source": "ctx._source.location=[ctx._source.longitude,ctx._source.latitude]"
# }
# }
filters: |
[FILTER]
Name kubernetes
Match nginx-ingress.*
Buffer_Size 128KB
Merge_Parser k8s-nginx-ingress
Merge_Log On
Keep_Log Off
K8S-Logging.Parser On
K8S-Logging.Exclude On
# ## https://docs.fluentbit.io/manual/pipeline/outputs
# outputs: |
# [OUTPUT]
# Name es
# Match kube.*
# Host elasticsearch.es.svc.cluster.local
# Port 9200
# Index logs-kube.%Y%m
# Retry_Limit 20
# Buffer_Size 64KB
# Trace_Error On
# Replace_Dots On
# Suppress_Type_Name On
# [OUTPUT]
# Name es
# Match host.*
# Host elasticsearch.es.svc.cluster.local
# Port 9200
# Index logs-node.%Y%m
# Retry_Limit 20
# Replace_Dots On
# Suppress_Type_Name On
[FILTER]
Name kubernetes
Match kube.*
Buffer_Size 128KB
Merge_Log On
Keep_Log Off
K8S-Logging.Parser On
K8S-Logging.Exclude On
# ## https://docs.fluentbit.io/manual/pipeline/parsers
# customParsers: |
# [PARSER]
# Name docker_json
# Format json
# Time_Keep Off
# Time_Key time
# Time_Format %Y-%m-%dT%H:%M:%S.%L
## filter order matteres
[FILTER]
Name geoip2
Match nginx-ingress.*
Database /mnt/GeoLite2-City.mmdb
Lookup_key host
Record city host %{city.names.en}
Record latitude host %{location.latitude}
Record longitude host %{location.longitude}
Record country host %{country.iso_code}
Record country_name host %{country.names.en}
# Record postal_code host %{postal.code}
Record region_code host %{subdivisions.0.iso_code}
Record region_name host %{subdivisions.0.names.en}
# # The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname"
# volumeMounts:
# - name: config
# mountPath: /fluent-bit/etc/fluent-bit.conf
# subPath: fluent-bit.conf
# - name: config
# mountPath: /fluent-bit/etc/custom_parsers.conf
# subPath: custom_parsers.conf
[FILTER]
Name lua
Match nginx-ingress.*
script /mnt/generate-location.lua
call generate_location
## https://docs.fluentbit.io/manual/pipeline/outputs
outputs: |
[OUTPUT]
Name es
Match nginx-ingress.*
Host elasticsearch.es.svc.cluster.local
Port 9200
Index geo-nginx-ingress.%Y
Retry_Limit 10
Buffer_Size 1024KB
Trace_Error On
Replace_Dots On
Suppress_Type_Name On
# daemonSetVolumes:
# - name: varlog
# hostPath:
# path: /var/log
# - name: varlibdockercontainers
# hostPath:
# path: /var/lib/docker/containers
# - name: etcmachineid
# hostPath:
# path: /etc/machine-id
# type: File
[OUTPUT]
Name es
Match kube.*
Host elasticsearch.es.svc.cluster.local
Port 9200
Index logs-kube.%Y%m
Retry_Limit 20
Buffer_Size 64KB
Trace_Error On
Replace_Dots On
Suppress_Type_Name On
[OUTPUT]
Name es
Match host.*
Host elasticsearch.es.svc.cluster.local
Port 9200
Index logs-node.%Y%m
Retry_Limit 20
Replace_Dots On
Suppress_Type_Name On
# daemonSetVolumeMounts:
# - name: varlog
# mountPath: /var/log
# - name: varlibdockercontainers
# mountPath: /var/lib/docker/containers
# readOnly: true
# - name: etcmachineid
# mountPath: /etc/machine-id
# readOnly: true
## https://docs.fluentbit.io/manual/pipeline/parsers
customParsers: |
[PARSER]
Name docker_json
Format json
Time_Keep Off
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
# args: []
# The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname"
volumeMounts:
- name: config
mountPath: /fluent-bit/etc/fluent-bit.conf
subPath: fluent-bit.conf
- name: config
mountPath: /fluent-bit/etc/custom_parsers.conf
subPath: custom_parsers.conf
# command: []
daemonSetVolumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: etcmachineid
hostPath:
path: /etc/machine-id
type: File
# initContainers:
# []
# # - name: do-something
# # image: busybox
# # command: ['do', 'something']
daemonSetVolumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: etcmachineid
mountPath: /etc/machine-id
readOnly: true
# destination:
# server: https://192.168.1.2:6443
# namespace: es
# syncPolicy:
# automated:
# prune: true
# selfHeal: true
args: []
command: []
initContainers:
[]
# - name: do-something
# image: busybox
# command: ['do', 'something']
destination:
server: https://192.168.1.2:6443
namespace: es
syncPolicy:
automated:
prune: true
selfHeal: true