update nginx and fluent bit

apply.sh

@@ -1,6 +1,6 @@
 #! /bin/bash -eu
 
-# deploy NFS Subdir External Provisioner to your cluster
+# deploy NFS Subdir External Provisioner to your cluster nfs-client
 # https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner
 
 helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/

floating-resources.yaml/k8s-share-pv.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: nfs-k8s-share
+spec:
+  capacity:
+    storage: 20Gi
+  accessModes:
+    - ReadWriteMany
+  nfs:
+    server: 192.168.1.4
+    path: "/k8s-share"
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  namespace: "es"
+  name: nfs-k8s-share
+spec:
+  storageClassName: ""
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 20Gi

fluentbit-values.yaml

@@ -34,7 +34,8 @@ podSecurityPolicy:
 podSecurityContext:
   {}
   # fsGroup: 2000
-dnsConfig: {}
+dnsConfig:
+  {}
   # nameservers:
   #   - 1.2.3.4
   # searches:
@@ -45,7 +46,8 @@ dnsConfig: {}
 #     value: "2"
 #   - name: edns0
 
-hostAliases: []
+hostAliases:
+  []
   # - ip: "1.2.3.4"
   #   hostnames:
   #   - "foo.local"
@@ -63,8 +65,7 @@ securityContext:
 service:
   type: ClusterIP
   port: 2020
-  labels:
-    {}
+  labels: {}
   annotations:
     {}
     # prometheus.io/path: "/api/v1/metrics/prometheus"
@@ -100,8 +101,8 @@ dashboards:
   labelKey: grafana_dashboard
   annotations: {}
 
-
-livenessProbe: {}
+livenessProbe:
+  {}
   # httpGet:
   #   path: /
   #   port: http
@@ -122,6 +123,8 @@ resources:
 nodeSelector: {}
 
 tolerations:
+  - effect: NoSchedule
+    operator: Exists
   - key: "node"
     operator: "Equal"
     value: "special"
@@ -139,22 +142,28 @@ env: []
 
 envFrom: []
 
-extraContainers: []
-  # - name: do-something
-  #   image: busybox
-  #   command: ['do', 'something']
-
 extraPorts: []
 #   - port: 5170
 #     containerPort: 5170
 #     protocol: TCP
 #     name: tcp
 
-extraVolumes: []
+extraContainers: []
+  # - name: toolbox
+  #   image: alpine
+  #   command: ['sleep', '31536000']
 
-extraVolumeMounts: []
+extraVolumes:
+  - name: plugins
+    persistentVolumeClaim:
+      claimName: nfs-k8s-share
 
-updateStrategy: {}
+extraVolumeMounts:
+  - name: plugins
+    mountPath: /mnt
+
+updateStrategy:
+  {}
   # type: RollingUpdate
   # rollingUpdate:
   #   maxUnavailable: 1
@@ -191,6 +200,9 @@ config:
         Tag kube.*
         Mem_Buf_Limit 64Mb
         Skip_Long_Lines On
+        Docker_Mode        On
+        Docker_Mode_Flush  5
+        Docker_Mode_Parser docker
 
     [INPUT]
         Name systemd
@@ -199,14 +211,48 @@ config:
         Read_From_Tail On
 
   ## https://docs.fluentbit.io/manual/pipeline/filters
+  ## https://www.maxmind.com/en/accounts/700873/geoip/downloads
+  # POST _reindex
+  # {
+  #   "source": {
+  #     "index": "logs-kube.*",
+  #     "query": {
+  #       "exists": {
+  #         "field": "country"
+  #       }
+  #     }
+  #   },
+  #   "dest": {
+  #     "index": "geo-nginx-ingress"
+  #   },
+  #   "script": {
+  #     "source": "ctx._source.location=[ctx._source.longitude,ctx._source.latitude]"
+  #   }
+  # }
   filters: |
     [FILTER]
         Name kubernetes
         Match kube.*
+        Buffer_Size 128KB
         Merge_Log On
         Keep_Log Off
         K8S-Logging.Parser On
         K8S-Logging.Exclude On
+    ## filter order matteres
+    [FILTER]
+        Name geoip2
+        Match kube.*
+        Database /mnt/GeoLite2-City.mmdb
+        Lookup_key host
+        Record city           host %{city.names.en}
+        Record latitude       host %{location.latitude}
+        Record longitude      host %{location.longitude}
+        Record country        host %{country.iso_code}
+        Record country_name   host %{country.names.en}
+        # Record postal_code    host %{postal.code}
+        Record region_code    host %{subdivisions.0.iso_code}
+        # Record region_name    host %{subdivisions.0.names.en}
+
 
   ## https://docs.fluentbit.io/manual/pipeline/outputs
   outputs: |
@@ -215,9 +261,7 @@ config:
         Match kube.*
         Host elasticsearch.es.svc.cluster.local
         Port 9200
-        Logstash_Format On
-        Logstash_Prefix logs-kube
-        Logstash_DateFormat %Y-%m
+        Index logs-kube.%Y%m
         Retry_Limit 20
         Buffer_Size 64KB
         Trace_Error On
@@ -228,9 +272,7 @@ config:
         Match host.*
         Host elasticsearch.es.svc.cluster.local
         Port 9200
-        Logstash_Format On
-        Logstash_Prefix node
-        Logstash_DateFormat %Y-%m
+        Index logs-node.%Y%m
         Retry_Limit 20
         Replace_Dots    On
         Suppress_Type_Name On
@@ -238,7 +280,7 @@ config:
   ## https://docs.fluentbit.io/manual/pipeline/parsers
   customParsers: |
     [PARSER]
-        Name docker_no_time
+        Name docker_json
         Format json
         Time_Keep Off
         Time_Key time
@@ -279,7 +321,8 @@ args: []
 
 command: []
 
-initContainers: []
+initContainers:
+  []
   # - name: do-something
   #   image: busybox
   #   command: ['do', 'something']

nginx-values.yaml

@@ -8,6 +8,9 @@ metrics:
 stats:
   enabled: true
 
+podAnnotations:
+  fluentbit.io/parser: k8s-nginx-ingress
+
 nodeSelector:
   gateway: "true"
 tolerations:
@@ -32,3 +35,7 @@ service:
 
 # config:
 #   "use-forwarded-headers": "true"
+
+defaultBackend:
+  podAnnotations:
+    fluentbit.io/exclude: "true"