Add es and update postgres with db metrics on

This commit is contained in:
2022-03-20 16:54:23 +11:00
parent 8ec516bebe
commit 02ae564453
5 changed files with 703 additions and 10 deletions
+4
View File
@@ -67,6 +67,10 @@ k apply -f postgres-pv.yml -n default
k create namespace db
# postgres
# export data south uri for exporting metrics
kubectl -n db create secret generic postgresql-data-source-uri --from-literal=DATA_SOURCE_NAME="postgresql://postgres:xxx@localhost:5432/?sslmode=disable"
helm upgrade --install new-postgres -n db -f postgres-values.yaml bitnami/postgresql --set global.postgresql.auth.postgresPassword=xxxxxx
# media applications
+5 -2
View File
@@ -142,6 +142,8 @@ metadata:
app: kibana
spec:
replicas: 1
strategy:
type: RollingUpdate
selector:
matchLabels:
app: kibana
@@ -167,8 +169,8 @@ spec:
mountPath: /usr/share/kibana/config
resources:
requests:
cpu: 100m
memory: 256Mi
cpu: 200m
memory: 512Mi
limits:
cpu: 500m
memory: 1024Mi
@@ -240,6 +242,7 @@ data:
xpack.security.encryptionKey: "MTExMXNzY2JiZ2dnc2RzZmRmdnIyMzhyKigpJjclCg=="
xpack.encryptedSavedObjects.encryptionKey: MTExMXNzY2JiQCMlXiZnZ2dzZHNmZGZ2cjIzOHIqKCkmNyUK
xpack.reporting.roles.enabled: false
xpack.reporting.encryptionKey: "MTExMXNzY2JiQCMlXiZnZ2dzZHNmMzlldG5zdnhkZnZyMjM4ciooKSY3JQo="
# xpack.monitoring.enabled: false
# xpack.monitoring.ui.container.elasticsearch.enabled: false
# xpack.monitoring.kibana.collection.enabled: false
+662
View File
@@ -0,0 +1,662 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: agent-node-datastreams
namespace: kube-system
labels:
k8s-app: elastic-agent
data:
agent.yml: |-
outputs:
default:
type: elasticsearch
hosts:
- >-
${ES_HOST}
username: ${ES_USERNAME}
password: ${ES_PASSWORD}
agent:
monitoring:
enabled: true
use_output: default
logs: true
metrics: true
providers.kubernetes:
node: ${NODE_NAME}
scope: node
inputs:
- name: kubernetes-cluster-metrics
condition: ${kubernetes_leaderelection.leader} == true
type: kubernetes/metrics
use_output: default
meta:
package:
name: kubernetes
version: 0.2.8
data_stream:
namespace: default
streams:
- data_stream:
dataset: kubernetes.apiserver
type: metrics
metricsets:
- apiserver
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.KUBERNETES_SERVICE_HOST}:${env.KUBERNETES_SERVICE_PORT}'
period: 30s
ssl.certificate_authorities:
- /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
- data_stream:
dataset: kubernetes.event
type: metrics
metricsets:
- event
period: 10s
add_metadata: true
- data_stream:
dataset: kubernetes.state_container
type: metrics
metricsets:
- state_container
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_cronjob
type: metrics
metricsets:
- state_cronjob
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_deployment
type: metrics
metricsets:
- state_deployment
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_job
type: metrics
metricsets:
- state_job
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_node
type: metrics
metricsets:
- state_node
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_persistentvolume
type: metrics
metricsets:
- state_persistentvolume
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_persistentvolumeclaim
type: metrics
metricsets:
- state_persistentvolumeclaim
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_pod
type: metrics
metricsets:
- state_pod
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_replicaset
type: metrics
metricsets:
- state_replicaset
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_resourcequota
type: metrics
metricsets:
- state_resourcequota
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_service
type: metrics
metricsets:
- state_service
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_statefulset
type: metrics
metricsets:
- state_statefulset
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- data_stream:
dataset: kubernetes.state_storageclass
type: metrics
metricsets:
- state_storageclass
add_metadata: true
hosts:
- 'kube-state-metrics:8080'
period: 10s
- name: system-logs
type: logfile
use_output: default
meta:
package:
name: system
version: 0.10.7
data_stream:
namespace: default
streams:
- data_stream:
dataset: system.auth
type: logs
paths:
- /var/log/auth.log*
- /var/log/secure*
exclude_files:
- .gz$
multiline:
pattern: ^\s
match: after
processors:
- add_fields:
target: ''
fields:
ecs.version: 1.12.0
- data_stream:
dataset: system.syslog
type: logs
paths:
- /var/log/messages*
- /var/log/syslog*
exclude_files:
- .gz$
multiline:
pattern: ^\s
match: after
processors:
- add_fields:
target: ''
fields:
ecs.version: 1.12.0
- name: container-log
type: logfile
use_output: default
meta:
package:
name: log
version: 0.4.6
data_stream:
namespace: default
streams:
- data_stream:
dataset: generic
symlinks: true
paths:
- /var/log/containers/*${kubernetes.container.id}.log
- name: system-metrics
type: system/metrics
use_output: default
meta:
package:
name: system
version: 0.10.9
data_stream:
namespace: default
streams:
- data_stream:
dataset: system.core
type: metrics
metricsets:
- core
core.metrics:
- percentages
- data_stream:
dataset: system.cpu
type: metrics
period: 10s
cpu.metrics:
- percentages
- normalized_percentages
metricsets:
- cpu
- data_stream:
dataset: system.diskio
type: metrics
period: 10s
diskio.include_devices: null
metricsets:
- diskio
- data_stream:
dataset: system.filesystem
type: metrics
period: 1m
metricsets:
- filesystem
processors:
- drop_event.when.regexp:
system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
- data_stream:
dataset: system.fsstat
type: metrics
period: 1m
metricsets:
- fsstat
processors:
- drop_event.when.regexp:
system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
- data_stream:
dataset: system.load
type: metrics
period: 10s
metricsets:
- load
- data_stream:
dataset: system.memory
type: metrics
period: 10s
metricsets:
- memory
- data_stream:
dataset: system.network
type: metrics
period: 10s
network.interfaces: null
metricsets:
- network
- data_stream:
dataset: system.process
type: metrics
process.include_top_n.by_memory: 5
period: 10s
processes:
- .*
process.include_top_n.by_cpu: 5
process.cgroups.enabled: false
process.cmdline.cache.enabled: true
metricsets:
- process
process.include_cpu_ticks: false
system.hostfs: /hostfs
- data_stream:
dataset: system.process_summary
type: metrics
period: 10s
metricsets:
- process_summary
system.hostfs: /hostfs
- data_stream:
dataset: system.socket_summary
type: metrics
period: 10s
metricsets:
- socket_summary
system.hostfs: /hostfs
- name: kubernetes-node-metrics
type: kubernetes/metrics
use_output: default
meta:
package:
name: kubernetes
version: 0.2.8
data_stream:
namespace: default
streams:
- data_stream:
dataset: kubernetes.controllermanager
type: metrics
metricsets:
- controllermanager
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${kubernetes.pod.ip}:10257'
period: 10s
ssl.verification_mode: none
condition: ${kubernetes.labels.component} == 'kube-controller-manager'
- data_stream:
dataset: kubernetes.scheduler
type: metrics
metricsets:
- scheduler
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${kubernetes.pod.ip}:10259'
period: 10s
ssl.verification_mode: none
condition: ${kubernetes.labels.component} == 'kube-scheduler'
- data_stream:
dataset: kubernetes.proxy
type: metrics
metricsets:
- proxy
hosts:
- 'localhost:10249'
period: 10s
- data_stream:
dataset: kubernetes.container
type: metrics
metricsets:
- container
add_metadata: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.NODE_NAME}:10250'
period: 10s
ssl.verification_mode: none
- data_stream:
dataset: kubernetes.node
type: metrics
metricsets:
- node
add_metadata: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.NODE_NAME}:10250'
period: 10s
ssl.verification_mode: none
- data_stream:
dataset: kubernetes.pod
type: metrics
metricsets:
- pod
add_metadata: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.NODE_NAME}:10250'
period: 10s
ssl.verification_mode: none
- data_stream:
dataset: kubernetes.system
type: metrics
metricsets:
- system
add_metadata: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.NODE_NAME}:10250'
period: 10s
ssl.verification_mode: none
- data_stream:
dataset: kubernetes.volume
type: metrics
metricsets:
- volume
add_metadata: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
hosts:
- 'https://${env.NODE_NAME}:10250'
period: 10s
ssl.verification_mode: none
# Add extra input blocks here, based on conditions
# so as to automatically identify targeted Pods and start monitoring them
# using a predefined integration. For instance:
#- name: redis
# type: redis/metrics
# use_output: default
# meta:
# package:
# name: redis
# version: 0.3.6
# data_stream:
# namespace: default
# streams:
# - data_stream:
# dataset: redis.info
# type: metrics
# metricsets:
# - info
# hosts:
# - '${kubernetes.pod.ip}:6379'
# idle_timeout: 20s
# maxconn: 10
# network: tcp
# period: 10s
# condition: ${kubernetes.labels.app} == 'redis'
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: elastic-agent
namespace: kube-system
labels:
app: elastic-agent
spec:
selector:
matchLabels:
app: elastic-agent
template:
metadata:
labels:
app: elastic-agent
spec:
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
serviceAccountName: elastic-agent
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: elastic-agent
image: docker.elastic.co/beats/elastic-agent:8.0.1
args: [
"-c", "/etc/agent.yml",
"-e",
]
env:
- name: ES_USERNAME
value: "elastic"
- name: ES_PASSWORD
value: ""
- name: ES_HOST
value: ""
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
securityContext:
runAsUser: 0
resources:
limits:
memory: 500Mi
requests:
cpu: 100m
memory: 200Mi
volumeMounts:
- name: datastreams
mountPath: /etc/agent.yml
readOnly: true
subPath: agent.yml
- name: proc
mountPath: /hostfs/proc
readOnly: true
- name: cgroup
mountPath: /hostfs/sys/fs/cgroup
readOnly: true
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
volumes:
- name: datastreams
configMap:
defaultMode: 0640
name: agent-node-datastreams
- name: proc
hostPath:
path: /proc
- name: cgroup
hostPath:
path: /sys/fs/cgroup
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: elastic-agent
subjects:
- kind: ServiceAccount
name: elastic-agent
namespace: kube-system
roleRef:
kind: ClusterRole
name: elastic-agent
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
namespace: kube-system
name: elastic-agent
subjects:
- kind: ServiceAccount
name: elastic-agent
namespace: kube-system
roleRef:
kind: Role
name: elastic-agent
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: elastic-agent-kubeadm-config
namespace: kube-system
subjects:
- kind: ServiceAccount
name: elastic-agent
namespace: kube-system
roleRef:
kind: Role
name: elastic-agent-kubeadm-config
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: elastic-agent
labels:
k8s-app: elastic-agent
rules:
- apiGroups: [""]
resources:
- nodes
- namespaces
- events
- pods
- services
- configmaps
verbs: ["get", "list", "watch"]
# Enable this rule only if planing to use kubernetes_secrets provider
#- apiGroups: [""]
# resources:
# - secrets
# verbs: ["get"]
- apiGroups: ["extensions"]
resources:
- replicasets
verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
resources:
- statefulsets
- deployments
- replicasets
verbs: ["get", "list", "watch"]
- apiGroups: ["batch"]
resources:
- jobs
verbs: ["get", "list", "watch"]
- apiGroups:
- ""
resources:
- nodes/stats
verbs:
- get
# required for apiserver
- nonResourceURLs:
- "/metrics"
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: elastic-agent
# should be the namespace where elastic-agent is running
namespace: kube-system
labels:
k8s-app: elastic-agent
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs: ["get", "create", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: elastic-agent-kubeadm-config
namespace: kube-system
labels:
k8s-app: elastic-agent
rules:
- apiGroups: [""]
resources:
- configmaps
resourceNames:
- kubeadm-config
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: elastic-agent
namespace: kube-system
labels:
k8s-app: elastic-agent
---
+2 -2
View File
@@ -614,8 +614,8 @@ spec:
mountPath: /app/filebrowser.db
resources:
requests:
cpu: 100m
memory: 128Mi
cpu: 150m
memory: 256Mi
limits:
cpu: 400m
memory: 1024Mi
+30 -6
View File
@@ -4,6 +4,11 @@
# postgresqlDataDir: /bitnami/postgresql
postgresqlDatabase: home
global:
postgresql:
auth:
database: junv-db
# TODO: need to UPDATE DATA before update this image tag
# error: DETAIL: The data directory was initialized by PostgreSQL version 11, which is not compatible with this version 14.1.
image:
@@ -14,15 +19,24 @@ volumePermissions:
securityContext:
fsGroup: 1000
runAsUser: 1000
service:
type: NodePort
nodePort: 32222
metrics:
enabled: false
enabled: true
extraEnvVars:
- name: PG_EXPORTER_AUTO_DISCOVER_DATABASES
value: "true"
- name: DATA_SOURCE_NAME
valueFrom:
secretKeyRef:
name: postgresql-data-source-name
key: DATA_SOURCE_NAME
serviceMonitor:
enabled: true
additionalLabels:
release: prometheus
namespace: db
selector:
app.kubernetes.io/instance: new-postgres
app.kubernetes.io/name: postgresql
app.kubernetes.io/component: primary
customLivenessProbe:
exec:
@@ -49,6 +63,16 @@ customRreadinessProbe:
failureThreshold: 6
primary:
labels:
role: primary
service:
type: NodePort
nodePort: 32222
terminationGracePeriodSeconds: 5
resources:
requests:
memory: 400Mi
cpu: 250m
persistence:
mountPath: /bitnami/postgresql
storageClass: local