#! /bin/bash -eu # deploy NFS Subdir External Provisioner to your cluster nfs-client # https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ helm install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ --set nfs.server=192.168.1.4 --set nfs.path=/k8s # taints kubectl taint nodes server-1 node=special:NoSchedule ## install nginx ingress controller grafana dashboards # https://github.com/kubernetes/ingress-nginx/tree/master/deploy/grafana/dashboards # in order to pull private docker image kubectl create secret docker-registry regcred --docker-server=https://docker.pkg.github.com --docker-username=wahyd4 --docker-password=344f51a3979759788914088c8bfe87736ffefb5b --docker-email=wahyd4@gmail.com kubectl -n kube-system apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml k apply -f prometheus-pv.yml -n default # kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml # kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml # kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml # kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml # kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml # kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml helm upgrade --install prometheus -n default -f prometheus-values.yaml prometheus-community/kube-prometheus-stack k create namespace nginx-ingress helm upgrade --install ingress bitnami/nginx-ingress-controller -n nginx-ingress -f nginx-values.yaml k apply -f lets.yaml # helm upgrade --install cert-manager --namespace cert-manager -f cert-manager-values.yaml jetstack/cert-manager # cert manager # kubectl get Issuers,ClusterIssuers,Certificates,CertificateRequests,Orders,Challenges --all-namespaces helm upgrade --install \ cert-manager jetstack/cert-manager \ --namespace cert-manager \ --create-namespace \ --version v1.17.2 \ --set prometheus.enabled=false \ --set installCRDs=true \ --set webhook.timeoutSeconds=4 k apply -f prometheus-ingress.yml -n default kubectl -n default create secret generic oauth2-proxy-creds \ --from-literal=cookie-secret=ZuThURd/AK9ouHNxg0Czhg== \ --from-literal=client-id=4f46385b435275c693ec \ --from-literal=client-secret=518d5b4693aaaee3e62ce5b8d9da6ec5cc160098 helm upgrade oauth2-proxy --install -n default stable/oauth2-proxy --values oauth2-proxy-config.yaml k apply -f postgres-pv.yml -n default k create namespace db # postgres # export data south uri for exporting metrics # more details please check https://github.com/prometheus-community/postgres_exporter kubectl -n db create secret generic postgresql-data-source-uri --from-literal=DATA_SOURCE_NAME="postgresql://postgres:xxx@localhost:5432/?sslmode=disable" helm upgrade --install new-postgres -n db -f postgres-values.yaml bitnami/postgresql --set global.postgresql.auth.postgresPassword=xxxxxx # media applications k apply -f media.yaml # influxdb helm upgrade --install influxdb bitnami/influxdb --namespace db \ --set adminUser.name=junv \ --set adminUser.pwd=Lovewifi2020! \ --set persistence.storageClass=manual \ --set persistence.existingClaim=influxdb \ --set influxdb.securityContext.fsGroup=1000 \ --set influxdb.securityContext.runAsUser=1000 # metabase k apply -f metabase.yaml -n home-apps # create secret kubectl -n apps create secret generic database-credentials --from-literal=password=<...> k apply -f pihole-pv.yaml -n home-apps kubectl -n home-apps create secret generic pihole-admin-password --from-literal=webpassword=Uj2s3iUkRubkJs4 k apply -f pihole.yaml -n home-apps helm upgrade --install telegraf -f telegraf-values.yaml stable/telegraf -n home-apps ### level=error ts=2020-06-25T14:19:22.581Z caller=klog.go:94 component=k8s_client_runtime func=ErrorDepth msg="/app/discovery/kubernetes/kubernetes.go:263: Failed to list *v1.Pod: pods is forbidden: User \"system:serviceaccount:default:default\" cannot list resource \"pods\" in API group \"\" in the namespace \"home-apps\"" # https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/getting-started.md # service account and role binding, one service account can have multiple role bindings kubectl create rolebinding apps-editor-binding --clusterrole=edit --serviceaccount=apps:cd-robot --namespace=apps # https://coreos.com/operators/prometheus/docs/latest/rbac.html kubectl create rolebinding apps-metrics-operator-binding --clusterrole=prometheus-prometheus-oper-operator --serviceaccount=apps:cd-robot --namespace=apps k apply -f prometheus-service-account.yaml -n default # create service monitor for telegraf k apply -f telegraf-service-monitor.yaml -n default # ddns k apply -f ddns.yaml -n home-apps # cluster issuers k apply -f lets.yaml ### badges kubectl -n apps create secret generic sentry-credentials --from-literal=dsn=xxx ### Nextcloud # kubectl -n apps create secret generic postgresql-credentials --from-literal=username=postgres --from-literal=password=xxx k apply -f nextcloud-pv.yaml -n media # datadog helm upgrade --install datadog -f datadog-values.yaml -n datadog --set datadog.site='datadoghq.com' --set datadog.apiKey=a78f4b0a6500fc28440a6c4274e6080e datadog/datadog # etcd and db backups k apply -f backup-cronjobs.yaml -n default # longhorn helm upgrade --install longhorn longhorn/longhorn -n longhorn -f longhorn-values.yaml # redis helm upgrade --install redis -n db -f redis-values.yaml bitnami/redis k create secret generic redis-password -n crawlab --from-literal=password=$REDIS_PASSWORD # superset # navigate to the codebase of superset and update TODO to provide db password manually helm upgrade --install -n home-apps --values ~/code/home-docker/superset-values.yaml superset helm/superset --set redis.password=$(kubectl get secret --namespace db redis -o jsonpath="{.data.redis-password}" | base64 --decode) --set postgresql.postgresqlPassword= --set postgresql.postgresHost=new-postgres-postgresql.db.svc.cluster.local # n8n k apply -f n8n.yaml -n home-apps # minio k apply -f minio-pv.yaml -n db # create secrets for longhorn kubectl -n media create secret generic minio --from-literal=AWS_ENDPOINTS=https://minio.junv.cc --from-literal=AWS_ACCESS_KEY_ID='' --from-literal=AWS_SECRET_ACCESS_KEY='' # fill s3 bucket name for longhorn s3://bucket-name@ap-southeast-2/ # helm kubectl -n db create secret generic minio-credentials --from-literal=root-password="xxx" --from-literal=root-user=junv helm upgrade --install minio -n db -f minio-values.yaml bitnami/minio --set accessKey.password="TODO" \ --set secretKey.password="TODO" # copy secret to another namespace kubectl get secret minio --namespace=db -o yaml | sed 's/namespace: db/namespace: longhorn/g' | kubectl create -f - # namecheap ddns kubectl -n home-apps create secret generic namecheap-password --from-literal=password=xxxx # es k create namespace es k apply -f es.yaml # fluent bit helm repo add fluent https://fluent.github.io/helm-charts helm upgrade --install fluent-bit fluent/fluent-bit -f fluentbit-values.yaml -n es # code server https://code.junv.cc k apply -f nfs-code-server.yaml -n apps k apply -f code-server.yaml # mongodb k create secret generic mongodb-password --from-literal=password=xxx --from-literal=username=junv helm upgrade --install mongodb -n db -f mongodb-values.yaml bitnami/mongodb ``` db.createUser( { user: "junv", pwd: "xxx", roles: [ { role: "userAdminAnyDatabase", db: "admin" }] } ) ``` # crawlab k apply -f crawlab.yaml -n crawlab # secuirty kubectl create namespace security kubectl apply -f https://raw.githubusercontent.com/aquasecurity/kube-bench/main/job.yaml -n security # NFS client storageclass provisioner https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner # wire guard k create namespace wireguard k apply -f wireguard.yaml -n wireguard # argo k create namespace argocd kubectl -n argocd create secret generic redis-credentials --from-literal=redis-password="xxx" helm upgrade --install argocd -n argocd -f argocd-values.yaml argo/argo-cd # k8s github docker image pull # GITHUB_TOKEN the github access token kubectl create secret docker-registry github-image-pull-secret --docker-server=https://ghcr.io --docker-username=wahyd4 --docker-password=$GITHUB_TOKEN --docker-email=wahyd4@gmail.com # dns issue # https://medium.com/geekculture/k8s-troubleshooting-how-to-debug-coredns-issues-724e8b973cfc#id_token=eyJhbGciOiJSUzI1NiIsImtpZCI6IjI1NWNjYTZlYzI4MTA2MDJkODBiZWM4OWU0NTZjNDQ5NWQ3NDE4YmIiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJodHRwczovL2FjY291bnRzLmdvb2dsZS5jb20iLCJuYmYiOjE2NzgyNTQ1MjIsImF1ZCI6IjIxNjI5NjAzNTgzNC1rMWs2cWUwNjBzMnRwMmEyamFtNGxqZGNtczAwc3R0Zy5hcHBzLmdvb2dsZXVzZXJjb250ZW50LmNvbSIsInN1YiI6IjExMTc1MDM4MjkxNDgyMjY1ODIzNyIsImVtYWlsIjoid2FoeWQ0QGdtYWlsLmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJhenAiOiIyMTYyOTYwMzU4MzQtazFrNnFlMDYwczJ0cDJhMmphbTRsamRjbXMwMHN0dGcuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJuYW1lIjoiSnVud2VpIiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hL0FHTm15eFpDRExHNkRlZ19wRlhyTG5RYzNEU05rZWQyY1h2R3hjWjdEMjh1TUZJPXM5Ni1jIiwiZ2l2ZW5fbmFtZSI6Ikp1bndlaSIsImlhdCI6MTY3ODI1NDgyMiwiZXhwIjoxNjc4MjU4NDIyLCJqdGkiOiJmMDI2YTgyMDcwMTU4YzM2OGM5Mjc0OTQ3ZGMzNzg2ZmI5NDhkOWM1In0.ChPrY2r-akr1-3KPz4Tz0pMz8I9sQBr4n4LAC5sZTmPUfl4CcRYKoDdgYuDK6F6DR4nr0WlC4Ay2SQNHrtXeQsGEOZPpI0hLaG-YJ0Lv4jI-MI8whfLYYCN7vTcg6fyNJ0SgsuciDeIR50g2C1cHsCm9lrYcBBI_E9s8ef5WpL-SR8U0BBnrALxRut5LYUadsenM7PPnl5agP1Wx_ntMHXhhWVpBJiZ_4jHY-DA4wjnCxaMr3GNjeHbQ4Sg6xi3NJqmCcOFcQyr486QXEKgKrw4EJF9tuFlDGAp6ouEgkZ0xVyFi-8a9I28D4-TM0FCvlJPvY3UAxPnPtpIODjIv1w # https://github.com/kubernetes/kubernetes/issues/24224 # https://stackoverflow.com/questions/41574846/kubernetes-pods-replying-with-unexpected-source-for-dns-queries ## Generate kubeconfig for a service account kubeadm kubeconfig user --client-name=kubernetes-admin --config=new-cluster.yaml ## install some version of a package sudo apt install kubeadm=1.28.7 ### k8s admin credentials location cat /etc/kubernetes/admin.conf ## ubuntu package repo list folder /etc/apt/sources.list.d ## check ssh login fails cat /var/log/auth.log | grep "Failed" # Check fail2ban status sudo systemctl status fail2ban sudo fail2ban-client status sshd # When port 53 is used, then check systemd-resolved sudo systemctl status systemd-resolved sudo systemctl disable systemd-resolved vim /etc/resolv.conf # and remove local nameserver # redis helm upgrade --install redis -n db -f db/redis-values.yaml bitnami/redis # downgrade apt sudo apt install kubeadm=1.30.11-1.1 # passless sudo echo "junv ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/junv # View last N lines: ssh junv@192.168.1.2 -p22422 "sudo journalctl -u k3s -n 100" # View logs since a specific time: ssh junv@192.168.1.2 -p22422 "sudo journalctl -u k3s --since '10 minutes ago'" # View logs with timestamps: ssh junv@192.168.1.2 -p22422 "sudo journalctl -u k3s -f -o short-precise"