update nginx and postgre to fix the issue, and add crawlab

2026-08-09 04:15:52 +10:00 · 2022-02-19 00:06:12 +11:00
parent bccd0475ac
commit 048c66abc8
6 changed files with 319 additions and 6 deletions
@@ -58,7 +58,8 @@ k apply -f postgres-pv.yml -n default

 k create namespace db

-helm upgrade --install new-postgres -n db -f postgres-values.yaml bitnami/postgresql --set postgresqlPassword=
+# postgres
+helm upgrade --install new-postgres -n db -f postgres-values.yaml bitnami/postgresql --set global.postgresql.auth.postgresPassword=xxxxxx

 # media applications

@@ -141,6 +142,8 @@ helm upgrade --install longhorn longhorn/longhorn  -n longhorn -f longhorn-value

 helm upgrade --install redis -n db -f redis-values.yaml bitnami/redis

+k create secret generic redis-password -n crawlab --from-literal=password=$REDIS_PASSWORD
+
 # superset
 # navigate to the codebase of superset and update TODO to provide db password manually

@@ -190,3 +193,22 @@ helm upgrade --install fluent-bit fluent/fluent-bit -f fluentbit-values.yaml
 k apply -f nfs-code-server.yaml -n apps

 k apply -f code-server.yaml
+
+# mongodb
+
+k create secret generic mongodb-password --from-literal=password=xxx --from-literal=username=junv
+
+helm upgrade --install mongodb -n db -f mongodb-values.yaml bitnami/mongodb
+
+```
+db.createUser(
+  {
+    user: "junv",
+    pwd:  "xxx",
+    roles: [ { role: "userAdminAnyDatabase", db: "admin" }]
+  }
+)
+```
+# crawlab
+
+k apply -f crawlab.yaml -n crawlab
@@ -0,0 +1,154 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crawlab
+  namespace: crawlab
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+  selector:
+    app: crawlab-master
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: crawlab-ingress
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    kubernetes.io/tls-acme: "true"
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    nginx.ingress.kubernetes.io/auth-url: "https://id.junv.me/oauth2/auth"
+    nginx.ingress.kubernetes.io/auth-signin: "https://id.junv.me/oauth2/start?rd=https%3A%2F%2F$host$request_uri"
+spec:
+  tls:
+  - hosts:
+    - crawlab.junv.me
+    secretName: crawlab-tls
+  rules:
+  - host: crawlab.junv.me
+    http:
+      paths:
+      - backend:
+          service:
+            name: crawlab
+            port:
+              number: 80
+        path: /
+        pathType: Prefix
+
+---
+# master
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: crawlab-master
+  namespace: crawlab
+spec:
+  serviceName: crawlab-master
+  selector:
+    matchLabels:
+      app: crawlab-master
+  template:
+    metadata:
+      labels:
+        app: crawlab-master
+    spec:
+      containers:
+      - image: tikazyq/crawlab:latest
+        imagePullPolicy: Always
+        name: crawlab
+        env:
+        - name: CRAWLAB_SERVER_MASTER
+          value: "Y"
+        - name: CRAWLAB_MONGO_HOST
+          value: "mongodb.db.svc.cluster.local"
+        - name: CRAWLAB_REDIS_ADDRESS
+          value: "redis-master.db.svc.cluster.local"
+        - name: CRAWLAB_REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: redis-password
+              key: password
+        - name: CRAWLAB_SETTING_ALLOWREGISTER
+          value: "Y"
+        - name: CRAWLAB_SERVER_LANG_NODE
+          value: "N"
+        - name: CRAWLAB_SERVER_LANG_JAVA
+          value: "N"
+        - name: CRAWLAB_SERVER_LANG_DOTNET
+          value: "N"
+        - name: CRAWLAB_SERVER_REGISTER_TYPE
+          value: "hostname"
+        - name: CRAWLAB_MONGO_DB
+          value: "crawlab"
+        - name: CRAWLAB_MONGO_USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: mongodb-password
+              key: username
+        - name: CRAWLAB_MONGO_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: mongodb-password
+              key: password
+        ports:
+        - containerPort: 8080
+          name: crawlab
+
+---
+# worker
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: crawlab-worker
+  namespace: crawlab
+spec:
+  serviceName: crawlab-worker
+  replicas: 2
+  selector:
+    matchLabels:
+      app: crawlab-worker
+  template:
+    metadata:
+      labels:
+        app: crawlab-worker
+    spec:
+      containers:
+      - image: tikazyq/crawlab:latest
+        imagePullPolicy: Always
+        name: crawlab
+        env:
+        - name: CRAWLAB_SERVER_MASTER
+          value: "N"
+        - name: CRAWLAB_MONGO_HOST
+          value: "mongodb.db.svc.cluster.local"
+        - name: CRAWLAB_MONGO_DB
+          value: "crawlab"
+        - name: CRAWLAB_MONGO_USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: mongodb-password
+              key: username
+        - name: CRAWLAB_MONGO_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: mongodb-password
+              key: password
+        - name: CRAWLAB_REDIS_ADDRESS
+          value: "redis-master.db.svc.cluster.local"
+        - name: CRAWLAB_REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: redis-password
+              key: password
+        - name: CRAWLAB_SERVER_LANG_NODE
+          value: "Y"
+        - name: CRAWLAB_SERVER_LANG_JAVA
+          value: "Y"
+        - name: CRAWLAB_SERVER_LANG_DOTNET
+          value: "Y"
+        - name: CRAWLAB_SERVER_REGISTER_TYPE
+          value: "hostname"
@@ -0,0 +1,92 @@
+## Enable persistence using Persistent Volume Claims
+## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
+##
+persistence:
+  ## @param persistence.enabled Enable MongoDB(&reg;) data persistence using PVC
+  ##
+  enabled: true
+  ## @param persistence.medium Provide a medium for `emptyDir` volumes.
+  ## Requires persistence.enabled: false
+  ##
+  medium: ""
+  ## @param persistence.existingClaim Provide an existing `PersistentVolumeClaim` (only when `architecture=standalone`)
+  ## Requires persistence.enabled: true
+  ## If defined, PVC must be created manually before volume will be bound
+  ## Ignored when mongodb.architecture=replicaset
+  ##
+  existingClaim: "mongodb-nfs"
+  ## @param persistence.storageClass PVC Storage Class for MongoDB(&reg;) data volume
+  ## If defined, storageClassName: <storageClass>
+  ## If set to "-", storageClassName: "", which disables dynamic provisioning
+  ## If undefined (the default) or set to null, no storageClassName spec is
+  ## set, choosing the default provisioner.
+  ##
+  storageClass: "nfs"
+  ## @param persistence.accessModes PV Access Mode
+  ##
+  accessModes:
+    - ReadWriteOnce
+  ## @param persistence.size PVC Storage Request for MongoDB(&reg;) data volume
+  ##
+  size: 50Gi
+  ## @param persistence.annotations PVC annotations
+  ##
+  annotations: {}
+  ## @param persistence.mountPath Path to mount the volume at
+  ## MongoDB(&reg;) images.
+  ##
+  mountPath: /bitnami/mongodb
+  ## @param persistence.subPath Subdirectory of the volume to mount at
+  ## and one PV for multiple services.
+  ##
+  subPath: ""
+  ## Fine tuning for volumeClaimTemplates
+  ##
+  volumeClaimTemplates:
+    ## @param persistence.volumeClaimTemplates.selector A label query over volumes to consider for binding (e.g. when using local volumes)
+    ## A label query over volumes to consider for binding (e.g. when using local volumes)
+    ## See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#labelselector-v1-meta for more details
+    ##
+    selector: {}
+    ## @param persistence.volumeClaimTemplates.requests Custom PVC requests attributes
+    ## Sometime cloud providers use additional requests attributes to provision custom storage instance
+    ## See https://cloud.ibm.com/docs/containers?topic=containers-file_storage#file_dynamic_statefulset
+    ##
+    requests: {}
+    ## @param persistence.volumeClaimTemplates.dataSource Add dataSource to the VolumeClaimTemplate
+    ##
+    dataSource: {}
+
+auth:
+  ## @param auth.enabled Enable authentication
+  ## ref: https://docs.mongodb.com/manual/tutorial/enable-authentication/
+  ##
+  enabled: true
+  ## @param auth.rootUser MongoDB(&reg;) root user
+  ##
+  rootUser: root
+  ## @param auth.rootPassword MongoDB(&reg;) root password
+  ## ref: https://github.com/bitnami/bitnami-docker-mongodb/blob/master/README.md#setting-the-root-password-on-first-run
+  ##
+  rootPassword: ""
+  ## MongoDB(&reg;) custom users and databases
+  ## ref: https://github.com/bitnami/bitnami-docker-mongodb/blob/master/README.md#creating-users-and-databases-on-first-run
+  ## @param auth.usernames List of custom users to be created during the initialization
+  ## @param auth.passwords List of passwords for the custom users set at `auth.usernames`
+  ## @param auth.databases List of custom databases to be created during the initialization
+  ##
+  usernames: [junv]
+  passwords: []
+  databases: [crawlab]
+  ## @param auth.replicaSetKey Key used for authentication in the replicaset (only when `architecture=replicaset`)
+  ##
+  replicaSetKey: ""
+  ## @param auth.existingSecret Existing secret with MongoDB(&reg;) credentials (keys: `mongodb-password`, `mongodb-root-password`, ` mongodb-replica-set-key`)
+  ## NOTE: When it's set the previous parameters are ignored.
+  ##
+  existingSecret: ""
+
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: mongodb-nfs
+spec:
+  capacity:
+    storage: 50Gi
+  accessModes:
+    - ReadWriteMany
+  nfs:
+    server: 192.168.1.4
+    path: "/mongodb"
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  namespace: "db"
+  name: "mongodb-nfs"
+spec:
+  storageClassName: ""
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 50Gi
@@ -15,6 +15,10 @@ tolerations:
    operator: Equal
    value: special
    effect: NoSchedule
+  - key: node-role.kubernetes.io/master
+    operator: Equal
+    value: ""
+    effect: NoSchedule
 replicaCount: 1

 service:
@@ -3,10 +3,12 @@
 #   tag: 11
 # postgresqlDataDir: /bitnami/postgresql
 postgresqlDatabase: home
-persistence:
-  mountPath: /bitnami/postgresql
-  storageClass: local
-  existingClaim: local-postgres-pvc
+
+# TODO: need to UPDATE DATA  before update this image tag
+# error:  DETAIL:  The data directory was initialized by PostgreSQL version 11, which is not compatible with this version 14.1.
+image:
+  tag: 11
+
 volumePermissions:
  enabled: true
 securityContext:
@@ -16,7 +18,7 @@ service:
  type: NodePort
  nodePort: 32222
 metrics:
-  enabled: true
+  enabled: false
  serviceMonitor:
    enabled: true
    additionalLabels:
@@ -47,6 +49,10 @@ customRreadinessProbe:
  failureThreshold: 6

 primary:
+  persistence:
+    mountPath: /bitnami/postgresql
+    storageClass: local
+    existingClaim: local-postgres-pvc
  nodeSelector:
    kubernetes.io/hostname: server-1
  tolerations:
@@ -54,3 +60,12 @@ primary:
      operator: "Equal"
      value: "special"
      effect: "NoSchedule"
+    - key: node-role.kubernetes.io/master
+      operator: Equal
+      value: ""
+      effect: NoSchedule
+
+readReplicas:
+  replicaCount: 0
+  persistence:
+    enabled: false