home-docker/archive/crawl4ai.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: crawl4ai-config-local-pv
  labels:
    type: local
    name: crawl4ai-config
spec:
  capacity:
    storage: 2Gi
  volumeMode: Filesystem
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: local-storage
  local:
    path: /mnt/k8s/crawl4ai/config
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - server-3
---

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  namespace: ai
  name: crawl4ai-config-local-pvc
spec:
  volumeName: crawl4ai-config-local-pv
  storageClassName: "local-storage"
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 2Gi
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: crawl4ai-user-data-local-pv
  labels:
    type: local
    name: crawl4ai-user-data
spec:
  capacity:
    storage: 10Gi
  volumeMode: Filesystem
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Delete
  storageClassName: local-storage
  local:
    path: /mnt/k8s/crawl4ai/user-data
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - server-3
---

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  namespace: ai
  name: crawl4ai-user-data-local-pvc
spec:
  volumeName: crawl4ai-user-data-local-pv
  storageClassName: "local-storage"
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: crawl4ai
  namespace: ai
  labels:
    app: crawl4ai
spec:
  replicas: 1
  revisionHistoryLimit: 2
  selector:
    matchLabels:
      app: crawl4ai
  template:
    metadata:
      labels:
        app: crawl4ai
        name: crawl4ai
    spec:
      containers:
        - name: crawl4ai
          image: mirror.gcr.io/unclecode/crawl4ai:0.7.4
          imagePullPolicy: IfNotPresent
          securityContext:
            # appuser: 999 set by the image
            runAsUser: 999
            runAsGroup: 999
          ports:
            - containerPort: 11235
              name: api
              protocol: TCP
          env:
            - name: TZ
              value: "Australia/Melbourne"
            # LLM API Keys - uncomment and add secrets as needed
            # - name: OPENAI_API_KEY
            #   valueFrom:
            #     secretKeyRef:
            #       name: llm-api-keys
            #       key: openai-api-key
            # - name: ANTHROPIC_API_KEY
            #   valueFrom:
            #     secretKeyRef:
            #       name: llm-api-keys
            #       key: anthropic-api-key
          volumeMounts:
            - name: user-data
              mountPath: "/home/appuser/.crawl4ai"
            - name: config
              mountPath: "/app/config.yml"
              subPath: "config.yml"
            - name: shm
              mountPath: "/dev/shm"
          resources:
            requests:
              cpu: 200m
              memory: 4Gi
            limits:
              cpu: 3000m
              memory: 8Gi
          livenessProbe:
            httpGet:
              path: /health
              port: 11235
            initialDelaySeconds: 30
            periodSeconds: 30
            timeoutSeconds: 10
          readinessProbe:
            httpGet:
              path: /
              port: 11235
            initialDelaySeconds: 5
            periodSeconds: 10
            timeoutSeconds: 5
      volumes:
        - name: config
          persistentVolumeClaim:
            claimName: crawl4ai-config-local-pvc
        - name: user-data
          persistentVolumeClaim:
            claimName: crawl4ai-user-data-local-pvc
        - name: shm
          emptyDir:
            medium: Memory
            sizeLimit: 1Gi

---

apiVersion: v1
kind: Service
metadata:
  name: crawl4ai
  namespace: ai
  labels:
    name: crawl4ai
spec:
  type: ClusterIP
  selector:
    app: crawl4ai
  ports:
    - name: api
      targetPort: api
      port: 80
      protocol: TCP

---

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: crawl4ai-ingress
  namespace: ai
  annotations:
    kubernetes.io/ingress.class: "nginx"
    kubernetes.io/tls-acme: "true"
    cert-manager.io/cluster-issuer: "letsencrypt-prod"
    nginx.ingress.kubernetes.io/auth-url: "https://pass.junv.cc/oauth2/auth"
    nginx.ingress.kubernetes.io/auth-signin: "https://pass.junv.cc/oauth2/start?rd=https://$host$escaped_request_uri"
    # Increase proxy timeouts for long-running crawl operations
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
    # Increase body size for large payloads
    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
spec:
  tls:
  - hosts:
    - crawl.junv.cc
    secretName: crawl-tls
  rules:
  - host: crawl.junv.cc
    http:
      paths:
      - backend:
          service:
            name: crawl4ai
            port:
              number: 80
        path: /
        pathType: Prefix

---

# Optional: Create a separate ingress without auth for API access
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: crawl-api-ingress
  namespace: ai
  annotations:
    kubernetes.io/ingress.class: "nginx"
    kubernetes.io/tls-acme: "true"
    cert-manager.io/cluster-issuer: "letsencrypt-prod"
    # No auth annotations for API access
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
    nginx.ingress.kubernetes.io/whitelist-source-range: "192.168.1.0/24"
spec:
  tls:
  - hosts:
    - crawl-api.junv.cc
    secretName: crawl-api-tls
  rules:
  - host: crawl-api.junv.cc
    http:
      paths:
      - backend:
          service:
            name: crawl4ai
            port:
              number: 80
        path: /
        pathType: Prefix