From 343e52ed9fbfc79eb517223f0926760f3369d2a9 Mon Sep 17 00:00:00 2001
From: Junwei Zhao <wahyd4@gmail.com>
Date: Fri, 20 Jun 2025 15:34:50 +1000
Subject: [PATCH] Add crawl4ai

---
 ai/crawl4ai.yaml | 183 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 183 insertions(+)
 create mode 100644 ai/crawl4ai.yaml

diff --git a/ai/crawl4ai.yaml b/ai/crawl4ai.yaml
new file mode 100644
index 0000000..7e6bcf0
--- /dev/null
+++ b/ai/crawl4ai.yaml
@@ -0,0 +1,183 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  namespace: home-apps
+  name: crawl4ai-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: nfs-client
+  resources:
+    requests:
+      storage: 1Gi
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crawl4ai
+  namespace: home-apps
+  labels:
+    app: crawl4ai
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: crawl4ai
+  template:
+    metadata:
+      labels:
+        app: crawl4ai
+        name: crawl4ai
+    spec:
+      containers:
+        - name: crawl4ai
+          image: mirror.gcr.io/unclecode/crawl4ai:latest
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            runAsUser: 0
+            runAsGroup: 0
+          ports:
+            - containerPort: 11235
+              name: api
+              protocol: TCP
+          env:
+            - name: TZ
+              value: "Australia/Melbourne"
+            # LLM API Keys - uncomment and add secrets as needed
+            # - name: OPENAI_API_KEY
+            #   valueFrom:
+            #     secretKeyRef:
+            #       name: llm-api-keys
+            #       key: openai-api-key
+            # - name: ANTHROPIC_API_KEY
+            #   valueFrom:
+            #     secretKeyRef:
+            #       name: llm-api-keys
+            #       key: anthropic-api-key
+          volumeMounts:
+            - name: config
+              mountPath: "/app/data"
+              subPath: "config.yml"
+            - name: shm
+              mountPath: "/dev/shm"
+          resources:
+            requests:
+              cpu: 500m
+              memory: 4Gi
+            limits:
+              cpu: 2000m
+              memory: 8Gi
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 11235
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 11235
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+      volumes:
+        - name: config
+          persistentVolumeClaim:
+            claimName: crawl4ai-pvc
+        - name: shm
+          emptyDir:
+            medium: Memory
+            sizeLimit: 1Gi
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: crawl4ai
+  namespace: home-apps
+  labels:
+    name: crawl4ai
+spec:
+  type: ClusterIP
+  selector:
+    app: crawl4ai
+  ports:
+    - name: api
+      targetPort: api
+      port: 80
+      protocol: TCP
+
+---
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: crawl4ai-ingress
+  namespace: home-apps
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    kubernetes.io/tls-acme: "true"
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    nginx.ingress.kubernetes.io/auth-url: "https://id.junv.cc/oauth2/auth"
+    nginx.ingress.kubernetes.io/auth-signin: "https://id.junv.cc/oauth2/start?rd=https%3A%2F%2F$host$request_uri"
+    # Increase proxy timeouts for long-running crawl operations
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
+    # Increase body size for large payloads
+    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
+spec:
+  tls:
+  - hosts:
+    - crawl.junv.cc
+    secretName: crawl-tls
+  rules:
+  - host: crawl.junv.cc
+    http:
+      paths:
+      - backend:
+          service:
+            name: crawl4ai
+            port:
+              number: 80
+        path: /
+        pathType: Prefix
+
+---
+
+# Optional: Create a separate ingress without auth for API access
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: crawl-api-ingress
+  namespace: home-apps
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    kubernetes.io/tls-acme: "true"
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    # No auth annotations for API access
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
+    nginx.ingress.kubernetes.io/whitelist-source-range: "192.168.1.0/24"
+spec:
+  tls:
+  - hosts:
+    - crawl-api.junv.cc
+    secretName: crawl-api-tls
+  rules:
+  - host: crawl-api.junv.cc
+    http:
+      paths:
+      - backend:
+          service:
+            name: crawl4ai
+            port:
+              number: 80
+        path: /
+        pathType: Prefix