From f69fdddf2c53716a9fd1a2dd3ee9f0baffb60e1a Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Sat, 9 Aug 2025 21:48:04 +1000 Subject: [PATCH] enrich ingress geo ip info --- GeoIP-Analytics-README.md | 206 ++++++++++ grafana-dashboards/nginx-geo-analytics.json | 417 ++++++++++++++++++++ terraform/argocd.tf | 16 + terraform/grafana.tf | 82 ++++ terraform/logging.tf | 316 +++++++++++++++ terraform/scripts/enricher.py | 245 ++++++++++++ 6 files changed, 1282 insertions(+) create mode 100644 GeoIP-Analytics-README.md create mode 100644 grafana-dashboards/nginx-geo-analytics.json create mode 100644 terraform/grafana.tf create mode 100644 terraform/logging.tf create mode 100644 terraform/scripts/enricher.py diff --git a/GeoIP-Analytics-README.md b/GeoIP-Analytics-README.md new file mode 100644 index 0000000..34ee447 --- /dev/null +++ b/GeoIP-Analytics-README.md @@ -0,0 +1,206 @@ +# Nginx Ingress GeoIP Analytics + +This solution provides geographic visualization of traffic to your Kubernetes nginx ingress controller by extracting IP addresses from access logs and displaying them on a world map in Grafana. + +## Architecture + +``` +┌─────────────────┐ ┌──────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Nginx Ingress │───▶│ Promtail │───▶│ Loki │───▶│ GeoIP Enricher │ +│ (Access Logs) │ │ (Log Scraper)│ │ (Log Storage) │ │ (IP Analysis) │ +└─────────────────┘ └──────────────┘ └─────────────────┘ └─────────────────┘ + │ + ▼ +┌─────────────────┐ ┌──────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Grafana │◀───│ Prometheus │◀───│ Pushgateway │◀───│ Geo Metrics │ +│ (Visualization)│ │ (Metrics) │ │ (Metrics Proxy) │ │ (Push) │ +└─────────────────┘ └──────────────┘ └─────────────────┘ └─────────────────┘ +``` + +## Components + +### 1. **Loki + Promtail** (Log Collection) +- **Loki**: Stores and indexes nginx access logs +- **Promtail**: Scrapes logs from nginx ingress pods and parses IP addresses + +### 2. **GeoIP Enricher** (Custom Service) +- Queries Loki for recent access logs +- Extracts unique IP addresses +- Enriches IPs with geolocation data using ip-api.com +- Pushes geographic metrics to Prometheus via Pushgateway + +### 3. **Prometheus + Pushgateway** (Metrics Storage) +- Stores geographic metrics with labels for country, city, latitude, longitude +- Provides time-series data for visualization + +### 4. **Grafana Dashboard** (Visualization) +- World map showing request origins +- Pie chart of requests by country +- Time series of request rates +- Detailed geographic breakdown table + +## Installation + +### Deploy with Terraform + +```bash +cd terraform + +# Initialize Terraform +terraform init + +# Plan the deployment +terraform plan + +# Apply the configuration +terraform apply +``` + +This will deploy: +- Loki and Promtail for log collection +- Prometheus Pushgateway for metrics +- GeoIP enricher service +- Grafana with pre-configured dashboard +- All necessary Kubernetes resources + +### Access the Services + +After deployment, you can access: + +- **Grafana**: https://grafana.junv.cc (admin/admin123) +- **Loki**: http://loki.logging.svc.cluster.local:3100 +- **Pushgateway**: http://prometheus-pushgateway.prometheus.svc.cluster.local:9091 + +## Verification + +```bash +# Check all components are running +kubectl -n logging get pods +kubectl -n prometheus get pods +kubectl -n grafana get pods + +# Check GeoIP enricher logs +kubectl -n logging logs -l app=geoip-enricher -f + +# Verify metrics are being pushed +kubectl -n prometheus port-forward svc/prometheus-pushgateway 9091:9091 +# Visit http://localhost:9091/metrics and search for "nginx_geo" +``` + +## Configuration + +### GeoIP Service Rate Limits + +The solution uses the free ip-api.com service with these limits: +- 45 requests per minute +- 1000 requests per day + +For production use, consider: +- Using MaxMind GeoLite2 database (local lookups) +- Implementing IP caching to reduce API calls +- Using paid geolocation services for higher limits + +### Log Format + +The nginx ingress is configured with a detailed log format: +``` +$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] [$proxy_alternative_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id +``` + +### Data Collection Frequency + +- GeoIP enricher runs every 5 minutes +- Queries last 5 minutes of logs from Loki +- Processes up to 5000 log entries per cycle + +## Troubleshooting + +### No Data in Grafana + +1. **Check if logs are being collected:** + ```bash + kubectl -n logging logs -l app.kubernetes.io/name=promtail + ``` + +2. **Verify Loki has data:** + ```bash + kubectl -n logging port-forward svc/loki 3100:3100 + # Visit http://localhost:3100 and query: {job="nginx-ingress"} + ``` + +3. **Check GeoIP enricher logs:** + ```bash + kubectl -n logging logs -l app=geoip-enricher -f + ``` + +4. **Verify metrics in Pushgateway:** + ```bash + kubectl -n prometheus port-forward svc/prometheus-pushgateway 9091:9091 + # Visit http://localhost:9091/metrics and search for "nginx_geo" + ``` + +### GeoIP Enricher Not Processing IPs + +1. **Check if nginx logs are in expected format:** + ```bash + kubectl -n ingress-nginx logs -l app.kubernetes.io/component=controller + ``` + +2. **Verify network connectivity to ip-api.com:** + ```bash + kubectl -n logging exec -it deployment/geoip-enricher -- curl -s "http://ip-api.com/json/8.8.8.8" + ``` + +### High Memory Usage + +1. **Adjust resource limits in geoip-enricher.yaml:** + ```yaml + resources: + limits: + memory: "512Mi" # Increase if needed + ``` + +2. **Reduce log query frequency:** + - Edit the sleep time in enricher.py (default: 300 seconds) + +## Security Considerations + +1. **Network Policies**: Restrict GeoIP enricher network access +2. **Resource Limits**: Set appropriate CPU/memory limits +3. **RBAC**: Create minimal service account permissions +4. **Data Retention**: Configure Loki retention policies + +## Monitoring + +Monitor the solution with these queries: + +```prometheus +# Enricher health +up{job="geoip-enricher"} + +# Processing rate +increase(nginx_geo_requests_total[5m]) + +# Unique countries detected +count by (country) (nginx_geo_requests_by_country_total) +``` + +## Scaling + +For high-traffic deployments: + +1. **Horizontal scaling**: Increase GeoIP enricher replicas +2. **Caching**: Implement Redis cache for IP lookups +3. **Batching**: Process IPs in larger batches +4. **Local database**: Use MaxMind GeoLite2 for offline lookups + +## Cost Optimization + +1. **IP filtering**: Skip private/internal IP ranges +2. **Deduplication**: Cache recent IP lookups +3. **Sampling**: Process only a percentage of requests +4. **Regional focus**: Limit processing to specific regions + +## License + +This solution is provided as-is for educational and operational use. Please ensure compliance with your organization's security and privacy policies. diff --git a/grafana-dashboards/nginx-geo-analytics.json b/grafana-dashboards/nginx-geo-analytics.json new file mode 100644 index 0000000..87927bc --- /dev/null +++ b/grafana-dashboards/nginx-geo-analytics.json @@ -0,0 +1,417 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "basemap": { + "config": {}, + "name": "Layer 0", + "type": "default" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": true, + "showDebug": false, + "showMeasure": false, + "showScale": false, + "showZoom": true + }, + "layers": [ + { + "config": { + "showLegend": true, + "style": { + "color": { + "field": "Value", + "fixed": "dark-green" + }, + "opacity": 0.4, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "field": "Value", + "fixed": 5, + "max": 15, + "min": 2 + }, + "symbol": { + "fixed": "img/icons/marker/circle.svg", + "mode": "fixed" + }, + "textConfig": { + "fontSize": 12, + "offsetX": 0, + "offsetY": 0, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "filterData": { + "id": "byRefId", + "options": "A" + }, + "location": { + "latitude": "lat", + "longitude": "lon", + "mode": "coords" + }, + "name": "Layer 1", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "coords", + "lat": 0, + "lon": 0, + "zoom": 2 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "nginx_geo_requests_total", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Geographic Distribution of Requests", + "type": "geomap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 2, + "options": { + "displayLabels": [ + "country" + ], + "legend": { + "displayMode": "visible", + "placement": "right" + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "nginx_geo_requests_by_country_total", + "instant": true, + "legendFormat": "{{country}}", + "range": false, + "refId": "A" + } + ], + "title": "Requests by Country", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(nginx_geo_requests_by_country_total[5m])", + "legendFormat": "{{country}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate by Country (5m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 4, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "pluginVersion": "9.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "nginx_geo_requests_total", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Detailed Geographic Breakdown", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "instance": true, + "job": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 37, + "style": "dark", + "tags": [ + "nginx", + "geo", + "network" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Nginx Ingress Geographic Analytics", + "uid": "nginx-geo-analytics", + "version": 1, + "weekStart": "" +} diff --git a/terraform/argocd.tf b/terraform/argocd.tf index aa9d561..127bbf7 100644 --- a/terraform/argocd.tf +++ b/terraform/argocd.tf @@ -93,6 +93,22 @@ resource "helm_release" "ingress_nginx" { value = "ingress-nginx" } + # Enable detailed access logs for GeoIP analysis + set { + name = "controller.config.log-format-escape-json" + value = "true" + } + + set { + name = "controller.config.log-format-upstream" + value = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" $request_length $request_time [$proxy_upstream_name] [$proxy_alternative_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id" + } + + set { + name = "controller.config.enable-real-ip" + value = "true" + } + } # Check for cert-manager CRDs before proceeding diff --git a/terraform/grafana.tf b/terraform/grafana.tf new file mode 100644 index 0000000..0709811 --- /dev/null +++ b/terraform/grafana.tf @@ -0,0 +1,82 @@ +# Grafana deployment (if not already exists) +resource "kubernetes_namespace" "grafana" { + metadata { + name = "grafana" + } +} + +resource "helm_release" "grafana" { + name = "grafana" + repository = "https://grafana.github.io/helm-charts" + chart = "grafana" + version = "7.3.7" + namespace = kubernetes_namespace.grafana.metadata[0].name + create_namespace = false + + values = [ + <<-EOT + adminPassword: admin123 + + service: + type: ClusterIP + port: 80 + + ingress: + enabled: true + ingressClassName: nginx + hosts: + - grafana.junv.cc + tls: + - secretName: grafana-tls + hosts: + - grafana.junv.cc + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + + persistence: + enabled: true + size: 10Gi + + datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + url: http://prometheus-server.prometheus.svc.cluster.local + access: proxy + isDefault: true + - name: Loki + type: loki + url: http://loki.logging.svc.cluster.local:3100 + access: proxy + + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + + dashboardsConfigMaps: + default: "grafana-dashboard-geoip" + + plugins: + - grafana-worldmap-panel + - grafana-piechart-panel + + env: + GF_FEATURE_TOGGLES_ENABLE: geomap + EOT + ] + + depends_on = [ + kubernetes_config_map.grafana_dashboard_geoip + ] +} diff --git a/terraform/logging.tf b/terraform/logging.tf new file mode 100644 index 0000000..9b90fcb --- /dev/null +++ b/terraform/logging.tf @@ -0,0 +1,316 @@ +resource "kubernetes_namespace" "logging" { + metadata { + name = "logging" + } +} + +# Install Loki for log aggregation +resource "helm_release" "loki" { + name = "loki" + repository = "https://grafana.github.io/helm-charts" + chart = "loki" + version = "5.36.1" + namespace = kubernetes_namespace.logging.metadata[0].name + create_namespace = false + + values = [ + <<-EOT + deploymentMode: SingleBinary + loki: + commonConfig: + replication_factor: 1 + storage: + type: 'filesystem' + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules + auth_enabled: false + server: + http_listen_port: 3100 + grpc_listen_port: 9095 + ingester: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 5m + chunk_retain_period: 30s + schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + limits_config: + retention_period: 24h + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + max_cache_freshness_per_query: 10m + split_queries_by_interval: 15m + per_stream_rate_limit: 512M + per_stream_rate_limit_burst: 1024M + ingestion_rate_mb: 512 + ingestion_burst_size_mb: 1024 + max_streams_per_user: 0 + max_line_size: 256000 + singleBinary: + replicas: 1 + persistence: + enabled: true + size: 10Gi + extraEnv: + - name: JAEGER_AGENT_HOST + value: "" + test: + enabled: false + monitoring: + serviceMonitor: + enabled: false + selfMonitoring: + enabled: false + grafanaAgent: + installOperator: false + lokiCanary: + enabled: false + gateway: + enabled: false + EOT + ] +} + +# Install Promtail for log collection +resource "helm_release" "promtail" { + name = "promtail" + repository = "https://grafana.github.io/helm-charts" + chart = "promtail" + version = "6.15.3" + namespace = kubernetes_namespace.logging.metadata[0].name + + values = [ + <<-EOT + config: + clients: + - url: http://loki:3100/loki/api/v1/push + scrapeConfigs: + - job_name: nginx-ingress + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - ingress-nginx + pipeline_stages: + - regex: + expression: '^(?P[^\s]+)\s+-\s+(?P[^\s]+)\s+\[(?P[^\]]+)\]\s+"(?P[^\s]+)\s+(?P[^\s]+)\s+(?P[^"]+)"\s+(?P[^\s]+)\s+(?P[^\s]+)\s+"(?P[^"]*)"\s+"(?P[^"]*)"\s+(?P[^\s]+)\s+(?P[^\s]+)\s+\[(?P[^\]]*)\]\s+\[(?P[^\]]*)\]\s+(?P[^\s]+)\s+(?P[^\s]+)\s+(?P[^\s]+)\s+(?P[^\s]+)\s+(?P[^\s]+)' + - labels: + remote_addr: remote_addr + method: method + status: status + upstream_addr: upstream_addr + - output: + source: remote_addr + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + target_label: app + regex: ingress-nginx + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component] + target_label: component + regex: controller + - source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + regex: controller + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + EOT + ] + + depends_on = [helm_release.loki] +} + +# Install Pushgateway for custom metrics +resource "helm_release" "prometheus_pushgateway" { + name = "prometheus-pushgateway" + repository = "https://prometheus-community.github.io/helm-charts" + chart = "prometheus-pushgateway" + version = "2.4.2" + namespace = "prometheus" + + set { + name = "serviceMonitor.enabled" + value = "true" + } + + set { + name = "serviceMonitor.additionalLabels.release" + value = "prometheus" + } +} + +# GeoIP Enricher ConfigMap +resource "kubernetes_config_map" "geoip_enricher_code" { + metadata { + name = "geoip-enricher-code" + namespace = kubernetes_namespace.logging.metadata[0].name + } + + data = { + "enricher.py" = file("${path.module}/scripts/enricher.py") + "requirements.txt" = <<-EOT + requests==2.31.0 + prometheus-client==0.19.0 + EOT + } +} + +# GeoIP Enricher Deployment +resource "kubernetes_deployment" "geoip_enricher" { + metadata { + name = "geoip-enricher" + namespace = kubernetes_namespace.logging.metadata[0].name + labels = { + app = "geoip-enricher" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = "geoip-enricher" + } + } + + template { + metadata { + labels = { + app = "geoip-enricher" + } + } + + spec { + container { + name = "geoip-enricher" + image = "python:3.11-slim" + + command = ["/bin/bash"] + args = [ + "-c", + "cd /app && pip install -r requirements.txt && python enricher.py" + ] + + port { + container_port = 8080 + name = "http" + } + + env { + name = "LOKI_URL" + value = "http://loki:3100" + } + + env { + name = "PROMETHEUS_PUSHGATEWAY" + value = "prometheus-pushgateway.prometheus.svc.cluster.local:9091" + } + + env { + name = "PYTHONUNBUFFERED" + value = "1" + } + + volume_mount { + name = "app-code" + mount_path = "/app" + } + + resources { + requests = { + memory = "128Mi" + cpu = "100m" + } + limits = { + memory = "256Mi" + cpu = "200m" + } + } + + liveness_probe { + exec { + command = ["python", "-c", "print('healthy')"] + } + initial_delay_seconds = 30 + period_seconds = 60 + } + } + + volume { + name = "app-code" + config_map { + name = kubernetes_config_map.geoip_enricher_code.metadata[0].name + } + } + + restart_policy = "Always" + } + } + } + + depends_on = [ + helm_release.loki, + helm_release.prometheus_pushgateway + ] +} + +# GeoIP Enricher Service +resource "kubernetes_service" "geoip_enricher" { + metadata { + name = "geoip-enricher" + namespace = kubernetes_namespace.logging.metadata[0].name + labels = { + app = "geoip-enricher" + } + } + + spec { + selector = { + app = "geoip-enricher" + } + + port { + port = 8080 + target_port = 8080 + protocol = "TCP" + name = "http" + } + + type = "ClusterIP" + } +} + +# Grafana Dashboard ConfigMap +resource "kubernetes_config_map" "grafana_dashboard_geoip" { + metadata { + name = "grafana-dashboard-geoip" + namespace = "grafana" + labels = { + grafana_dashboard = "1" + } + } + + data = { + "nginx-geo-analytics.json" = file("${path.module}/../grafana-dashboards/nginx-geo-analytics.json") + } + + depends_on = [ + kubernetes_namespace.grafana + ] +} diff --git a/terraform/scripts/enricher.py b/terraform/scripts/enricher.py new file mode 100644 index 0000000..0042ade --- /dev/null +++ b/terraform/scripts/enricher.py @@ -0,0 +1,245 @@ +import requests +import json +import time +import logging +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway +import os +from urllib.parse import urljoin + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +def get_geo_data(ip): + """Get geolocation data for an IP address using ip-api.com""" + try: + # Skip private IPs + if ip.startswith(('10.', '172.', '192.168.', '127.', '169.254.')): + return None + + # Rate limit: ip-api.com allows 45 requests per minute for free + response = requests.get(f"http://ip-api.com/json/{ip}?fields=status,message,country,countryCode,city,lat,lon,isp,org", timeout=5) + data = response.json() + + if data.get('status') == 'success': + return { + 'country': data.get('country', 'Unknown'), + 'country_code': data.get('countryCode', 'UN'), + 'city': data.get('city', 'Unknown'), + 'lat': float(data.get('lat', 0)), + 'lon': float(data.get('lon', 0)), + 'isp': data.get('isp', 'Unknown'), + 'org': data.get('org', 'Unknown') + } + else: + logger.warning(f"Failed to get geo data for {ip}: {data.get('message', 'Unknown error')}") + return None + except Exception as e: + logger.error(f"Error getting geo data for {ip}: {e}") + return None + +def query_loki_logs(): + """Query Loki for recent nginx ingress logs""" + loki_url = os.getenv('LOKI_URL', 'http://loki:3100') + + # Query for nginx ingress logs with remote_addr label + query = '{job="nginx-ingress"} |= "" | logfmt | __error__=""' + + # Query last 5 minutes + end_time = int(time.time()) + start_time = end_time - 300 + + url = urljoin(loki_url, '/loki/api/v1/query_range') + params = { + 'query': query, + 'start': start_time * 1000000000, # nanoseconds + 'end': end_time * 1000000000, + 'limit': 5000 + } + + try: + logger.info(f"Querying Loki: {url}") + response = requests.get(url, params=params, timeout=30) + response.raise_for_status() + return response.json() + except Exception as e: + logger.error(f"Error querying Loki: {e}") + return None + +def extract_ips_from_logs(logs_data): + """Extract unique IP addresses from log data""" + unique_ips = set() + + if not logs_data or 'data' not in logs_data: + return unique_ips + + results = logs_data['data'].get('result', []) + logger.info(f"Processing {len(results)} log streams") + + for stream in results: + values = stream.get('values', []) + logger.info(f"Processing stream with {len(values)} log entries") + + for entry in values: + if len(entry) < 2: + continue + + log_line = entry[1] + try: + # Try to parse as JSON first (if log-format-escape-json is enabled) + if log_line.startswith('{'): + log_json = json.loads(log_line) + remote_addr = log_json.get('remote_addr') + else: + # Parse the standard nginx log format + parts = log_line.split(' ') + if parts: + remote_addr = parts[0] + else: + continue + + if remote_addr and remote_addr != '-' and not remote_addr.startswith('127.'): + unique_ips.add(remote_addr) + + except (json.JSONDecodeError, IndexError) as e: + # Skip malformed log entries + continue + + logger.info(f"Extracted {len(unique_ips)} unique IP addresses") + return unique_ips + +def push_geo_metrics(ip_geo_data): + """Push geographic metrics to Prometheus via pushgateway""" + if not ip_geo_data: + logger.info("No geo data to push") + return + + registry = CollectorRegistry() + + # Create metrics + geo_requests_total = Gauge( + 'nginx_geo_requests_total', + 'Total number of requests by geographic location', + ['country', 'country_code', 'city', 'lat', 'lon'], + registry=registry + ) + + geo_requests_by_country = Gauge( + 'nginx_geo_requests_by_country_total', + 'Total number of requests by country', + ['country', 'country_code'], + registry=registry + ) + + geo_unique_ips = Gauge( + 'nginx_geo_unique_ips_total', + 'Number of unique IP addresses by geographic location', + ['country', 'country_code', 'city', 'lat', 'lon'], + registry=registry + ) + + # Aggregate by location + location_counts = {} + country_counts = {} + + for ip, geo in ip_geo_data.items(): + if geo: + # Location-based aggregation + location_key = ( + geo['country'], + geo['country_code'], + geo['city'], + str(geo['lat']), + str(geo['lon']) + ) + location_counts[location_key] = location_counts.get(location_key, 0) + 1 + + # Country-based aggregation + country_key = (geo['country'], geo['country_code']) + country_counts[country_key] = country_counts.get(country_key, 0) + 1 + + # Set metrics + for (country, country_code, city, lat, lon), count in location_counts.items(): + geo_requests_total.labels( + country=country, + country_code=country_code, + city=city, + lat=lat, + lon=lon + ).set(count) + + geo_unique_ips.labels( + country=country, + country_code=country_code, + city=city, + lat=lat, + lon=lon + ).set(1) # Each location represents at least 1 unique IP in this batch + + for (country, country_code), count in country_counts.items(): + geo_requests_by_country.labels( + country=country, + country_code=country_code + ).set(count) + + # Push to Prometheus Pushgateway + pushgateway_url = os.getenv('PROMETHEUS_PUSHGATEWAY', 'prometheus-pushgateway.prometheus.svc.cluster.local:9091') + try: + push_to_gateway(pushgateway_url, job='geoip-enricher', registry=registry) + logger.info(f"Successfully pushed metrics for {len(location_counts)} locations to {pushgateway_url}") + except Exception as e: + logger.error(f"Failed to push metrics to pushgateway: {e}") + +def main(): + """Main loop for the GeoIP enricher""" + logger.info("Starting GeoIP enricher service...") + + while True: + try: + logger.info("Starting new enrichment cycle...") + + # Query logs from Loki + logs_data = query_loki_logs() + if not logs_data: + logger.warning("No log data received from Loki") + time.sleep(60) + continue + + # Extract unique IPs + unique_ips = extract_ips_from_logs(logs_data) + if not unique_ips: + logger.info("No unique IPs found in logs") + time.sleep(60) + continue + + logger.info(f"Processing {len(unique_ips)} unique IP addresses") + + # Enrich with geo data (with rate limiting) + ip_geo_data = {} + for i, ip in enumerate(unique_ips): + if i > 0 and i % 10 == 0: # Progress logging + logger.info(f"Processed {i}/{len(unique_ips)} IPs") + + geo_data = get_geo_data(ip) + if geo_data: + ip_geo_data[ip] = geo_data + + # Rate limit: 45 requests per minute = ~1.3 seconds between requests + time.sleep(1.5) + + # Push metrics to Prometheus + if ip_geo_data: + push_geo_metrics(ip_geo_data) + logger.info(f"Enrichment cycle completed. Processed {len(ip_geo_data)} IPs with geo data") + else: + logger.info("No geo data collected this cycle") + + except Exception as e: + logger.error(f"Error in main loop: {e}") + + # Wait 5 minutes before next cycle + logger.info("Waiting 5 minutes before next cycle...") + time.sleep(300) + +if __name__ == "__main__": + main()