From e89825a114b2d5be9b3e7dd06c5bd430a2639888 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Wed, 1 Apr 2026 21:33:07 +1100 Subject: [PATCH] Add auto ban feature! --- .gitignore | 2 + nginxmon/fetcher.py | 267 ++++++++++++++++---- nginxmon/forms.py | 3 + nginxmon/migrations/0004_auto_ban_fields.py | 33 +++ nginxmon/migrations/0005_banned_ip_model.py | 29 +++ nginxmon/models.py | 41 +++ nginxmon/templates/nginxmon/_live_logs.html | 3 + nginxmon/templates/nginxmon/dashboard.html | 63 +++++ nginxmon/urls.py | 1 + nginxmon/views.py | 25 +- scripts/analyze_ips.py | 27 ++ scripts/run_autoban.py | 25 ++ 12 files changed, 472 insertions(+), 47 deletions(-) create mode 100644 nginxmon/migrations/0004_auto_ban_fields.py create mode 100644 nginxmon/migrations/0005_banned_ip_model.py create mode 100644 scripts/analyze_ips.py create mode 100644 scripts/run_autoban.py diff --git a/.gitignore b/.gitignore index e64b67d..73cfdab 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,5 @@ DerivedData/ .swiftpm/ .build/ data/db.sqlite3 + +.playwright-mcp/ diff --git a/nginxmon/fetcher.py b/nginxmon/fetcher.py index 8e28998..8ccb525 100644 --- a/nginxmon/fetcher.py +++ b/nginxmon/fetcher.py @@ -78,6 +78,10 @@ def fetch_and_store() -> int: NginxAccessLog.objects.bulk_create(new_logs, ignore_conflicts=True) logger.info('nginxmon: inserted %d new log entries', len(new_logs)) enrich_geo_batch(new_logs) + if settings.auto_ban_enabled: + auto_ban_auth_scanners(settings) + auto_ban_php_scanners(settings) + auto_ban_404_flood(settings) _touch(settings) return len(new_logs) @@ -132,10 +136,219 @@ def ingest_raw(text: str) -> int: NginxAccessLog.objects.bulk_create(new_logs, ignore_conflicts=True) logger.info('nginxmon: ingested %d log entries from raw text', len(new_logs)) enrich_geo_batch(new_logs) + if settings.auto_ban_enabled: + auto_ban_auth_scanners(settings) + auto_ban_php_scanners(settings) + auto_ban_404_flood(settings) return len(new_logs) +# Auth-probe paths that signal someone trying to log in / brute-force OAuth +_AUTH_PROBE_PREFIXES = ('/oauth2/', '/authorize') + + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + +def _push_bans(settings: NginxSettings, candidate_ips: list[str], + ban_source: str, make_reason, make_note, + window_start) -> list[str]: + """ + Common routine: + 1. Filter out excluded / private IPs. + 2. Load ConfigMap, find new offenders, write back. + 3. Create BannedIP records. + 4. Annotate matching log rows. + Returns list of newly banned IPs. + """ + from django.db.models import Q + from .models import BannedIP + + excluder = _build_excluder(settings.excluded_ips) + candidate_ips = [ip for ip in candidate_ips + if not excluder(ip) and not _is_private_ip(ip)] + if not candidate_ips: + return [] + + try: + from links.mini_apps_views import _read_blocked_ips, _write_blocked_ips + blocked = _read_blocked_ips() + except Exception as exc: + logger.error('nginxmon auto_ban: could not read ConfigMap: %s', exc) + return [] + + already_in_db = set( + BannedIP.objects.filter(ip__in=candidate_ips).values_list('ip', flat=True) + ) + newly_banned = [ip for ip in candidate_ips + if ip not in blocked and ip not in already_in_db] + if not newly_banned: + return [] + + # Push to ConfigMap + try: + _write_blocked_ips(blocked + newly_banned) + logger.info('nginxmon auto_ban [%s]: banned %d IPs: %s', + ban_source, len(newly_banned), newly_banned) + except Exception as exc: + logger.error('nginxmon auto_ban: could not write ConfigMap: %s', exc) + return [] + + # Persist BannedIP records and annotate log rows + for ip in newly_banned: + reason = make_reason(ip, window_start) + note = f'Auto banned by system. Reason: {reason}.' + + # Geo from cache + from .models import IPGeoCache + geo = IPGeoCache.objects.filter(ip=ip).first() + BannedIP.objects.get_or_create( + ip=ip, + defaults=dict( + ban_source=ban_source, + reason=reason, + request_count=NginxAccessLog.objects.filter( + remote_addr=ip, timestamp__gte=window_start).count(), + country=geo.country if geo else '', + city=geo.city if geo else '', + ), + ) + + # Annotate all matching log rows for this IP in the window + NginxAccessLog.objects.filter( + remote_addr=ip, timestamp__gte=window_start, + ).update(note=note) + + return newly_banned + + +# --------------------------------------------------------------------------- +# Detector: repeated auth-probe failures +# --------------------------------------------------------------------------- + +def auto_ban_auth_scanners(settings: NginxSettings) -> list[str]: + """ + Ban IPs with > threshold *failed* (4xx/5xx) requests to /oauth2/ or /authorize + within the rolling window. + """ + from django.db.models import Count, Q + from datetime import timedelta + + window_start = dj_tz.now() - timedelta(hours=settings.auto_ban_window_hours) + + probe_filter = Q() + for prefix in _AUTH_PROBE_PREFIXES: + probe_filter |= Q(request_uri__startswith=prefix) + + candidates = list( + NginxAccessLog.objects + .filter(probe_filter, timestamp__gte=window_start, status__gte=400) + .values('remote_addr') + .annotate(cnt=Count('id')) + .filter(cnt__gt=settings.auto_ban_threshold) + .values_list('remote_addr', flat=True) + ) + + def make_reason(ip, ws): + paths = list( + NginxAccessLog.objects.filter( + probe_filter, remote_addr=ip, timestamp__gte=ws, status__gte=400, + ).values_list('request_uri', flat=True)[:20] + ) + hit = sorted({p for prefix in _AUTH_PROBE_PREFIXES for p in paths if p.startswith(prefix)}) + return 'repeated failed auth-probe requests to: ' + (', '.join(hit) or ', '.join(_AUTH_PROBE_PREFIXES)) + + return _push_bans(settings, candidates, 'auto_auth_probe', make_reason, None, window_start) + + +# --------------------------------------------------------------------------- +# Detector: PHP webshell / backdoor scanner +# --------------------------------------------------------------------------- + +def auto_ban_php_scanners(settings: NginxSettings) -> list[str]: + """ + Ban IPs probing random .php paths (classic webshell/backdoor scanners). + Threshold reuses auto_ban_threshold (default 5). Any IP making > threshold + requests to *.php paths (excluding the known auth endpoints) gets banned. + """ + from django.db.models import Count, Q + from datetime import timedelta + + window_start = dj_tz.now() - timedelta(hours=settings.auto_ban_window_hours) + + # Match paths ending in .php or containing .php? — but NOT auth endpoints + auth_filter = Q() + for prefix in _AUTH_PROBE_PREFIXES: + auth_filter |= Q(request_uri__startswith=prefix) + + php_filter = Q(request_uri__iregex=r'\.php(\?|$|/)') + + candidates = list( + NginxAccessLog.objects + .filter(php_filter, timestamp__gte=window_start) + .exclude(auth_filter) + .values('remote_addr') + .annotate(cnt=Count('id')) + .filter(cnt__gt=settings.auto_ban_threshold) + .values_list('remote_addr', flat=True) + ) + + def make_reason(ip, ws): + sample = list( + NginxAccessLog.objects.filter( + php_filter, remote_addr=ip, timestamp__gte=ws, + ).values_list('request_uri', flat=True)[:5] + ) + cnt = NginxAccessLog.objects.filter( + php_filter, remote_addr=ip, timestamp__gte=ws, + ).count() + return (f'PHP webshell/backdoor scanner — {cnt} probes to .php paths, ' + f'e.g.: {", ".join(sample[:3])}') + + return _push_bans(settings, candidates, 'auto_php_scan', make_reason, None, window_start) + + +# --------------------------------------------------------------------------- +# Detector: 404 flood (scraper / content scanner) +# --------------------------------------------------------------------------- + +def auto_ban_404_flood(settings: NginxSettings) -> list[str]: + """ + Ban IPs generating an abnormally high number of 404s (badge scrapers, + content scanners, etc.). Threshold = 5 × auto_ban_threshold. + """ + from django.db.models import Count + from datetime import timedelta + + window_start = dj_tz.now() - timedelta(hours=settings.auto_ban_window_hours) + threshold = settings.auto_ban_threshold * 5 # harsher: 25 by default + + candidates = list( + NginxAccessLog.objects + .filter(status=404, timestamp__gte=window_start) + .values('remote_addr') + .annotate(cnt=Count('id')) + .filter(cnt__gt=threshold) + .values_list('remote_addr', flat=True) + ) + + def make_reason(ip, ws): + cnt = NginxAccessLog.objects.filter( + remote_addr=ip, status=404, timestamp__gte=ws, + ).count() + sample = list( + NginxAccessLog.objects.filter( + remote_addr=ip, status=404, timestamp__gte=ws, + ).values_list('request_uri', flat=True)[:3] + ) + return (f'404 flood — {cnt} consecutive 404 responses, ' + f'e.g.: {", ".join(sample)}') + + return _push_bans(settings, candidates, 'auto_404_flood', make_reason, None, window_start) + + def _k8s_logs(settings: NginxSettings) -> str | None: """Fetch pod logs via the Kubernetes Python client (works in-cluster and locally).""" since = settings.fetch_interval_seconds + _OVERLAP @@ -207,6 +420,14 @@ def cleanup_old_logs(days: int = 7): logger.info('nginxmon: pruned %d old log entries (>%d days)', deleted, days) +def _is_private_ip(ip: str) -> bool: + """Return True for RFC-1918, loopback, link-local, and other private ranges.""" + try: + return ipaddress.ip_address(ip).is_private + except ValueError: + return False + + def _build_excluder(excluded_ips_text: str): """ Build and return a callable(ip: str) -> bool that returns True when the @@ -251,49 +472,3 @@ def _build_excluder(excluded_ips_text: str): return False return _is_excluded - - -def _build_excluder(excluded_ips_text: str): - """ - Build and return a callable(ip: str) -> bool that returns True when the - given IP should be excluded from ingestion. - - Supported entry formats (one per line): - 192.168.1.218 exact IP - 192.168.1.x wildcard — any IP whose first 3 octets match - 192.168.1.0/24 CIDR range - """ - exact: set[str] = set() - wildcards: list[str] = [] # prefixes like '192.168.1.' - networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [] - - for raw_line in (excluded_ips_text or '').splitlines(): - entry = raw_line.strip() - if not entry or entry.startswith('#'): - continue - if entry.endswith('.x') or entry.endswith('.*'): - # Wildcard: treat everything before .x as a prefix - wildcards.append(entry[:-1]) # keep trailing '.' - elif '/' in entry: - try: - networks.append(ipaddress.ip_network(entry, strict=False)) - except ValueError: - logger.warning('nginxmon: invalid CIDR in excluded_ips: %r', entry) - else: - exact.add(entry) - - def _is_excluded(ip: str) -> bool: - if ip in exact: - return True - for prefix in wildcards: - if ip.startswith(prefix): - return True - if networks: - try: - addr = ipaddress.ip_address(ip) - return any(addr in net for net in networks) - except ValueError: - pass - return False - - return _is_excluded diff --git a/nginxmon/forms.py b/nginxmon/forms.py index 8f33368..562997e 100644 --- a/nginxmon/forms.py +++ b/nginxmon/forms.py @@ -13,6 +13,9 @@ class NginxSettingsForm(forms.ModelForm): 'log_file_path', 'enabled', 'excluded_ips', + 'auto_ban_enabled', + 'auto_ban_threshold', + 'auto_ban_window_hours', ] widgets = { 'excluded_ips': forms.Textarea(attrs={ diff --git a/nginxmon/migrations/0004_auto_ban_fields.py b/nginxmon/migrations/0004_auto_ban_fields.py new file mode 100644 index 0000000..da6f1ce --- /dev/null +++ b/nginxmon/migrations/0004_auto_ban_fields.py @@ -0,0 +1,33 @@ +# Generated by Django 5.2.12 on 2026-04-01 10:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nginxmon', '0003_add_excluded_ips'), + ] + + operations = [ + migrations.AddField( + model_name='nginxaccesslog', + name='note', + field=models.TextField(blank=True, default=''), + ), + migrations.AddField( + model_name='nginxsettings', + name='auto_ban_enabled', + field=models.BooleanField(default=True, help_text='Automatically ban IPs that repeatedly probe auth endpoints.'), + ), + migrations.AddField( + model_name='nginxsettings', + name='auto_ban_threshold', + field=models.IntegerField(default=5, help_text='Number of auth-probe requests from one IP within the window before auto-ban.'), + ), + migrations.AddField( + model_name='nginxsettings', + name='auto_ban_window_hours', + field=models.IntegerField(default=24, help_text='Rolling window (hours) used to count auth-probe requests.'), + ), + ] diff --git a/nginxmon/migrations/0005_banned_ip_model.py b/nginxmon/migrations/0005_banned_ip_model.py new file mode 100644 index 0000000..0c37886 --- /dev/null +++ b/nginxmon/migrations/0005_banned_ip_model.py @@ -0,0 +1,29 @@ +# Generated by Django 5.2.12 on 2026-04-01 10:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nginxmon', '0004_auto_ban_fields'), + ] + + operations = [ + migrations.CreateModel( + name='BannedIP', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ip', models.GenericIPAddressField(db_index=True, unique=True)), + ('ban_source', models.CharField(choices=[('manual', 'Manual'), ('auto_auth_probe', 'Auto: Auth Probe'), ('auto_php_scan', 'Auto: PHP Scanner'), ('auto_404_flood', 'Auto: 404 Flood')], default='manual', max_length=30)), + ('reason', models.TextField()), + ('request_count', models.IntegerField(default=0)), + ('banned_at', models.DateTimeField(auto_now_add=True)), + ('country', models.CharField(blank=True, max_length=100)), + ('city', models.CharField(blank=True, max_length=100)), + ], + options={ + 'ordering': ['-banned_at'], + }, + ), + ] diff --git a/nginxmon/models.py b/nginxmon/models.py index 1d1556e..401dd3c 100644 --- a/nginxmon/models.py +++ b/nginxmon/models.py @@ -39,6 +39,18 @@ class NginxSettings(models.Model): 'Matching requests will not be ingested.' ), ) + auto_ban_enabled = models.BooleanField( + default=True, + help_text='Automatically ban IPs that repeatedly probe auth endpoints.', + ) + auto_ban_threshold = models.IntegerField( + default=5, + help_text='Number of auth-probe requests from one IP within the window before auto-ban.', + ) + auto_ban_window_hours = models.IntegerField( + default=24, + help_text='Rolling window (hours) used to count auth-probe requests.', + ) class Meta: verbose_name = 'Nginx Settings' @@ -107,6 +119,8 @@ class NginxAccessLog(models.Model): upstream_response_time = models.FloatField(null=True, blank=True) upstream_status = models.IntegerField(null=True, blank=True) request_id = models.CharField(max_length=100, blank=True, db_index=True) + # Auto-ban annotation + note = models.TextField(blank=True, default='') # Geo (populated after insert) country = models.CharField(max_length=100, blank=True) country_code = models.CharField(max_length=10, blank=True) @@ -176,3 +190,30 @@ class ThreatAlert(models.Model): @property def error_rate(self): return self.error_count / self.request_count if self.request_count else 0 + + +class BannedIP(models.Model): + """ + Permanent record of every IP banned by the system (auto or manual). + The IP is also pushed to the Kubernetes nginx ConfigMap block list. + """ + BAN_SOURCES = [ + ('manual', 'Manual'), + ('auto_auth_probe', 'Auto: Auth Probe'), + ('auto_php_scan', 'Auto: PHP Scanner'), + ('auto_404_flood', 'Auto: 404 Flood'), + ] + + ip = models.GenericIPAddressField(unique=True, db_index=True) + ban_source = models.CharField(max_length=30, choices=BAN_SOURCES, default='manual') + reason = models.TextField() + request_count = models.IntegerField(default=0) + banned_at = models.DateTimeField(auto_now_add=True) + country = models.CharField(max_length=100, blank=True) + city = models.CharField(max_length=100, blank=True) + + class Meta: + ordering = ['-banned_at'] + + def __str__(self): + return f'{self.ip} [{self.get_ban_source_display()}] @ {self.banned_at:%Y-%m-%d %H:%M}' diff --git a/nginxmon/templates/nginxmon/_live_logs.html b/nginxmon/templates/nginxmon/_live_logs.html index db3b304..5e509ed 100644 --- a/nginxmon/templates/nginxmon/_live_logs.html +++ b/nginxmon/templates/nginxmon/_live_logs.html @@ -25,6 +25,9 @@ {{ log.request_time|floatformat:3 }}s + + {% if log.note %} {{ log.note }}{% endif %} + {% empty %} No logs yet — use "Fetch Now" or wait for the scheduler. diff --git a/nginxmon/templates/nginxmon/dashboard.html b/nginxmon/templates/nginxmon/dashboard.html index be2773e..44fc8b9 100644 --- a/nginxmon/templates/nginxmon/dashboard.html +++ b/nginxmon/templates/nginxmon/dashboard.html @@ -251,6 +251,7 @@ Service ⇅ Status ⇅ Time(s) ⇅ + Note @@ -260,6 +261,68 @@ + +
+
+
+ +

Banned IPs

+ {{ banned_ips|length }} +
+
+ {% if banned_ips %} +
+ + + + + + + + + + + + + + {% for ban in banned_ips %} + + + + + + + + + + {% endfor %} + +
IPSourceReasonRequests
+ + + + {{ ban.get_ban_source_display }} + + {{ ban.reason }}{{ ban.request_count }} +
+ {% csrf_token %} + +
+
+
+ {% else %} +
No IPs banned yet.
+ {% endif %} +
+