diff --git a/nginxmon/fetcher.py b/nginxmon/fetcher.py index cb6b8cf..67ee20f 100644 --- a/nginxmon/fetcher.py +++ b/nginxmon/fetcher.py @@ -2,6 +2,7 @@ Fetch nginx ingress logs — from the Kubernetes API (production) or a local file (dev/testing). Deduplicates by request_id so overlapping windows don't double-insert. """ +import ipaddress import logging from datetime import timedelta @@ -35,6 +36,7 @@ def fetch_and_store() -> int: _touch(settings) return 0 + excluder = _build_excluder(settings.excluded_ips) since_seconds = settings.fetch_interval_seconds + _OVERLAP existing_ids = set( NginxAccessLog.objects.filter( @@ -88,11 +90,16 @@ def ingest_raw(text: str) -> int: if not entries: return 0 + settings = NginxSettings.get() + excluder = _build_excluder(settings.excluded_ips) + existing_ids = set( NginxAccessLog.objects.values_list('request_id', flat=True) ) new_logs, seen = [], set() for e in entries: + if excluder(e['remote_addr']): + continue key = e['request_id'] or ( f"{e['timestamp'].isoformat()}|{e['remote_addr']}|{e['request_uri']}" ) @@ -196,3 +203,95 @@ def cleanup_old_logs(days: int = 7): deleted, _ = NginxAccessLog.objects.filter(timestamp__lt=cutoff).delete() if deleted: logger.info('nginxmon: pruned %d old log entries (>%d days)', deleted, days) + + +def _build_excluder(excluded_ips_text: str): + """ + Build and return a callable(ip: str) -> bool that returns True when the + given IP should be excluded from ingestion. + + Supported entry formats (one per line): + 192.168.1.218 exact IP + 192.168.1.x wildcard — any IP whose first 3 octets match + 192.168.1.0/24 CIDR range + """ + exact: set[str] = set() + wildcards: list[str] = [] # prefixes like '192.168.1.' + networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [] + + for raw_line in (excluded_ips_text or '').splitlines(): + entry = raw_line.strip() + if not entry or entry.startswith('#'): + continue + if entry.endswith('.x') or entry.endswith('.*'): + # Wildcard: everything before .x/.* becomes a dotted prefix + wildcards.append(entry[:-1]) # keep trailing '.' + elif '/' in entry: + try: + networks.append(ipaddress.ip_network(entry, strict=False)) + except ValueError: + logger.warning('nginxmon: invalid CIDR in excluded_ips: %r', entry) + else: + exact.add(entry) + + def _is_excluded(ip: str) -> bool: + if ip in exact: + return True + for prefix in wildcards: + if ip.startswith(prefix): + return True + if networks: + try: + addr = ipaddress.ip_address(ip) + return any(addr in net for net in networks) + except ValueError: + pass + return False + + return _is_excluded + + +def _build_excluder(excluded_ips_text: str): + """ + Build and return a callable(ip: str) -> bool that returns True when the + given IP should be excluded from ingestion. + + Supported entry formats (one per line): + 192.168.1.218 exact IP + 192.168.1.x wildcard — any IP whose first 3 octets match + 192.168.1.0/24 CIDR range + """ + exact: set[str] = set() + wildcards: list[str] = [] # prefixes like '192.168.1.' + networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [] + + for raw_line in (excluded_ips_text or '').splitlines(): + entry = raw_line.strip() + if not entry or entry.startswith('#'): + continue + if entry.endswith('.x') or entry.endswith('.*'): + # Wildcard: treat everything before .x as a prefix + wildcards.append(entry[:-1]) # keep trailing '.' + elif '/' in entry: + try: + networks.append(ipaddress.ip_network(entry, strict=False)) + except ValueError: + logger.warning('nginxmon: invalid CIDR in excluded_ips: %r', entry) + else: + exact.add(entry) + + def _is_excluded(ip: str) -> bool: + if ip in exact: + return True + for prefix in wildcards: + if ip.startswith(prefix): + return True + if networks: + try: + addr = ipaddress.ip_address(ip) + return any(addr in net for net in networks) + except ValueError: + pass + return False + + return _is_excluded diff --git a/nginxmon/forms.py b/nginxmon/forms.py index c6a4ca6..8f33368 100644 --- a/nginxmon/forms.py +++ b/nginxmon/forms.py @@ -12,7 +12,15 @@ class NginxSettingsForm(forms.ModelForm): 'fetch_interval_seconds', 'log_file_path', 'enabled', + 'excluded_ips', ] + widgets = { + 'excluded_ips': forms.Textarea(attrs={ + 'rows': 4, + 'placeholder': '192.168.1.218\n192.168.1.x\n192.168.1.0/24', + 'class': 'font-mono text-xs', + }), + } class NginxAlertProfileForm(forms.ModelForm): diff --git a/nginxmon/migrations/0003_add_excluded_ips.py b/nginxmon/migrations/0003_add_excluded_ips.py new file mode 100644 index 0000000..d3377e9 --- /dev/null +++ b/nginxmon/migrations/0003_add_excluded_ips.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.12 on 2026-04-01 08:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nginxmon', '0002_covering_index'), + ] + + operations = [ + migrations.AddField( + model_name='nginxsettings', + name='excluded_ips', + field=models.TextField(blank=True, default='', help_text='One entry per line. Supports exact IPs (192.168.1.218), wildcards (192.168.1.x), and CIDR ranges (192.168.1.0/24). Matching requests will not be ingested.'), + ), + ] diff --git a/nginxmon/models.py b/nginxmon/models.py index 5a16869..1d1556e 100644 --- a/nginxmon/models.py +++ b/nginxmon/models.py @@ -30,6 +30,15 @@ class NginxSettings(models.Model): ) enabled = models.BooleanField(default=True) last_fetch_at = models.DateTimeField(null=True, blank=True) + excluded_ips = models.TextField( + blank=True, + default='', + help_text=( + 'One entry per line. Supports exact IPs (192.168.1.218), ' + 'wildcards (192.168.1.x), and CIDR ranges (192.168.1.0/24). ' + 'Matching requests will not be ingested.' + ), + ) class Meta: verbose_name = 'Nginx Settings'