From 5b1df2ee5ea045fb1f65c572c9bf163feea806a2 Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Thu, 16 Apr 2026 14:01:43 +1000 Subject: [PATCH] Add router monitor --- core/apps.py | 20 + core/settings.py | 1 + core/urls.py | 1 + k8s/manifest.template.yaml | 4 + k8s/manifest.yaml | 4 + routermon/__init__.py | 0 routermon/admin.py | 22 + routermon/apps.py | 10 + routermon/forms.py | 20 + routermon/management/__init__.py | 0 routermon/management/commands/__init__.py | 0 routermon/migrations/0001_initial.py | 62 +++ routermon/migrations/__init__.py | 0 routermon/models.py | 87 ++++ routermon/parser.py | 156 +++++++ routermon/receiver.py | 301 +++++++++++++ routermon/tasks.py | 23 + .../templates/routermon/_live_queries.html | 45 ++ routermon/templates/routermon/dashboard.html | 413 ++++++++++++++++++ routermon/templates/routermon/settings.html | 183 ++++++++ routermon/urls.py | 15 + routermon/views.py | 307 +++++++++++++ templates/base.html | 10 + 23 files changed, 1684 insertions(+) create mode 100644 routermon/__init__.py create mode 100644 routermon/admin.py create mode 100644 routermon/apps.py create mode 100644 routermon/forms.py create mode 100644 routermon/management/__init__.py create mode 100644 routermon/management/commands/__init__.py create mode 100644 routermon/migrations/0001_initial.py create mode 100644 routermon/migrations/__init__.py create mode 100644 routermon/models.py create mode 100644 routermon/parser.py create mode 100644 routermon/receiver.py create mode 100644 routermon/tasks.py create mode 100644 routermon/templates/routermon/_live_queries.html create mode 100644 routermon/templates/routermon/dashboard.html create mode 100644 routermon/templates/routermon/settings.html create mode 100644 routermon/urls.py create mode 100644 routermon/views.py diff --git a/core/apps.py b/core/apps.py index ea6bcce..68bd57f 100644 --- a/core/apps.py +++ b/core/apps.py @@ -70,3 +70,23 @@ class CoreConfig(AppConfig): replace_existing=True, ) logger.info("Scheduled periodic task: flush_click_buffer (every 60s)") + + # ── routermon ────────────────────────────────────────────────────────── + try: + from routermon.tasks import cleanup_old_queries + from routermon.models import RouterMonSettings + from routermon.receiver import start_receiver + + rm_settings = RouterMonSettings.get() + if rm_settings.enabled: + start_receiver(rm_settings.syslog_port) + + scheduler.add_job( + cleanup_old_queries, + trigger=IntervalTrigger(hours=6), + id='routermon_cleanup', + replace_existing=True, + ) + logger.info("routermon: scheduled cleanup job (every 6h)") + except Exception as exc: + logger.warning("routermon: startup error (non-fatal): %s", exc) diff --git a/core/settings.py b/core/settings.py index e253962..fc9ee78 100644 --- a/core/settings.py +++ b/core/settings.py @@ -22,6 +22,7 @@ INSTALLED_APPS = [ 'markdown', # 只需要基本的markdown包 'netscan', 'nginxmon', + 'routermon', ] ROOT_URLCONF = 'core.urls' diff --git a/core/urls.py b/core/urls.py index b09f086..0c82af8 100644 --- a/core/urls.py +++ b/core/urls.py @@ -49,6 +49,7 @@ urlpatterns = [ # Include netscan and files BEFORE links.urls to prevent the alias catch-all from intercepting them path('ui/netscan/', include('netscan.urls')), path('ui/nginxmon/', include('nginxmon.urls')), + path('ui/routermon/', include('routermon.urls')), path('ui/files/', include('links.file_urls')), # Import external image by URL — /import/images/ (also plural alias) diff --git a/k8s/manifest.template.yaml b/k8s/manifest.template.yaml index e842553..20646c8 100644 --- a/k8s/manifest.template.yaml +++ b/k8s/manifest.template.yaml @@ -175,6 +175,10 @@ spec: - containerPort: 8000 name: links-port protocol: TCP + - containerPort: 5514 + name: syslog-udp + protocol: UDP + hostPort: 5514 resources: requests: cpu: 200m diff --git a/k8s/manifest.yaml b/k8s/manifest.yaml index b4099e4..3c1008b 100644 --- a/k8s/manifest.yaml +++ b/k8s/manifest.yaml @@ -175,6 +175,10 @@ spec: - containerPort: 8000 name: links-port protocol: TCP + - containerPort: 5514 + name: syslog-udp + protocol: UDP + hostPort: 5514 resources: requests: cpu: 200m diff --git a/routermon/__init__.py b/routermon/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/routermon/admin.py b/routermon/admin.py new file mode 100644 index 0000000..490e917 --- /dev/null +++ b/routermon/admin.py @@ -0,0 +1,22 @@ +from django.contrib import admin +from .models import RouterMonSettings, DnsQuery, DhcpLease + + +@admin.register(RouterMonSettings) +class RouterMonSettingsAdmin(admin.ModelAdmin): + list_display = ['syslog_port', 'enabled', 'retention_days', 'last_received_at'] + + +@admin.register(DnsQuery) +class DnsQueryAdmin(admin.ModelAdmin): + list_display = ['timestamp', 'client_ip', 'client_name', 'domain', 'query_type', 'is_nxdomain', 'resolved_ip', 'country'] + list_filter = ['is_nxdomain', 'query_type'] + search_fields = ['domain', 'client_ip', 'client_name'] + ordering = ['-timestamp'] + + +@admin.register(DhcpLease) +class DhcpLeaseAdmin(admin.ModelAdmin): + list_display = ['ip', 'mac', 'hostname', 'last_seen'] + search_fields = ['ip', 'mac', 'hostname'] + ordering = ['-last_seen'] diff --git a/routermon/apps.py b/routermon/apps.py new file mode 100644 index 0000000..5408942 --- /dev/null +++ b/routermon/apps.py @@ -0,0 +1,10 @@ +from django.apps import AppConfig +import logging + +logger = logging.getLogger(__name__) + + +class RouterMonConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'routermon' + verbose_name = 'Router Monitor' diff --git a/routermon/forms.py b/routermon/forms.py new file mode 100644 index 0000000..5c08c86 --- /dev/null +++ b/routermon/forms.py @@ -0,0 +1,20 @@ +from django import forms +from .models import RouterMonSettings + + +class RouterMonSettingsForm(forms.ModelForm): + class Meta: + model = RouterMonSettings + fields = ['enabled', 'syslog_port', 'excluded_clients', 'retention_days'] + widgets = { + 'excluded_clients': forms.Textarea(attrs={'rows': 4, 'class': 'font-mono text-sm'}), + } + help_texts = { + 'syslog_port': ( + 'UDP port the server listens on (default 5514). ' + 'Configure your router to send syslog here. ' + 'If your router only supports port 514, add an iptables redirect on the k3s node.' + ), + 'excluded_clients': 'One LAN IP per line. Queries from these devices will be ignored.', + 'retention_days': 'DNS query log records older than this will be automatically deleted.', + } diff --git a/routermon/management/__init__.py b/routermon/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/routermon/management/commands/__init__.py b/routermon/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/routermon/migrations/0001_initial.py b/routermon/migrations/0001_initial.py new file mode 100644 index 0000000..6e818e1 --- /dev/null +++ b/routermon/migrations/0001_initial.py @@ -0,0 +1,62 @@ +# Generated by Django 5.2.12 on 2026-04-16 02:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='DhcpLease', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ip', models.GenericIPAddressField(db_index=True, unique=True)), + ('mac', models.CharField(blank=True, max_length=17)), + ('hostname', models.CharField(blank=True, max_length=200)), + ('last_seen', models.DateTimeField(auto_now=True)), + ], + options={ + 'ordering': ['ip'], + }, + ), + migrations.CreateModel( + name='RouterMonSettings', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enabled', models.BooleanField(default=True)), + ('syslog_port', models.IntegerField(default=5514, help_text='UDP port to listen on for syslog datagrams. Default: 5514 (non-privileged). If your router only supports port 514, add an iptables redirect on the k3s node: iptables -t nat -A PREROUTING -p udp --dport 514 -j REDIRECT --to-port 5514')), + ('excluded_clients', models.TextField(blank=True, default='', help_text='One LAN IP per line. DNS queries from these clients will not be stored.')), + ('retention_days', models.IntegerField(default=7, help_text='Delete DNS query records older than this many days.')), + ('last_received_at', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'verbose_name': 'Router Monitor Settings', + 'verbose_name_plural': 'Router Monitor Settings', + }, + ), + migrations.CreateModel( + name='DnsQuery', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(db_index=True)), + ('client_ip', models.GenericIPAddressField(db_index=True)), + ('client_name', models.CharField(blank=True, help_text='Hostname from DHCP lease at query time', max_length=200)), + ('domain', models.CharField(db_index=True, max_length=500)), + ('query_type', models.CharField(default='A', max_length=20)), + ('is_nxdomain', models.BooleanField(db_index=True, default=False)), + ('resolved_ip', models.GenericIPAddressField(blank=True, null=True)), + ('country', models.CharField(blank=True, max_length=100)), + ('country_code', models.CharField(blank=True, max_length=10)), + ('city', models.CharField(blank=True, max_length=100)), + ], + options={ + 'ordering': ['-timestamp'], + 'indexes': [models.Index(fields=['client_ip', 'timestamp'], name='routermon_d_client__2d31fb_idx'), models.Index(fields=['domain', 'timestamp'], name='routermon_d_domain_ca0a82_idx'), models.Index(fields=['timestamp', 'is_nxdomain'], name='routermon_d_timesta_74c854_idx')], + }, + ), + ] diff --git a/routermon/migrations/__init__.py b/routermon/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/routermon/models.py b/routermon/models.py new file mode 100644 index 0000000..64ed561 --- /dev/null +++ b/routermon/models.py @@ -0,0 +1,87 @@ +from django.db import models + + +class RouterMonSettings(models.Model): + """Singleton (pk=1) — router syslog receiver configuration.""" + + enabled = models.BooleanField(default=True) + syslog_port = models.IntegerField( + default=5514, + help_text=( + 'UDP port to listen on for syslog datagrams. Default: 5514 (non-privileged). ' + 'If your router only supports port 514, add an iptables redirect on the k3s node: ' + 'iptables -t nat -A PREROUTING -p udp --dport 514 -j REDIRECT --to-port 5514' + ), + ) + excluded_clients = models.TextField( + blank=True, + default='', + help_text='One LAN IP per line. DNS queries from these clients will not be stored.', + ) + retention_days = models.IntegerField( + default=7, + help_text='Delete DNS query records older than this many days.', + ) + last_received_at = models.DateTimeField(null=True, blank=True) + + class Meta: + verbose_name = 'Router Monitor Settings' + verbose_name_plural = 'Router Monitor Settings' + + def __str__(self): + status = 'enabled' if self.enabled else 'disabled' + return f'RouterMon Settings ({status}, port {self.syslog_port})' + + @classmethod + def get(cls): + obj, _ = cls.objects.get_or_create(pk=1) + return obj + + +class DnsQuery(models.Model): + """One DNS query event received from the router's dnsmasq syslog.""" + + timestamp = models.DateTimeField(db_index=True) + client_ip = models.GenericIPAddressField(db_index=True) + client_name = models.CharField(max_length=200, blank=True, help_text='Hostname from DHCP lease at query time') + domain = models.CharField(max_length=500, db_index=True) + query_type = models.CharField(max_length=20, default='A') + is_nxdomain = models.BooleanField(default=False, db_index=True) + # Best-effort: first public A/AAAA record from the corresponding reply line. + resolved_ip = models.GenericIPAddressField(null=True, blank=True) + # Geo for resolved_ip (populated asynchronously) + country = models.CharField(max_length=100, blank=True) + country_code = models.CharField(max_length=10, blank=True) + city = models.CharField(max_length=100, blank=True) + + class Meta: + ordering = ['-timestamp'] + indexes = [ + models.Index(fields=['client_ip', 'timestamp']), + models.Index(fields=['domain', 'timestamp']), + models.Index(fields=['timestamp', 'is_nxdomain']), + ] + + def __str__(self): + flag = ' [NX]' if self.is_nxdomain else '' + return f'{self.client_ip} → {self.domain} ({self.query_type}){flag} @ {self.timestamp:%Y-%m-%d %H:%M:%S}' + + @property + def geo_display(self): + parts = [p for p in [self.city, self.country] if p] + return ', '.join(parts) if parts else '—' + + +class DhcpLease(models.Model): + """Latest-known DHCP lease: maps LAN IP to MAC + hostname.""" + + ip = models.GenericIPAddressField(unique=True, db_index=True) + mac = models.CharField(max_length=17, blank=True) + hostname = models.CharField(max_length=200, blank=True) + last_seen = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ['ip'] + + def __str__(self): + return f'{self.ip} ({self.hostname or self.mac or "unknown"})' diff --git a/routermon/parser.py b/routermon/parser.py new file mode 100644 index 0000000..f1a5656 --- /dev/null +++ b/routermon/parser.py @@ -0,0 +1,156 @@ +""" +Parse syslog datagrams from AsusWRT-Merlin's dnsmasq. + +Handles RFC 3164 syslog envelope: + Mmm DD HH:MM:SS hostname dnsmasq[PID]: message + or with PRI prefix: Mmm DD HH:MM:SS hostname dnsmasq[PID]: message + +Recognised message formats: + query[TYPE] domain from IP — DNS query event + reply domain is IP — resolved to an IP + reply domain is NXDOMAIN — domain does not exist + reply domain is NODATA — exists but no records of requested type + DHCPACK(iface) IP MAC [hostname] — DHCP lease acknowledged +""" +import re +import logging +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional + +logger = logging.getLogger(__name__) + +# ── Compiled regexes ────────────────────────────────────────────────────────── + +# Strip optional RFC 3164 PRI prefix: +_RE_PRI = re.compile(r'^<\d+>') + +# Full syslog line: optional PRI + "Mmm DD HH:MM:SS hostname process[pid]: msg" +# We capture the process token and the message body. +_RE_SYSLOG = re.compile( + r'(?:<\d+>)?' # optional PRI + r'\w{3}\s+\d+\s+\d{2}:\d{2}:\d{2}' # timestamp (Mmm DD HH:MM:SS) + r'\s+\S+' # hostname + r'\s+(dnsmasq(?:-dhcp)?)\[\d+\]:' # process (group 1) + r'\s*(.*)', # message body (group 2) + re.DOTALL, +) + +# dnsmasq DNS query: query[TYPE] domain from IP +_RE_QUERY = re.compile(r'^query\[(\w+)\]\s+([\w.\-]+)\s+from\s+([\d.a-fA-F:]+)') + +# dnsmasq reply: reply domain is ANSWER +# ANSWER may be: an IP address, NXDOMAIN, NODATA, NODATA-IPv4, NODATA-IPv6, etc. +_RE_REPLY = re.compile(r'^reply\s+([\w.\-]+)\s+is\s+(\S+)') + +# dnsmasq DHCP ack: DHCPACK(iface) IP MAC [hostname] +_RE_DHCP = re.compile(r'^DHCPACK\(\S+\)\s+([\d.]+)\s+([\da-fA-F:]+)\s*(\S*)') + +# Simple IPv4/IPv6 check (enough to distinguish from NXDOMAIN/NODATA strings) +_RE_IP = re.compile(r'^[\d.a-fA-F:]+$') + +# Private IP ranges to skip for geo (same check as nginxmon) +import ipaddress +_PRIVATE_NETS = ( + ipaddress.ip_network('10.0.0.0/8'), + ipaddress.ip_network('172.16.0.0/12'), + ipaddress.ip_network('192.168.0.0/16'), + ipaddress.ip_network('127.0.0.0/8'), + ipaddress.ip_network('::1/128'), + ipaddress.ip_network('fc00::/7'), +) + + +def _is_public_ip(addr: str) -> bool: + try: + ip = ipaddress.ip_address(addr) + return not any(ip in net for net in _PRIVATE_NETS) + except ValueError: + return False + + +# ── Result dataclasses ──────────────────────────────────────────────────────── + +@dataclass +class QueryEvent: + client_ip: str + domain: str + query_type: str + + +@dataclass +class ReplyEvent: + domain: str + answer: str # raw answer (IP, NXDOMAIN, NODATA, …) + is_nxdomain: bool + resolved_ip: Optional[str] # public IP only, or None + + +@dataclass +class DhcpEvent: + ip: str + mac: str + hostname: str + + +@dataclass +class ParseResult: + query: Optional[QueryEvent] = None + reply: Optional[ReplyEvent] = None + dhcp: Optional[DhcpEvent] = None + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def parse_line(raw: str) -> Optional[ParseResult]: + """ + Parse a single syslog datagram string. + Returns a ParseResult if a recognised event was found, else None. + """ + raw = raw.strip() + if not raw: + return None + + m = _RE_SYSLOG.match(raw) + if not m: + # Some Merlin builds omit the standard syslog prefix; try matching + # dnsmasq message body directly. + body = raw + else: + body = m.group(2).strip() + + # DNS query + mq = _RE_QUERY.match(body) + if mq: + return ParseResult(query=QueryEvent( + client_ip=mq.group(3), + domain=mq.group(2).rstrip('.'), + query_type=mq.group(1), + )) + + # DNS reply + mr = _RE_REPLY.match(body) + if mr: + domain = mr.group(1).rstrip('.') + answer = mr.group(2) + is_nxdomain = answer.upper() in ('NXDOMAIN', 'NODATA', 'NODATA-IPV4', 'NODATA-IPV6') + resolved_ip = None + if not is_nxdomain and _RE_IP.match(answer) and _is_public_ip(answer): + resolved_ip = answer + return ParseResult(reply=ReplyEvent( + domain=domain, + answer=answer, + is_nxdomain=is_nxdomain, + resolved_ip=resolved_ip, + )) + + # DHCP ack + md = _RE_DHCP.match(body) + if md: + return ParseResult(dhcp=DhcpEvent( + ip=md.group(1), + mac=md.group(2).lower(), + hostname=md.group(3), + )) + + return None diff --git a/routermon/receiver.py b/routermon/receiver.py new file mode 100644 index 0000000..fd7004c --- /dev/null +++ b/routermon/receiver.py @@ -0,0 +1,301 @@ +""" +UDP syslog receiver for routermon. + +Architecture: + - One background thread owns the UDP socket and enqueues parsed events. + - A separate writer thread drains the queue and bulk-saves to the DB every + FLUSH_INTERVAL seconds (or when the queue reaches FLUSH_SIZE). + - Only one Gunicorn worker binds the socket (EADDRINUSE is silently ignored + in the others — matches the project's existing in-process APScheduler style). + +dnsmasq reply correlation: + - A small in-memory dict `_pending` maps `domain` → (pk, timestamp) for + outstanding query rows awaiting a reply. Entries expire after REPLY_TTL + seconds so stale pending records don't accumulate. +""" +import errno +import logging +import queue +import socket +import threading +import time +from datetime import timedelta + +from django.utils import timezone + +logger = logging.getLogger(__name__) + +FLUSH_INTERVAL = 2 # seconds between writer flushes +FLUSH_SIZE = 100 # flush immediately when queue reaches this size +REPLY_TTL = 10 # seconds to keep a query in _pending for reply matching +MAX_QUEUE = 10_000 # drop oldest items if queue exceeds this (backpressure) + +_receiver_thread: threading.Thread | None = None +_writer_thread: threading.Thread | None = None +_sock: socket.socket | None = None +_queue: queue.Queue = queue.Queue(maxsize=MAX_QUEUE) + +# in-memory pending dict: domain → (query_pk, created_at) +_pending: dict = {} +_pending_lock = threading.Lock() + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def start_receiver(port: int): + """ + Attempt to start the UDP receiver on `port`. Safe to call from every + Gunicorn worker — only the first to bind succeeds; others silently skip. + """ + global _receiver_thread, _writer_thread, _sock + + if _receiver_thread and _receiver_thread.is_alive(): + return # already running in this process + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('0.0.0.0', port)) + sock.settimeout(1.0) + _sock = sock + logger.info('routermon: UDP syslog receiver bound on port %d', port) + except OSError as exc: + if exc.errno == errno.EADDRINUSE: + logger.debug('routermon: port %d already bound (another worker), skipping', port) + else: + logger.error('routermon: failed to bind UDP port %d: %s', port, exc) + return + + _writer_thread = threading.Thread(target=_writer_loop, daemon=True, name='routermon-writer') + _writer_thread.start() + + _receiver_thread = threading.Thread( + target=_receiver_loop, args=(sock,), daemon=True, name='routermon-receiver' + ) + _receiver_thread.start() + + +def stop_receiver(): + global _sock + if _sock: + try: + _sock.close() + except Exception: + pass + _sock = None + + +# ── Socket receiver loop ────────────────────────────────────────────────────── + +def _receiver_loop(sock: socket.socket): + from .parser import parse_line + + logger.info('routermon: receiver loop started') + while True: + try: + data, _ = sock.recvfrom(4096) + except socket.timeout: + continue + except OSError: + # Socket closed (shutdown) + break + try: + line = data.decode('utf-8', errors='replace').rstrip('\n\r') + result = parse_line(line) + if result is not None: + _enqueue(result) + except Exception as exc: + logger.debug('routermon: parse error: %s', exc) + + logger.info('routermon: receiver loop stopped') + + +def _enqueue(result): + try: + _queue.put_nowait(result) + except queue.Full: + # Drop oldest item to make room (keep most recent) + try: + _queue.get_nowait() + _queue.put_nowait(result) + except Exception: + pass + + +# ── Writer loop ─────────────────────────────────────────────────────────────── + +def _writer_loop(): + logger.info('routermon: writer loop started') + batch = [] + last_flush = time.monotonic() + + while True: + # Collect items until flush threshold or timeout + deadline = last_flush + FLUSH_INTERVAL + while time.monotonic() < deadline and len(batch) < FLUSH_SIZE: + try: + item = _queue.get(timeout=max(0.1, deadline - time.monotonic())) + batch.append(item) + except queue.Empty: + break + + if batch: + try: + _flush(batch) + except Exception as exc: + logger.error('routermon: flush error: %s', exc, exc_info=True) + batch = [] + + last_flush = time.monotonic() + + logger.info('routermon: writer loop stopped') + + +def _flush(batch: list): + """Persist a batch of ParseResult objects to the database.""" + from .models import RouterMonSettings, DnsQuery, DhcpLease + + settings = RouterMonSettings.get() + if not settings.enabled: + return + + excluded = _build_excluded_set(settings.excluded_clients) + now = timezone.now() + + # --- DHCP leases (upsert immediately) --- + dhcp_events = [r.dhcp for r in batch if r.dhcp] + for ev in dhcp_events: + DhcpLease.objects.update_or_create( + ip=ev.ip, + defaults={'mac': ev.mac, 'hostname': ev.hostname}, + ) + + # --- DNS queries --- + hostname_cache = { + lease.ip: lease.hostname + for lease in DhcpLease.objects.filter( + ip__in={r.query.client_ip for r in batch if r.query} + ) + } + + query_objs = [] + for r in batch: + if not r.query: + continue + q = r.query + if q.client_ip in excluded: + continue + obj = DnsQuery( + timestamp=now, + client_ip=q.client_ip, + client_name=hostname_cache.get(q.client_ip, ''), + domain=q.domain, + query_type=q.query_type, + ) + query_objs.append(obj) + + if query_objs: + created = DnsQuery.objects.bulk_create(query_objs) + # Register created queries in pending dict for reply correlation + with _pending_lock: + for obj in created: + _pending[obj.domain] = (obj.pk, now) + + # --- DNS replies: correlate with pending queries --- + reply_events = [r.reply for r in batch if r.reply] + if reply_events: + _apply_replies(reply_events, now) + + # Expire old pending entries + _expire_pending(now) + + # Touch last_received_at once per flush (coalesced) + RouterMonSettings.objects.filter(pk=1).update(last_received_at=now) + + # Trigger geo enrichment in background (lazy: only for rows with resolved_ip that have no geo yet) + pks_needing_geo = list( + DnsQuery.objects.filter( + resolved_ip__isnull=False, country='', timestamp__gte=now - timedelta(minutes=5) + ).values_list('pk', flat=True)[:200] + ) + if pks_needing_geo: + threading.Thread( + target=_enrich_geo, args=(pks_needing_geo,), daemon=True + ).start() + + +def _apply_replies(reply_events, now): + from .models import DnsQuery + + with _pending_lock: + for ev in reply_events: + entry = _pending.pop(ev.domain, None) + if entry is None: + continue + pk, _ = entry + updates = {} + if ev.is_nxdomain: + updates['is_nxdomain'] = True + if ev.resolved_ip: + updates['resolved_ip'] = ev.resolved_ip + if updates: + DnsQuery.objects.filter(pk=pk).update(**updates) + + +def _expire_pending(now): + cutoff = now - timedelta(seconds=REPLY_TTL) + with _pending_lock: + expired = [domain for domain, (_, ts) in _pending.items() if ts < cutoff] + for domain in expired: + del _pending[domain] + + +def _enrich_geo(pks: list[int]): + """Geo-enrich DnsQuery rows by resolved_ip using nginxmon's IPGeoCache.""" + try: + from nginxmon.models import IPGeoCache + from routermon.models import DnsQuery + import ipaddress + + rows = list(DnsQuery.objects.filter(pk__in=pks).values('pk', 'resolved_ip')) + ips = list({r['resolved_ip'] for r in rows if r['resolved_ip']}) + if not ips: + return + + # Fetch missing IPs from ip-api.com + cached = {c.ip: c for c in IPGeoCache.objects.filter(ip__in=ips)} + missing = [ip for ip in ips if ip not in cached] + if missing: + from nginxmon.geo import _lookup_batch + api_results = _lookup_batch(missing) + for ip, data in api_results.items(): + obj, _ = IPGeoCache.objects.get_or_create(ip=ip) + obj.country = data.get('country', '') + obj.country_code = data.get('countryCode', '') + obj.region = data.get('regionName', '') + obj.city = data.get('city', '') + obj.lat = data.get('lat') + obj.lon = data.get('lon') + obj.isp = data.get('isp', '') + obj.save() + cached[ip] = obj + + for row in rows: + geo = cached.get(row['resolved_ip']) + if geo and geo.country: + DnsQuery.objects.filter(pk=row['pk'], country='').update( + country=geo.country, + country_code=geo.country_code, + city=geo.city, + ) + except Exception as exc: + logger.warning('routermon: geo enrichment error: %s', exc) + + +def _build_excluded_set(text: str) -> set: + result = set() + for line in (text or '').splitlines(): + entry = line.strip() + if entry and not entry.startswith('#'): + result.add(entry) + return result diff --git a/routermon/tasks.py b/routermon/tasks.py new file mode 100644 index 0000000..06d274a --- /dev/null +++ b/routermon/tasks.py @@ -0,0 +1,23 @@ +"""APScheduler periodic tasks for routermon.""" +import logging + +logger = logging.getLogger(__name__) + + +def cleanup_old_queries(): + """Delete DnsQuery rows older than retention_days. Run periodically.""" + try: + from datetime import timedelta + from django.utils import timezone + from .models import RouterMonSettings, DnsQuery + + settings = RouterMonSettings.get() + if settings.retention_days <= 0: + return + + cutoff = timezone.now() - timedelta(days=settings.retention_days) + deleted, _ = DnsQuery.objects.filter(timestamp__lt=cutoff).delete() + if deleted: + logger.info('routermon: pruned %d old DNS query records (>%d days)', deleted, settings.retention_days) + except Exception as exc: + logger.error('routermon: cleanup_old_queries error: %s', exc, exc_info=True) diff --git a/routermon/templates/routermon/_live_queries.html b/routermon/templates/routermon/_live_queries.html new file mode 100644 index 0000000..9aafa30 --- /dev/null +++ b/routermon/templates/routermon/_live_queries.html @@ -0,0 +1,45 @@ + + + + + + + + + + + + + {% for q in queries %} + + + + + + + + + {% empty %} + + + + {% endfor %} + +
TimeClientDomainTypeResolvedGeo
{{ q.timestamp|date:"H:i:s" }} + + {% if q.client_name %}{{ q.client_name }}{% endif %} + + + {{ q.query_type }} + {% if q.is_nxdomain %} + NXDOMAIN + {% elif q.resolved_ip %} + {{ q.resolved_ip }} + {% else %} + — + {% endif %} + {{ q.geo_display }}
+ No DNS queries yet. Make sure your router is forwarding syslog to this server. +
diff --git a/routermon/templates/routermon/dashboard.html b/routermon/templates/routermon/dashboard.html new file mode 100644 index 0000000..009a000 --- /dev/null +++ b/routermon/templates/routermon/dashboard.html @@ -0,0 +1,413 @@ +{% extends 'base.html' %} +{% load i18n %} + +{% block content %} + + + + +
+ + +
+
+

+ + Router DNS Monitor + + + {% if settings.enabled %}Live{% else %}Paused{% endif %} + +

+

+ ASUS GT-AX6000 · dnsmasq syslog · UDP :{{ settings.syslog_port }} +

+
+ {% if settings.last_received_at %}Last: {{ settings.last_received_at|timesince }} ago{% else %}No data yet{% endif %} + · + — +
+
+
+ +
+ {% for rkey, rlabel in all_ranges %} + + {{ rlabel|slice:"5:" }} + + {% endfor %} +
+
+ + + Settings + +
+
+
+ + + {% for msg in messages %} +
+ {{ msg }} +
+ {% endfor %} + + +
+ {% with s=stats %} +
+
DNS Queries
+
{{ s.total }}
+
+
+
Unique Domains
+
{{ s.unique_domains }}
+
+
+
Devices
+
{{ s.unique_clients }}
+
+
+
NXDOMAIN
+
+ {{ s.nxdomain_count }} +
+
+
+
NX Rate
+
+ {{ s.nxdomain_pct }}% +
+
+ {% endwith %} +
+ + +
+ +
+
{{ range_label }}
+
+
+ +
+
Top Domains — {{ range_label }}
+
+
+
+ + +
+
+
Resolved IP Geography — {{ range_label }}
+
+
+
+
+ + +
+ +
+
+ Top Queried Domains — {{ range_label }} +
+ + + + + + + + + + + {% for d in top_domains %} + + + + + + + {% empty %} + + {% endfor %} + +
DomainCountryQueriesNX
+ + + {% if d.country %} + {% if d.country_code %}{{ d.country_code }}{% endif %} + {{ d.country }} + {% else %}—{% endif %} + {{ d.count }} + {{ d.nxcount }}
No data yet
+
+ + +
+
+ Top Clients — {{ range_label }} +
+ + + + + + + + + + + {% for c in top_clients %} + + + + + + + {% empty %} + + {% endfor %} + +
ClientHostnameQueriesNX
+ + {{ c.hostname|default:"—" }}{{ c.count }} + {{ c.nxcount }}
No data yet
+
+
+ + +
+
+
Live DNS Queries
+ +
+ + + + + +
+
+ +
+
Loading…
+
+
+ +
+ + +{% endblock %} diff --git a/routermon/templates/routermon/settings.html b/routermon/templates/routermon/settings.html new file mode 100644 index 0000000..0168585 --- /dev/null +++ b/routermon/templates/routermon/settings.html @@ -0,0 +1,183 @@ +{% extends 'base.html' %} +{% load i18n %} + +{% block content %} +
+ + +
+
+

+ Router Monitor Settings +

+

UDP syslog receiver configuration

+
+ + Dashboard + +
+ + {% for msg in messages %} +
+ {{ msg }} +
+ {% endfor %} + + +
+
+

Receiver Configuration

+

+ The server listens for UDP syslog datagrams from your ASUS router's dnsmasq daemon. +

+
+
+ {% csrf_token %} + {% for field in form %} +
+ + {{ field }} + {% if field.help_text %}

{{ field.help_text }}

{% endif %} + {% for error in field.errors %}

{{ error }}

{% endfor %} +
+ {% endfor %} +
+ +
+
+
+ + +
+
+ +
+

Router Setup — AsusWRT-Merlin 3006.x (SSH method)

+

+ SSH into your ASUS GT-AX6000 and run the commands below. This works on all Merlin 3006.x builds. +

+
+
+
+ + +
+ Prerequisite: Enable SSH in the Merlin UI first — + Administration → System → SSH Daemon, set to LAN only, then save. + Also enable JFFS custom scripts and configs on the same page if not already enabled. +
+ +
+

+ 1 + Enable DNS query logging in dnsmasq +

+

+ SSH into the router and add a dnsmasq option that persists across reboots via JFFS: +

+ 
ssh admin@192.168.1.1
+
+# Add log-queries to dnsmasq (persistent via JFFS)
+echo "log-queries" >> /jffs/configs/dnsmasq.conf.add
+
+# Apply immediately (no reboot needed)
+service restart_dnsmasq
+
+ +
+

+ 2 + Configure remote syslog forwarding +

+

+ Set nvram variables to forward syslog to this server, then restart the syslog daemon: +

+ 
# Still in the SSH session:
+nvram set log_remote=1
+nvram set log_ipaddr=192.168.1.2
+nvram set log_port={{ settings.syslog_port }}
+nvram commit
+
+service restart_syslog
+

+ DNS queries should appear on the dashboard within a few seconds of the next DNS lookup on your network. +

+
+ +
+

+ 3 + Verify syslog is arriving +

+ 
# On the k3s node (192.168.1.2) — listen for UDP packets:
+nc -ulk {{ settings.syslog_port }}
+
+# Or watch with tcpdump:
+tcpdump -i any -A udp port {{ settings.syslog_port }}
+
+# You should see lines like:
+# dnsmasq[1234]: query[A] google.com from 192.168.1.x
+# dnsmasq[1234]: reply google.com is 142.250.80.46
+
+ + +
+

+ + If port {{ settings.syslog_port }} is not reachable from the router +

+

+ If the router can only send to the standard syslog port (514), redirect it on the k3s node: +

+ 
# On the k3s node, run as root — then change log_port to 514 in nvram above
+iptables -t nat -A PREROUTING -p udp --dport 514 -j REDIRECT --to-port {{ settings.syslog_port }}
+
+# Persist across reboots (Debian/Ubuntu):
+apt install iptables-persistent && netfilter-persistent save
+
+ + +
+

To disable / revert

+ 
ssh admin@192.168.1.1
+
+# Remove dnsmasq log-queries line
+sed -i '/^log-queries$/d' /jffs/configs/dnsmasq.conf.add
+
+# Disable remote syslog
+nvram set log_remote=0
+nvram commit
+
+service restart_dnsmasq
+service restart_syslog
+
+ +
+
+ + +
+

Receiver Status

+
+
Status
+
+ {% if settings.enabled %}Enabled (listening on UDP :{{ settings.syslog_port }}){% else %}Disabled{% endif %} +
+
Last received
+
+ {% if settings.last_received_at %}{{ settings.last_received_at|date:"Y-m-d H:i:s" }}{% else %}Never{% endif %} +
+
Retention
+
{{ settings.retention_days }} days
+
+
+ +
+{% endblock %} diff --git a/routermon/urls.py b/routermon/urls.py new file mode 100644 index 0000000..3d2bfe9 --- /dev/null +++ b/routermon/urls.py @@ -0,0 +1,15 @@ +from django.urls import path +from . import views + +urlpatterns = [ + path('', views.DashboardView.as_view(), name='routermon-dashboard'), + path('settings/', views.SettingsView.as_view(), name='routermon-settings'), + # HTMX partials + path('partials/queries/', views.LiveQueriesPartialView.as_view(), name='routermon-queries-partial'), + # JSON APIs + path('api/chart/', views.ChartDataView.as_view(), name='routermon-chart'), + path('api/geo/', views.GeoStatsView.as_view(), name='routermon-geo'), + # Actions + path('toggle/', views.ToggleView.as_view(), name='routermon-toggle'), + path('status/', views.StatusStreamView.as_view(), name='routermon-status'), +] diff --git a/routermon/views.py b/routermon/views.py new file mode 100644 index 0000000..878db7d --- /dev/null +++ b/routermon/views.py @@ -0,0 +1,307 @@ +import json +import logging +import time +from datetime import timedelta + +from django.contrib import messages +from django.db.models import Count, Q +from django.db.models.functions import TruncDay, TruncHour, TruncMinute +from django.http import JsonResponse, StreamingHttpResponse +from django.shortcuts import redirect, render +from django.utils import timezone +from django.views.generic import TemplateView, View + +from .forms import RouterMonSettingsForm +from .models import DhcpLease, DnsQuery, RouterMonSettings + +logger = logging.getLogger(__name__) + +# ── Time range config (mirrors nginxmon) ───────────────────────────────────── + +_RANGES = { + '5m': {'delta': timedelta(minutes=5), 'trunc': TruncMinute, 'group_n': 1, 'label': 'Last 5 min', 'fmt': '%H:%M'}, + '30m': {'delta': timedelta(minutes=30), 'trunc': TruncMinute, 'group_n': 2, 'label': 'Last 30 min', 'fmt': '%H:%M'}, + '6h': {'delta': timedelta(hours=6), 'trunc': TruncMinute, 'group_n': 15, 'label': 'Last 6 hours', 'fmt': '%H:%M'}, + '1d': {'delta': timedelta(hours=24), 'trunc': TruncHour, 'group_n': 1, 'label': 'Last 24 hours', 'fmt': '%H:%M'}, + '7d': {'delta': timedelta(days=7), 'trunc': TruncHour, 'group_n': 4, 'label': 'Last 7 days', 'fmt': '%m/%d %H:%M'}, + '30d': {'delta': timedelta(days=30), 'trunc': TruncDay, 'group_n': 1, 'label': 'Last 30 days', 'fmt': '%m/%d'}, +} +_DEFAULT_RANGE = '30m' + + +def _get_range(request): + key = request.GET.get('range', _DEFAULT_RANGE) + if key not in _RANGES: + key = _DEFAULT_RANGE + cfg = _RANGES[key] + return key, cfg, timezone.now() - cfg['delta'] + + +def _floor_bucket_key(dt, trunc_fn, group_n, fmt): + if trunc_fn is TruncMinute: + m = (dt.minute // group_n) * group_n + aligned = dt.replace(minute=m, second=0, microsecond=0) + elif trunc_fn is TruncHour: + h = (dt.hour // group_n) * group_n + aligned = dt.replace(hour=h, minute=0, second=0, microsecond=0) + else: + from datetime import date + epoch = date(2020, 1, 1) + d = dt.date() if hasattr(dt, 'date') else dt + days = (d - epoch).days + floored = epoch + timedelta(days=(days // group_n) * group_n) + aligned = dt.replace( + year=floored.year, month=floored.month, day=floored.day, + hour=0, minute=0, second=0, microsecond=0, + ) + return aligned.strftime(fmt) + + +# ── Dashboard ───────────────────────────────────────────────────────────────── + +class DashboardView(TemplateView): + template_name = 'routermon/dashboard.html' + + def get_context_data(self, **kwargs): + ctx = super().get_context_data(**kwargs) + range_key, cfg, since = _get_range(self.request) + settings = RouterMonSettings.get() + + qs = DnsQuery.objects.filter(timestamp__gte=since) + total = qs.count() + nxdomain_count = qs.filter(is_nxdomain=True).count() + + stats = { + 'total': total, + 'unique_domains': qs.values('domain').distinct().count(), + 'unique_clients': qs.values('client_ip').distinct().count(), + 'nxdomain_count': nxdomain_count, + 'nxdomain_pct': round(nxdomain_count / total * 100, 1) if total else 0, + } + + top_domains = list( + qs.values('domain') + .annotate( + count=Count('id'), + nxcount=Count('id', filter=Q(is_nxdomain=True)), + ) + .order_by('-count')[:20] + ) + + # Attach geo (country) to top domains via resolved_ip + domain_geo = {} + for row in qs.filter(resolved_ip__isnull=False, country__gt='').values('domain', 'country_code', 'country').distinct()[:100]: + if row['domain'] not in domain_geo: + domain_geo[row['domain']] = {'country': row['country'], 'country_code': row['country_code']} + for d in top_domains: + geo = domain_geo.get(d['domain'], {}) + d['country'] = geo.get('country', '') + d['country_code'] = geo.get('country_code', '') + + # Build client hostname map from DhcpLease + client_ips = list(qs.values_list('client_ip', flat=True).distinct()[:50]) + hostname_map = { + lease.ip: lease.hostname + for lease in DhcpLease.objects.filter(ip__in=client_ips) + if lease.hostname + } + + top_clients = list( + qs.values('client_ip', 'client_name') + .annotate(count=Count('id'), nxcount=Count('id', filter=Q(is_nxdomain=True))) + .order_by('-count')[:15] + ) + for c in top_clients: + c['hostname'] = hostname_map.get(c['client_ip'], c['client_name'] or '') + + ctx.update({ + 'settings': settings, + 'stats': stats, + 'top_domains': top_domains, + 'top_clients': top_clients, + 'current_range': range_key, + 'range_label': cfg['label'], + 'all_ranges': [(k, v['label']) for k, v in _RANGES.items()], + 'top_domains_json': json.dumps(top_domains, default=str), + }) + return ctx + + +# ── Settings ────────────────────────────────────────────────────────────────── + +class SettingsView(View): + template_name = 'routermon/settings.html' + + def get(self, request): + obj = RouterMonSettings.get() + return render(request, self.template_name, { + 'form': RouterMonSettingsForm(instance=obj), + 'settings': obj, + }) + + def post(self, request): + obj = RouterMonSettings.get() + form = RouterMonSettingsForm(request.POST, instance=obj) + if form.is_valid(): + form.save() + messages.success(request, 'Router Monitor settings saved.') + return redirect('routermon-settings') + return render(request, self.template_name, {'form': form, 'settings': obj}) + + +# ── HTMX partials ───────────────────────────────────────────────────────────── + +class LiveQueriesPartialView(View): + _SORT_MAP = { + 'timestamp': 'timestamp', + 'client': 'client_ip', + 'domain': 'domain', + 'type': 'query_type', + } + + def get(self, request): + qs = DnsQuery.objects.all() + + client_f = request.GET.get('client', '').strip() + domain_f = request.GET.get('domain', '').strip() + type_f = request.GET.get('type', '').strip() + nx_f = request.GET.get('nxdomain', '').strip() + sort_col = request.GET.get('sort', 'timestamp') + sort_ord = request.GET.get('order', 'desc') + + if client_f: + qs = qs.filter(Q(client_ip=client_f) | Q(client_name__icontains=client_f)) + if domain_f: + qs = qs.filter(domain__icontains=domain_f) + if type_f: + qs = qs.filter(query_type__iexact=type_f) + if nx_f == '1': + qs = qs.filter(is_nxdomain=True) + + db_col = self._SORT_MAP.get(sort_col, 'timestamp') + qs = qs.order_by(f'{"" if sort_ord == "asc" else "-"}{db_col}') + + return render(request, 'routermon/_live_queries.html', {'queries': qs[:60]}) + + +class GeoStatsView(View): + """Country-level aggregates + bubble data for the geo map.""" + + def get(self, request): + range_key, cfg, since = _get_range(request) + qs = DnsQuery.objects.filter(timestamp__gte=since) + + countries = list( + qs.exclude(country_code='') + .values('country_code', 'country') + .annotate(total=Count('id'), nxcount=Count('id', filter=Q(is_nxdomain=True))) + .order_by('-total') + ) + + # Bubble data from IPGeoCache for resolved IPs + ip_agg = list( + qs.filter(resolved_ip__isnull=False) + .values('resolved_ip') + .annotate(total=Count('id')) + .order_by('-total')[:300] + ) + ip_set = [r['resolved_ip'] for r in ip_agg] + + try: + from nginxmon.models import IPGeoCache + geo_cache = { + c.ip: (c.lat, c.lon, c.country, c.city) + for c in IPGeoCache.objects.filter(ip__in=ip_set, lat__isnull=False, is_private=False) + } + except Exception: + geo_cache = {} + + bubbles = [] + for row in ip_agg: + c = geo_cache.get(row['resolved_ip']) + if c and c[0] is not None and c[1] is not None: + bubbles.append({ + 'lat': round(float(c[0]), 2), + 'lon': round(float(c[1]), 2), + 'total': row['total'], + 'label': c[3] or c[2] or row['resolved_ip'], + }) + + return JsonResponse({'countries': countries, 'bubbles': bubbles, 'range': range_key}) + + +# ── Chart data ──────────────────────────────────────────────────────────────── + +class ChartDataView(View): + def get(self, request): + range_key, cfg, since = _get_range(request) + trunc_fn = cfg['trunc'] + group_n = cfg['group_n'] + fmt = cfg['fmt'] + + rows = list( + DnsQuery.objects + .filter(timestamp__gte=since) + .annotate(bucket=trunc_fn('timestamp')) + .values('bucket') + .annotate( + total=Count('id'), + nxcount=Count('id', filter=Q(is_nxdomain=True)), + ) + .order_by('bucket') + ) + + buckets: dict = {} + for row in rows: + bkt = row['bucket'] + if bkt is None: + continue + key = _floor_bucket_key(bkt, trunc_fn, group_n, fmt) + if key not in buckets: + buckets[key] = {'total': 0, 'nxcount': 0} + buckets[key]['total'] += row['total'] + buckets[key]['nxcount'] += row['nxcount'] + + return JsonResponse({ + 'labels': list(buckets), + 'total': [v['total'] for v in buckets.values()], + 'nxcount': [v['nxcount'] for v in buckets.values()], + 'range': range_key, + 'label': cfg['label'], + }) + + +# ── Toggle + SSE status stream ───────────────────────────────────────────────── + +class ToggleView(View): + def post(self, request): + settings = RouterMonSettings.get() + settings.enabled = not settings.enabled + settings.save(update_fields=['enabled']) + return JsonResponse({'enabled': settings.enabled}) + + +class StatusStreamView(View): + """SSE — pushes receiver status JSON every 3 seconds.""" + + def get(self, request): + def event_stream(): + while True: + try: + settings = RouterMonSettings.get() + total = DnsQuery.objects.count() + last = settings.last_received_at.isoformat() if settings.last_received_at else None + payload = json.dumps({ + 'enabled': settings.enabled, + 'last_received_at': last, + 'total_queries': total, + }) + yield f'data: {payload}\n\n' + except Exception: + pass + time.sleep(3) + + resp = StreamingHttpResponse(event_stream(), content_type='text/event-stream') + resp['Cache-Control'] = 'no-cache' + resp['X-Accel-Buffering'] = 'no' + return resp diff --git a/templates/base.html b/templates/base.html index f081645..9493983 100644 --- a/templates/base.html +++ b/templates/base.html @@ -151,6 +151,16 @@ + +
+ + + + {% trans "Router Monitor" %} +
+ +