Files
junvandCopilot cfca2c2ed7 routermon: add WAN geo map + fix batch geo enrichment
- Add WanGeoStatsView (api/wan/geo/) returning country choropleth +
  bubble data for WAN source IPs using IPGeoCache lat/lon
- Add wanGeoMap (420px) in dashboard between ports chart and live table:
  red colour scale by country, rose bubbles sized by attempt count,
  top-5 country legend
- Rewrite _geo_worker_loop to drain up to 200 queue items per cycle,
  bulk DB cache check, batch API calls (100 IPs per request) instead
  of 1 request per IP — fixes ip-api.com timeout pile-up under scan
  volumes
- Add _geo_skip in-memory set: IPs that return no geo data get an
  empty IPGeoCache placeholder saved to DB so they are never retried;
  skip set also prevents re-queuing known-empty IPs during flush
- Remove _fetch_geo / _enrich_dns_geo / _enrich_wan_geo helpers
  (logic consolidated into the new batch worker)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-16 16:13:30 +10:00

446 lines
16 KiB
Python

import json
import logging
import time
from datetime import timedelta
from django.contrib import messages
from django.db.models import Count, Q
from django.db.models.functions import TruncDay, TruncHour, TruncMinute
from django.http import JsonResponse, StreamingHttpResponse
from django.shortcuts import redirect, render
from django.utils import timezone
from django.views.generic import TemplateView, View
from .forms import RouterMonSettingsForm
from .models import DhcpLease, DnsQuery, RouterMonSettings, WanEvent
logger = logging.getLogger(__name__)
# ── Time range config (mirrors nginxmon) ─────────────────────────────────────
_RANGES = {
'5m': {'delta': timedelta(minutes=5), 'trunc': TruncMinute, 'group_n': 1, 'label': 'Last 5 min', 'fmt': '%H:%M'},
'30m': {'delta': timedelta(minutes=30), 'trunc': TruncMinute, 'group_n': 2, 'label': 'Last 30 min', 'fmt': '%H:%M'},
'6h': {'delta': timedelta(hours=6), 'trunc': TruncMinute, 'group_n': 15, 'label': 'Last 6 hours', 'fmt': '%H:%M'},
'1d': {'delta': timedelta(hours=24), 'trunc': TruncHour, 'group_n': 1, 'label': 'Last 24 hours', 'fmt': '%H:%M'},
'7d': {'delta': timedelta(days=7), 'trunc': TruncHour, 'group_n': 4, 'label': 'Last 7 days', 'fmt': '%m/%d %H:%M'},
'30d': {'delta': timedelta(days=30), 'trunc': TruncDay, 'group_n': 1, 'label': 'Last 30 days', 'fmt': '%m/%d'},
}
_DEFAULT_RANGE = '30m'
def _get_range(request):
key = request.GET.get('range', _DEFAULT_RANGE)
if key not in _RANGES:
key = _DEFAULT_RANGE
cfg = _RANGES[key]
return key, cfg, timezone.now() - cfg['delta']
def _floor_bucket_key(dt, trunc_fn, group_n, fmt):
if trunc_fn is TruncMinute:
m = (dt.minute // group_n) * group_n
aligned = dt.replace(minute=m, second=0, microsecond=0)
elif trunc_fn is TruncHour:
h = (dt.hour // group_n) * group_n
aligned = dt.replace(hour=h, minute=0, second=0, microsecond=0)
else:
from datetime import date
epoch = date(2020, 1, 1)
d = dt.date() if hasattr(dt, 'date') else dt
days = (d - epoch).days
floored = epoch + timedelta(days=(days // group_n) * group_n)
aligned = dt.replace(
year=floored.year, month=floored.month, day=floored.day,
hour=0, minute=0, second=0, microsecond=0,
)
return aligned.strftime(fmt)
# ── Dashboard ─────────────────────────────────────────────────────────────────
class DashboardView(TemplateView):
template_name = 'routermon/dashboard.html'
def get_context_data(self, **kwargs):
ctx = super().get_context_data(**kwargs)
range_key, cfg, since = _get_range(self.request)
settings = RouterMonSettings.get()
qs = DnsQuery.objects.filter(timestamp__gte=since)
total = qs.count()
nxdomain_count = qs.filter(is_nxdomain=True).count()
stats = {
'total': total,
'unique_domains': qs.values('domain').distinct().count(),
'unique_clients': qs.values('client_ip').distinct().count(),
'nxdomain_count': nxdomain_count,
'nxdomain_pct': round(nxdomain_count / total * 100, 1) if total else 0,
}
top_domains = list(
qs.values('domain')
.annotate(
count=Count('id'),
nxcount=Count('id', filter=Q(is_nxdomain=True)),
)
.order_by('-count')[:20]
)
# Attach geo (country) to top domains via resolved_ip
domain_geo = {}
for row in qs.filter(resolved_ip__isnull=False, country__gt='').values('domain', 'country_code', 'country').distinct()[:100]:
if row['domain'] not in domain_geo:
domain_geo[row['domain']] = {'country': row['country'], 'country_code': row['country_code']}
for d in top_domains:
geo = domain_geo.get(d['domain'], {})
d['country'] = geo.get('country', '')
d['country_code'] = geo.get('country_code', '')
# Build client hostname map from DhcpLease
client_ips = list(qs.values_list('client_ip', flat=True).distinct()[:50])
hostname_map = {
lease.ip: lease.hostname
for lease in DhcpLease.objects.filter(ip__in=client_ips)
if lease.hostname
}
top_clients = list(
qs.values('client_ip', 'client_name')
.annotate(count=Count('id'), nxcount=Count('id', filter=Q(is_nxdomain=True)))
.order_by('-count')[:15]
)
for c in top_clients:
c['hostname'] = hostname_map.get(c['client_ip'], c['client_name'] or '')
wan_since = timezone.now() - timedelta(hours=24)
wan_qs = WanEvent.objects.filter(timestamp__gte=wan_since)
wan_total = wan_qs.count()
top_attacked_ports = list(
wan_qs.filter(dst_port__isnull=False)
.values('dst_port', 'protocol')
.annotate(count=Count('id'))
.order_by('-count')[:10]
)
top_wan_sources = list(
wan_qs.values('src_ip', 'country', 'country_code')
.annotate(count=Count('id'))
.order_by('-count')[:10]
)
ctx.update({
'settings': settings,
'stats': stats,
'top_domains': top_domains,
'top_clients': top_clients,
'current_range': range_key,
'range_label': cfg['label'],
'all_ranges': [(k, v['label']) for k, v in _RANGES.items()],
'top_domains_json': json.dumps(top_domains, default=str),
'wan_total_24h': wan_total,
'top_attacked_ports': top_attacked_ports,
'top_attacked_ports_json': json.dumps(top_attacked_ports, default=str),
'top_wan_sources': top_wan_sources,
})
return ctx
# ── Settings ──────────────────────────────────────────────────────────────────
class SettingsView(View):
template_name = 'routermon/settings.html'
def get(self, request):
obj = RouterMonSettings.get()
return render(request, self.template_name, {
'form': RouterMonSettingsForm(instance=obj),
'settings': obj,
})
def post(self, request):
obj = RouterMonSettings.get()
form = RouterMonSettingsForm(request.POST, instance=obj)
if form.is_valid():
form.save()
messages.success(request, 'Router Monitor settings saved.')
return redirect('routermon-settings')
return render(request, self.template_name, {'form': form, 'settings': obj})
# ── HTMX partials ─────────────────────────────────────────────────────────────
class LiveQueriesPartialView(View):
_SORT_MAP = {
'timestamp': 'timestamp',
'client': 'client_ip',
'domain': 'domain',
'type': 'query_type',
}
def get(self, request):
qs = DnsQuery.objects.all()
client_f = request.GET.get('client', '').strip()
domain_f = request.GET.get('domain', '').strip()
type_f = request.GET.get('type', '').strip()
nx_f = request.GET.get('nxdomain', '').strip()
sort_col = request.GET.get('sort', 'timestamp')
sort_ord = request.GET.get('order', 'desc')
if client_f:
qs = qs.filter(Q(client_ip=client_f) | Q(client_name__icontains=client_f))
if domain_f:
qs = qs.filter(domain__icontains=domain_f)
if type_f:
qs = qs.filter(query_type__iexact=type_f)
if nx_f == '1':
qs = qs.filter(is_nxdomain=True)
db_col = self._SORT_MAP.get(sort_col, 'timestamp')
qs = qs.order_by(f'{"" if sort_ord == "asc" else "-"}{db_col}')
return render(request, 'routermon/_live_queries.html', {'queries': qs[:60]})
class GeoStatsView(View):
"""Country-level aggregates + bubble data for the geo map."""
def get(self, request):
range_key, cfg, since = _get_range(request)
qs = DnsQuery.objects.filter(timestamp__gte=since)
countries = list(
qs.exclude(country_code='')
.values('country_code', 'country')
.annotate(total=Count('id'), nxcount=Count('id', filter=Q(is_nxdomain=True)))
.order_by('-total')
)
# Bubble data from IPGeoCache for resolved IPs
ip_agg = list(
qs.filter(resolved_ip__isnull=False)
.values('resolved_ip')
.annotate(total=Count('id'))
.order_by('-total')[:300]
)
ip_set = [r['resolved_ip'] for r in ip_agg]
try:
from nginxmon.models import IPGeoCache
geo_cache = {
c.ip: (c.lat, c.lon, c.country, c.city)
for c in IPGeoCache.objects.filter(ip__in=ip_set, lat__isnull=False, is_private=False)
}
except Exception:
geo_cache = {}
bubbles = []
for row in ip_agg:
c = geo_cache.get(row['resolved_ip'])
if c and c[0] is not None and c[1] is not None:
bubbles.append({
'lat': round(float(c[0]), 2),
'lon': round(float(c[1]), 2),
'total': row['total'],
'label': c[3] or c[2] or row['resolved_ip'],
})
return JsonResponse({'countries': countries, 'bubbles': bubbles, 'range': range_key})
# ── Chart data ────────────────────────────────────────────────────────────────
class ChartDataView(View):
def get(self, request):
range_key, cfg, since = _get_range(request)
trunc_fn = cfg['trunc']
group_n = cfg['group_n']
fmt = cfg['fmt']
rows = list(
DnsQuery.objects
.filter(timestamp__gte=since)
.annotate(bucket=trunc_fn('timestamp'))
.values('bucket')
.annotate(
total=Count('id'),
nxcount=Count('id', filter=Q(is_nxdomain=True)),
)
.order_by('bucket')
)
buckets: dict = {}
for row in rows:
bkt = row['bucket']
if bkt is None:
continue
key = _floor_bucket_key(bkt, trunc_fn, group_n, fmt)
if key not in buckets:
buckets[key] = {'total': 0, 'nxcount': 0}
buckets[key]['total'] += row['total']
buckets[key]['nxcount'] += row['nxcount']
return JsonResponse({
'labels': list(buckets),
'total': [v['total'] for v in buckets.values()],
'nxcount': [v['nxcount'] for v in buckets.values()],
'range': range_key,
'label': cfg['label'],
})
# ── Toggle + SSE status stream ─────────────────────────────────────────────────
class ToggleView(View):
def post(self, request):
settings = RouterMonSettings.get()
settings.enabled = not settings.enabled
settings.save(update_fields=['enabled'])
return JsonResponse({'enabled': settings.enabled})
class StatusStreamView(View):
"""SSE — pushes receiver status JSON every 3 seconds."""
def get(self, request):
def event_stream():
while True:
try:
settings = RouterMonSettings.get()
total = DnsQuery.objects.count()
last = settings.last_received_at.isoformat() if settings.last_received_at else None
payload = json.dumps({
'enabled': settings.enabled,
'last_received_at': last,
'total_queries': total,
})
yield f'data: {payload}\n\n'
except Exception:
pass
time.sleep(3)
resp = StreamingHttpResponse(event_stream(), content_type='text/event-stream')
resp['Cache-Control'] = 'no-cache'
resp['X-Accel-Buffering'] = 'no'
return resp
# ── WAN views ─────────────────────────────────────────────────────────────────
class WanLivePartialView(View):
"""HTMX partial: recent WAN events table."""
_SORT_MAP = {
'timestamp': 'timestamp',
'src_ip': 'src_ip',
'dst_port': 'dst_port',
'protocol': 'protocol',
'country': 'country',
}
def get(self, request):
qs = WanEvent.objects.all()
src_f = request.GET.get('src_ip', '').strip()
port_f = request.GET.get('dst_port', '').strip()
proto_f = request.GET.get('protocol', '').strip()
country_f = request.GET.get('country', '').strip()
sort_col = request.GET.get('sort', 'timestamp')
sort_ord = request.GET.get('order', 'desc')
if src_f:
qs = qs.filter(src_ip__icontains=src_f)
if port_f:
try:
qs = qs.filter(dst_port=int(port_f))
except ValueError:
pass
if proto_f:
qs = qs.filter(protocol__iexact=proto_f)
if country_f:
qs = qs.filter(Q(country__icontains=country_f) | Q(country_code__iexact=country_f))
db_col = self._SORT_MAP.get(sort_col, 'timestamp')
qs = qs.order_by(f'{"" if sort_ord == "asc" else "-"}{db_col}')
return render(request, 'routermon/_live_wan.html', {'wan_events': qs[:60]})
class WanGeoStatsView(View):
"""Country-level aggregates + bubble data for WAN incoming events."""
def get(self, request):
range_key, cfg, since = _get_range(request)
qs = WanEvent.objects.filter(timestamp__gte=since)
countries = list(
qs.exclude(country_code='')
.values('country_code', 'country')
.annotate(total=Count('id'))
.order_by('-total')
)
ip_agg = list(
qs.values('src_ip')
.annotate(total=Count('id'))
.order_by('-total')[:300]
)
ip_set = [r['src_ip'] for r in ip_agg]
try:
from nginxmon.models import IPGeoCache
geo_cache = {
c.ip: (c.lat, c.lon, c.country, c.city)
for c in IPGeoCache.objects.filter(ip__in=ip_set, lat__isnull=False, is_private=False)
}
except Exception:
geo_cache = {}
bubbles = []
for row in ip_agg:
c = geo_cache.get(row['src_ip'])
if c and c[0] is not None and c[1] is not None:
bubbles.append({
'lat': round(float(c[0]), 2),
'lon': round(float(c[1]), 2),
'total': row['total'],
'label': c[3] or c[2] or row['src_ip'],
})
return JsonResponse({'countries': countries, 'bubbles': bubbles, 'range': range_key})
class WanChartDataView(View):
"""JSON — WAN event counts over time (for chart)."""
def get(self, request):
range_key, cfg, since = _get_range(request)
trunc_fn = cfg['trunc']
group_n = cfg['group_n']
fmt = cfg['fmt']
rows = list(
WanEvent.objects
.filter(timestamp__gte=since)
.annotate(bucket=trunc_fn('timestamp'))
.values('bucket')
.annotate(total=Count('id'))
.order_by('bucket')
)
buckets: dict = {}
for row in rows:
bkt = row['bucket']
if bkt is None:
continue
key = _floor_bucket_key(bkt, trunc_fn, group_n, fmt)
buckets[key] = buckets.get(key, 0) + row['total']
return JsonResponse({
'labels': list(buckets),
'total': list(buckets.values()),
'range': range_key,
'label': cfg['label'],
})