mirror of
https://github.com/wahyd4/links.git
synced 2026-08-11 14:16:05 +10:00
101 lines
3.2 KiB
Python
101 lines
3.2 KiB
Python
"""
|
|
Geo-IP enrichment using ip-api.com batch endpoint (free, no key required).
|
|
Results are cached in IPGeoCache to minimise outbound requests.
|
|
"""
|
|
import ipaddress
|
|
import logging
|
|
|
|
import requests
|
|
|
|
from .models import IPGeoCache, NginxAccessLog
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_BATCH_URL = 'http://ip-api.com/batch'
|
|
_BATCH_FIELDS = 'country,countryCode,regionName,city,lat,lon,isp,status,query'
|
|
_PRIVATE_RANGES = (
|
|
ipaddress.ip_network('10.0.0.0/8'),
|
|
ipaddress.ip_network('172.16.0.0/12'),
|
|
ipaddress.ip_network('192.168.0.0/16'),
|
|
ipaddress.ip_network('127.0.0.0/8'),
|
|
ipaddress.ip_network('::1/128'),
|
|
ipaddress.ip_network('fc00::/7'),
|
|
)
|
|
|
|
|
|
def _is_private(ip: str) -> bool:
|
|
try:
|
|
addr = ipaddress.ip_address(ip)
|
|
return any(addr in net for net in _PRIVATE_RANGES)
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def _lookup_batch(ips: list[str]) -> dict[str, dict]:
|
|
"""Query ip-api.com for up to 100 IPs. Returns {ip: geo_dict}."""
|
|
payload = [{'query': ip, 'fields': _BATCH_FIELDS} for ip in ips[:100]]
|
|
try:
|
|
resp = requests.post(_BATCH_URL, json=payload, timeout=10)
|
|
resp.raise_for_status()
|
|
results = {}
|
|
for item in resp.json():
|
|
q = item.get('query', '')
|
|
if item.get('status') == 'success':
|
|
results[q] = item
|
|
return results
|
|
except Exception as exc:
|
|
logger.warning('ip-api.com batch lookup failed: %s', exc)
|
|
return {}
|
|
|
|
|
|
def enrich_geo_batch(log_objs: list[NginxAccessLog]):
|
|
"""
|
|
Look up geo data for IPs in `log_objs`, update cache and log records.
|
|
Uses the local cache first; only queries the API for unknown IPs.
|
|
"""
|
|
unique_ips = {obj.remote_addr for obj in log_objs}
|
|
|
|
# Split private vs public
|
|
private_ips = {ip for ip in unique_ips if _is_private(ip)}
|
|
public_ips = unique_ips - private_ips
|
|
|
|
# Ensure private IPs are in cache
|
|
for ip in private_ips:
|
|
IPGeoCache.objects.get_or_create(ip=ip, defaults={'is_private': True})
|
|
|
|
# Load cached public IPs
|
|
cached = {
|
|
g.ip: g
|
|
for g in IPGeoCache.objects.filter(ip__in=public_ips)
|
|
}
|
|
missing = [ip for ip in public_ips if ip not in cached]
|
|
|
|
# Fetch missing from API in batches of 100
|
|
api_results: dict[str, dict] = {}
|
|
for i in range(0, len(missing), 100):
|
|
api_results.update(_lookup_batch(missing[i:i + 100]))
|
|
|
|
# Upsert cache
|
|
for ip, data in api_results.items():
|
|
obj, _ = IPGeoCache.objects.get_or_create(ip=ip)
|
|
obj.country = data.get('country', '')
|
|
obj.country_code = data.get('countryCode', '')
|
|
obj.region = data.get('regionName', '')
|
|
obj.city = data.get('city', '')
|
|
obj.lat = data.get('lat')
|
|
obj.lon = data.get('lon')
|
|
obj.isp = data.get('isp', '')
|
|
obj.save()
|
|
cached[ip] = obj
|
|
|
|
# Apply geo back via per-IP queryset update (avoids needing PKs on in-memory objects)
|
|
for ip, geo in cached.items():
|
|
NginxAccessLog.objects.filter(
|
|
remote_addr=ip, country=''
|
|
).update(
|
|
country=geo.country or '',
|
|
country_code=geo.country_code or '',
|
|
region=geo.region or '',
|
|
city=geo.city or '',
|
|
)
|