mirror of
https://github.com/wahyd4/links.git
synced 2026-08-08 21:04:53 +10:00
84 lines
3.3 KiB
Python
84 lines
3.3 KiB
Python
"""
|
|
Parse nginx ingress-controller log lines.
|
|
|
|
Log format:
|
|
<pod> <container> <remote_addr> - - [<time_local>] "<method> <uri> <proto>" <status>
|
|
<bytes> "<referer>" "<user_agent>" <req_len> <req_time> [<service>] []
|
|
<upstream_addr> <upstream_resp_len> <upstream_resp_time> <upstream_status> <req_id>
|
|
"""
|
|
import re
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# The leading \S* tolerates stray chars (e.g. 'z') sometimes prepended by log shippers.
|
|
_LOG_RE = re.compile(
|
|
r'\S*ingress-nginx\S+\s+' # pod name
|
|
r'(?P<container>\S+)\s+' # container
|
|
r'(?P<remote_addr>[\d.a-fA-F:]+)\s+-\s+-\s+' # IP - -
|
|
r'\[(?P<time_local>[^\]]+)\]\s+' # [timestamp]
|
|
r'"(?P<method>\S+)\s+(?P<request_uri>\S+)\s+' # "METHOD /path
|
|
r'(?P<protocol>[^"]+)"\s+' # PROTO"
|
|
r'(?P<status>\d+)\s+' # status
|
|
r'(?P<body_bytes_sent>\d+)\s+' # bytes
|
|
r'"(?P<http_referer>[^"]*)"\s+' # "referer"
|
|
r'"(?P<http_user_agent>[^"]*)"\s+' # "ua"
|
|
r'(?P<request_length>\d+)\s+' # req_len
|
|
r'(?P<request_time>[\d.]+)\s+' # req_time
|
|
r'\[(?P<service>[^\]]*)\]\s+' # [service]
|
|
r'\[(?P<extra>[^\]]*)\]\s+' # []
|
|
r'(?P<upstream_addr>\S+)\s+' # upstream addr
|
|
r'(?P<upstream_response_length>\d+)\s+' # upstream bytes
|
|
r'(?P<upstream_response_time>[\d.]+)\s+' # upstream time
|
|
r'(?P<upstream_status>\d+)\s+' # upstream status
|
|
r'(?P<request_id>\S+)' # request id
|
|
)
|
|
|
|
_TIME_FMT = '%d/%b/%Y:%H:%M:%S %z'
|
|
|
|
|
|
def parse_line(line: str) -> dict | None:
|
|
"""Return a parsed dict or None if the line doesn't match."""
|
|
m = _LOG_RE.search(line)
|
|
if not m:
|
|
return None
|
|
d = m.groupdict()
|
|
try:
|
|
ts = datetime.strptime(d['time_local'], _TIME_FMT)
|
|
except ValueError:
|
|
logger.debug('Bad timestamp: %s', d['time_local'])
|
|
return None
|
|
return {
|
|
'timestamp': ts,
|
|
'remote_addr': d['remote_addr'],
|
|
'method': d['method'],
|
|
'request_uri': d['request_uri'],
|
|
'protocol': d['protocol'].strip(),
|
|
'status': int(d['status']),
|
|
'body_bytes_sent': int(d['body_bytes_sent']),
|
|
'http_referer': d['http_referer'],
|
|
'http_user_agent': d['http_user_agent'],
|
|
'request_length': int(d['request_length']),
|
|
'request_time': float(d['request_time']),
|
|
'service': d['service'],
|
|
'upstream_addr': d['upstream_addr'],
|
|
'upstream_response_time': float(d['upstream_response_time']),
|
|
'upstream_status': int(d['upstream_status']),
|
|
'request_id': d['request_id'],
|
|
}
|
|
|
|
|
|
def parse_lines(text: str) -> list[dict]:
|
|
results = []
|
|
for line in text.splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
parsed = parse_line(line)
|
|
if parsed:
|
|
results.append(parsed)
|
|
else:
|
|
logger.debug('Skipped unparseable line: %.120s', line)
|
|
return results
|