moritz/infrapuzzle
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

monitoring: optimize alerts and mitigate fritzbox-exporter timeouts

Browse Source
This commit is contained in:
Moritz Graf
2026-07-11 07:40:06 +02:00
parent c5458821a7
commit bdabc56afe
10 changed files with 269 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/monitoring/alertmanagerconfig.secret.yaml
BIN
View File
Binary file not shown.
k8s/monitoring/fritzbox-exporter.yaml
+29 -1
View File
@@ -15,6 +15,34 @@ spec:
labels:
app: fritzbox-exporter
spec:
initContainers:
# This initContainer implements the Egress Reject Workaround (Option B).
# The FRITZ!Box TR-064 API redirects login_sid.lua calls to its public WAN IP because
# we query it from a different subnet (192.168.11.1). When the WAN IP is dynamic or port
# 80/443 are closed, the HTTP client hangs for the default 2-minute Linux TCP timeout,
# failing the entire Prometheus scrape.
# This container injects iptables rules into the pod's network namespace to instantly
# reject (TCP RST) egress to any public WAN IPs on port 80/443. The exporter will immediately
# fail the login step (in ms) and continue scraping TR-064 metrics on port 49000 (which is allowed).
- name: init-iptables
image: alpine:latest
securityContext:
capabilities:
add:
- NET_ADMIN
command: ["/bin/sh", "-c"]
args:
- |
apk add --no-cache iptables
# Allow all local RFC1918 subnets and loopback
iptables -A OUTPUT -d 127.0.0.1/32 -j ACCEPT
iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT
iptables -A OUTPUT -d 172.16.0.0/12 -j ACCEPT
iptables -A OUTPUT -d 192.168.0.0/16 -j ACCEPT
# Reject port 80 and 443 egress to any public IP (non-RFC1918)
iptables -A OUTPUT -p tcp --dport 80 -j REJECT --reject-with tcp-reset
iptables -A OUTPUT -p tcp --dport 443 -j REJECT --reject-with tcp-reset
echo "Egress iptables REJECT rules injected successfully"
containers:
- name: fritzbox-exporter
image: ghcr.io/sberk42/fritzbox_exporter/fritzbox_exporter:latest
@@ -95,7 +123,7 @@ spec:
rules:
- alert: FritzBoxExporterOffline
expr: up{job="fritzbox-exporter"} == 0
for: 5m
for: 1h
labels:
severity: warning
annotations:
k8s/monitoring/scripts/analyze_alerts.py
Executable
+168
View File
@@ -0,0 +1,168 @@
#!/usr/bin/env python3
import subprocess
import json
import urllib.parse
import datetime
import os
import sys
def get_grafana_pod():
cmd = ["kubectl", "get", "pods", "-n", "monitoring", "-l", "app.kubernetes.io/name=grafana", "-o", "jsonpath={.items[0].metadata.name}"]
res = subprocess.run(cmd, capture_output=True, text=True)
if res.returncode == 0 and res.stdout.strip():
return res.stdout.strip()
return "prometheus-operator-grafana-7f66b47fc6-dtdtg"
GRAFANA_POD = get_grafana_pod()
def query_range_kubectl(query, start_time, end_time, step):
params = {
"query": query,
"start": start_time.isoformat(),
"end": end_time.isoformat(),
"step": step
}
query_str = urllib.parse.urlencode(params)
url = f"http://prometheus-operated:9090/api/v1/query_range?{query_str}"
cmd = [
"kubectl", "-n", "monitoring", "exec",
GRAFANA_POD,
"-c", "grafana", "--",
"curl", "-s", url
]
try:
res = subprocess.run(cmd, capture_output=True, text=True, check=True)
data = json.loads(res.stdout)
if data.get("status") == "success":
return data.get("data", {}).get("result", [])
else:
print(f"Prometheus API error: {data}", file=sys.stderr)
except Exception as e:
print(f"Failed to query via kubectl: {e}", file=sys.stderr)
return []
def format_duration(seconds):
days = int(seconds // (24 * 3600))
seconds %= (24 * 3600)
hours = int(seconds // 3600)
seconds %= 3600
minutes = int(seconds // 60)
seconds = int(seconds % 60)
parts = []
if days > 0: parts.append(f"{days}d")
if hours > 0: parts.append(f"{hours}h")
if minutes > 0: parts.append(f"{minutes}m")
if seconds > 0 or not parts: parts.append(f"{seconds}s")
return " ".join(parts)
def parse_periods(values, step_seconds):
if not values:
return []
sorted_values = sorted([(float(ts), int(float(val))) for ts, val in values], key=lambda x: x[0])
periods = []
current_start = None
last_ts = None
for ts, val in sorted_values:
if val != 1:
if current_start is not None:
periods.append({
"start": current_start,
"end": last_ts,
"duration": last_ts - current_start + step_seconds,
"ongoing": False
})
current_start = None
continue
if current_start is None:
current_start = ts
else:
if ts - last_ts > step_seconds * 3:
periods.append({
"start": current_start,
"end": last_ts,
"duration": last_ts - current_start + step_seconds,
"ongoing": False
})
current_start = ts
last_ts = ts
if current_start is not None:
periods.append({
"start": current_start,
"end": last_ts,
"duration": last_ts - current_start + step_seconds,
"ongoing": True
})
return periods
def main():
print(f"Querying Prometheus from pod {GRAFANA_POD}...")
now = datetime.datetime.now(datetime.timezone.utc)
start_time = now - datetime.timedelta(days=14)
step = "2m"
step_seconds = 120
print(f"Analyzing alert history from {start_time.strftime('%Y-%m-%d %H:%M:%S')} to {now.strftime('%Y-%m-%d %H:%M:%S')} UTC...")
# Query ALERTS metric
results = query_range_kubectl('ALERTS', start_time, now, step)
if not results:
print("No alerts found in the specified range.")
return
all_events = []
for result in results:
metric = result.get("metric", {})
values = result.get("values", [])
alert_name = metric.get("alertname", "Unknown")
alert_state = metric.get("alertstate", "Unknown")
# Extract target labels that distinguish this alert instance
ignored_keys = {"__name__", "alertname", "alertstate"}
labels = {k: v for k, v in metric.items() if k not in ignored_keys}
periods = parse_periods(values, step_seconds)
for p in periods:
all_events.append({
"alertname": alert_name,
"alertstate": alert_state,
"labels": labels,
"start": p["start"],
"end": p["end"],
"duration": p["duration"],
"ongoing": p.get("ongoing", False)
})
# Sort events by start time descending
all_events.sort(key=lambda x: x["start"], reverse=True)
print(f"\nFound {len(all_events)} alert events in the last 14 days:")
print("-" * 100)
# Print summary table
for i, ev in enumerate(all_events):
start_dt = datetime.datetime.fromtimestamp(ev["start"], datetime.timezone.utc)
end_dt = datetime.datetime.fromtimestamp(ev["end"], datetime.timezone.utc)
ongoing_str = " (Ongoing)" if ev["ongoing"] else ""
labels_str = ", ".join([f"{k}={v}" for k, v in ev["labels"].items()])
print(f"[{i+1}] {ev['alertname']} ({ev['alertstate']})")
print(f" Duration: {format_duration(ev['duration'])}{ongoing_str}")
print(f" Timeline: {start_dt.strftime('%Y-%m-%d %H:%M:%S')} -> {end_dt.strftime('%Y-%m-%d %H:%M:%S')} UTC")
print(f" Labels: {labels_str}")
print("-" * 100)
if __name__ == "__main__":
main()
k8s/monitoring/taupi-fan.yml
+3 -3
View File
@@ -59,7 +59,7 @@ spec:
rules:
- alert: TaupiFanOffline
expr: up{job="taupi-fan"} == 0
for: 8h
for: 1h
labels:
severity: critical
annotations:
@@ -67,7 +67,7 @@ spec:
description: "The Shelly plug at 192.168.10.168 is unreachable by Prometheus. Check Wi-Fi connection or power status."
- alert: TaupiFanSensorsStale
expr: taupi_lost_connection_seconds_innen > 600 or taupi_lost_connection_seconds_aussen > 600
for: 5m
for: 1h
labels:
severity: warning
annotations:
@@ -75,7 +75,7 @@ spec:
description: "The BLE sensors for temperature and humidity inside or outside have not sent new data for over 10 minutes."
- alert: TaupiFanMoldDanger
expr: taupi_is_critical == 1
for: 30m
for: 1h
labels:
severity: warning
annotations: