#!/usr/bin/env python3 import subprocess import json import urllib.parse import datetime import os import sys def get_grafana_pod(): cmd = ["kubectl", "get", "pods", "-n", "monitoring", "-l", "app.kubernetes.io/name=grafana", "-o", "jsonpath={.items[0].metadata.name}"] res = subprocess.run(cmd, capture_output=True, text=True) if res.returncode == 0 and res.stdout.strip(): return res.stdout.strip() return "prometheus-operator-grafana-7f66b47fc6-dtdtg" GRAFANA_POD = get_grafana_pod() def query_range_kubectl(query, start_time, end_time, step): params = { "query": query, "start": start_time.isoformat(), "end": end_time.isoformat(), "step": step } query_str = urllib.parse.urlencode(params) url = f"http://prometheus-operated:9090/api/v1/query_range?{query_str}" cmd = [ "kubectl", "-n", "monitoring", "exec", GRAFANA_POD, "-c", "grafana", "--", "curl", "-s", url ] try: res = subprocess.run(cmd, capture_output=True, text=True, check=True) data = json.loads(res.stdout) if data.get("status") == "success": return data.get("data", {}).get("result", []) else: print(f"Prometheus API error: {data}", file=sys.stderr) except Exception as e: print(f"Failed to query via kubectl: {e}", file=sys.stderr) return [] def format_duration(seconds): days = int(seconds // (24 * 3600)) seconds %= (24 * 3600) hours = int(seconds // 3600) seconds %= 3600 minutes = int(seconds // 60) seconds = int(seconds % 60) parts = [] if days > 0: parts.append(f"{days}d") if hours > 0: parts.append(f"{hours}h") if minutes > 0: parts.append(f"{minutes}m") if seconds > 0 or not parts: parts.append(f"{seconds}s") return " ".join(parts) def parse_periods(values, step_seconds): if not values: return [] sorted_values = sorted([(float(ts), int(float(val))) for ts, val in values], key=lambda x: x[0]) periods = [] current_start = None last_ts = None for ts, val in sorted_values: if val != 1: if current_start is not None: periods.append({ "start": current_start, "end": last_ts, "duration": last_ts - current_start + step_seconds, "ongoing": False }) current_start = None continue if current_start is None: current_start = ts else: if ts - last_ts > step_seconds * 3: periods.append({ "start": current_start, "end": last_ts, "duration": last_ts - current_start + step_seconds, "ongoing": False }) current_start = ts last_ts = ts if current_start is not None: periods.append({ "start": current_start, "end": last_ts, "duration": last_ts - current_start + step_seconds, "ongoing": True }) return periods def main(): print(f"Querying Prometheus from pod {GRAFANA_POD}...") now = datetime.datetime.now(datetime.timezone.utc) start_time = now - datetime.timedelta(days=14) step = "2m" step_seconds = 120 print(f"Analyzing alert history from {start_time.strftime('%Y-%m-%d %H:%M:%S')} to {now.strftime('%Y-%m-%d %H:%M:%S')} UTC...") # Query ALERTS metric results = query_range_kubectl('ALERTS', start_time, now, step) if not results: print("No alerts found in the specified range.") return all_events = [] for result in results: metric = result.get("metric", {}) values = result.get("values", []) alert_name = metric.get("alertname", "Unknown") alert_state = metric.get("alertstate", "Unknown") # Extract target labels that distinguish this alert instance ignored_keys = {"__name__", "alertname", "alertstate"} labels = {k: v for k, v in metric.items() if k not in ignored_keys} periods = parse_periods(values, step_seconds) for p in periods: all_events.append({ "alertname": alert_name, "alertstate": alert_state, "labels": labels, "start": p["start"], "end": p["end"], "duration": p["duration"], "ongoing": p.get("ongoing", False) }) # Sort events by start time descending all_events.sort(key=lambda x: x["start"], reverse=True) print(f"\nFound {len(all_events)} alert events in the last 14 days:") print("-" * 100) # Print summary table for i, ev in enumerate(all_events): start_dt = datetime.datetime.fromtimestamp(ev["start"], datetime.timezone.utc) end_dt = datetime.datetime.fromtimestamp(ev["end"], datetime.timezone.utc) ongoing_str = " (Ongoing)" if ev["ongoing"] else "" labels_str = ", ".join([f"{k}={v}" for k, v in ev["labels"].items()]) print(f"[{i+1}] {ev['alertname']} ({ev['alertstate']})") print(f" Duration: {format_duration(ev['duration'])}{ongoing_str}") print(f" Timeline: {start_dt.strftime('%Y-%m-%d %H:%M:%S')} -> {end_dt.strftime('%Y-%m-%d %H:%M:%S')} UTC") print(f" Labels: {labels_str}") print("-" * 100) if __name__ == "__main__": main()