infrapuzzle/k8s/monitoring/scripts/analyze_alerts.py

#!/usr/bin/env python3
import subprocess
import json
import urllib.parse
import datetime
import os
import sys

def get_grafana_pod():
    cmd = ["kubectl", "get", "pods", "-n", "monitoring", "-l", "app.kubernetes.io/name=grafana", "-o", "jsonpath={.items[0].metadata.name}"]
    res = subprocess.run(cmd, capture_output=True, text=True)
    if res.returncode == 0 and res.stdout.strip():
        return res.stdout.strip()
    return "prometheus-operator-grafana-7f66b47fc6-dtdtg"

GRAFANA_POD = get_grafana_pod()

def query_range_kubectl(query, start_time, end_time, step):
    params = {
        "query": query,
        "start": start_time.isoformat(),
        "end": end_time.isoformat(),
        "step": step
    }
    query_str = urllib.parse.urlencode(params)
    url = f"http://prometheus-operated:9090/api/v1/query_range?{query_str}"
    
    cmd = [
        "kubectl", "-n", "monitoring", "exec",
        GRAFANA_POD,
        "-c", "grafana", "--",
        "curl", "-s", url
    ]
    try:
        res = subprocess.run(cmd, capture_output=True, text=True, check=True)
        data = json.loads(res.stdout)
        if data.get("status") == "success":
            return data.get("data", {}).get("result", [])
        else:
            print(f"Prometheus API error: {data}", file=sys.stderr)
    except Exception as e:
        print(f"Failed to query via kubectl: {e}", file=sys.stderr)
    return []

def format_duration(seconds):
    days = int(seconds // (24 * 3600))
    seconds %= (24 * 3600)
    hours = int(seconds // 3600)
    seconds %= 3600
    minutes = int(seconds // 60)
    seconds = int(seconds % 60)
    
    parts = []
    if days > 0: parts.append(f"{days}d")
    if hours > 0: parts.append(f"{hours}h")
    if minutes > 0: parts.append(f"{minutes}m")
    if seconds > 0 or not parts: parts.append(f"{seconds}s")
    return " ".join(parts)

def parse_periods(values, step_seconds):
    if not values:
        return []
    
    sorted_values = sorted([(float(ts), int(float(val))) for ts, val in values], key=lambda x: x[0])
    
    periods = []
    current_start = None
    last_ts = None
    
    for ts, val in sorted_values:
        if val != 1:
            if current_start is not None:
                periods.append({
                    "start": current_start,
                    "end": last_ts,
                    "duration": last_ts - current_start + step_seconds,
                    "ongoing": False
                })
                current_start = None
            continue
            
        if current_start is None:
            current_start = ts
        else:
            if ts - last_ts > step_seconds * 3:
                periods.append({
                    "start": current_start,
                    "end": last_ts,
                    "duration": last_ts - current_start + step_seconds,
                    "ongoing": False
                })
                current_start = ts
        last_ts = ts
        
    if current_start is not None:
        periods.append({
            "start": current_start,
            "end": last_ts,
            "duration": last_ts - current_start + step_seconds,
            "ongoing": True
        })
        
    return periods

def main():
    print(f"Querying Prometheus from pod {GRAFANA_POD}...")
    
    now = datetime.datetime.now(datetime.timezone.utc)
    start_time = now - datetime.timedelta(days=14)
    step = "2m"
    step_seconds = 120
    
    print(f"Analyzing alert history from {start_time.strftime('%Y-%m-%d %H:%M:%S')} to {now.strftime('%Y-%m-%d %H:%M:%S')} UTC...")
    
    # Query ALERTS metric
    results = query_range_kubectl('ALERTS', start_time, now, step)
    
    if not results:
        print("No alerts found in the specified range.")
        return
        
    all_events = []
    
    for result in results:
        metric = result.get("metric", {})
        values = result.get("values", [])
        
        alert_name = metric.get("alertname", "Unknown")
        alert_state = metric.get("alertstate", "Unknown")
        
        # Extract target labels that distinguish this alert instance
        ignored_keys = {"__name__", "alertname", "alertstate"}
        labels = {k: v for k, v in metric.items() if k not in ignored_keys}
        
        periods = parse_periods(values, step_seconds)
        for p in periods:
            all_events.append({
                "alertname": alert_name,
                "alertstate": alert_state,
                "labels": labels,
                "start": p["start"],
                "end": p["end"],
                "duration": p["duration"],
                "ongoing": p.get("ongoing", False)
            })
            
    # Sort events by start time descending
    all_events.sort(key=lambda x: x["start"], reverse=True)
    
    print(f"\nFound {len(all_events)} alert events in the last 14 days:")
    print("-" * 100)
    
    # Print summary table
    for i, ev in enumerate(all_events):
        start_dt = datetime.datetime.fromtimestamp(ev["start"], datetime.timezone.utc)
        end_dt = datetime.datetime.fromtimestamp(ev["end"], datetime.timezone.utc)
        ongoing_str = " (Ongoing)" if ev["ongoing"] else ""
        
        labels_str = ", ".join([f"{k}={v}" for k, v in ev["labels"].items()])
        
        print(f"[{i+1}] {ev['alertname']} ({ev['alertstate']})")
        print(f"    Duration: {format_duration(ev['duration'])}{ongoing_str}")
        print(f"    Timeline: {start_dt.strftime('%Y-%m-%d %H:%M:%S')} -> {end_dt.strftime('%Y-%m-%d %H:%M:%S')} UTC")
        print(f"    Labels:   {labels_str}")
        print("-" * 100)

if __name__ == "__main__":
    main()