Moritz Graf
dd60a095c6
- Upgrade OpenClaw primary model from deepseek-v4.1-flash to deepseek-v4.1-flash-1017 across all interactive agents, heartbeats, compaction, and cron jobs. - Configure hardened OpenRouter provider routing for 1017 (deepinfra, inference-net, parasail, cloudflare) with strict fp8/bf16 quantization and tool_choice verification. - Retain deepseek-v4.1-flash (0910), z-ai/glm-5.3-flash, and qwen3.8-flash in the automated fallback chain. - Update ADR 0002 with 1017 economics (30% price reduction on prompt/cache, +60% intelligence benchmark increase) and a 7-point provider qualification playbook. - Add operational scripts and tooling for session compaction, subagent vector deduplication, gateway preflight, and triage diagnostics in k8s/openclaw. - Update DuckDB backend engine, git-sync logic, and test suites. - Enhance Mila tracking UI components and add container build configuration. - Sync Terraform infrastructure state for Authentik.
337 lines
15 KiB
Python
337 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Nightly multi-agent session maintenance and compaction script for OpenClaw.
|
|
1. Pre-warms OS page cache for SQLite databases on spinning disks.
|
|
2. Recovers any wedged or locked sessions via recover_sessions.py.
|
|
3. Prunes abandoned isolated cron sessions (>48h old) to prevent token/state bloat.
|
|
4. Tier A Hard-Reset: Resets koch, hebamme, and shopping topics to 0 tokens (clean archive).
|
|
5. Tier B Smart-Compaction: Compacts active sessions exceeding token thresholds (5s pacing).
|
|
6. Checkpoints SQLite WAL journals across all agent databases to disk.
|
|
7. Optimizes SQLite query planner without blocking full VACUUM.
|
|
"""
|
|
import glob
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
THRESHOLD_TOKENS = 15000
|
|
THRESHOLD_TRANSCRIPT_BYTES = 200000
|
|
THRESHOLD_EVENT_COUNT = 30
|
|
CRON_MAX_AGE_HOURS = 48
|
|
RPC_TIMEOUT_MS = 180000
|
|
|
|
# Tracking Channels: Deterministic sliding-window (mila_tracker, nutrition_tracker).
|
|
# Strict sliding window of ~5-8 events (2-3 turns max), 0 LLM tokens, instant truncation.
|
|
TRACKING_TARGETS = [
|
|
("mila_tracker", "agent:mila_tracker:telegram:group:-1004326921510:topic:2796"),
|
|
("nutrition_tracker", "agent:nutrition_tracker:telegram:group:-1004326921510:topic:2797"),
|
|
]
|
|
MAX_TRACKING_EVENTS = 8
|
|
MAX_TRACKING_LINES = 8
|
|
|
|
# Tier A: High-frequency interaction topics (koch, hebamme, shopping).
|
|
# Deep-compaction keeps a compact summary + latest 2 turns without completely wiping conversational memory.
|
|
TIER_A_TARGETS = [
|
|
("koch", "agent:koch:telegram:group:-1004326921510:topic:21"),
|
|
("hebamme", "agent:hebamme:telegram:group:-1004326921510:topic:22"),
|
|
("shopping", "agent:shopping:telegram:group:-1004326921510:topic:115"),
|
|
]
|
|
|
|
def prewarm_page_cache():
|
|
print("=== Pre-warming SQLite databases and Knowledge Wiki into OS page cache ===")
|
|
t0 = time.time()
|
|
try:
|
|
subprocess.run(
|
|
"cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 || true; "
|
|
"find /home/node/.openclaw/wiki/main /home/node/.openclaw/workspace* -maxdepth 3 -name '*.md' -exec cat {} + > /dev/null 2>&1 || true",
|
|
shell=True,
|
|
timeout=90.0
|
|
)
|
|
print(f"Page cache pre-warmed in {time.time() - t0:.2f}s")
|
|
except Exception as e:
|
|
print(f"Page cache pre-warming warning: {e}")
|
|
|
|
def prune_stale_cron_sessions():
|
|
print(f"=== Pruning isolated cron sessions older than {CRON_MAX_AGE_HOURS}h ===")
|
|
cutoff_ms = int(time.time() * 1000) - (CRON_MAX_AGE_HOURS * 3600 * 1000)
|
|
total_pruned = 0
|
|
total_tokens_reclaimed = 0
|
|
|
|
for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")):
|
|
agent = db.split("/")[5]
|
|
try:
|
|
conn = sqlite3.connect(db, timeout=30.0)
|
|
conn.execute("PRAGMA foreign_keys = ON;")
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT name FROM sqlite_master WHERE type=\"table\"")
|
|
all_tables = set(r[0] for r in cursor.fetchall())
|
|
|
|
cursor.execute("SELECT session_key, current_session_id, updated_at, entry_json FROM session_nodes WHERE session_key LIKE \"%:cron:%\"")
|
|
stale = []
|
|
for k, sid, up, ej in cursor.fetchall():
|
|
if up and up < cutoff_ms:
|
|
tok = 0
|
|
if ej:
|
|
try:
|
|
tok = json.loads(ej).get("totalTokens", 0) or 0
|
|
except Exception:
|
|
pass
|
|
stale.append((k, sid, tok))
|
|
|
|
if stale:
|
|
tokens_reclaimed = sum(t[2] for t in stale)
|
|
for k, sid, _ in stale:
|
|
for t in ["trajectory_runtime_events", "transcript_events", "transcript_event_identities", "session_transcript_active_events", "transcript_rewrite_watermarks", "session_transcript_index_state", "session_transcript_fts", "session_conversations"]:
|
|
if t in all_tables and sid:
|
|
cursor.execute(f"DELETE FROM {t} WHERE session_id = ?", (sid,))
|
|
for t in ["session_participants", "heartbeat_outcomes", "session_progress_cards", "board_widgets", "board_tabs"]:
|
|
if t in all_tables:
|
|
cursor.execute(f"DELETE FROM {t} WHERE session_key = ?", (k,))
|
|
if "memory_entry_origins" in all_tables:
|
|
cursor.execute("DELETE FROM memory_entry_origins WHERE session_key = ? OR session_id = ?", (k, sid or ""))
|
|
if "session_windows" in all_tables:
|
|
cursor.execute("DELETE FROM session_windows WHERE session_key = ? OR session_id = ?", (k, sid or ""))
|
|
cursor.execute("DELETE FROM session_nodes WHERE session_key = ?", (k,))
|
|
conn.commit()
|
|
total_pruned += len(stale)
|
|
total_tokens_reclaimed += tokens_reclaimed
|
|
print(f" [{agent}] Pruned {len(stale)} dead cron sessions ({tokens_reclaimed} tokens reclaimed)")
|
|
|
|
# Unconditionally resolve any dangling foreign key references across all tables
|
|
while True:
|
|
cursor.execute("PRAGMA foreign_key_check;")
|
|
violations = cursor.fetchall()
|
|
if not violations:
|
|
break
|
|
for table, rowid, parent, fkid in violations:
|
|
cursor.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
|
|
conn.commit()
|
|
conn.close()
|
|
except Exception as e:
|
|
print(f" [{agent}] Pruning error: {e}")
|
|
|
|
# Clear resolved quarantine entries if all databases are healthy
|
|
try:
|
|
q_path = "/home/node/.openclaw/state/openclaw-quarantine.sqlite"
|
|
if os.path.exists(q_path):
|
|
q_conn = sqlite3.connect(q_path)
|
|
q_cur = q_conn.cursor()
|
|
q_cur.execute("DELETE FROM quarantined_databases")
|
|
q_conn.commit()
|
|
q_conn.close()
|
|
except Exception as e:
|
|
print(f"Quarantine cleanup error: {e}")
|
|
|
|
print(f"Cron pruning complete: {total_pruned} sessions pruned, {total_tokens_reclaimed} tokens reclaimed.")
|
|
|
|
def compact_tracking_sessions():
|
|
print("=== Tracking Channels Sliding-Window Maintenance (mila_tracker, nutrition_tracker) ===")
|
|
for agent, key in TRACKING_TARGETS:
|
|
db_path = f"/home/node/.openclaw/agents/{agent}/agent/openclaw-agent.sqlite"
|
|
if not os.path.exists(db_path):
|
|
continue
|
|
try:
|
|
con = sqlite3.connect(db_path, timeout=5.0)
|
|
row = con.execute("SELECT current_session_id FROM session_nodes WHERE session_key=?", (key,)).fetchone()
|
|
if not row or not row[0]:
|
|
con.close()
|
|
print(f" [{agent}] No active session node found for {key}.")
|
|
continue
|
|
sid = row[0]
|
|
cnt = con.execute("SELECT count(*) FROM transcript_events WHERE session_id=?", (sid,)).fetchone()[0]
|
|
con.close()
|
|
|
|
if cnt > MAX_TRACKING_EVENTS:
|
|
print(f" [{agent}] Session {key} has {cnt} events (> {MAX_TRACKING_EVENTS}). Truncating to last {MAX_TRACKING_LINES} lines...")
|
|
cmd = [
|
|
"openclaw", "sessions", "compact", key,
|
|
"--agent", agent,
|
|
"--max-lines", str(MAX_TRACKING_LINES),
|
|
"--timeout", "30000"
|
|
]
|
|
r = subprocess.run(cmd, capture_output=True, text=True)
|
|
if r.returncode == 0:
|
|
print(f" ✓ Truncated successfully: {r.stdout.strip()}")
|
|
else:
|
|
print(f" ✗ Truncation failed: {r.stderr.strip()}")
|
|
else:
|
|
print(f" [{agent}] Session {key} has {cnt} events (<= {MAX_TRACKING_EVENTS}). Clean.")
|
|
except Exception as e:
|
|
print(f" ✗ [{agent}] Error during tracking compaction: {e}")
|
|
|
|
def compact_tier_a_sessions():
|
|
print("=== Tier A: Deep-Compaction (koch, hebamme, shopping) ===")
|
|
for agent, key in TIER_A_TARGETS:
|
|
db_path = f"/home/node/.openclaw/agents/{agent}/agent/openclaw-agent.sqlite"
|
|
if not os.path.exists(db_path):
|
|
continue
|
|
try:
|
|
con = sqlite3.connect(db_path, timeout=10.0)
|
|
row = con.execute("SELECT current_session_id FROM session_nodes WHERE session_key=?", (key,)).fetchone()
|
|
con.close()
|
|
if not row or not row[0]:
|
|
print(f" [Tier A] {agent} ({key}) already at 0 (no active session).")
|
|
continue
|
|
ev_cnt, ev_bytes = get_session_metrics(agent, key, row[0])
|
|
if ev_cnt <= 2 and ev_bytes < 20000:
|
|
print(f" [Tier A] {agent} ({key}) has only {ev_cnt} events ({ev_bytes} bytes), skipping.")
|
|
continue
|
|
print(f" [Tier A] Deep-compacting {agent} ({key}) [{ev_cnt} events, {ev_bytes//1024}KB]...")
|
|
r = subprocess.run(
|
|
["openclaw", "sessions", "compact", key, "--agent", agent, "--timeout", str(RPC_TIMEOUT_MS)],
|
|
capture_output=True, text=True
|
|
)
|
|
if r.returncode == 0:
|
|
print(f" ✓ Deep-compacted: {r.stdout.strip()}")
|
|
elif "Already compacted" in r.stderr or "Already compacted" in r.stdout:
|
|
print(f" ✓ Already compacted.")
|
|
else:
|
|
print(f" ✗ Standard compact error: {r.stderr.strip()}; retrying with --max-lines 40...")
|
|
fb = subprocess.run(
|
|
["openclaw", "sessions", "compact", key, "--agent", agent, "--max-lines", "40", "--timeout", str(RPC_TIMEOUT_MS)],
|
|
capture_output=True, text=True
|
|
)
|
|
if fb.returncode == 0:
|
|
print(f" ✓ Truncation fallback success: {fb.stdout.strip()}")
|
|
else:
|
|
print(f" ✗ Truncation fallback failed: {fb.stderr.strip()}")
|
|
except Exception as e:
|
|
print(f" ✗ Exception deep-compacting {agent}: {e}")
|
|
time.sleep(2)
|
|
|
|
def get_session_metrics(agent, key, sid=None):
|
|
db_path = f"/home/node/.openclaw/agents/{agent}/agent/openclaw-agent.sqlite"
|
|
if not os.path.exists(db_path):
|
|
return 0, 0
|
|
try:
|
|
con = sqlite3.connect(db_path, timeout=5.0)
|
|
cur = con.cursor()
|
|
if not sid:
|
|
row = cur.execute("SELECT current_session_id FROM session_nodes WHERE session_key=?", (key,)).fetchone()
|
|
sid = row[0] if row else None
|
|
if not sid:
|
|
con.close()
|
|
return 0, 0
|
|
ev_cnt, ev_bytes = cur.execute(
|
|
"SELECT count(*), coalesce(sum(length(event_json)), 0) FROM transcript_events WHERE session_id=?",
|
|
(sid,)
|
|
).fetchone()
|
|
con.close()
|
|
return ev_cnt, ev_bytes
|
|
except Exception:
|
|
return 0, 0
|
|
|
|
def compact_active_sessions():
|
|
excluded_keys = set(t[1] for t in TIER_A_TARGETS) | set(t[1] for t in TRACKING_TARGETS)
|
|
print(f"=== Tier B: Running Multi-Agent Compaction (threshold: {THRESHOLD_TOKENS} tokens / {THRESHOLD_TRANSCRIPT_BYTES//1024}KB, LLM summarization) ===")
|
|
res = subprocess.run(["openclaw", "sessions", "--all-agents", "--json"], capture_output=True, text=True)
|
|
if res.returncode != 0:
|
|
print(f"Failed to list sessions: {res.stderr}")
|
|
return
|
|
|
|
sessions = json.loads(res.stdout).get("sessions", [])
|
|
compacted = 0
|
|
skipped = 0
|
|
|
|
for s in sessions:
|
|
key = s.get("key", "")
|
|
agent = s.get("agentId") or "main"
|
|
tokens = s.get("totalTokens") or 0
|
|
sid = s.get("sessionId")
|
|
|
|
# Skip Tier A and Tracking agents as they have dedicated routines
|
|
if key in excluded_keys:
|
|
continue
|
|
|
|
# Target main sessions and all telegram sessions (topics and direct)
|
|
is_target = (
|
|
key == "agent:main:main" or
|
|
":telegram:" in key or
|
|
(key.startswith("agent:") and key.endswith(":main"))
|
|
)
|
|
if is_target and not key.endswith(":heartbeat"):
|
|
ev_cnt, ev_bytes = get_session_metrics(agent, key, sid)
|
|
needs_compaction = (
|
|
tokens >= THRESHOLD_TOKENS or
|
|
ev_bytes >= THRESHOLD_TRANSCRIPT_BYTES or
|
|
(tokens == 0 and ev_cnt >= THRESHOLD_EVENT_COUNT)
|
|
)
|
|
if needs_compaction:
|
|
stat_str = f"{tokens} tokens" if tokens else f"{ev_bytes//1024}KB / {ev_cnt} events"
|
|
print(f"Compacting {agent} ({stat_str}) -> {key}...")
|
|
c_res = subprocess.run(
|
|
["openclaw", "sessions", "compact", key, "--agent", agent, "--timeout", str(RPC_TIMEOUT_MS)],
|
|
capture_output=True, text=True
|
|
)
|
|
if c_res.returncode == 0:
|
|
print(f" ✓ {c_res.stdout.strip()}")
|
|
compacted += 1
|
|
elif "Already compacted" in c_res.stderr or "Already compacted" in c_res.stdout:
|
|
print(f" ✓ Already compacted.")
|
|
skipped += 1
|
|
else:
|
|
print(f" ✗ {c_res.stderr.strip()}; retrying with --max-lines 80 fallback...")
|
|
fb_res = subprocess.run(
|
|
["openclaw", "sessions", "compact", key, "--agent", agent, "--max-lines", "80", "--timeout", str(RPC_TIMEOUT_MS)],
|
|
capture_output=True, text=True
|
|
)
|
|
if fb_res.returncode == 0:
|
|
print(f" ✓ (fallback) {fb_res.stdout.strip()}")
|
|
compacted += 1
|
|
else:
|
|
print(f" ✗ Fallback failed: {fb_res.stderr.strip()}")
|
|
time.sleep(5) # 5s pause to avoid rotational HDD I/O saturation
|
|
else:
|
|
skipped += 1
|
|
|
|
print(f"Compaction complete: {compacted} compacted, {skipped} skipped.")
|
|
|
|
def checkpoint_wal_journals():
|
|
print("=== Checkpointing SQLite WAL journals across all agent databases ===")
|
|
dbs = ["/home/node/.openclaw/state/openclaw.sqlite"] + glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")
|
|
wal_freed = 0
|
|
for db in dbs:
|
|
wal = db + "-wal"
|
|
if os.path.exists(wal):
|
|
sz = os.path.getsize(wal)
|
|
try:
|
|
con = sqlite3.connect(db, timeout=30.0)
|
|
con.execute("PRAGMA wal_checkpoint(TRUNCATE);")
|
|
con.close()
|
|
wal_freed += sz
|
|
except Exception as e:
|
|
print(f" ✗ Error checkpointing {db}: {e}")
|
|
print(f"WAL Checkpoint complete: {wal_freed / (1024*1024):.2f} MB merged to disk.")
|
|
|
|
def vacuum_and_optimize():
|
|
print("=== Running SQLite OPTIMIZE across all databases ===")
|
|
dbs = ["/home/node/.openclaw/state/openclaw.sqlite"] + glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")
|
|
for db in dbs:
|
|
try:
|
|
con = sqlite3.connect(db, timeout=30.0)
|
|
con.execute("PRAGMA optimize;")
|
|
con.close()
|
|
except Exception as e:
|
|
print(f" ✗ Error optimizing {db}: {e}")
|
|
print("SQLite optimization complete.")
|
|
|
|
def main():
|
|
prewarm_page_cache()
|
|
try:
|
|
subprocess.run(["python3", "/home/node/.openclaw/scripts/recover_sessions.py"], check=False)
|
|
except Exception:
|
|
pass
|
|
prune_stale_cron_sessions()
|
|
compact_tracking_sessions()
|
|
compact_tier_a_sessions()
|
|
compact_active_sessions()
|
|
checkpoint_wal_journals()
|
|
vacuum_and_optimize()
|
|
prewarm_page_cache()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|