feat(openclaw): upgrade fleet to DeepSeek V4.1 Flash 1017 and harden operations
- Upgrade OpenClaw primary model from deepseek-v4.1-flash to deepseek-v4.1-flash-1017 across all interactive agents, heartbeats, compaction, and cron jobs. - Configure hardened OpenRouter provider routing for 1017 (deepinfra, inference-net, parasail, cloudflare) with strict fp8/bf16 quantization and tool_choice verification. - Retain deepseek-v4.1-flash (0910), z-ai/glm-5.3-flash, and qwen3.8-flash in the automated fallback chain. - Update ADR 0002 with 1017 economics (30% price reduction on prompt/cache, +60% intelligence benchmark increase) and a 7-point provider qualification playbook. - Add operational scripts and tooling for session compaction, subagent vector deduplication, gateway preflight, and triage diagnostics in k8s/openclaw. - Update DuckDB backend engine, git-sync logic, and test suites. - Enhance Mila tracking UI components and add container build configuration. - Sync Terraform infrastructure state for Authentik.
This commit is contained in:
Moritz Graf
@@ -41,11 +41,12 @@ TIER_A_TARGETS = [
|
||||
]
|
||||
|
||||
def prewarm_page_cache():
|
||||
print("=== Pre-warming SQLite databases into OS page cache ===")
|
||||
print("=== Pre-warming SQLite databases and Knowledge Wiki into OS page cache ===")
|
||||
t0 = time.time()
|
||||
try:
|
||||
subprocess.run(
|
||||
"cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 || true",
|
||||
"cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 || true; "
|
||||
"find /home/node/.openclaw/wiki/main /home/node/.openclaw/workspace* -maxdepth 3 -name '*.md' -exec cat {} + > /dev/null 2>&1 || true",
|
||||
shell=True,
|
||||
timeout=90.0
|
||||
)
|
||||
@@ -329,6 +330,7 @@ def main():
|
||||
compact_active_sessions()
|
||||
checkpoint_wal_journals()
|
||||
vacuum_and_optimize()
|
||||
prewarm_page_cache()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -1,34 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
import glob, os, sqlite3, sys
|
||||
#!/usr/bin/env python3
|
||||
import glob, json, os, sqlite3, sys
|
||||
|
||||
vec_path = "/app/node_modules/sqlite-vec-linux-x64/vec0.so"
|
||||
for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")):
|
||||
agent = os.path.basename(os.path.dirname(os.path.dirname(db)))
|
||||
if agent == "main":
|
||||
continue
|
||||
try:
|
||||
conn = sqlite3.connect(db, timeout=10.0)
|
||||
if os.path.exists(vec_path):
|
||||
conn.enable_load_extension(True)
|
||||
conn.load_extension(vec_path)
|
||||
cur = conn.cursor()
|
||||
chk = cur.execute("SELECT count(*) FROM memory_index_chunks").fetchone()[0]
|
||||
if chk > 0:
|
||||
print(f"[dedup_subagents] Purging {chk} duplicated vector chunks from agent {agent}...")
|
||||
for t in [
|
||||
"memory_index_sources",
|
||||
"memory_index_chunks",
|
||||
"memory_index_chunk_recall_metadata",
|
||||
"memory_index_chunk_provenance",
|
||||
"memory_entry_origins",
|
||||
"memory_embedding_cache",
|
||||
"memory_index_chunks_vec"
|
||||
]:
|
||||
try: cur.execute(f"DELETE FROM {t}")
|
||||
except Exception: pass
|
||||
conn.commit()
|
||||
cur.execute("VACUUM")
|
||||
print(f"[dedup_subagents] Reclaimed disk space for agent {agent}")
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
print(f"[dedup_subagents] Notice for {agent}: {e}", file=sys.stderr)
|
||||
cfg_path = "/home/node/.openclaw/openclaw.json"
|
||||
if os.path.exists(cfg_path):
|
||||
try:
|
||||
with open(cfg_path, "r") as f:
|
||||
cfg = json.load(f)
|
||||
mem_search = cfg.get("memory", {}).get("search", {})
|
||||
if mem_search.get("extraPaths"):
|
||||
mem_search["extraPaths"] = []
|
||||
with open(cfg_path, "w") as f:
|
||||
json.dump(cfg, f, indent=2)
|
||||
print("[dedup_subagents] Cleared top-level memory.search.extraPaths from openclaw.json (scoped to main only)")
|
||||
except Exception as e:
|
||||
print(f"[dedup_subagents] Warning checking openclaw.json: {e}", file=sys.stderr)
|
||||
|
||||
vec_path = "/app/node_modules/sqlite-vec-linux-x64/vec0.so"
|
||||
for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")):
|
||||
agent = os.path.basename(os.path.dirname(os.path.dirname(db)))
|
||||
if agent == "main":
|
||||
continue
|
||||
try:
|
||||
conn = sqlite3.connect(db, timeout=10.0)
|
||||
if os.path.exists(vec_path):
|
||||
conn.enable_load_extension(True)
|
||||
conn.load_extension(vec_path)
|
||||
cur = conn.cursor()
|
||||
chk = cur.execute("SELECT count(*) FROM memory_index_chunks").fetchone()[0]
|
||||
if chk > 0:
|
||||
print(f"[dedup_subagents] Purging {chk} duplicated vector chunks from agent {agent}...")
|
||||
for t in [
|
||||
"memory_index_sources",
|
||||
"memory_index_chunks",
|
||||
"memory_index_chunk_recall_metadata",
|
||||
"memory_index_chunk_provenance",
|
||||
"memory_entry_origins",
|
||||
"memory_embedding_cache",
|
||||
"memory_index_chunks_vec"
|
||||
]:
|
||||
try: cur.execute(f"DELETE FROM {t}")
|
||||
except Exception: pass
|
||||
conn.commit()
|
||||
cur.execute("VACUUM")
|
||||
print(f"[dedup_subagents] Reclaimed disk space for agent {agent}")
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
print(f"[dedup_subagents] Notice for {agent}: {e}", file=sys.stderr)
|
||||
|
||||
@@ -1,78 +1,78 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { execSync, spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { execSync, spawn } from "node:child_process";
|
||||
|
||||
if (!process.env.__OPENCLAW_PREFLIGHT_DONE) {
|
||||
process.env.__OPENCLAW_PREFLIGHT_DONE = "1";
|
||||
if (!process.env.__OPENCLAW_PREFLIGHT_DONE) {
|
||||
process.env.__OPENCLAW_PREFLIGHT_DONE = "1";
|
||||
|
||||
const isGateway = process.argv.some(arg => arg.endsWith("openclaw.mjs") || arg.endsWith("openclaw")) && process.argv.includes("gateway");
|
||||
const isGateway = process.argv.some(arg => arg.endsWith("openclaw.mjs") || arg.endsWith("openclaw")) && process.argv.includes("gateway");
|
||||
|
||||
if (isGateway) {
|
||||
const t0 = Date.now();
|
||||
console.log("[gateway-preflight] Initializing container self-healing preflight...");
|
||||
if (isGateway) {
|
||||
const t0 = Date.now();
|
||||
console.log("[gateway-preflight] Initializing container self-healing preflight...");
|
||||
|
||||
// 1. Clean up stale gateway lock files from ungraceful kills
|
||||
for (const dir of ["/tmp/openclaw-1000", "/tmp"]) {
|
||||
if (fs.existsSync(dir)) {
|
||||
try {
|
||||
const files = fs.readdirSync(dir);
|
||||
for (const file of files) {
|
||||
if (file.startsWith("gateway") && file.endsWith(".lock")) {
|
||||
const fullPath = path.join(dir, file);
|
||||
fs.unlinkSync(fullPath);
|
||||
console.log(`[gateway-preflight] Removed stale lock: ${fullPath}`);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(`[gateway-preflight] Lock cleanup warning in ${dir}:`, e.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Clear false database quarantines
|
||||
const qPath = "/home/node/.openclaw/state/openclaw-quarantine.sqlite";
|
||||
if (fs.existsSync(qPath)) {
|
||||
try {
|
||||
execSync(`/home/node/.openclaw/.local/bin/python3 -c "import sqlite3; c=sqlite3.connect(\x27${qPath}\x27); c.execute(\x27DELETE FROM quarantined_databases\x27); c.commit(); c.close()"`, { stdio: "ignore" });
|
||||
console.log("[gateway-preflight] Cleared database quarantine.");
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
// 3. Deduplicate subagent vector databases (keeps non-main agent DBs < 15MB)
|
||||
try {
|
||||
execSync("/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/dedup_subagent_vectors.py", { stdio: "inherit" });
|
||||
} catch (e) {
|
||||
console.warn("[gateway-preflight] Vector deduplication warning:", e.message);
|
||||
}
|
||||
|
||||
// 4. Pre-warm page cache sequentially (transforms random 70 IOPS disk reads into sequential streaming)
|
||||
try {
|
||||
const pw0 = Date.now();
|
||||
execSync("cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 || true");
|
||||
console.log(`[gateway-preflight] Pre-warmed SQLite databases in ${Date.now() - pw0}ms`);
|
||||
} catch (_) {}
|
||||
|
||||
// 5. Recover any wedged running sessions left over from ungraceful container restart
|
||||
try {
|
||||
execSync("/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/recover_sessions.py", { stdio: "inherit" });
|
||||
} catch (e) {
|
||||
console.warn("[gateway-preflight] Session recovery warning:", e.message);
|
||||
}
|
||||
|
||||
// 6. Spawn background turn replay daemon if interrupted turns were detected
|
||||
try {
|
||||
if (fs.existsSync("/tmp/openclaw-interrupted-turns.json")) {
|
||||
console.log("[gateway-preflight] Spawning background turn replay daemon...");
|
||||
const child = spawn("/home/node/.openclaw/.local/bin/python3", ["/home/node/.openclaw/scripts/replay_interrupted_turns.py"], {
|
||||
detached: true,
|
||||
stdio: "ignore"
|
||||
});
|
||||
child.unref();
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn("[gateway-preflight] Could not spawn turn replay daemon:", e.message);
|
||||
}
|
||||
|
||||
console.log(`[gateway-preflight] Self-healing completed in ${Date.now() - t0}ms.`);
|
||||
// 1. Clean up stale gateway lock files from ungraceful kills
|
||||
for (const dir of ["/tmp/openclaw-1000", "/tmp"]) {
|
||||
if (fs.existsSync(dir)) {
|
||||
try {
|
||||
const files = fs.readdirSync(dir);
|
||||
for (const file of files) {
|
||||
if (file.startsWith("gateway") && file.endsWith(".lock")) {
|
||||
const fullPath = path.join(dir, file);
|
||||
fs.unlinkSync(fullPath);
|
||||
console.log(`[gateway-preflight] Removed stale lock: ${fullPath}`);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(`[gateway-preflight] Lock cleanup warning in ${dir}:`, e.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Clear false database quarantines
|
||||
const qPath = "/home/node/.openclaw/state/openclaw-quarantine.sqlite";
|
||||
if (fs.existsSync(qPath)) {
|
||||
try {
|
||||
execSync(`/home/node/.openclaw/.local/bin/python3 -c "import sqlite3; c=sqlite3.connect(\x27${qPath}\x27); c.execute(\x27DELETE FROM quarantined_databases\x27); c.commit(); c.close()"`, { stdio: "ignore" });
|
||||
console.log("[gateway-preflight] Cleared database quarantine.");
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
// 3. Deduplicate subagent vector databases (keeps non-main agent DBs < 15MB)
|
||||
try {
|
||||
execSync("/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/dedup_subagent_vectors.py", { stdio: "inherit" });
|
||||
} catch (e) {
|
||||
console.warn("[gateway-preflight] Vector deduplication warning:", e.message);
|
||||
}
|
||||
|
||||
// 4. Pre-warm page cache sequentially (transforms random 70 IOPS disk reads into sequential streaming)
|
||||
try {
|
||||
const pw0 = Date.now();
|
||||
execSync("cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 || true; find /home/node/.openclaw/wiki/main /home/node/.openclaw/workspace* -maxdepth 3 -name '*.md' -exec cat {} + > /dev/null 2>&1 || true");
|
||||
console.log(`[gateway-preflight] Pre-warmed SQLite databases and Wiki markdown in ${Date.now() - pw0}ms`);
|
||||
} catch (_) {}
|
||||
|
||||
// 5. Recover any wedged running sessions left over from ungraceful container restart
|
||||
try {
|
||||
execSync("/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/recover_sessions.py", { stdio: "inherit" });
|
||||
} catch (e) {
|
||||
console.warn("[gateway-preflight] Session recovery warning:", e.message);
|
||||
}
|
||||
|
||||
// 6. Spawn background turn replay daemon if interrupted turns were detected
|
||||
try {
|
||||
if (fs.existsSync("/tmp/openclaw-interrupted-turns.json")) {
|
||||
console.log("[gateway-preflight] Spawning background turn replay daemon...");
|
||||
const child = spawn("/home/node/.openclaw/.local/bin/python3", ["/home/node/.openclaw/scripts/replay_interrupted_turns.py"], {
|
||||
detached: true,
|
||||
stdio: "ignore"
|
||||
});
|
||||
child.unref();
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn("[gateway-preflight] Could not spawn turn replay daemon:", e.message);
|
||||
}
|
||||
|
||||
console.log(`[gateway-preflight] Self-healing completed in ${Date.now() - t0}ms.`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ def triage_agent_session(agent, topic_id=None, limit=10, db_root="/home/node/.op
|
||||
|
||||
events.reverse()
|
||||
print("\n📜 RECENT TRANSCRIPT EVENTS (Chronological):")
|
||||
|
||||
|
||||
last_role = None
|
||||
for seq, ca_ms, ej_str in events:
|
||||
try:
|
||||
|
||||
@@ -1,80 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
import glob, sqlite3, json, os, sys, time
|
||||
#!/usr/bin/env python3
|
||||
import glob, sqlite3, json, os, sys, time
|
||||
|
||||
t0 = time.time()
|
||||
recovered = 0
|
||||
interrupted_turns = []
|
||||
t0 = time.time()
|
||||
recovered = 0
|
||||
interrupted_turns = []
|
||||
|
||||
def extract_prompt(msg):
|
||||
if not msg: return ""
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str): return content
|
||||
if isinstance(content, list):
|
||||
parts = []
|
||||
for c in content:
|
||||
if isinstance(c, dict): parts.append(c.get("text") or c.get("content") or "")
|
||||
elif isinstance(c, str): parts.append(c)
|
||||
return "\n".join(filter(None, parts))
|
||||
return str(content or "")
|
||||
def extract_prompt(msg):
|
||||
if not msg: return ""
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str): return content
|
||||
if isinstance(content, list):
|
||||
parts = []
|
||||
for c in content:
|
||||
if isinstance(c, dict): parts.append(c.get("text") or c.get("content") or "")
|
||||
elif isinstance(c, str): parts.append(c)
|
||||
return "\n".join(filter(None, parts))
|
||||
return str(content or "")
|
||||
|
||||
for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")):
|
||||
try:
|
||||
conn = sqlite3.connect(db, timeout=10.0)
|
||||
cursor = conn.cursor()
|
||||
while True:
|
||||
cursor.execute("PRAGMA foreign_key_check;")
|
||||
violations = cursor.fetchall()
|
||||
if not violations: break
|
||||
for table, rowid, parent, fkid in violations:
|
||||
cursor.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
|
||||
cursor.execute("SELECT session_key, entry_json, current_session_id FROM session_nodes WHERE status = \"running\"")
|
||||
rows = cursor.fetchall()
|
||||
agent_id = os.path.basename(os.path.dirname(os.path.dirname(db)))
|
||||
for sk, ej, csid in rows:
|
||||
try:
|
||||
p = json.loads(ej) if ej else {}
|
||||
sid = csid or p.get("sessionId")
|
||||
if sid:
|
||||
ev = cursor.execute("SELECT seq, event_json FROM transcript_events WHERE session_id = ? ORDER BY seq DESC LIMIT 1", (sid,)).fetchone()
|
||||
if ev:
|
||||
ev_seq, ev_raw = ev
|
||||
try:
|
||||
ev_data = json.loads(ev_raw)
|
||||
msg = ev_data.get("message") or ev_data
|
||||
if msg.get("role") == "user":
|
||||
prompt_text = extract_prompt(msg)
|
||||
ctx = p.get("restartRecoveryDeliveryContext") or p.get("delivery", {}).get("context") or p.get("delivery", {}).get("route", {}).get("context") or {}
|
||||
interrupted_turns.append({
|
||||
"agent": agent_id,
|
||||
"session_key": sk,
|
||||
"session_id": sid,
|
||||
"context": ctx,
|
||||
"prompt": prompt_text
|
||||
})
|
||||
cursor.execute("PRAGMA foreign_keys = ON")
|
||||
cursor.execute("DELETE FROM transcript_events WHERE session_id = ? AND seq = ?", (sid, ev_seq))
|
||||
except Exception as ex:
|
||||
print(f"[recover_sessions] Error parsing event: {ex}", file=sys.stderr)
|
||||
p["status"] = "done"
|
||||
p["activeWriterRunId"] = None
|
||||
p["activeClaimRunId"] = None
|
||||
p["restartRecoveryDeliveryRunId"] = None
|
||||
cursor.execute("UPDATE session_nodes SET status = \"done\", entry_json = ? WHERE session_key = ?", (json.dumps(p), sk))
|
||||
cursor.execute("UPDATE session_nodes SET entry_valid = 1 WHERE session_key = ?", (sk,))
|
||||
recovered += 1
|
||||
except Exception: pass
|
||||
cursor.execute("UPDATE session_windows SET status = \"done\" WHERE status = \"running\"")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
print(f"[recover_sessions] Warning for {db}: {e}", file=sys.stderr)
|
||||
for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")):
|
||||
try:
|
||||
conn = sqlite3.connect(db, timeout=10.0)
|
||||
cursor = conn.cursor()
|
||||
while True:
|
||||
cursor.execute("PRAGMA foreign_key_check;")
|
||||
violations = cursor.fetchall()
|
||||
if not violations: break
|
||||
for table, rowid, parent, fkid in violations:
|
||||
cursor.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
|
||||
cursor.execute("SELECT session_key, entry_json, current_session_id, status FROM session_nodes WHERE status IN (\"running\", \"failed\", \"error\")")
|
||||
rows = cursor.fetchall()
|
||||
agent_id = os.path.basename(os.path.dirname(os.path.dirname(db)))
|
||||
for sk, ej, csid, st in rows:
|
||||
try:
|
||||
p = json.loads(ej) if ej else {}
|
||||
sid = csid or p.get("sessionId")
|
||||
if sid:
|
||||
ev = cursor.execute("SELECT seq, event_json FROM transcript_events WHERE session_id = ? ORDER BY seq DESC LIMIT 1", (sid,)).fetchone()
|
||||
if ev:
|
||||
ev_seq, ev_raw = ev
|
||||
try:
|
||||
ev_data = json.loads(ev_raw)
|
||||
msg = ev_data.get("message") or ev_data
|
||||
if st == "running" and msg.get("role") == "user":
|
||||
prompt_text = extract_prompt(msg)
|
||||
ctx = p.get("restartRecoveryDeliveryContext") or p.get("delivery", {}).get("context") or p.get("delivery", {}).get("route", {}).get("context") or {}
|
||||
interrupted_turns.append({
|
||||
"agent": agent_id,
|
||||
"session_key": sk,
|
||||
"session_id": sid,
|
||||
"context": ctx,
|
||||
"prompt": prompt_text
|
||||
})
|
||||
cursor.execute("PRAGMA foreign_keys = ON")
|
||||
cursor.execute("DELETE FROM transcript_events WHERE session_id = ? AND seq = ?", (sid, ev_seq))
|
||||
elif msg.get("role") == "assistant" and msg.get("stopReason") == "error":
|
||||
cursor.execute("PRAGMA foreign_keys = ON")
|
||||
cursor.execute("DELETE FROM transcript_events WHERE session_id = ? AND seq = ?", (sid, ev_seq))
|
||||
except Exception as ex:
|
||||
print(f"[recover_sessions] Error parsing event: {ex}", file=sys.stderr)
|
||||
p["status"] = "done"
|
||||
p["activeWriterRunId"] = None
|
||||
p["activeClaimRunId"] = None
|
||||
p["restartRecoveryDeliveryRunId"] = None
|
||||
cursor.execute("UPDATE session_nodes SET status = \"done\", entry_json = ? WHERE session_key = ?", (json.dumps(p), sk))
|
||||
cursor.execute("UPDATE session_nodes SET entry_valid = 1 WHERE session_key = ?", (sk,))
|
||||
recovered += 1
|
||||
except Exception: pass
|
||||
cursor.execute("UPDATE session_nodes SET entry_valid = 1 WHERE entry_valid != 1")
|
||||
cursor.execute("UPDATE session_windows SET status = \"done\" WHERE status IN (\"running\", \"failed\", \"error\")")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
print(f"[recover_sessions] Warning for {db}: {e}", file=sys.stderr)
|
||||
|
||||
if interrupted_turns:
|
||||
try:
|
||||
with open("/tmp/openclaw-interrupted-turns.json", "w") as f:
|
||||
json.dump(interrupted_turns, f, indent=2)
|
||||
print(f"[recover_sessions] Detected {len(interrupted_turns)} interrupted user turn(s) saved to /tmp/openclaw-interrupted-turns.json")
|
||||
except Exception as e:
|
||||
print(f"[recover_sessions] Error writing interrupted turns: {e}", file=sys.stderr)
|
||||
state_db = "/home/node/.openclaw/state/openclaw.sqlite"
|
||||
if os.path.exists(state_db):
|
||||
try:
|
||||
sconn = sqlite3.connect(state_db, timeout=10.0)
|
||||
scur = sconn.cursor()
|
||||
scur.execute("UPDATE channel_ingress_events SET attempts = 0, last_attempt_at = NULL, last_error = NULL WHERE status = 'pending' AND attempts > 0")
|
||||
sconn.commit()
|
||||
sconn.close()
|
||||
except Exception as e:
|
||||
print(f"[recover_sessions] Warning resetting ingress backoff: {e}", file=sys.stderr)
|
||||
|
||||
if interrupted_turns:
|
||||
try:
|
||||
with open("/tmp/openclaw-interrupted-turns.json", "w") as f:
|
||||
json.dump(interrupted_turns, f, indent=2)
|
||||
print(f"[recover_sessions] Detected {len(interrupted_turns)} interrupted user turn(s) saved to /tmp/openclaw-interrupted-turns.json")
|
||||
except Exception as e:
|
||||
print(f"[recover_sessions] Error writing interrupted turns: {e}", file=sys.stderr)
|
||||
|
||||
print(f"[recover_sessions] Completed in {time.time() - t0:.2f}s ({recovered} wedged sessions recovered)")
|
||||
|
||||
print(f"[recover_sessions] Completed in {time.time() - t0:.2f}s ({recovered} wedged sessions recovered)")
|
||||
|
||||
@@ -1,97 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
import json, os, subprocess, sys, time, urllib.request
|
||||
#!/usr/bin/env python3
|
||||
import json, os, subprocess, sys, time, urllib.request
|
||||
|
||||
INTERRUPTED_FILE = "/tmp/openclaw-interrupted-turns.json"
|
||||
INTERRUPTED_FILE = "/tmp/openclaw-interrupted-turns.json"
|
||||
|
||||
if not os.path.exists(INTERRUPTED_FILE):
|
||||
sys.exit(0)
|
||||
if not os.path.exists(INTERRUPTED_FILE):
|
||||
sys.exit(0)
|
||||
|
||||
try:
|
||||
with open(INTERRUPTED_FILE, "r") as f:
|
||||
turns = json.load(f)
|
||||
except Exception as e:
|
||||
print(f"[replay_interrupted_turns] Failed to read {INTERRUPTED_FILE}: {e}", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
try:
|
||||
with open(INTERRUPTED_FILE, "r") as f:
|
||||
turns = json.load(f)
|
||||
except Exception as e:
|
||||
print(f"[replay_interrupted_turns] Failed to read {INTERRUPTED_FILE}: {e}", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
|
||||
if not turns:
|
||||
sys.exit(0)
|
||||
if not turns:
|
||||
sys.exit(0)
|
||||
|
||||
print(f"[replay_interrupted_turns] Found {len(turns)} interrupted turn(s). Waiting for gateway /healthz...")
|
||||
print(f"[replay_interrupted_turns] Found {len(turns)} interrupted turn(s). Waiting for gateway /healthz...")
|
||||
|
||||
gateway_ready = False
|
||||
for _ in range(60):
|
||||
try:
|
||||
with urllib.request.urlopen("http://127.0.0.1:18790/healthz", timeout=3) as resp:
|
||||
if resp.status == 200:
|
||||
gateway_ready = True
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(2)
|
||||
gateway_ready = False
|
||||
for _ in range(60):
|
||||
try:
|
||||
with urllib.request.urlopen("http://127.0.0.1:18790/healthz", timeout=3) as resp:
|
||||
if resp.status == 200:
|
||||
gateway_ready = True
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(2)
|
||||
|
||||
if not gateway_ready:
|
||||
print("[replay_interrupted_turns] Gateway did not become healthy in 120s. Aborting.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not gateway_ready:
|
||||
print("[replay_interrupted_turns] Gateway did not become healthy in 120s. Aborting.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Allow 5s for Telegram ingress to bind
|
||||
time.sleep(5)
|
||||
# Allow 5s for Telegram ingress to bind
|
||||
time.sleep(5)
|
||||
|
||||
for turn in turns:
|
||||
sk = turn.get("session_key")
|
||||
sid = turn.get("session_id")
|
||||
agent_id = turn.get("agent")
|
||||
ctx = turn.get("context") or {}
|
||||
prompt = turn.get("prompt") or ""
|
||||
|
||||
channel = ctx.get("channel") or "telegram"
|
||||
to_target = ctx.get("to") or ""
|
||||
thread_id = ctx.get("threadId")
|
||||
|
||||
target = to_target
|
||||
if target.startswith("telegram:"):
|
||||
parts = target.split(":")
|
||||
target = parts[1] if len(parts) > 1 else target
|
||||
for turn in turns:
|
||||
sk = turn.get("session_key")
|
||||
sid = turn.get("session_id")
|
||||
agent_id = turn.get("agent")
|
||||
ctx = turn.get("context") or {}
|
||||
prompt = turn.get("prompt") or ""
|
||||
|
||||
print(f"[replay_interrupted_turns] Processing agent '{agent_id}' on session '{sk}' target='{target}' thread='{thread_id}'")
|
||||
|
||||
# 1. Send status notification to Telegram if target exists
|
||||
if target:
|
||||
notice = "⚠️ OpenClaw wurde während der Bearbeitung neu gestartet – deine Anfrage wird jetzt nachgeholt..."
|
||||
notify_cmd = [
|
||||
"openclaw", "message", "send",
|
||||
"--channel", channel,
|
||||
"--target", str(target),
|
||||
"-m", notice
|
||||
]
|
||||
if thread_id is not None:
|
||||
notify_cmd.extend(["--thread-id", str(thread_id)])
|
||||
try:
|
||||
subprocess.run(notify_cmd, check=False, timeout=30)
|
||||
print(f"[replay_interrupted_turns] Sent recovery notice to {target}")
|
||||
except Exception as e:
|
||||
print(f"[replay_interrupted_turns] Notice failed: {e}", file=sys.stderr)
|
||||
channel = ctx.get("channel") or "telegram"
|
||||
to_target = ctx.get("to") or ""
|
||||
thread_id = ctx.get("threadId")
|
||||
|
||||
# 2. Replay agent turn
|
||||
if sk and prompt:
|
||||
p_short = (prompt[:60] + "...") if len(prompt) > 60 else prompt
|
||||
print(f"[replay_interrupted_turns] Replaying agent turn: {p_short}")
|
||||
agent_cmd = [
|
||||
"openclaw", "agent",
|
||||
"--session-key", sk,
|
||||
"-m", prompt,
|
||||
"--deliver"
|
||||
]
|
||||
try:
|
||||
res = subprocess.run(agent_cmd, capture_output=True, text=True, timeout=300)
|
||||
if res.returncode == 0:
|
||||
print(f"[replay_interrupted_turns] Turn replay succeeded for {sk}")
|
||||
else:
|
||||
print(f"[replay_interrupted_turns] Turn replay exited with code {res.returncode}: {res.stderr[:200]}", file=sys.stderr)
|
||||
except Exception as e:
|
||||
print(f"[replay_interrupted_turns] Replay execution error: {e}", file=sys.stderr)
|
||||
target = to_target
|
||||
if target.startswith("telegram:"):
|
||||
parts = target.split(":")
|
||||
target = parts[1] if len(parts) > 1 else target
|
||||
|
||||
try:
|
||||
os.remove(INTERRUPTED_FILE)
|
||||
print(f"[replay_interrupted_turns] Cleaned up {INTERRUPTED_FILE}")
|
||||
except Exception:
|
||||
pass
|
||||
print(f"[replay_interrupted_turns] Processing agent '{agent_id}' on session '{sk}' target='{target}' thread='{thread_id}'")
|
||||
|
||||
# 1. Send status notification to Telegram if target exists
|
||||
if target:
|
||||
notice = "⚠️ OpenClaw wurde während der Bearbeitung neu gestartet – deine Anfrage wird jetzt nachgeholt..."
|
||||
notify_cmd = [
|
||||
"openclaw", "message", "send",
|
||||
"--channel", channel,
|
||||
"--target", str(target),
|
||||
"-m", notice
|
||||
]
|
||||
if thread_id is not None:
|
||||
notify_cmd.extend(["--thread-id", str(thread_id)])
|
||||
try:
|
||||
subprocess.run(notify_cmd, check=False, timeout=30)
|
||||
print(f"[replay_interrupted_turns] Sent recovery notice to {target}")
|
||||
except Exception as e:
|
||||
print(f"[replay_interrupted_turns] Notice failed: {e}", file=sys.stderr)
|
||||
|
||||
# 2. Replay agent turn
|
||||
if sk and prompt:
|
||||
p_short = (prompt[:60] + "...") if len(prompt) > 60 else prompt
|
||||
print(f"[replay_interrupted_turns] Replaying agent turn: {p_short}")
|
||||
agent_cmd = [
|
||||
"openclaw", "agent",
|
||||
"--session-key", sk,
|
||||
"-m", prompt,
|
||||
"--deliver"
|
||||
]
|
||||
try:
|
||||
res = subprocess.run(agent_cmd, capture_output=True, text=True, timeout=300)
|
||||
if res.returncode == 0:
|
||||
print(f"[replay_interrupted_turns] Turn replay succeeded for {sk}")
|
||||
else:
|
||||
print(f"[replay_interrupted_turns] Turn replay exited with code {res.returncode}: {res.stderr[:200]}", file=sys.stderr)
|
||||
except Exception as e:
|
||||
print(f"[replay_interrupted_turns] Replay execution error: {e}", file=sys.stderr)
|
||||
|
||||
try:
|
||||
os.remove(INTERRUPTED_FILE)
|
||||
print(f"[replay_interrupted_turns] Cleaned up {INTERRUPTED_FILE}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user