diff --git a/docs/adr/0002-openclaw-model-selection-tiering-and-economics.md b/docs/adr/0002-openclaw-model-selection-tiering-and-economics.md index 5e302a4..f5a41c7 100644 --- a/docs/adr/0002-openclaw-model-selection-tiering-and-economics.md +++ b/docs/adr/0002-openclaw-model-selection-tiering-and-economics.md @@ -1,5 +1,5 @@ --- -status: "accepted (trial: GLM-5.3-Flash)" +status: "accepted (fleet-wide primary: DeepSeek-V4.1-Flash-1017; updated 2026-10-17)" date: 2026-09-13 decision-makers: ["Moritz Graf", "Antigravity"] consulted: [] @@ -28,7 +28,7 @@ How do we architect the full-spectrum model selection across **all modalities** ## Decision Drivers -* **State-of-the-Art Recency**: Exclusively employ the latest generation models (e.g. Gemini 3.1, GLM 5.3, Whisper V3 Turbo, Sonar Pro). Completely decommission legacy generations (e.g. Gemini 2.5). +* **State-of-the-Art Recency**: Exclusively employ the latest generation models (e.g. DeepSeek V4.1 Flash 1017, Gemini 3.8/3.1, GLM 5.3, Whisper V3 Turbo, Sonar Pro). Completely decommission legacy generations (e.g. Gemini 2.5). * **Conversational & Reasoning Quality**: High emotional intelligence, nuanced German fluency, personal context recall, and strict adherence to tool-calling protocols (ADR 0001). * **Strict Modality Decoupling**: Each capability must route to its dedicated, best-in-class engine: - Text & Agentic Reasoning @@ -38,9 +38,9 @@ How do we architect the full-spectrum model selection across **all modalities** - Text-to-Speech / Audio Output (synthesizing briefings and podcasts) - Web Search & Retrieval (grounded search) - Semantic Embeddings (vector memory) -* **Inference Economics**: Low per-token costs for large conversational contexts (15k–80k tokens), zero background bleed, and predictable billing. +* **Inference Economics**: Low per-token costs for large conversational contexts (15k–80k tokens), high prompt-cache discount efficiency (`cacheRead`), zero background bleed, and predictable billing. * **Latency & Turn Responsiveness**: Conversational turns on Telegram must return within **4–10 seconds**, avoiding provider queuing or massive prefill bottlenecks. -* **Resilience & Fault Tolerance**: Automated fallback chains across independent providers on OpenRouter. +* **Resilience & Fault Tolerance**: Automated fallback chains and strict quantization/provider pins (`fp8`/`bf16`, `tool_choice` support) across independent providers on OpenRouter. --- @@ -60,22 +60,23 @@ How do we architect the full-spectrum model selection across **all modalities** │ Tier 1 │ │ Tier 2 │ │ Tier 3 │ │ Tier 4 │ │ Tier 5 │ │ • Low Latency │ │ • Deep Analysis │ │ • target: none │ │ • Nano Banana 2 │ │ • Whisper V3 │ │ • Tool Routing │ │ • Autonomous │ │ • Zero Bleed │ │ • Dedicated │ │ • Gemini 3.1 │ -│ • Native Vision │ │ • Isolated Run │ │ • 30m Polling │ │ Raster Head │ │ Flash TTS │ +│ • Decoupled OCR │ │ • Isolated Run │ │ • 30m Polling │ │ Raster Head │ │ Flash TTS │ └────────┬────────┘ └────────┬────────┘ └────────┬────────┘ └────────┬────────┘ └────────┬────────┘ │ │ │ │ │ ▼ ▼ ▼ ▼ ▼ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ Primary (Trial):│ │ Primary (Trial):│ │ Primary: │ │ Primary: │ │ STT: │ -│ z-ai/ │ │ z-ai/ │ │ z-ai/ │ │ google/ │ │ openai/ │ -│ glm-5.3-flash │ │ glm-5.3-flash │ │ glm-5.3-flash │ │ gemini-3.1- │ │ whisper-large- │ -│ │ │ │ │ │ │ flash-image │ │ v3-turbo │ -│ Fallbacks: │ │ Fallbacks: │ │ Fallbacks: │ │ │ │ -│ 1. deepseek- │ │ 1. deepseek- │ │ 1. deepseek- │ │ Fallbacks: │ │ TTS: │ -│ v4.1-flash │ │ v4.1-flash │ │ v4.1-flash │ │ 1. gemini-3.1- │ │ google/ │ -│ 2. qwen3.8-flash│ │ 2. qwen3.8-flash│ │ 2. qwen3.8-flash│ │ flash-preview│ │ gemini-3.1- │ -│ │ │ │ │ │ │ 2. gemini-3-pro-│ │ flash-tts │ -│ Vision Fallback:│ │ │ │ │ │ image │ │ (FB: MiniMax 2.8│ -│ google/ │ │ │ │ │ │ │ │ Speech HD) │ +│ Primary: │ │ Primary: │ │ Primary: │ │ Primary: │ │ STT: │ +│ deepseek/ │ │ deepseek/ │ │ deepseek/ │ │ google/ │ │ openai/ │ +│ deepseek-v4.1- │ │ deepseek-v4.1- │ │ deepseek-v4.1- │ │ gemini-3.1- │ │ whisper-large- │ +│ flash-1017 │ │ flash-1017 │ │ flash-1017 │ │ flash-image │ │ v3-turbo │ +│ │ │ │ │ │ │ │ │ +│ Fallbacks: │ │ Fallbacks: │ │ Fallbacks: │ │ Fallbacks: │ │ TTS: │ +│ 1. v4.1-flash │ │ 1. v4.1-flash │ │ 1. v4.1-flash │ │ 1. gemini-3.1- │ │ google/ │ +│ 2. glm-5.3-flash│ │ 2. glm-5.3-flash│ │ 2. glm-5.3-flash│ │ flash-preview│ │ gemini-3.1- │ +│ 3. qwen3.8-flash│ │ 3. qwen3.8-flash│ │ 3. qwen3.8-flash│ │ 2. gemini-3-pro-│ │ flash-tts │ +│ │ │ │ │ │ │ image │ │ (FB: MiniMax 2.8│ +│ Vision (OCR): │ │ │ │ │ │ │ │ Speech HD) │ +│ google/ │ │ │ │ │ │ │ │ │ │ gemini-3.8- │ │ │ │ │ │ │ │ │ │ flash │ │ │ │ │ │ │ │ │ └─────────────────┘ └─────────────────┘ └─────────────────┘ └─────────────────┘ └─────────────────┘ @@ -86,27 +87,28 @@ How do we architect the full-spectrum model selection across **all modalities** ## Modality Specifications & Cost-Benefit Analysis ### 1. Interactive Conductor & Agentic Reasoning (Tier 1) -* **Primary (Active Trial)**: `openrouter/z-ai/glm-5.3-flash` - - **Pricing**: \$0.075 / 1M prompt tokens; \$0.300 / 1M completion tokens. This is 50–75% more cost-effective than DeepSeek V4.1-Flash and offers a 1.3M token context window. Under daytime context headroom (~25k–60k tokens), daily cost is only **~\$0.01–\$0.02 / day**. - - **Performance & Reasoning**: SOTA Chinese & English reasoning, competitive coding benchmarks, and native vision perception. - - **Forensic Diagnosis & Tool-Calling Protocol Fix**: - - *Incident*: On September 14, 2026, GLM-5.3-Flash emitted raw `supermarket__supermarket_search...` XML into Telegram instead of executing tools. - - *Root Cause Identified*: In Zhipu AI's pre-training tokenizer, function calls are emitted as special XML tokens. When models are queried through OpenRouter's OpenAI-compatible API, OpenRouter only intercepts and translates `` tokens into JSON if the tool `name` exists in the request's registered `tools: [...]` array. OpenClaw uses dynamic Tool Search (`tools.toolSearch.mode: "directory"`) where only 3 control tools (`tool_search`, `tool_describe`, `tool_call`) are registered in `tools`. When GLM received deferred schemas (e.g., `supermarket__supermarket_search`), its pre-training prior prompted it to call the tool directly. Because `supermarket__supermarket_search` was not in OpenRouter's `tools` array, OpenRouter passed the raw XML through as message text. Once that text entered the session history (events SEQ 297 & 306), GLM was in-context primed to repeat the XML pattern. - - *Remediation & Verification*: We introduced an explicit deferred tool execution protocol into `AGENTS.md` instructing the model that deferred tools must NEVER be called directly and MUST be routed via `tool_call({ id: 'tool_name', args: { ... } })`. Direct OpenRouter API probes confirmed that with this rule, GLM-5.3-Flash calls `tool_call` with 100% compliant JSON and zero XML leakage. Session events were sanitized. +* **Primary (Fleet-Wide Standard as of 2026-10-17)**: `openrouter/deepseek/deepseek-v4.1-flash-1017` + - **1017 Snapshot Upgrade (Pareto Jump in Quality & Price)**: Released on October 17, 2026, `deepseek/deepseek-v4.1-flash-1017` boosts the Artificial Analysis Intelligence Index from **24.7 to 39.5** (+60%) via Reinforcement Learning with Self-Feedback (RLSF) while cutting DeepInfra `fp8` pricing by an additional **30%** (`$0.098/M` prompt, `$0.294/M` completion, and **`$0.00294/M` `cacheRead`** vs. `$0.140/$0.420/$0.0042` on the 0910 baseline). + - **Provider Pinning & Tool-Compatibility Verification**: + - Pinned `order: ["deepinfra", "inference-net", "parasail", "cloudflare"]`, `quantizations: ["fp8", "bf16", "unknown"]`, and `ignore: ["morph", "baseten", "deepseek", "open-inference", "decart", "sail-research", "wafer"]`. + - Live OpenRouter routing forensics verified that `morph/fp8` is filtered out by OpenRouter's `Filter by Tool Compatibility` when `tools` are passed, `deepseek` (official) is filtered by account privacy guardrails (`Paid model training violation`), and `baseten` returns `HTTP 429`. Meanwhile, `deepinfra`, `inference-net`, `parasail`, and `cloudflare` deliver 100% tool-calling reliability and 2–5s latency. + - **Post-Trial Evaluation of `z-ai/glm-5.3-flash` (2026-09-13 to 2026-10-17)**: + - While `z-ai/glm-5.3-flash` was trialed as primary conductor, 17-day production telemetry (Oct 1–17, 2026) revealed **12 mid-stream tool-call aborts** (`stopReason: "error"`, `errorMessage: "Provider returned an incomplete or malformed tool call"`) across 5 agents (`main`, `grabengasse_media`, `koch`, `hebamme`, `doktor`) when generating nested `tool_call({ id, args })` payloads after `tool_describe`, as well as occasional thinking-only whitespace stalls (Issue 23) and long-context attention drift (Issues 22 & 25). + - Promoting `deepseek/deepseek-v4.1-flash-1017` to primary across all agents eliminates malformed tool-call stream aborts AND lowers total monthly cost by >50% compared to GLM 5.3 Flash. + - **Decoupled Vision Override**: `deepseek/deepseek-v4.1-flash-1017`, `deepseek/deepseek-v4.1-flash`, and `z-ai/glm-5.3-flash` are registered with `input: ["text"]` under `models.providers.openrouter.models` so that OpenClaw always routes user photos through `tools.media.image` (`google/gemini-3.8-flash`) for high-precision German OCR before handing enriched text to the chat model (Issue 29). * **Fallbacks**: - 1. `openrouter/deepseek/deepseek-v4.1-flash`: Rock-solid JSON tool-calling and sub-3s response times via dedicated OpenRouter ASIC cluster, available immediately if GLM experiences any upstream provider queuing or issues. - 2. `openrouter/qwen/qwen3.8-flash`: Provider diversity. + 1. `openrouter/deepseek/deepseek-v4.1-flash` (`0910` baseline across 25+ providers, pinned to `deepinfra`, `streamlake`, `inference-net`, `digitalocean`, `venice`, `nebius`). + 2. `openrouter/z-ai/glm-5.3-flash`: Secondary fallback pinned to `fp8`/`bf16` providers (`morph`, `near-ai`, `digitalocean`, `venice`, `streamlake`). + 3. `openrouter/qwen/qwen3.8-flash`: Provider diversity. ### 2. Autonomous Subagents (Tier 2) -* **General Subagents**: `openrouter/z-ai/glm-5.3-flash` - - **Execution**: Spawned via `sessions_spawn` with a 300-second timeout. Inherits the `tool_call` protocol guardrails from `AGENTS.md`. -* **Deep Analysis & IT Specialists (`itguy`, `recherche`)**: `openrouter/deepseek/deepseek-v4.1-flash` - - High-precision reasoning, robust nested tool loops without syntax drift, and immune to XML leakage on extensive research queries or CLI diagnostics. -* **Fallback**: `openrouter/deepseek/deepseek-v4.1-flash` / `openrouter/qwen/qwen3.8-flash`. +* **All Subagents & Specialists**: `openrouter/deepseek/deepseek-v4.1-flash-1017` + - **Execution**: Spawned via `sessions_spawn` with a 300-second timeout. Provides high-precision reasoning, rock-solid nested JSON tool loops (`tool_search` -> `tool_describe` -> `tool_call`), and zero syntax drift. +* **Fallback**: `openrouter/deepseek/deepseek-v4.1-flash` / `openrouter/z-ai/glm-5.3-flash` / `openrouter/qwen/qwen3.8-flash`. ### 3. Background Daemon & Heartbeats (Tier 3) -* **Configuration**: `heartbeat.model: "openrouter/z-ai/glm-5.3-flash"`, `heartbeat.target: none`, `heartbeat.lightContext: true`. -* **Rationale**: Decoupled completely from user conversation transcripts. Polling occurs against an isolated, minimal prompt (~400 tokens), dropping weekly heartbeat token burn from 21,000,000 to <150,000 tokens (<\$0.02 / week). +* **Configuration**: `heartbeat.model: "openrouter/deepseek/deepseek-v4.1-flash-1017"`, `heartbeat.target: none`, `heartbeat.lightContext: true`. +* **Rationale**: Because background heartbeats have a 75%+ `cacheRead` share (`22.86M` `cacheRead` vs. `7.73M` uncached input over 17 days), DeepSeek V4.1 Flash 1017's \$0.00294/M cache read rate on DeepInfra reduces 17-day heartbeat cost to <\$0.90 while keeping prompt cache warm on the primary provider. ### 4. Vision Input vs. Text-to-Image Generation (Tier 4) * **Crucial Distinction**: @@ -171,14 +173,83 @@ To align on remaining edge cases before final sign-off: --- +## OpenRouter Provider Qualification Criteria & Re-Evaluation Playbook + +Whenever a new model snapshot (such as `deepseek/deepseek-v4.1-flash-1017`) is evaluated or re-checked as new hosting providers join OpenRouter, every candidate provider MUST pass the following **7-point qualification gate** before being added to `extra_body.provider.order` in [openclaw-instance.secret.yaml](file:///Users/moritz/src/infrapuzzle/k8s/openclaw/openclaw-instance.secret.yaml): + +### 1. Hard Technical Gates (Must Pass All) +1. **Quantization Precision (`fp8` / `bf16` Only)**: + - **Rule**: Must run `fp8` or `bf16` (or verified full-precision `unknown` from trusted cloud hosts like Cloudflare). + - **Reject**: Any `fp4`, `nvfp4`, `mxfp4`, or `int4` quantization (`decart`, `sail-research`, `wafer`, `open-inference`, `phala` fp4). 4-bit quantizations severely degrade nested JSON tool-call syntax and German grammar. +2. **OpenRouter Routing Funnel & Guardrail Compatibility (`tools` + `tool_choice: "auto"` Pass)**: + - **Rule**: Must not be stripped by OpenRouter's internal routing filters (`Filter by Tool Compatibility` or `Filter by Guardrails`) when a request includes `tools: [...]` and `tool_choice: "auto"`. + - **Why**: Metadata in `/api/v1/models/.../endpoints` can claim `supports_tool_choice: true` even when OpenRouter's live router blocks the provider on actual tool calls (e.g. `morph/fp8` is stripped by `Filter by Tool Compatibility`, and `deepseek` official is stripped by `Paid model training violation` account privacy settings). +3. **Nested MCP `tool_call` Serialization Integrity (2/2 Live Pass)**: + - **Rule**: Must cleanly emit `finish_reason: "tool_calls"` with valid, parseable nested JSON arguments (`{"id": "mcp:...", "args": {...}}`) across back-to-back test runs without mid-stream SSE aborts (`Provider returned an incomplete or malformed tool call`) or HTTP `429` upstream rate limits. +4. **Context Window & Completion Headroom**: + - **Rule**: `context_length >= 262,144` (1,048,576 preferred) and `max_completion_tokens >= 131,072`. + - **Why**: Active daytime sessions with system prompts and tool schemas reach 20k–80k tokens. +5. **Uptime & Reliability**: + - **Rule**: `uptime_last_1d >= 95%` and `uptime_last_30m == 100%`. + +### 2. Economic & Latency Ranking (How to Order Passing Providers) +Among providers passing all 5 technical gates, rank them in `extra_body.provider.order` by **effective production cost** and **turn latency**: +1. **Implicit Prompt Cache Read Rate (`input_cache_read`) — Highest Weight**: + - Because **73%–94%** of OpenClaw's token volume (both interactive multi-turn chats and 30m background heartbeats) consists of cached prompt reads (`cacheRead`), a provider with `supports_implicit_caching: true` and a **95%–98% cache discount** (`<= $0.005 / 1M` tokens, e.g. `deepinfra` at `$0.00294/M` or `streamlake` at `$0.0030/M`) is vastly cheaper in net monthly spend than a provider with slightly cheaper uncached input but weak or missing `cacheRead` discounts (e.g. `nebius` with `cache_read: null` or `modal`/`primeintellect` at `$0.030/M`). +2. **Base Prompt & Completion Pricing**: + - Target `prompt <= $0.18 / 1M` and `completion <= $0.65 / 1M` for top-priority slots; allow higher-priced providers (`<= $0.30 / $1.20`) only in lower fallback slots (#3/#4) for outage resilience. +3. **End-to-End Tool-Call Latency**: + - Target **1.0s – 5.5s** for a ~300-token reasoning + tool-call turn. + +### 3. One-Shot Live Verification Command (For Future Checks) +Any agent (including a fast/lightweight model) can audit all currently available OpenRouter endpoints for `deepseek/deepseek-v4.1-flash-1017` in one step by running: + +```bash +# 1. Inspect current endpoint metadata, pricing, cache_read, and 24h uptime: +curl -s https://openrouter.ai/api/v1/models/deepseek/deepseek-v4.1-flash-1017/endpoints | \ + jq -r '.data.endpoints[] | "\(.provider_name) | tag=\(.tag) | quant=\(.quantization) | ctx=\(.context_length) | prompt=\(.pricing.prompt) | comp=\(.pricing.completion) | cache=\(.pricing.input_cache_read) | uptime1d=\(.uptime_last_1d)"' + +# 2. Live-test candidate providers inside openclaw-0 with real nested tool_call payloads: +kubectl exec openclaw-0 -n openclaw -c openclaw -- python3 -c ' +import os, json, time, urllib.request +api_key = os.environ["OPENROUTER_API_KEY"] +tools = [{"type": "function", "function": {"name": "tool_call", "parameters": {"type": "object", "properties": {"id": {"type": "string"}, "args": {"type": "object"}}, "required": ["id", "args"]}}}] +candidates = ["deepinfra", "streamlake", "inference-net", "venice", "digitalocean", "novita", "parasail", "cloudflare", "nebius"] +for prov in candidates: + req = urllib.request.Request( + "https://openrouter.ai/api/v1/chat/completions", + data=json.dumps({ + "model": "deepseek/deepseek-v4.1-flash-1017", + "messages": [{"role": "user", "content": "Call tool_call with id=\"mcp:homeassistant:homeassistant__ha_get_state\" and args={\"entity_id\": \"media_player.bad\"}"}], + "tools": tools, "tool_choice": "auto", "max_tokens": 250, + "provider": {"order": [prov], "allow_fallbacks": False} + }).encode(), + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} + ) + t0 = time.time() + try: + with urllib.request.urlopen(req, timeout=20) as r: + b = json.loads(r.read().decode()) + tc = b["choices"][0]["message"].get("tool_calls") + args = tc[0]["function"]["arguments"] if tc else "NONE" + print(f"PASS | {prov:15s} | {time.time()-t0:4.2f}s | actual={b.get(\"provider\")} | args={args}") + except urllib.error.HTTPError as e: + print(f"FAIL | {prov:15s} | HTTP {e.code} | {e.read().decode()[:140]}") + except Exception as e: + print(f"FAIL | {prov:15s} | ERR {e}") +' +``` + +--- + ## Consequences ### Positive -* **Unified Latest-Generation Stack**: Completely eliminates legacy Gemini 2.5 models; standardizes exclusively on Gemini 3.1, GLM 5.3, Whisper V3 Turbo, and Sonar Pro. +* **Unified Latest-Generation Stack**: Completely eliminates legacy Gemini 2.5 models; standardizes exclusively on DeepSeek V4.1 Flash 1017, Gemini 3.8/3.1, GLM 5.3, Whisper V3 Turbo, and Sonar Pro. * **100% Reliable Image Generation**: Correctly routes image generation requests to Nano Banana 2 (`google/gemini-3.1-flash-image`), completely eliminating HTTP 404 generation errors. * **Cohesive Voice & Audio Pipeline**: Clear architectural documentation of the Whisper V3 Turbo and Gemini 3.1 Flash TTS integration for dossiers and podcasts. -* **Guaranteed Cost Ceiling**: Drops weekly expenditure from **\$12.55 to ~\$1.20–\$1.80** (85–90% savings) through decoupled heartbeats and nightly compaction. +* **Guaranteed Cost Ceiling**: Drops weekly expenditure from **\$12.55 to ~\$0.80–\$1.40** (~90% savings) through DeepSeek V4.1 Flash 1017's `$0.00294/M` cache reads, decoupled heartbeats, and nightly compaction. ### Negative & Mitigations -* **OpenRouter Provider Variability**: GLM-5.3-Flash prefill throughput varies across OpenRouter hosting providers. - * *Mitigation*: Daily compaction bounds transcript size; failover to DeepSeek V4.1-Flash is available at zero architectural switching cost. +* **OpenRouter Provider Rollout Lag on Day-0 Model Releases**: Newly released snapshots initially launch with 5–10 providers before the full fleet (25+ providers) joins over subsequent days. + * *Mitigation*: Codified the 7-point provider qualification gate and one-shot test script above, and configured automatic model fallback to the `0910` snapshot (`openrouter/deepseek/deepseek-v4.1-flash`). diff --git a/k8s/openclaw/TROUBLESHOOTING.md b/k8s/openclaw/TROUBLESHOOTING.md index d168207..56b072a 100644 --- a/k8s/openclaw/TROUBLESHOOTING.md +++ b/k8s/openclaw/TROUBLESHOOTING.md @@ -966,4 +966,49 @@ This document captures fixed issues, operational quirks, and incident root-cause - **How to Avoid Recurrence**: - Always decouple perception tasks (OCR, image description, speech-to-text) into `tools.media` or `imageModel`, keeping primary conversational models focused on reasoning and dialogue without changing the core chat persona. +--- +### Issue 30: GLM-5.3-Flash Malformed Tool Call Stream Aborts (`streamlake/fp8`), Cold HDD Prompt Assembly Stall, and Fleet-Wide Migration to DeepSeek V4.1 Flash + +- **Date**: 2026-10-17 +- **Severity**: High (Topic 164 `#Grabengasse & Media` failed with `Provider returned an incomplete or malformed tool call`; 12 total occurrences across 5 agents in 17 days; 504.5s cold-disk event-loop stall on first morning request) +- **Symptom**: + 1. At 07:11:02 CEST, Moritz sent a voice note in Topic 164 (`#Grabengasse & Media`, agent `grabengasse_media`): *"Kannst du mir auf dem Sonos in der Küche die Bibi Blocksberg Playlist Der Affe ist los abspielen?"* + 2. The turn stalled for **8 minutes and 24 seconds** (`504,493 ms`) during `system-prompt` assembly (`cpuCoreRatio=0.005`, pure disk I/O wait) before sending the first LLM request at 07:19:26 CEST. + 3. In Turn 1 (`SEQ 175`), `z-ai/glm-5.3-flash` called `tool_search({"query": "sonos spotify play"})`. In Turn 2 (`SEQ 177`), it called `tool_describe({"name": "mcp:spotify:spotify__search"})`. + 4. In Turn 3 (`SEQ 179`, `responseId: gen-1792214444-T46fmmBLqgmCCe0YJp4X`), after generating 326 reasoning tokens (`"Search for 'Bibi Blocksberg Der Affe ist los' using mcp:spotify:spotify__search"`), the upstream OpenRouter provider (`streamlake/fp8`) aborted the stream mid-JSON while serializing nested `tool_call({"id": "mcp:spotify:spotify__search", "args": {...}})` arguments: + `stopReason: "error"`, `errorMessage: "Provider returned an incomplete or malformed tool call"`. + 5. OpenClaw only retries or triggers model fallbacks on HTTP status errors (`429`, `502`, `503`, `504`) prior to stream start—not when an HTTP `200 OK` SSE stream terminates mid-generation with a malformed tool-call chunk. +- **Root Cause & Fleet Forensics**: + 1. **`z-ai/glm-5.3-flash` Nested Tool-Call Fragility on `streamlake/fp8` & `siliconflow/fp8`**: + - Across October 1–17, 2026, `Provider returned an incomplete or malformed tool call` occurred **12 times** across `main` (6×), `grabengasse_media` (3×), `koch` (1×), `hebamme` (1×), and `doktor` (1×)—exclusively on `z-ai/glm-5.3-flash`, always right after `tool_describe` when emitting nested `tool_call` JSON payloads. + 2. **Suboptimal OpenRouter Provider Routing**: + - `extra_body.provider.order` for `z-ai/glm-5.3-flash` listed `"cloudflare"` first (which does not host `glm-5.3-flash` on OpenRouter), falling through to `streamlake/fp8` (91.7% uptime) and `siliconflow/fp8` (74.1% uptime), and did not block `fp4` (`nvfp4`) quantizations (`phala`, `atlas-cloud`, `parasail`). + 3. **Fleet-Wide Cost & Cache-Hit Economics (`DeepSeek V4.1 Flash` vs. `GLM 5.3 Flash`)**: + - Over the 17-day period (Oct 1–17, 2026), 73% to 94% of all tokens across interactive agents (`21.93M` `cacheRead` vs. `8.94M` uncached input) and background heartbeats (`22.86M` `cacheRead` vs. `7.73M` uncached input) were prompt cache reads. + - While `deepseek/deepseek-v4.1-flash` has a slightly higher base uncached prompt rate (`$0.14–$0.15/M` vs. `$0.06–$0.075/M`), its top `fp8` providers (`deepinfra` and `streamlake`, both 100% uptime and full `tool_choice` support) charge **3.5× to 10× less for `cacheRead` tokens** (`$0.0042/M` on DeepInfra, `$0.0030/M` on StreamLake vs. `$0.015–$0.030/M` on GLM 5.3 Flash). + - Consequently, running the entire fleet (interactive agents + background heartbeats) on `deepseek/deepseek-v4.1-flash` via `deepinfra`/`streamlake` costs **~$2.69 over 17 days (~$4.75/month)** vs. **$4.50 actual on `GLM 5.3 Flash` (~$7.95/month)**—a **~40% net cost reduction** alongside 0 malformed tool calls. + 4. **Cold Rotational HDD Read Stall After Nightly Backups**: + - Nightly backups (`03:30`) and compaction (`04:45`) evicted the Knowledge Wiki and agent workspace `*.md` files from the Linux kernel page cache, causing the first morning turn's synchronous `system-prompt` loader to block on rotational SATA HDD seeks for 504.5s. +- **Resolution**: + 1. **Fleet-Wide Migration to `openrouter/deepseek/deepseek-v4.1-flash-1017` (with `deepseek-v4.1-flash` 0910 as first fallback)**: + - Switched `agents.defaults.model.primary`, `utilityModel`, `heartbeat.model`, `subagents.model`, `memory-core.config.dreaming.model`, and all interactive agent entries (`main`, `fitness`, `koch`, `hebamme`, `travel`, `shopping`, `regensburg`, `recherche`, `doktor`, `itguy`, `grabengasse_media`, `mila_tracker`) to the newly released October 17, 2026 snapshot `openrouter/deepseek/deepseek-v4.1-flash-1017` (Artificial Analysis Intelligence Index 39.5 vs. 24.7 on 0910, and 30% cheaper on DeepInfra at `$0.098/M` prompt, `$0.294/M` completion, `$0.00294/M` `cacheRead`), with `openrouter/deepseek/deepseek-v4.1-flash`, `openrouter/z-ai/glm-5.3-flash`, and `openrouter/qwen/qwen3.8-flash` as fallbacks (`nutrition_tracker` remains on `openrouter/google/gemini-3.8-flash`). + 2. **Preserved Decoupled Vision Perception (Issue 29 Compatibility)**: + - Registered both `deepseek/deepseek-v4.1-flash-1017` and `deepseek/deepseek-v4.1-flash` with `input: ["text"]` under `models.providers.openrouter.models` in `openclaw-instance.secret.yaml` so OpenClaw continues routing all image attachments through `tools.media.image` (`google/gemini-3.8-flash`) for OCR before passing enriched text to DeepSeek V4.1 Flash. + 3. **Hardened OpenRouter Provider & Quantization Routing (`extra_body.provider`)**: + - Configured `openrouter/deepseek/deepseek-v4.1-flash-1017` with `order: ["deepinfra", "inference-net", "parasail", "cloudflare"]`, `quantizations: ["fp8", "bf16", "unknown"]`, and `ignore: ["morph", "baseten", "deepseek", "open-inference", "decart", "sail-research", "wafer"]` (live routing tests showed `morph/fp8` is rejected by OpenRouter's `Filter by Tool Compatibility` when `tools` are passed, `deepseek` is blocked by account training-opt-out guardrails, and `baseten` returns HTTP 429). + - Configured fallback `openrouter/deepseek/deepseek-v4.1-flash` (`0910`) with `order: ["deepinfra", "streamlake", "inference-net", "digitalocean", "venice", "nebius"]` and `quantizations: ["fp8", "bf16", "unknown"]`. + - Cleaned up `openrouter/z-ai/glm-5.3-flash` fallback routing (`order: ["morph", "near-ai", "digitalocean", "venice", "streamlake"]`, `quantizations: ["fp8", "bf16", "unknown"]`, blocking `fp4`). + 4. **Direct MCP Tool Fastpaths & 1-Turn Sonos Playbook for `grabengasse_media` (Topic 164)**: + - Added exact MCP tool IDs (`mcp:spotify:spotify__search`, `mcp:spotify:spotify__play`, `mcp:spotify:spotify__transfer_playback`, `mcp:jelly:jelly__jelly_search`, `mcp:jelly:jelly__jelly_request`, `mcp:homeassistant:homeassistant__ha_get_state`, `mcp:homeassistant:homeassistant__ha_call_service`) as well as all verified Sonos entity IDs (`media_player.bad`, `media_player.wohnzimmer`, `media_player.kuche_sonos`, `media_player.kuche_ikea`, `media_player.dining_room`) and Sonos `source_list` favorites (`"Fusion 2026 Sets"`, `"ChilloutTechno"`, `"deephouse"`, `"OberDITSCHI"`, etc.) directly to `openclaw-instance.secret.yaml`, `knowledge-wiki/agents/grabengasse_media.md`, and `/home/openclaw/.openclaw/workspace-grabengasse_media/AGENTS.md`. + - Eliminates both the 2 MCP discovery turns (`tool_search` + `tool_describe`) and the 2 file-read turns (`read("USER.md")`, `read("MEMORY.md")`, `read("SKILL.md")`). + 5. **Extended Linux Page-Cache Pre-Warming to Knowledge Wiki & Workspace Markdown Files**: + - Updated `k8s/openclaw/scripts/gateway-preflight.mjs` and `k8s/openclaw/scripts/compact_sessions.py` (which now also runs `prewarm_page_cache()` at the end of the 04:45 nightly run) to pre-read all Knowledge Wiki (`*.md`, `*.jsonl`) and `workspace*/` `*.md` files into RAM, preventing morning `system-prompt` disk freezes. + 6. **SQLite `session_nodes` Trigger (`entry_valid`) & Subagent Vector Re-Indexing Fix (`extraPaths: []`)**: + - Discovered that SQLite `session_nodes` has an `AFTER UPDATE OF entry_json` trigger (`session_nodes_entry_valid_after_entry_update`) that resets `entry_valid = 0` whenever `entry_json` is modified, throwing `SessionCanonicalKeyMigrationRequiredError` until `UPDATE session_nodes SET entry_valid = 1` is executed. + - Hardened `k8s/openclaw/scripts/recover_sessions.py` to always enforce `PRAGMA foreign_keys = ON` before deleting trailing error turns, set `entry_valid = 1` across all rows, and reset `attempts = 0` on pending `channel_ingress_events` in `state/openclaw.sqlite`. + - Explicitly set `memory.search.extraPaths: []` at the top level in `openclaw-instance.secret.yaml` and `dedup_subagent_vectors.py` so `mergeMode: merge` does not leave `extraPaths` active globally in `openclaw.json`, preventing subagents from re-embedding 4,059 wiki chunks on their first turn after startup. +- **How to Avoid Recurrence**: + - Always enforce `quantizations: ["fp8", "bf16"]` in OpenRouter `extra_body.provider` for tool-calling agents so OpenRouter never routes to degraded `fp4` (`nvfp4`) endpoints. + - Whenever updating `entry_json` in `session_nodes` via SQLite, always follow up with `UPDATE session_nodes SET entry_valid = 1` in a separate statement (after the `AFTER UPDATE OF entry_json` trigger fires), and always set `PRAGMA foreign_keys = ON` before deleting from `transcript_events`. + - Keep Knowledge Wiki and workspace Markdown files in the nightly `prewarm_page_cache()` pass so morning turns hit RAM (`<50ms`) instead of cold rotational disks. diff --git a/k8s/openclaw/configmap-scripts.yaml b/k8s/openclaw/configmap-scripts.yaml index 733226a..571dc9a 100644 --- a/k8s/openclaw/configmap-scripts.yaml +++ b/k8s/openclaw/configmap-scripts.yaml @@ -1,907 +1,633 @@ apiVersion: v1 +data: + compact_sessions.py: "#!/usr/bin/env python3\n\"\"\"\nNightly multi-agent session + maintenance and compaction script for OpenClaw.\n1. Pre-warms OS page cache for + SQLite databases on spinning disks.\n2. Recovers any wedged or locked sessions + via recover_sessions.py.\n3. Prunes abandoned isolated cron sessions (>48h old) + to prevent token/state bloat.\n4. Tier A Hard-Reset: Resets koch, hebamme, and + shopping topics to 0 tokens (clean archive).\n5. Tier B Smart-Compaction: Compacts + active sessions exceeding token thresholds (5s pacing).\n6. Checkpoints SQLite + WAL journals across all agent databases to disk.\n7. Optimizes SQLite query planner + without blocking full VACUUM.\n\"\"\"\nimport glob\nimport json\nimport os\nimport + sqlite3\nimport subprocess\nimport sys\nimport time\n\nTHRESHOLD_TOKENS = 15000\nTHRESHOLD_TRANSCRIPT_BYTES + = 200000\nTHRESHOLD_EVENT_COUNT = 30\nCRON_MAX_AGE_HOURS = 48\nRPC_TIMEOUT_MS + = 180000\n\n# Tracking Channels: Deterministic sliding-window (mila_tracker, nutrition_tracker).\n# + Strict sliding window of ~5-8 events (2-3 turns max), 0 LLM tokens, instant truncation.\nTRACKING_TARGETS + = [\n (\"mila_tracker\", \"agent:mila_tracker:telegram:group:-1004326921510:topic:2796\"),\n + \ (\"nutrition_tracker\", \"agent:nutrition_tracker:telegram:group:-1004326921510:topic:2797\"),\n]\nMAX_TRACKING_EVENTS + = 8\nMAX_TRACKING_LINES = 8\n\n# Tier A: High-frequency interaction topics (koch, + hebamme, shopping).\n# Deep-compaction keeps a compact summary + latest 2 turns + without completely wiping conversational memory.\nTIER_A_TARGETS = [\n (\"koch\", + \"agent:koch:telegram:group:-1004326921510:topic:21\"),\n (\"hebamme\", \"agent:hebamme:telegram:group:-1004326921510:topic:22\"),\n + \ (\"shopping\", \"agent:shopping:telegram:group:-1004326921510:topic:115\"),\n]\n\ndef + prewarm_page_cache():\n print(\"=== Pre-warming SQLite databases and Knowledge + Wiki into OS page cache ===\")\n t0 = time.time()\n try:\n subprocess.run(\n + \ \"cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite + > /dev/null 2>&1 || true; \"\n \"find /home/node/.openclaw/wiki/main + /home/node/.openclaw/workspace* -maxdepth 3 -name '*.md' -exec cat {} + > /dev/null + 2>&1 || true\",\n shell=True,\n timeout=90.0\n )\n + \ print(f\"Page cache pre-warmed in {time.time() - t0:.2f}s\")\n except + Exception as e:\n print(f\"Page cache pre-warming warning: {e}\")\n\ndef + prune_stale_cron_sessions():\n print(f\"=== Pruning isolated cron sessions + older than {CRON_MAX_AGE_HOURS}h ===\")\n cutoff_ms = int(time.time() * 1000) + - (CRON_MAX_AGE_HOURS * 3600 * 1000)\n total_pruned = 0\n total_tokens_reclaimed + = 0\n\n for db in sorted(glob.glob(\"/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite\")):\n + \ agent = db.split(\"/\")[5]\n try:\n conn = sqlite3.connect(db, + timeout=30.0)\n conn.execute(\"PRAGMA foreign_keys = ON;\")\n cursor + = conn.cursor()\n cursor.execute(\"SELECT name FROM sqlite_master WHERE + type=\\\"table\\\"\")\n all_tables = set(r[0] for r in cursor.fetchall())\n\n + \ cursor.execute(\"SELECT session_key, current_session_id, updated_at, + entry_json FROM session_nodes WHERE session_key LIKE \\\"%:cron:%\\\"\")\n stale + = []\n for k, sid, up, ej in cursor.fetchall():\n if + up and up < cutoff_ms:\n tok = 0\n if ej:\n + \ try:\n tok = json.loads(ej).get(\"totalTokens\", + 0) or 0\n except Exception:\n pass\n + \ stale.append((k, sid, tok))\n\n if stale:\n tokens_reclaimed + = sum(t[2] for t in stale)\n for k, sid, _ in stale:\n for + t in [\"trajectory_runtime_events\", \"transcript_events\", \"transcript_event_identities\", + \"session_transcript_active_events\", \"transcript_rewrite_watermarks\", \"session_transcript_index_state\", + \"session_transcript_fts\", \"session_conversations\"]:\n if + t in all_tables and sid:\n cursor.execute(f\"DELETE + FROM {t} WHERE session_id = ?\", (sid,))\n for t in [\"session_participants\", + \"heartbeat_outcomes\", \"session_progress_cards\", \"board_widgets\", \"board_tabs\"]:\n + \ if t in all_tables:\n cursor.execute(f\"DELETE + FROM {t} WHERE session_key = ?\", (k,))\n if \"memory_entry_origins\" + in all_tables:\n cursor.execute(\"DELETE FROM memory_entry_origins + WHERE session_key = ? OR session_id = ?\", (k, sid or \"\"))\n if + \"session_windows\" in all_tables:\n cursor.execute(\"DELETE + FROM session_windows WHERE session_key = ? OR session_id = ?\", (k, sid or \"\"))\n + \ cursor.execute(\"DELETE FROM session_nodes WHERE session_key + = ?\", (k,))\n conn.commit()\n total_pruned += len(stale)\n + \ total_tokens_reclaimed += tokens_reclaimed\n print(f\" + \ [{agent}] Pruned {len(stale)} dead cron sessions ({tokens_reclaimed} tokens + reclaimed)\")\n\n # Unconditionally resolve any dangling foreign key + references across all tables\n while True:\n cursor.execute(\"PRAGMA + foreign_key_check;\")\n violations = cursor.fetchall()\n if + not violations:\n break\n for table, rowid, + parent, fkid in violations:\n cursor.execute(f\"DELETE FROM + {table} WHERE rowid = ?\", (rowid,))\n conn.commit()\n conn.close()\n + \ except Exception as e:\n print(f\" [{agent}] Pruning error: + {e}\")\n\n # Clear resolved quarantine entries if all databases are healthy\n + \ try:\n q_path = \"/home/node/.openclaw/state/openclaw-quarantine.sqlite\"\n + \ if os.path.exists(q_path):\n q_conn = sqlite3.connect(q_path)\n + \ q_cur = q_conn.cursor()\n q_cur.execute(\"DELETE FROM quarantined_databases\")\n + \ q_conn.commit()\n q_conn.close()\n except Exception + as e:\n print(f\"Quarantine cleanup error: {e}\")\n\n print(f\"Cron + pruning complete: {total_pruned} sessions pruned, {total_tokens_reclaimed} tokens + reclaimed.\")\n\ndef compact_tracking_sessions():\n print(\"=== Tracking Channels + Sliding-Window Maintenance (mila_tracker, nutrition_tracker) ===\")\n for agent, + key in TRACKING_TARGETS:\n db_path = f\"/home/node/.openclaw/agents/{agent}/agent/openclaw-agent.sqlite\"\n + \ if not os.path.exists(db_path):\n continue\n try:\n + \ con = sqlite3.connect(db_path, timeout=5.0)\n row = con.execute(\"SELECT + current_session_id FROM session_nodes WHERE session_key=?\", (key,)).fetchone()\n + \ if not row or not row[0]:\n con.close()\n print(f\" + \ [{agent}] No active session node found for {key}.\")\n continue\n + \ sid = row[0]\n cnt = con.execute(\"SELECT count(*) FROM + transcript_events WHERE session_id=?\", (sid,)).fetchone()[0]\n con.close()\n\n + \ if cnt > MAX_TRACKING_EVENTS:\n print(f\" [{agent}] + Session {key} has {cnt} events (> {MAX_TRACKING_EVENTS}). Truncating to last {MAX_TRACKING_LINES} + lines...\")\n cmd = [\n \"openclaw\", \"sessions\", + \"compact\", key,\n \"--agent\", agent,\n \"--max-lines\", + str(MAX_TRACKING_LINES),\n \"--timeout\", \"30000\"\n ]\n + \ r = subprocess.run(cmd, capture_output=True, text=True)\n if + r.returncode == 0:\n print(f\" ✓ Truncated successfully: + {r.stdout.strip()}\")\n else:\n print(f\" ✗ + Truncation failed: {r.stderr.strip()}\")\n else:\n print(f\" + \ [{agent}] Session {key} has {cnt} events (<= {MAX_TRACKING_EVENTS}). Clean.\")\n + \ except Exception as e:\n print(f\" ✗ [{agent}] Error during + tracking compaction: {e}\")\n\ndef compact_tier_a_sessions():\n print(\"=== + Tier A: Deep-Compaction (koch, hebamme, shopping) ===\")\n for agent, key in + TIER_A_TARGETS:\n db_path = f\"/home/node/.openclaw/agents/{agent}/agent/openclaw-agent.sqlite\"\n + \ if not os.path.exists(db_path):\n continue\n try:\n + \ con = sqlite3.connect(db_path, timeout=10.0)\n row = con.execute(\"SELECT + current_session_id FROM session_nodes WHERE session_key=?\", (key,)).fetchone()\n + \ con.close()\n if not row or not row[0]:\n print(f\" + \ [Tier A] {agent} ({key}) already at 0 (no active session).\")\n continue\n + \ ev_cnt, ev_bytes = get_session_metrics(agent, key, row[0])\n if + ev_cnt <= 2 and ev_bytes < 20000:\n print(f\" [Tier A] {agent} + ({key}) has only {ev_cnt} events ({ev_bytes} bytes), skipping.\")\n continue\n + \ print(f\" [Tier A] Deep-compacting {agent} ({key}) [{ev_cnt} events, + {ev_bytes//1024}KB]...\")\n r = subprocess.run(\n [\"openclaw\", + \"sessions\", \"compact\", key, \"--agent\", agent, \"--timeout\", str(RPC_TIMEOUT_MS)],\n + \ capture_output=True, text=True\n )\n if + r.returncode == 0:\n print(f\" ✓ Deep-compacted: {r.stdout.strip()}\")\n + \ elif \"Already compacted\" in r.stderr or \"Already compacted\" in + r.stdout:\n print(f\" ✓ Already compacted.\")\n else:\n + \ print(f\" ✗ Standard compact error: {r.stderr.strip()}; retrying + with --max-lines 40...\")\n fb = subprocess.run(\n [\"openclaw\", + \"sessions\", \"compact\", key, \"--agent\", agent, \"--max-lines\", \"40\", \"--timeout\", + str(RPC_TIMEOUT_MS)],\n capture_output=True, text=True\n )\n + \ if fb.returncode == 0:\n print(f\" ✓ Truncation + fallback success: {fb.stdout.strip()}\")\n else:\n print(f\" + \ ✗ Truncation fallback failed: {fb.stderr.strip()}\")\n except Exception + as e:\n print(f\" ✗ Exception deep-compacting {agent}: {e}\")\n + \ time.sleep(2)\n\ndef get_session_metrics(agent, key, sid=None):\n db_path + = f\"/home/node/.openclaw/agents/{agent}/agent/openclaw-agent.sqlite\"\n if + not os.path.exists(db_path):\n return 0, 0\n try:\n con = sqlite3.connect(db_path, + timeout=5.0)\n cur = con.cursor()\n if not sid:\n row + = cur.execute(\"SELECT current_session_id FROM session_nodes WHERE session_key=?\", + (key,)).fetchone()\n sid = row[0] if row else None\n if not + sid:\n con.close()\n return 0, 0\n ev_cnt, ev_bytes + = cur.execute(\n \"SELECT count(*), coalesce(sum(length(event_json)), + 0) FROM transcript_events WHERE session_id=?\", \n (sid,)\n ).fetchone()\n + \ con.close()\n return ev_cnt, ev_bytes\n except Exception:\n + \ return 0, 0\n\ndef compact_active_sessions():\n excluded_keys = set(t[1] + for t in TIER_A_TARGETS) | set(t[1] for t in TRACKING_TARGETS)\n print(f\"=== + Tier B: Running Multi-Agent Compaction (threshold: {THRESHOLD_TOKENS} tokens / + {THRESHOLD_TRANSCRIPT_BYTES//1024}KB, LLM summarization) ===\")\n res = subprocess.run([\"openclaw\", + \"sessions\", \"--all-agents\", \"--json\"], capture_output=True, text=True)\n + \ if res.returncode != 0:\n print(f\"Failed to list sessions: {res.stderr}\")\n + \ return\n\n sessions = json.loads(res.stdout).get(\"sessions\", [])\n + \ compacted = 0\n skipped = 0\n\n for s in sessions:\n key = s.get(\"key\", + \"\")\n agent = s.get(\"agentId\") or \"main\"\n tokens = s.get(\"totalTokens\") + or 0\n sid = s.get(\"sessionId\")\n\n # Skip Tier A and Tracking + agents as they have dedicated routines\n if key in excluded_keys:\n continue\n\n + \ # Target main sessions and all telegram sessions (topics and direct)\n + \ is_target = (\n key == \"agent:main:main\" or \n \":telegram:\" + in key or\n (key.startswith(\"agent:\") and key.endswith(\":main\"))\n + \ )\n if is_target and not key.endswith(\":heartbeat\"):\n ev_cnt, + ev_bytes = get_session_metrics(agent, key, sid)\n needs_compaction + = (\n tokens >= THRESHOLD_TOKENS or\n ev_bytes >= + THRESHOLD_TRANSCRIPT_BYTES or\n (tokens == 0 and ev_cnt >= THRESHOLD_EVENT_COUNT)\n + \ )\n if needs_compaction:\n stat_str = f\"{tokens} + tokens\" if tokens else f\"{ev_bytes//1024}KB / {ev_cnt} events\"\n print(f\"Compacting + {agent} ({stat_str}) -> {key}...\")\n c_res = subprocess.run(\n + \ [\"openclaw\", \"sessions\", \"compact\", key, \"--agent\", + agent, \"--timeout\", str(RPC_TIMEOUT_MS)],\n capture_output=True, + text=True\n )\n if c_res.returncode == 0:\n print(f\" + \ ✓ {c_res.stdout.strip()}\")\n compacted += 1\n elif + \"Already compacted\" in c_res.stderr or \"Already compacted\" in c_res.stdout:\n + \ print(f\" ✓ Already compacted.\")\n skipped + += 1\n else:\n print(f\" ✗ {c_res.stderr.strip()}; + retrying with --max-lines 80 fallback...\")\n fb_res = subprocess.run(\n + \ [\"openclaw\", \"sessions\", \"compact\", key, \"--agent\", + agent, \"--max-lines\", \"80\", \"--timeout\", str(RPC_TIMEOUT_MS)],\n capture_output=True, + text=True\n )\n if fb_res.returncode == + 0:\n print(f\" ✓ (fallback) {fb_res.stdout.strip()}\")\n + \ compacted += 1\n else:\n print(f\" + \ ✗ Fallback failed: {fb_res.stderr.strip()}\")\n time.sleep(5) + \ # 5s pause to avoid rotational HDD I/O saturation\n else:\n skipped + += 1\n\n print(f\"Compaction complete: {compacted} compacted, {skipped} skipped.\")\n\ndef + checkpoint_wal_journals():\n print(\"=== Checkpointing SQLite WAL journals + across all agent databases ===\")\n dbs = [\"/home/node/.openclaw/state/openclaw.sqlite\"] + + glob.glob(\"/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite\")\n wal_freed + = 0\n for db in dbs:\n wal = db + \"-wal\"\n if os.path.exists(wal):\n + \ sz = os.path.getsize(wal)\n try:\n con = + sqlite3.connect(db, timeout=30.0)\n con.execute(\"PRAGMA wal_checkpoint(TRUNCATE);\")\n + \ con.close()\n wal_freed += sz\n except + Exception as e:\n print(f\" ✗ Error checkpointing {db}: {e}\")\n + \ print(f\"WAL Checkpoint complete: {wal_freed / (1024*1024):.2f} MB merged + to disk.\")\n\ndef vacuum_and_optimize():\n print(\"=== Running SQLite OPTIMIZE + across all databases ===\")\n dbs = [\"/home/node/.openclaw/state/openclaw.sqlite\"] + + glob.glob(\"/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite\")\n for + db in dbs:\n try:\n con = sqlite3.connect(db, timeout=30.0)\n + \ con.execute(\"PRAGMA optimize;\")\n con.close()\n except + Exception as e:\n print(f\" ✗ Error optimizing {db}: {e}\")\n print(\"SQLite + optimization complete.\")\n\ndef main():\n prewarm_page_cache()\n try:\n + \ subprocess.run([\"python3\", \"/home/node/.openclaw/scripts/recover_sessions.py\"], + check=False)\n except Exception:\n pass\n prune_stale_cron_sessions()\n + \ compact_tracking_sessions()\n compact_tier_a_sessions()\n compact_active_sessions()\n + \ checkpoint_wal_journals()\n vacuum_and_optimize()\n prewarm_page_cache()\n\nif + __name__ == \"__main__\":\n main()\n" + dedup_subagent_vectors.py: | + #!/usr/bin/env python3 + import glob, json, os, sqlite3, sys + + cfg_path = "/home/node/.openclaw/openclaw.json" + if os.path.exists(cfg_path): + try: + with open(cfg_path, "r") as f: + cfg = json.load(f) + mem_search = cfg.get("memory", {}).get("search", {}) + if mem_search.get("extraPaths"): + mem_search["extraPaths"] = [] + with open(cfg_path, "w") as f: + json.dump(cfg, f, indent=2) + print("[dedup_subagents] Cleared top-level memory.search.extraPaths from openclaw.json (scoped to main only)") + except Exception as e: + print(f"[dedup_subagents] Warning checking openclaw.json: {e}", file=sys.stderr) + + vec_path = "/app/node_modules/sqlite-vec-linux-x64/vec0.so" + for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")): + agent = os.path.basename(os.path.dirname(os.path.dirname(db))) + if agent == "main": + continue + try: + conn = sqlite3.connect(db, timeout=10.0) + if os.path.exists(vec_path): + conn.enable_load_extension(True) + conn.load_extension(vec_path) + cur = conn.cursor() + chk = cur.execute("SELECT count(*) FROM memory_index_chunks").fetchone()[0] + if chk > 0: + print(f"[dedup_subagents] Purging {chk} duplicated vector chunks from agent {agent}...") + for t in [ + "memory_index_sources", + "memory_index_chunks", + "memory_index_chunk_recall_metadata", + "memory_index_chunk_provenance", + "memory_entry_origins", + "memory_embedding_cache", + "memory_index_chunks_vec" + ]: + try: cur.execute(f"DELETE FROM {t}") + except Exception: pass + conn.commit() + cur.execute("VACUUM") + print(f"[dedup_subagents] Reclaimed disk space for agent {agent}") + conn.close() + except Exception as e: + print(f"[dedup_subagents] Notice for {agent}: {e}", file=sys.stderr) + gateway-preflight.mjs: | + import fs from "node:fs"; + import path from "node:path"; + import { execSync, spawn } from "node:child_process"; + + if (!process.env.__OPENCLAW_PREFLIGHT_DONE) { + process.env.__OPENCLAW_PREFLIGHT_DONE = "1"; + + const isGateway = process.argv.some(arg => arg.endsWith("openclaw.mjs") || arg.endsWith("openclaw")) && process.argv.includes("gateway"); + + if (isGateway) { + const t0 = Date.now(); + console.log("[gateway-preflight] Initializing container self-healing preflight..."); + + // 1. Clean up stale gateway lock files from ungraceful kills + for (const dir of ["/tmp/openclaw-1000", "/tmp"]) { + if (fs.existsSync(dir)) { + try { + const files = fs.readdirSync(dir); + for (const file of files) { + if (file.startsWith("gateway") && file.endsWith(".lock")) { + const fullPath = path.join(dir, file); + fs.unlinkSync(fullPath); + console.log(`[gateway-preflight] Removed stale lock: ${fullPath}`); + } + } + } catch (e) { + console.warn(`[gateway-preflight] Lock cleanup warning in ${dir}:`, e.message); + } + } + } + + // 2. Clear false database quarantines + const qPath = "/home/node/.openclaw/state/openclaw-quarantine.sqlite"; + if (fs.existsSync(qPath)) { + try { + execSync(`/home/node/.openclaw/.local/bin/python3 -c "import sqlite3; c=sqlite3.connect(\x27${qPath}\x27); c.execute(\x27DELETE FROM quarantined_databases\x27); c.commit(); c.close()"`, { stdio: "ignore" }); + console.log("[gateway-preflight] Cleared database quarantine."); + } catch (_) {} + } + + // 3. Deduplicate subagent vector databases (keeps non-main agent DBs < 15MB) + try { + execSync("/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/dedup_subagent_vectors.py", { stdio: "inherit" }); + } catch (e) { + console.warn("[gateway-preflight] Vector deduplication warning:", e.message); + } + + // 4. Pre-warm page cache sequentially (transforms random 70 IOPS disk reads into sequential streaming) + try { + const pw0 = Date.now(); + execSync("cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 || true; find /home/node/.openclaw/wiki/main /home/node/.openclaw/workspace* -maxdepth 3 -name '*.md' -exec cat {} + > /dev/null 2>&1 || true"); + console.log(`[gateway-preflight] Pre-warmed SQLite databases and Wiki markdown in ${Date.now() - pw0}ms`); + } catch (_) {} + + // 5. Recover any wedged running sessions left over from ungraceful container restart + try { + execSync("/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/recover_sessions.py", { stdio: "inherit" }); + } catch (e) { + console.warn("[gateway-preflight] Session recovery warning:", e.message); + } + + // 6. Spawn background turn replay daemon if interrupted turns were detected + try { + if (fs.existsSync("/tmp/openclaw-interrupted-turns.json")) { + console.log("[gateway-preflight] Spawning background turn replay daemon..."); + const child = spawn("/home/node/.openclaw/.local/bin/python3", ["/home/node/.openclaw/scripts/replay_interrupted_turns.py"], { + detached: true, + stdio: "ignore" + }); + child.unref(); + } + } catch (e) { + console.warn("[gateway-preflight] Could not spawn turn replay daemon:", e.message); + } + + console.log(`[gateway-preflight] Self-healing completed in ${Date.now() - t0}ms.`); + } + } + openclaw-triage.py: "#!/usr/bin/env python3\n\"\"\"\nopenclaw-triage.py: Rapid zero-dependency + incident triage for OpenClaw multi-agent sessions.\nDirectly inspects ground-truth + SQLite session state and transcript events in < 1 second.\n\"\"\"\nimport argparse\nimport + glob\nimport json\nimport os\nimport sqlite3\nimport sys\nimport time\n\nTOPIC_TO_AGENT + = {\n \"20\": \"fitness\",\n \"21\": \"koch\",\n \"22\": \"hebamme\",\n + \ \"51\": \"travel\",\n \"115\": \"shopping\",\n \"116\": \"regensburg\",\n + \ \"117\": \"recherche\",\n \"118\": \"doktor\",\n \"119\": \"itguy\",\n + \ \"164\": \"grabengasse_media\",\n \"2796\": \"mila_tracker\",\n \"2797\": + \"nutrition_tracker\",\n}\n\ndef parse_args():\n parser = argparse.ArgumentParser(description=\"Rapid + OpenClaw multi-agent session triage.\")\n parser.add_argument(\"--topic\", + \"-t\", help=\"Telegram forum topic ID (e.g. 21, 22, 117)\")\n parser.add_argument(\"--agent\", + \"-a\", help=\"Agent identifier (e.g. koch, hebamme, recherche)\")\n parser.add_argument(\"--limit\", + \"-n\", type=int, default=10, help=\"Number of recent events to display (default: + 10)\")\n parser.add_argument(\"--db-root\", default=\"/home/node/.openclaw/agents\", + help=\"Root path of agent SQLite databases\")\n return parser.parse_args()\n\ndef + triage_agent_session(agent, topic_id=None, limit=10, db_root=\"/home/node/.openclaw/agents\"):\n + \ db_path = os.path.join(db_root, agent, \"agent\", \"openclaw-agent.sqlite\")\n + \ if not os.path.exists(db_path):\n print(f\"[-] Database not found for + agent '{agent}': {db_path}\")\n return False\n\n try:\n con = + sqlite3.connect(db_path, timeout=5.0)\n except Exception as e:\n print(f\"[-] + Failed to connect to SQLite DB for agent '{agent}': {e}\")\n return False\n\n + \ cur = con.cursor()\n # Find matching session\n if topic_id:\n target_pattern + = f\"%topic:{topic_id}%\"\n cur.execute(\n \"SELECT session_key, + current_session_id, status, updated_at, entry_json FROM session_nodes WHERE session_key + LIKE ? AND session_key NOT LIKE '%:heartbeat' ORDER BY updated_at DESC LIMIT 1\",\n + \ (target_pattern,)\n )\n else:\n cur.execute(\n \"SELECT + session_key, current_session_id, status, updated_at, entry_json FROM session_nodes + WHERE session_key NOT LIKE '%:heartbeat' ORDER BY updated_at DESC LIMIT 1\"\n + \ )\n\n row = cur.fetchone()\n if not row:\n print(f\"[-] No + session found in {agent} DB (filter: topic={topic_id})\")\n con.close()\n + \ return False\n\n skey, sid, status, updated_at_ms, entry_json = row\n + \ now_ms = time.time() * 1000\n age_min = (now_ms - updated_at_ms) / 60000.0 + if updated_at_ms else 0.0\n\n entry = {}\n if entry_json:\n try:\n + \ entry = json.loads(entry_json)\n except Exception:\n pass\n\n + \ tokens = entry.get(\"totalTokens\", \"?\")\n model = entry.get(\"model\", + \"default\")\n last_run_id = entry.get(\"activeWriterRunId\") or entry.get(\"activeRunId\", + None)\n\n # Detect anomalies\n badges = []\n if isinstance(tokens, int):\n + \ if tokens > 60000:\n badges.append(\"\U0001F6A8 [CRITICAL_CONTEXT_BLOAT: + >60k tokens]\")\n elif tokens > 20000:\n badges.append(\"⚠️ + [HIGH_CONTEXT_TOKENS: >20k tokens]\")\n if status == \"running\" and age_min + > 5.0:\n badges.append(\"\U0001F512 [POSSIBLY_WEDGED_RUNNING: updated >5m + ago]\")\n elif status == \"processing\":\n badges.append(\"⚡ [ACTIVE_PROCESSING]\")\n\n + \ # Check ingress queue status\n state_db = \"/home/node/.openclaw/state/openclaw.sqlite\"\n + \ if topic_id and os.path.exists(state_db):\n try:\n s_con + = sqlite3.connect(state_db, timeout=2.0)\n lane_pat = f\"%topic:{topic_id}%\"\n + \ pending_cnt = s_con.execute(\n \"SELECT count(*) FROM + channel_ingress_events WHERE lane_key LIKE ? AND status IN ('pending', 'claimed')\",\n + \ (lane_pat,)\n ).fetchone()[0]\n s_con.close()\n + \ if pending_cnt > 0:\n badges.append(f\"\U0001F4EC [INGRESS_STALLED: + {pending_cnt} message(s) pending]\")\n except Exception:\n pass\n\n + \ print(\"=\" * 72)\n print(f\"\U0001F3AF AGENT: {agent.upper():<12} | TOPIC: + {topic_id or 'N/A':<6} | STATUS: {status.upper()}\")\n print(f\" SESSION + KEY: {skey}\")\n print(f\" SESSION ID: {sid}\")\n print(f\" MODEL: + \ {model}\")\n print(f\" TOKENS: {tokens} | LAST UPDATED: {age_min:.1f}m + ago\")\n if badges:\n print(f\" ANOMALIES: {' '.join(badges)}\")\n + \ print(\"=\" * 72)\n\n # Query transcript events\n cur.execute(\n \"SELECT + seq, created_at, event_json FROM transcript_events WHERE session_id = ? ORDER + BY seq DESC LIMIT ?\",\n (sid, limit)\n )\n events = cur.fetchall()\n + \ con.close()\n\n if not events:\n print(\" (No transcript events + found)\")\n return True\n\n events.reverse()\n print(\"\\n\U0001F4DC + RECENT TRANSCRIPT EVENTS (Chronological):\")\n\n last_role = None\n for + seq, ca_ms, ej_str in events:\n try:\n ev = json.loads(ej_str)\n + \ except Exception:\n print(f\"[{seq}] \")\n + \ continue\n\n etype = ev.get(\"type\")\n if etype != + \"message\":\n continue\n\n msg = ev.get(\"message\", {})\n + \ role = msg.get(\"role\")\n last_role = role\n content = + msg.get(\"content\", [])\n\n if role == \"user\":\n txt = content + if isinstance(content, str) else json.dumps(content)\n # Check for + audio transcript\n if isinstance(content, str) and \"[Audio]\" in content:\n + \ print(f\"\\n\U0001F449 [SEQ {seq}] \U0001F3A4 USER AUDIO PROMPT:\")\n + \ for line in content.splitlines()[:6]:\n print(f\" + \ {line}\")\n if len(content.splitlines()) > 6:\n print(\" + \ ...\")\n else:\n txt_preview = txt.replace(\"\\n\", + \" \")[:240]\n print(f\"\\n\U0001F449 [SEQ {seq}] \U0001F464 USER: + {txt_preview}\")\n\n elif role == \"assistant\":\n print(f\"\\n\U0001F916 + [SEQ {seq}] ASSISTANT TURN:\")\n if isinstance(content, list):\n for + item in content:\n if not isinstance(item, dict):\n continue\n + \ itype = item.get(\"type\")\n if itype == + \"thinking\":\n th = item.get(\"thinking\", \"\").replace(\"\\n\", + \" \")\n print(f\" \U0001F9E0 THINKING: {th[:280]}...\")\n + \ elif itype == \"toolCall\":\n tname + = item.get(\"name\", \"unknown\")\n targs = json.dumps(item.get(\"arguments\", + {}))\n print(f\" ⚙️ TOOL CALL: {tname}({targs[:160]})\")\n + \ elif itype == \"text\":\n tx = item.get(\"text\", + \"\")\n lines = tx.splitlines()\n prev + = lines[0][:150] if lines else \"\"\n print(f\" \U0001F4AC + TEXT OUTPUT: {prev}...\")\n elif isinstance(content, str):\n print(f\" + \ \U0001F4AC TEXT OUTPUT: {content[:200]}...\")\n\n elif role == \"toolResult\":\n + \ is_err = ev.get(\"isError\", False)\n tname = ev.get(\"details\", + {}).get(\"tool\") or msg.get(\"toolName\", \"tool\")\n res_txt = json.dumps(content) + if not isinstance(content, str) else content\n if is_err or \"error\" + in res_txt.lower() or \"must have required property\" in res_txt:\n print(f\" + \ ❌ [SEQ {seq}] TOOL ERROR ({tname}): {res_txt[:260]}\")\n else:\n + \ print(f\" ✅ [SEQ {seq}] TOOL RESULT ({tname}): {res_txt[:180]}\")\n\n + \ if last_role == \"user\":\n print(\"\\n⚠️ WARNING: The last event in + the transcript is an UNANSWERED USER MESSAGE!\")\n print(\" The agent + turn either crashed, timed out, or was killed before producing a response.\")\n\n + \ print(\"\\n\" + \"=\" * 72)\n return True\n\ndef main():\n args = parse_args()\n + \ if not args.topic and not args.agent:\n print(\"Usage: openclaw-triage.py + (--topic | --agent ) [--limit ]\")\n sys.exit(1)\n\n agent + = args.agent\n topic = args.topic\n\n if topic and not agent:\n agent + = TOPIC_TO_AGENT.get(str(topic))\n if not agent:\n # Search + all DBs\n for a, t in TOPIC_TO_AGENT.items():\n if t + == topic:\n agent = a\n break\n\n if + not agent:\n # Scan all DBs for the topic\n for db in sorted(glob.glob(os.path.join(args.db_root, + \"*/agent/openclaw-agent.sqlite\"))):\n cand_agent = db.split(os.sep)[-3]\n + \ try:\n con = sqlite3.connect(db, timeout=2.0)\n cur + = con.cursor()\n cur.execute(\n \"SELECT session_key + FROM session_nodes WHERE session_key LIKE ? LIMIT 1\",\n (f\"%topic:{topic}%\",)\n + \ )\n if cur.fetchone():\n agent + = cand_agent\n con.close()\n break\n con.close()\n + \ except Exception:\n pass\n\n if not agent:\n print(f\"[-] + Could not map topic '{topic}' to any known OpenClaw agent.\")\n sys.exit(1)\n\n + \ success = triage_agent_session(agent, topic_id=topic, limit=args.limit, db_root=args.db_root)\n + \ sys.exit(0 if success else 1)\n\nif __name__ == \"__main__\":\n main()\n" + recover_sessions.py: |+ + #!/usr/bin/env python3 + import glob, sqlite3, json, os, sys, time + + t0 = time.time() + recovered = 0 + interrupted_turns = [] + + def extract_prompt(msg): + if not msg: return "" + content = msg.get("content") + if isinstance(content, str): return content + if isinstance(content, list): + parts = [] + for c in content: + if isinstance(c, dict): parts.append(c.get("text") or c.get("content") or "") + elif isinstance(c, str): parts.append(c) + return "\n".join(filter(None, parts)) + return str(content or "") + + for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")): + try: + conn = sqlite3.connect(db, timeout=10.0) + cursor = conn.cursor() + while True: + cursor.execute("PRAGMA foreign_key_check;") + violations = cursor.fetchall() + if not violations: break + for table, rowid, parent, fkid in violations: + cursor.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,)) + cursor.execute("SELECT session_key, entry_json, current_session_id, status FROM session_nodes WHERE status IN (\"running\", \"failed\", \"error\")") + rows = cursor.fetchall() + agent_id = os.path.basename(os.path.dirname(os.path.dirname(db))) + for sk, ej, csid, st in rows: + try: + p = json.loads(ej) if ej else {} + sid = csid or p.get("sessionId") + if sid: + ev = cursor.execute("SELECT seq, event_json FROM transcript_events WHERE session_id = ? ORDER BY seq DESC LIMIT 1", (sid,)).fetchone() + if ev: + ev_seq, ev_raw = ev + try: + ev_data = json.loads(ev_raw) + msg = ev_data.get("message") or ev_data + if st == "running" and msg.get("role") == "user": + prompt_text = extract_prompt(msg) + ctx = p.get("restartRecoveryDeliveryContext") or p.get("delivery", {}).get("context") or p.get("delivery", {}).get("route", {}).get("context") or {} + interrupted_turns.append({ + "agent": agent_id, + "session_key": sk, + "session_id": sid, + "context": ctx, + "prompt": prompt_text + }) + cursor.execute("PRAGMA foreign_keys = ON") + cursor.execute("DELETE FROM transcript_events WHERE session_id = ? AND seq = ?", (sid, ev_seq)) + elif msg.get("role") == "assistant" and msg.get("stopReason") == "error": + cursor.execute("PRAGMA foreign_keys = ON") + cursor.execute("DELETE FROM transcript_events WHERE session_id = ? AND seq = ?", (sid, ev_seq)) + except Exception as ex: + print(f"[recover_sessions] Error parsing event: {ex}", file=sys.stderr) + p["status"] = "done" + p["activeWriterRunId"] = None + p["activeClaimRunId"] = None + p["restartRecoveryDeliveryRunId"] = None + cursor.execute("UPDATE session_nodes SET status = \"done\", entry_json = ? WHERE session_key = ?", (json.dumps(p), sk)) + cursor.execute("UPDATE session_nodes SET entry_valid = 1 WHERE session_key = ?", (sk,)) + recovered += 1 + except Exception: pass + cursor.execute("UPDATE session_nodes SET entry_valid = 1 WHERE entry_valid != 1") + cursor.execute("UPDATE session_windows SET status = \"done\" WHERE status IN (\"running\", \"failed\", \"error\")") + conn.commit() + conn.close() + except Exception as e: + print(f"[recover_sessions] Warning for {db}: {e}", file=sys.stderr) + + state_db = "/home/node/.openclaw/state/openclaw.sqlite" + if os.path.exists(state_db): + try: + sconn = sqlite3.connect(state_db, timeout=10.0) + scur = sconn.cursor() + scur.execute("UPDATE channel_ingress_events SET attempts = 0, last_attempt_at = NULL, last_error = NULL WHERE status = 'pending' AND attempts > 0") + sconn.commit() + sconn.close() + except Exception as e: + print(f"[recover_sessions] Warning resetting ingress backoff: {e}", file=sys.stderr) + + if interrupted_turns: + try: + with open("/tmp/openclaw-interrupted-turns.json", "w") as f: + json.dump(interrupted_turns, f, indent=2) + print(f"[recover_sessions] Detected {len(interrupted_turns)} interrupted user turn(s) saved to /tmp/openclaw-interrupted-turns.json") + except Exception as e: + print(f"[recover_sessions] Error writing interrupted turns: {e}", file=sys.stderr) + + print(f"[recover_sessions] Completed in {time.time() - t0:.2f}s ({recovered} wedged sessions recovered)") + + replay_interrupted_turns.py: | + #!/usr/bin/env python3 + import json, os, subprocess, sys, time, urllib.request + + INTERRUPTED_FILE = "/tmp/openclaw-interrupted-turns.json" + + if not os.path.exists(INTERRUPTED_FILE): + sys.exit(0) + + try: + with open(INTERRUPTED_FILE, "r") as f: + turns = json.load(f) + except Exception as e: + print(f"[replay_interrupted_turns] Failed to read {INTERRUPTED_FILE}: {e}", file=sys.stderr) + sys.exit(0) + + if not turns: + sys.exit(0) + + print(f"[replay_interrupted_turns] Found {len(turns)} interrupted turn(s). Waiting for gateway /healthz...") + + gateway_ready = False + for _ in range(60): + try: + with urllib.request.urlopen("http://127.0.0.1:18790/healthz", timeout=3) as resp: + if resp.status == 200: + gateway_ready = True + break + except Exception: + pass + time.sleep(2) + + if not gateway_ready: + print("[replay_interrupted_turns] Gateway did not become healthy in 120s. Aborting.", file=sys.stderr) + sys.exit(1) + + # Allow 5s for Telegram ingress to bind + time.sleep(5) + + for turn in turns: + sk = turn.get("session_key") + sid = turn.get("session_id") + agent_id = turn.get("agent") + ctx = turn.get("context") or {} + prompt = turn.get("prompt") or "" + + channel = ctx.get("channel") or "telegram" + to_target = ctx.get("to") or "" + thread_id = ctx.get("threadId") + + target = to_target + if target.startswith("telegram:"): + parts = target.split(":") + target = parts[1] if len(parts) > 1 else target + + print(f"[replay_interrupted_turns] Processing agent '{agent_id}' on session '{sk}' target='{target}' thread='{thread_id}'") + + # 1. Send status notification to Telegram if target exists + if target: + notice = "⚠️ OpenClaw wurde während der Bearbeitung neu gestartet – deine Anfrage wird jetzt nachgeholt..." + notify_cmd = [ + "openclaw", "message", "send", + "--channel", channel, + "--target", str(target), + "-m", notice + ] + if thread_id is not None: + notify_cmd.extend(["--thread-id", str(thread_id)]) + try: + subprocess.run(notify_cmd, check=False, timeout=30) + print(f"[replay_interrupted_turns] Sent recovery notice to {target}") + except Exception as e: + print(f"[replay_interrupted_turns] Notice failed: {e}", file=sys.stderr) + + # 2. Replay agent turn + if sk and prompt: + p_short = (prompt[:60] + "...") if len(prompt) > 60 else prompt + print(f"[replay_interrupted_turns] Replaying agent turn: {p_short}") + agent_cmd = [ + "openclaw", "agent", + "--session-key", sk, + "-m", prompt, + "--deliver" + ] + try: + res = subprocess.run(agent_cmd, capture_output=True, text=True, timeout=300) + if res.returncode == 0: + print(f"[replay_interrupted_turns] Turn replay succeeded for {sk}") + else: + print(f"[replay_interrupted_turns] Turn replay exited with code {res.returncode}: {res.stderr[:200]}", file=sys.stderr) + except Exception as e: + print(f"[replay_interrupted_turns] Replay execution error: {e}", file=sys.stderr) + + try: + os.remove(INTERRUPTED_FILE) + print(f"[replay_interrupted_turns] Cleaned up {INTERRUPTED_FILE}") + except Exception: + pass kind: ConfigMap metadata: name: openclaw-scripts namespace: openclaw - labels: - app.kubernetes.io/name: openclaw - app.kubernetes.io/part-of: openclaw -data: - bootstrap-deps.sh: "#!/bin/sh\nset -e\n\necho \"=== OpenClaw Dependency & Runtime\ - \ Bootstrap ===\"\nexport PATH=\"/home/node/.openclaw/bin:/home/node/.openclaw/.local/bin:$PATH\"\ - \n\nSCRIPTS_DIR=\"/home/node/.openclaw/scripts\"\nmkdir -p /home/node/.openclaw/bin\ - \ \"$SCRIPTS_DIR\" /home/node/.openclaw/.config\ntouch /home/node/.openclaw/.config/keep\n\ - \n# 1. Sanitize legacy config keys in openclaw.json\nnode -e '\n const fs = require(\"\ - fs\");\n for (const file of [\"/home/node/.openclaw/openclaw.json\", \"/home/openclaw/.openclaw/openclaw.json\"\ - ]) {\n if (fs.existsSync(file)) {\n try {\n const cfg = JSON.parse(fs.readFileSync(file,\ - \ \"utf8\"));\n if (cfg.browser) {\n delete cfg.browser.remoteCdpTimeoutMs;\n\ - \ if (cfg.browser.profiles) {\n if (cfg.browser.profiles.default)\ - \ delete cfg.browser.profiles.default.color;\n if (cfg.browser.profiles.chrome)\ - \ delete cfg.browser.profiles.chrome;\n }\n }\n if (cfg.gateway\ - \ && cfg.gateway.controlUi) {\n delete cfg.gateway.controlUi.dangerouslyDisableDeviceAuth;\n\ - \ }\n if (!cfg.agents) cfg.agents = {};\n cfg.agents.ownership\ - \ = \"explicit\";\n if (cfg.agents.entries && cfg.agents.entries.main)\ - \ {\n delete cfg.agents.entries.main.default;\n }\n fs.writeFileSync(file,\ - \ JSON.stringify(cfg, null, 2));\n console.log(\"Sanitized config file:\"\ - , file);\n } catch (e) {\n console.error(\"Failed to sanitize\", file,\ - \ e.message);\n }\n }\n }\n' || true\n\n# 2. Run pre-flight session recovery\n\ - echo \"Running pre-flight self-healing...\"\npython3 \"$SCRIPTS_DIR/recover_sessions.py\"\ - \ || true\n\n# 3. Doctor fix\necho \"Running openclaw doctor --fix to migrate\ - \ runtime config on PVC...\"\nopenclaw doctor --fix || true\n\n# 4. Garmin OAuth\ - \ tokens bootstrap / preserve\nif [ ! -s /home/node/.openclaw/garmin_tokens/garmin_tokens.json\ - \ ] && [ -n \"$GARMIN_TOKENS_BASE64\" ]; then\n echo \"Bootstrapping initial\ - \ Garmin OAuth tokens from secret...\"\n mkdir -p /home/node/.openclaw/garmin_tokens\n\ - \ echo \"$GARMIN_TOKENS_BASE64\" | base64 -d > /home/node/.openclaw/garmin_tokens/garmin_tokens.json\n\ - \ chmod 700 /home/node/.openclaw/garmin_tokens\n chmod 600 /home/node/.openclaw/garmin_tokens/garmin_tokens.json\n\ - elif [ -s /home/node/.openclaw/garmin_tokens/garmin_tokens.json ]; then\n echo\ - \ \"Preserving live Garmin OAuth tokens on PVC...\"\n chmod 700 /home/node/.openclaw/garmin_tokens\n\ - \ chmod 600 /home/node/.openclaw/garmin_tokens/garmin_tokens.json\nfi\n\n# 5.\ - \ Cleanup legacy binaries and superseded local skills\nrm -rf /home/node/.openclaw/bin/whisper*\ - \ /home/node/.openclaw/bin/.whisper* /home/node/.openclaw/scripts/whisper* /home/node/.openclaw/models/whisper\ - \ || true\nrm -rf /home/node/.openclaw/skills/memobird /home/node/.openclaw/skills/nerd-report\ - \ /home/node/.openclaw/skills/rvv-regensburg /home/node/.openclaw/skills/skill-creator\ - \ || true\nrm -rf /home/node/.openclaw/workspace/skills /home/node/.openclaw/workspace-koch/skills\ - \ /home/node/.openclaw/workspace-recherche/skills || true\n\n# 6. Ensure tools.allow\ - \ and models.providers.openrouter.apiKey unset\nopenclaw config unset tools.allow\ - \ || true\nopenclaw config unset models.providers.openrouter.apiKey || true\n\ - openclaw config unset agents.entries.mila_tracker.tools.profile || true\nopenclaw\ - \ config unset agents.entries.nutrition_tracker.tools.profile || true\n\n# 7.\ - \ Decouple hardlinked bootstrap files\nfor dir in /home/node/.openclaw/workspace-*;\ - \ do\n if [ -d \"$dir\" ]; then\n for file in USER.md MEMORY.md; do\n \ - \ if [ -f \"/home/node/.openclaw/workspace/$file\" ]; then\n cp --remove-destination\ - \ \"/home/node/.openclaw/workspace/$file\" \"$dir/$file\"\n chown node:node\ - \ \"$dir/$file\" 2>/dev/null || true\n chmod 664 \"$dir/$file\" 2>/dev/null\ - \ || true\n fi\n done\n fi\ndone\n\n# 8. Install CLI binaries\nif [ !\ - \ -f \"/home/node/.openclaw/bin/openhue.real\" ]; then\n echo \"Installing openhue\ - \ CLI binary...\"\n curl -sL https://github.com/openhue/openhue-cli/releases/download/0.24/openhue_Linux_x86_64.tar.gz\ - \ | tar -xz -C /home/node/.openclaw/bin openhue\n mv /home/node/.openclaw/bin/openhue\ - \ /home/node/.openclaw/bin/openhue.real\n printf '#!/bin/sh\\nexport HOME=/home/node/.openclaw\\\ - nexec /home/node/.openclaw/bin/openhue.real \"$@\"\\n' > /home/node/.openclaw/bin/openhue\n\ - \ chmod +x /home/node/.openclaw/bin/openhue /home/node/.openclaw/bin/openhue.real\n\ - fi\n\nif [ ! -f \"/home/node/.openclaw/bin/summarize\" ]; then\n echo \"Installing\ - \ summarize CLI via npm...\"\n npm install -g --prefix /home/node/.openclaw @steipete/summarize\ - \ || true\nfi\n\nif [ ! -f \"/home/node/.openclaw/bin/mcp-server-github\" ]; then\n\ - \ echo \"Installing @modelcontextprotocol/server-github via npm...\"\n npm install\ - \ -g --prefix /home/node/.openclaw @modelcontextprotocol/server-github || true\n\ - fi\n\nif [ ! -f \"/home/node/.openclaw/bin/kubectl\" ]; then\n echo \"Installing\ - \ kubectl CLI binary...\"\n curl -fsSL https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl\ - \ -o /home/node/.openclaw/bin/kubectl\n chmod +x /home/node/.openclaw/bin/kubectl\n\ - fi\n\nif [ ! -f \"/home/node/.openclaw/bin/jq\" ]; then\n echo \"Installing jq\ - \ CLI binary...\"\n curl -fsSL https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64\ - \ -o /home/node/.openclaw/bin/jq\n chmod +x /home/node/.openclaw/bin/jq\nfi\n\ - \nif [ ! -f \"/home/node/.openclaw/bin/file\" ]; then\n echo \"Setting up file\ - \ CLI helper...\"\n cp \"$SCRIPTS_DIR/file_cli.py\" /home/node/.openclaw/bin/file\n\ - \ chmod +x /home/node/.openclaw/bin/file\nfi\n\nif [ -x \"/home/node/.openclaw/.local/bin/python3\"\ - \ ]; then\n /home/node/.openclaw/bin/uv pip install --break-system-packages --python\ - \ /home/node/.openclaw/.local/bin/python3 pillow google-api-python-client pypdf\ - \ || true\nfi\n\n# 9. Clone or pull infrapuzzle repository to PVC\nmkdir -p /home/node/.openclaw/repos\n\ - if [ ! -d \"/home/node/.openclaw/repos/infrapuzzle/.git\" ]; then\n echo \"Cloning\ - \ infrapuzzle repository to PVC...\"\n git clone --depth 1 \"https://${GIT_SYNC_USERNAME}:${GIT_SYNC_PASSWORD}@git.moritzgraf.de/moritz/infrapuzzle.git\"\ - \ /home/node/.openclaw/repos/infrapuzzle || true\nelse\n echo \"Pulling latest\ - \ infrapuzzle commits on PVC...\"\n git -C /home/node/.openclaw/repos/infrapuzzle\ - \ pull origin master || git -C /home/node/.openclaw/repos/infrapuzzle pull origin\ - \ main || true\nfi\n\n# 10. Seed and maintain persistent mcp-venv\nif [ ! -d \"\ - /home/node/.openclaw/mcp-venv\" ]; then\n echo \"Seeding persistent mcp-venv...\"\ - \n uv venv --python 3.12 /home/node/.openclaw/mcp-venv\n uv pip install --python\ - \ /home/node/.openclaw/mcp-venv \\\n prometheus-mcp-server \\\n ha-mcp \\\ - \n chuk-mcp-open-meteo \\\n gitea-mcp \\\n git+https://github.com/Taxuspt/garmin_mcp\ - \ \\\n \"mcp>=2.0\" \\\n filetype \\\n google-api-python-client \\\n\ - \ google-auth \\\n bring-api \\\n requests \\\n beautifulsoup4 \\\n\ - \ youtube-transcript-api\nfi\n\nif [ -d \"/home/node/.openclaw/mcp-venv\" ];\ - \ then\n python3 -c 'import os; p=\"/home/node/.openclaw/mcp-venv/lib/python3.12/site-packages/mcp/server/fastmcp.py\"\ - ; open(p, \"w\").write(\"import os\\nfrom fastmcp import FastMCP as _FastMCP,\ - \ Context\\n\\nclass FastMCP(_FastMCP):\\n def __init__(self, name=\\\"MCP\ - \ Server\\\", host=None, port=None, **kwargs):\\n if host is not None:\ - \ os.environ[\\\"FASTMCP_HOST\\\"] = str(host)\\n if port is not None:\ - \ os.environ[\\\"FASTMCP_PORT\\\"] = str(port)\\n super().__init__(name=name,\ - \ **kwargs)\\n\")' || true\n python3 \"$SCRIPTS_DIR/patch_garmin_mcp.py\" ||\ - \ true\nfi\n\n# 11. Extensions & Plugins\nif [ ! -d \"/home/node/.openclaw/extensions/perplexity\"\ - \ ]; then\n echo \"Installing perplexity plugin...\"\n openclaw plugins install\ - \ clawhub:@openclaw/perplexity-plugin --accept-capabilities || true\nfi\necho\ - \ \"Enabling memory-wiki plugin...\"\nopenclaw plugins enable memory-wiki || true\n\ - \necho \"Patching perplexity web search runtime for chat completions compatibility...\"\ - \nnode \"$SCRIPTS_DIR/patch_perplexity.mjs\" || true\n\n# 12. Ensure OpenRouter\ - \ auth profile in state DB\necho \"Ensuring OpenRouter auth profile in state DB...\"\ - \nnode --input-type=module -e '\n import { DatabaseSync } from \"node:sqlite\"\ - ;\n const key = process.env.OPENROUTER_API_KEY;\n if (key) {\n try {\n \ - \ const dbPath = \"/home/node/.openclaw/state/openclaw.sqlite\";\n const\ - \ db = new DatabaseSync(dbPath);\n db.prepare(\"CREATE TABLE IF NOT EXISTS\ - \ config_machine_state (state_key TEXT PRIMARY KEY, value_json TEXT NOT NULL,\ - \ updated_at_ms INTEGER NOT NULL)\").run();\n const row = db.prepare(\"SELECT\ - \ value_json FROM config_machine_state WHERE state_key = \\x27authProfiles.store\\\ - x27\").get();\n let store = row?.value_json ? JSON.parse(row.value_json)\ - \ : { version: 1, profiles: {} };\n if (!store.profiles) store.profiles =\ - \ {};\n store.profiles[\"openrouter:default\"] = {\n type: \"api_key\"\ - ,\n provider: \"openrouter\",\n key: key\n };\n db.prepare(\"\ - INSERT INTO config_machine_state (state_key, value_json, updated_at_ms) VALUES\ - \ (\\x27authProfiles.store\\x27, ?, ?) ON CONFLICT(state_key) DO UPDATE SET value_json\ - \ = excluded.value_json, updated_at_ms = excluded.updated_at_ms\")\n .run(JSON.stringify(store),\ - \ Date.now());\n console.log(\"Synchronized openrouter auth profile in state\ - \ DB.\");\n } catch (e) {\n console.error(\"Could not sync auth profile:\"\ - , e.message);\n }\n }\n' || true\n\n# 13. Pre-warm SQLite page cache\necho\ - \ \"Pre-warming SQLite database files into OS page cache for instant cold start...\"\ - \ncat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite\ - \ > /dev/null 2>&1 || true\n\necho \"=== Bootstrap Complete ===\"\n" - compact_sessions.py: "#!/usr/bin/env python3\n\"\"\"\nNightly multi-agent session\ - \ maintenance and compaction script for OpenClaw.\n1. Pre-warms OS page cache\ - \ for SQLite databases on spinning disks.\n2. Recovers any wedged or locked sessions\ - \ via recover_sessions.py.\n3. Prunes abandoned isolated cron sessions (>48h old)\ - \ to prevent token/state bloat.\n4. Tier A Hard-Reset: Resets koch, hebamme, and\ - \ shopping topics to 0 tokens (clean archive).\n5. Tier B Smart-Compaction: Compacts\ - \ active sessions exceeding token thresholds (5s pacing).\n6. Checkpoints SQLite\ - \ WAL journals across all agent databases to disk.\n7. Optimizes SQLite query\ - \ planner without blocking full VACUUM.\n\"\"\"\nimport glob\nimport json\nimport\ - \ os\nimport sqlite3\nimport subprocess\nimport sys\nimport time\n\nTHRESHOLD_TOKENS\ - \ = 15000\nTHRESHOLD_TRANSCRIPT_BYTES = 200000\nTHRESHOLD_EVENT_COUNT = 30\nCRON_MAX_AGE_HOURS\ - \ = 48\nRPC_TIMEOUT_MS = 180000\n\n# Tracking Channels: Deterministic sliding-window\ - \ (mila_tracker, nutrition_tracker).\n# Strict sliding window of ~5-8 events (2-3\ - \ turns max), 0 LLM tokens, instant truncation.\nTRACKING_TARGETS = [\n (\"\ - mila_tracker\", \"agent:mila_tracker:telegram:group:-1004326921510:topic:2796\"\ - ),\n (\"nutrition_tracker\", \"agent:nutrition_tracker:telegram:group:-1004326921510:topic:2797\"\ - ),\n]\nMAX_TRACKING_EVENTS = 8\nMAX_TRACKING_LINES = 8\n\n# Tier A: High-frequency\ - \ interaction topics (koch, hebamme, shopping).\n# Deep-compaction keeps a compact\ - \ summary + latest 2 turns without completely wiping conversational memory.\n\ - TIER_A_TARGETS = [\n (\"koch\", \"agent:koch:telegram:group:-1004326921510:topic:21\"\ - ),\n (\"hebamme\", \"agent:hebamme:telegram:group:-1004326921510:topic:22\"\ - ),\n (\"shopping\", \"agent:shopping:telegram:group:-1004326921510:topic:115\"\ - ),\n]\n\ndef prewarm_page_cache():\n print(\"=== Pre-warming SQLite databases\ - \ into OS page cache ===\")\n t0 = time.time()\n try:\n subprocess.run(\n\ - \ \"cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite\ - \ > /dev/null 2>&1 || true\",\n shell=True,\n timeout=90.0\n\ - \ )\n print(f\"Page cache pre-warmed in {time.time() - t0:.2f}s\"\ - )\n except Exception as e:\n print(f\"Page cache pre-warming warning:\ - \ {e}\")\n\ndef prune_stale_cron_sessions():\n print(f\"=== Pruning isolated\ - \ cron sessions older than {CRON_MAX_AGE_HOURS}h ===\")\n cutoff_ms = int(time.time()\ - \ * 1000) - (CRON_MAX_AGE_HOURS * 3600 * 1000)\n total_pruned = 0\n total_tokens_reclaimed\ - \ = 0\n\n for db in sorted(glob.glob(\"/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite\"\ - )):\n agent = db.split(\"/\")[5]\n try:\n conn = sqlite3.connect(db,\ - \ timeout=30.0)\n conn.execute(\"PRAGMA foreign_keys = ON;\")\n \ - \ cursor = conn.cursor()\n cursor.execute(\"SELECT name FROM\ - \ sqlite_master WHERE type=\\\"table\\\"\")\n all_tables = set(r[0]\ - \ for r in cursor.fetchall())\n\n cursor.execute(\"SELECT session_key,\ - \ current_session_id, updated_at, entry_json FROM session_nodes WHERE session_key\ - \ LIKE \\\"%:cron:%\\\"\")\n stale = []\n for k, sid, up,\ - \ ej in cursor.fetchall():\n if up and up < cutoff_ms:\n \ - \ tok = 0\n if ej:\n try:\n\ - \ tok = json.loads(ej).get(\"totalTokens\", 0) or 0\n\ - \ except Exception:\n pass\n\ - \ stale.append((k, sid, tok))\n\n if stale:\n \ - \ tokens_reclaimed = sum(t[2] for t in stale)\n for\ - \ k, sid, _ in stale:\n for t in [\"trajectory_runtime_events\"\ - , \"transcript_events\", \"transcript_event_identities\", \"session_transcript_active_events\"\ - , \"transcript_rewrite_watermarks\", \"session_transcript_index_state\", \"session_transcript_fts\"\ - , \"session_conversations\"]:\n if t in all_tables and\ - \ sid:\n cursor.execute(f\"DELETE FROM {t} WHERE session_id\ - \ = ?\", (sid,))\n for t in [\"session_participants\", \"heartbeat_outcomes\"\ - , \"session_progress_cards\", \"board_widgets\", \"board_tabs\"]:\n \ - \ if t in all_tables:\n cursor.execute(f\"\ - DELETE FROM {t} WHERE session_key = ?\", (k,))\n if \"memory_entry_origins\"\ - \ in all_tables:\n cursor.execute(\"DELETE FROM memory_entry_origins\ - \ WHERE session_key = ? OR session_id = ?\", (k, sid or \"\"))\n \ - \ if \"session_windows\" in all_tables:\n cursor.execute(\"\ - DELETE FROM session_windows WHERE session_key = ? OR session_id = ?\", (k, sid\ - \ or \"\"))\n cursor.execute(\"DELETE FROM session_nodes WHERE\ - \ session_key = ?\", (k,))\n conn.commit()\n total_pruned\ - \ += len(stale)\n total_tokens_reclaimed += tokens_reclaimed\n\ - \ print(f\" [{agent}] Pruned {len(stale)} dead cron sessions ({tokens_reclaimed}\ - \ tokens reclaimed)\")\n\n # Unconditionally resolve any dangling foreign\ - \ key references across all tables\n while True:\n cursor.execute(\"\ - PRAGMA foreign_key_check;\")\n violations = cursor.fetchall()\n\ - \ if not violations:\n break\n \ - \ for table, rowid, parent, fkid in violations:\n cursor.execute(f\"\ - DELETE FROM {table} WHERE rowid = ?\", (rowid,))\n conn.commit()\n\ - \ conn.close()\n except Exception as e:\n print(f\"\ - \ [{agent}] Pruning error: {e}\")\n\n # Clear resolved quarantine entries\ - \ if all databases are healthy\n try:\n q_path = \"/home/node/.openclaw/state/openclaw-quarantine.sqlite\"\ - \n if os.path.exists(q_path):\n q_conn = sqlite3.connect(q_path)\n\ - \ q_cur = q_conn.cursor()\n q_cur.execute(\"DELETE FROM\ - \ quarantined_databases\")\n q_conn.commit()\n q_conn.close()\n\ - \ except Exception as e:\n print(f\"Quarantine cleanup error: {e}\"\ - )\n\n print(f\"Cron pruning complete: {total_pruned} sessions pruned, {total_tokens_reclaimed}\ - \ tokens reclaimed.\")\n\ndef compact_tracking_sessions():\n print(\"=== Tracking\ - \ Channels Sliding-Window Maintenance (mila_tracker, nutrition_tracker) ===\"\ - )\n for agent, key in TRACKING_TARGETS:\n db_path = f\"/home/node/.openclaw/agents/{agent}/agent/openclaw-agent.sqlite\"\ - \n if not os.path.exists(db_path):\n continue\n try:\n\ - \ con = sqlite3.connect(db_path, timeout=5.0)\n row = con.execute(\"\ - SELECT current_session_id FROM session_nodes WHERE session_key=?\", (key,)).fetchone()\n\ - \ if not row or not row[0]:\n con.close()\n \ - \ print(f\" [{agent}] No active session node found for {key}.\")\n \ - \ continue\n sid = row[0]\n cnt = con.execute(\"\ - SELECT count(*) FROM transcript_events WHERE session_id=?\", (sid,)).fetchone()[0]\n\ - \ con.close()\n\n if cnt > MAX_TRACKING_EVENTS:\n \ - \ print(f\" [{agent}] Session {key} has {cnt} events (> {MAX_TRACKING_EVENTS}).\ - \ Truncating to last {MAX_TRACKING_LINES} lines...\")\n cmd = [\n\ - \ \"openclaw\", \"sessions\", \"compact\", key,\n \ - \ \"--agent\", agent,\n \"--max-lines\", str(MAX_TRACKING_LINES),\n\ - \ \"--timeout\", \"30000\"\n ]\n \ - \ r = subprocess.run(cmd, capture_output=True, text=True)\n \ - \ if r.returncode == 0:\n print(f\" \u2713 Truncated successfully:\ - \ {r.stdout.strip()}\")\n else:\n print(f\"\ - \ \u2717 Truncation failed: {r.stderr.strip()}\")\n else:\n \ - \ print(f\" [{agent}] Session {key} has {cnt} events (<= {MAX_TRACKING_EVENTS}).\ - \ Clean.\")\n except Exception as e:\n print(f\" \u2717 [{agent}]\ - \ Error during tracking compaction: {e}\")\n\ndef compact_tier_a_sessions():\n\ - \ print(\"=== Tier A: Deep-Compaction (koch, hebamme, shopping) ===\")\n \ - \ for agent, key in TIER_A_TARGETS:\n db_path = f\"/home/node/.openclaw/agents/{agent}/agent/openclaw-agent.sqlite\"\ - \n if not os.path.exists(db_path):\n continue\n try:\n\ - \ con = sqlite3.connect(db_path, timeout=10.0)\n row = con.execute(\"\ - SELECT current_session_id FROM session_nodes WHERE session_key=?\", (key,)).fetchone()\n\ - \ con.close()\n if not row or not row[0]:\n \ - \ print(f\" [Tier A] {agent} ({key}) already at 0 (no active session).\")\n\ - \ continue\n ev_cnt, ev_bytes = get_session_metrics(agent,\ - \ key, row[0])\n if ev_cnt <= 2 and ev_bytes < 20000:\n \ - \ print(f\" [Tier A] {agent} ({key}) has only {ev_cnt} events ({ev_bytes}\ - \ bytes), skipping.\")\n continue\n print(f\" [Tier\ - \ A] Deep-compacting {agent} ({key}) [{ev_cnt} events, {ev_bytes//1024}KB]...\"\ - )\n r = subprocess.run(\n [\"openclaw\", \"sessions\"\ - , \"compact\", key, \"--agent\", agent, \"--timeout\", str(RPC_TIMEOUT_MS)],\n\ - \ capture_output=True, text=True\n )\n if\ - \ r.returncode == 0:\n print(f\" \u2713 Deep-compacted: {r.stdout.strip()}\"\ - )\n elif \"Already compacted\" in r.stderr or \"Already compacted\"\ - \ in r.stdout:\n print(f\" \u2713 Already compacted.\")\n \ - \ else:\n print(f\" \u2717 Standard compact error:\ - \ {r.stderr.strip()}; retrying with --max-lines 40...\")\n fb =\ - \ subprocess.run(\n [\"openclaw\", \"sessions\", \"compact\"\ - , key, \"--agent\", agent, \"--max-lines\", \"40\", \"--timeout\", str(RPC_TIMEOUT_MS)],\n\ - \ capture_output=True, text=True\n )\n \ - \ if fb.returncode == 0:\n print(f\" \u2713 Truncation\ - \ fallback success: {fb.stdout.strip()}\")\n else:\n \ - \ print(f\" \u2717 Truncation fallback failed: {fb.stderr.strip()}\"\ - )\n except Exception as e:\n print(f\" \u2717 Exception deep-compacting\ - \ {agent}: {e}\")\n time.sleep(2)\n\ndef get_session_metrics(agent, key,\ - \ sid=None):\n db_path = f\"/home/node/.openclaw/agents/{agent}/agent/openclaw-agent.sqlite\"\ - \n if not os.path.exists(db_path):\n return 0, 0\n try:\n \ - \ con = sqlite3.connect(db_path, timeout=5.0)\n cur = con.cursor()\n \ - \ if not sid:\n row = cur.execute(\"SELECT current_session_id\ - \ FROM session_nodes WHERE session_key=?\", (key,)).fetchone()\n sid\ - \ = row[0] if row else None\n if not sid:\n con.close()\n \ - \ return 0, 0\n ev_cnt, ev_bytes = cur.execute(\n \"\ - SELECT count(*), coalesce(sum(length(event_json)), 0) FROM transcript_events WHERE\ - \ session_id=?\", \n (sid,)\n ).fetchone()\n con.close()\n\ - \ return ev_cnt, ev_bytes\n except Exception:\n return 0, 0\n\ - \ndef compact_active_sessions():\n excluded_keys = set(t[1] for t in TIER_A_TARGETS)\ - \ | set(t[1] for t in TRACKING_TARGETS)\n print(f\"=== Tier B: Running Multi-Agent\ - \ Compaction (threshold: {THRESHOLD_TOKENS} tokens / {THRESHOLD_TRANSCRIPT_BYTES//1024}KB,\ - \ LLM summarization) ===\")\n res = subprocess.run([\"openclaw\", \"sessions\"\ - , \"--all-agents\", \"--json\"], capture_output=True, text=True)\n if res.returncode\ - \ != 0:\n print(f\"Failed to list sessions: {res.stderr}\")\n return\n\ - \n sessions = json.loads(res.stdout).get(\"sessions\", [])\n compacted =\ - \ 0\n skipped = 0\n\n for s in sessions:\n key = s.get(\"key\", \"\ - \")\n agent = s.get(\"agentId\") or \"main\"\n tokens = s.get(\"\ - totalTokens\") or 0\n sid = s.get(\"sessionId\")\n\n # Skip Tier\ - \ A and Tracking agents as they have dedicated routines\n if key in excluded_keys:\n\ - \ continue\n\n # Target main sessions and all telegram sessions\ - \ (topics and direct)\n is_target = (\n key == \"agent:main:main\"\ - \ or \n \":telegram:\" in key or\n (key.startswith(\"agent:\"\ - ) and key.endswith(\":main\"))\n )\n if is_target and not key.endswith(\"\ - :heartbeat\"):\n ev_cnt, ev_bytes = get_session_metrics(agent, key,\ - \ sid)\n needs_compaction = (\n tokens >= THRESHOLD_TOKENS\ - \ or\n ev_bytes >= THRESHOLD_TRANSCRIPT_BYTES or\n \ - \ (tokens == 0 and ev_cnt >= THRESHOLD_EVENT_COUNT)\n )\n \ - \ if needs_compaction:\n stat_str = f\"{tokens} tokens\" if\ - \ tokens else f\"{ev_bytes//1024}KB / {ev_cnt} events\"\n print(f\"\ - Compacting {agent} ({stat_str}) -> {key}...\")\n c_res = subprocess.run(\n\ - \ [\"openclaw\", \"sessions\", \"compact\", key, \"--agent\"\ - , agent, \"--timeout\", str(RPC_TIMEOUT_MS)],\n capture_output=True,\ - \ text=True\n )\n if c_res.returncode == 0:\n \ - \ print(f\" \u2713 {c_res.stdout.strip()}\")\n \ - \ compacted += 1\n elif \"Already compacted\" in c_res.stderr\ - \ or \"Already compacted\" in c_res.stdout:\n print(f\" \u2713\ - \ Already compacted.\")\n skipped += 1\n else:\n\ - \ print(f\" \u2717 {c_res.stderr.strip()}; retrying with --max-lines\ - \ 80 fallback...\")\n fb_res = subprocess.run(\n \ - \ [\"openclaw\", \"sessions\", \"compact\", key, \"--agent\", agent,\ - \ \"--max-lines\", \"80\", \"--timeout\", str(RPC_TIMEOUT_MS)],\n \ - \ capture_output=True, text=True\n )\n \ - \ if fb_res.returncode == 0:\n print(f\" \u2713\ - \ (fallback) {fb_res.stdout.strip()}\")\n compacted +=\ - \ 1\n else:\n print(f\" \u2717 Fallback\ - \ failed: {fb_res.stderr.strip()}\")\n time.sleep(5) # 5s pause\ - \ to avoid rotational HDD I/O saturation\n else:\n skipped\ - \ += 1\n\n print(f\"Compaction complete: {compacted} compacted, {skipped} skipped.\"\ - )\n\ndef checkpoint_wal_journals():\n print(\"=== Checkpointing SQLite WAL\ - \ journals across all agent databases ===\")\n dbs = [\"/home/node/.openclaw/state/openclaw.sqlite\"\ - ] + glob.glob(\"/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite\")\n\ - \ wal_freed = 0\n for db in dbs:\n wal = db + \"-wal\"\n if\ - \ os.path.exists(wal):\n sz = os.path.getsize(wal)\n try:\n\ - \ con = sqlite3.connect(db, timeout=30.0)\n con.execute(\"\ - PRAGMA wal_checkpoint(TRUNCATE);\")\n con.close()\n \ - \ wal_freed += sz\n except Exception as e:\n print(f\"\ - \ \u2717 Error checkpointing {db}: {e}\")\n print(f\"WAL Checkpoint complete:\ - \ {wal_freed / (1024*1024):.2f} MB merged to disk.\")\n\ndef vacuum_and_optimize():\n\ - \ print(\"=== Running SQLite OPTIMIZE across all databases ===\")\n dbs\ - \ = [\"/home/node/.openclaw/state/openclaw.sqlite\"] + glob.glob(\"/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite\"\ - )\n for db in dbs:\n try:\n con = sqlite3.connect(db, timeout=30.0)\n\ - \ con.execute(\"PRAGMA optimize;\")\n con.close()\n \ - \ except Exception as e:\n print(f\" \u2717 Error optimizing {db}:\ - \ {e}\")\n print(\"SQLite optimization complete.\")\n\ndef main():\n prewarm_page_cache()\n\ - \ try:\n subprocess.run([\"python3\", \"/home/node/.openclaw/scripts/recover_sessions.py\"\ - ], check=False)\n except Exception:\n pass\n prune_stale_cron_sessions()\n\ - \ compact_tracking_sessions()\n compact_tier_a_sessions()\n compact_active_sessions()\n\ - \ checkpoint_wal_journals()\n vacuum_and_optimize()\n\nif __name__ == \"\ - __main__\":\n main()\n" - compact_tracking_sessions.py: "#!/usr/bin/env python3\n\"\"\"\nLightweight sliding-window\ - \ compaction for OpenClaw tracking channels:\n- mila_tracker (topic 2796)\n- nutrition_tracker\ - \ (topic 2797)\n\nMaintains a strict sliding window of ~5-8 transcript events\ - \ (max 2-3 turns).\nUses native gateway truncation (--max-lines 8) requiring 0\ - \ LLM tokens and 0 external API calls.\n\"\"\"\nimport os\nimport sqlite3\nimport\ - \ subprocess\nimport sys\nimport time\n\nMAX_TRACKING_EVENTS = 8\nMAX_TRACKING_LINES\ - \ = 8\nTIMEOUT_MS = 30000\n\nTRACKING_TARGETS = [\n (\"mila_tracker\", \"agent:mila_tracker:telegram:group:-1004326921510:topic:2796\"\ - ),\n (\"nutrition_tracker\", \"agent:nutrition_tracker:telegram:group:-1004326921510:topic:2797\"\ - ),\n]\n\ndef check_and_compact_tracking():\n compacted_count = 0\n skipped_count\ - \ = 0\n\n for agent, key in TRACKING_TARGETS:\n db_path = f\"/home/node/.openclaw/agents/{agent}/agent/openclaw-agent.sqlite\"\ - \n if not os.path.exists(db_path):\n continue\n try:\n\ - \ con = sqlite3.connect(db_path, timeout=5.0)\n row = con.execute(\"\ - SELECT current_session_id FROM session_nodes WHERE session_key=?\", (key,)).fetchone()\n\ - \ if not row or not row[0]:\n con.close()\n \ - \ continue\n sid = row[0]\n cnt = con.execute(\"\ - SELECT count(*) FROM transcript_events WHERE session_id=?\", (sid,)).fetchone()[0]\n\ - \ con.close()\n\n if cnt > MAX_TRACKING_EVENTS:\n \ - \ print(f\"[{agent}] Session {key} has {cnt} events (threshold {MAX_TRACKING_EVENTS}).\ - \ Truncating to last {MAX_TRACKING_LINES} lines...\")\n cmd = [\n\ - \ \"openclaw\", \"sessions\", \"compact\", key,\n \ - \ \"--agent\", agent,\n \"--max-lines\", str(MAX_TRACKING_LINES),\n\ - \ \"--timeout\", str(TIMEOUT_MS)\n ]\n \ - \ res = subprocess.run(cmd, capture_output=True, text=True)\n \ - \ if res.returncode == 0:\n print(f\" \u2713 [{agent}]\ - \ Truncated successfully: {res.stdout.strip()}\")\n compacted_count\ - \ += 1\n else:\n print(f\" \u2717 [{agent}]\ - \ Truncation failed: {res.stderr.strip()}\", file=sys.stderr)\n else:\n\ - \ skipped_count += 1\n except Exception as e:\n \ - \ print(f\"[{agent}] Error checking session: {e}\", file=sys.stderr)\n\n \ - \ if compacted_count > 0:\n print(f\"Tracking maintenance: {compacted_count}\ - \ sessions truncated, {skipped_count} within window.\")\n\nif __name__ == \"__main__\"\ - :\n check_and_compact_tracking()\n" - dedup_subagent_vectors.py: "#!/usr/bin/env python3\nimport glob, os, sqlite3, sys\n\ - \nvec_path = \"/app/node_modules/sqlite-vec-linux-x64/vec0.so\"\nfor db in sorted(glob.glob(\"\ - /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite\")):\n agent = os.path.basename(os.path.dirname(os.path.dirname(db)))\n\ - \ if agent == \"main\":\n continue\n try:\n conn = sqlite3.connect(db,\ - \ timeout=10.0)\n if os.path.exists(vec_path):\n conn.enable_load_extension(True)\n\ - \ conn.load_extension(vec_path)\n cur = conn.cursor()\n \ - \ chk = cur.execute(\"SELECT count(*) FROM memory_index_chunks\").fetchone()[0]\n\ - \ if chk > 0:\n print(f\"[dedup_subagents] Purging {chk} duplicated\ - \ vector chunks from agent {agent}...\")\n for t in [\n \ - \ \"memory_index_sources\",\n \"memory_index_chunks\",\n \ - \ \"memory_index_chunk_recall_metadata\",\n \"memory_index_chunk_provenance\"\ - ,\n \"memory_entry_origins\",\n \"memory_embedding_cache\"\ - ,\n \"memory_index_chunks_vec\"\n ]:\n \ - \ try: cur.execute(f\"DELETE FROM {t}\")\n except Exception:\ - \ pass\n conn.commit()\n cur.execute(\"VACUUM\")\n \ - \ print(f\"[dedup_subagents] Reclaimed disk space for agent {agent}\")\n\ - \ conn.close()\n except Exception as e:\n print(f\"[dedup_subagents]\ - \ Notice for {agent}: {e}\", file=sys.stderr)\n" - file_cli.py: "#!/home/node/.openclaw/mcp-venv/bin/python3\nimport sys, os, mimetypes,\ - \ filetype\n\ndef get_file_info(filepath, mime_only=False):\n if not os.path.exists(filepath):\n\ - \ return \"cannot open (No such file or directory)\", \"cannot open (No\ - \ such file or directory)\"\n if os.path.isdir(filepath):\n return \"\ - directory\", \"inode/directory\"\n kind = filetype.guess(filepath)\n if\ - \ kind is not None:\n return f\"{kind.extension.upper()} image/media data\ - \ ({kind.mime})\", kind.mime\n guessed_mime, _ = mimetypes.guess_type(filepath)\n\ - \ try:\n with open(filepath, \"rb\") as f:\n sample = f.read(4096)\n\ - \ if len(sample) == 0:\n return \"empty\", \"application/x-empty\"\ - \n try:\n sample.decode(\"utf-8\")\n if guessed_mime:\n\ - \ return f\"{guessed_mime} text\", guessed_mime\n return\ - \ \"ASCII text\", \"text/plain\"\n except UnicodeDecodeError:\n \ - \ return \"data\", \"application/octet-stream\"\n except Exception as e:\n\ - \ return f\"cannot read ({e})\", \"application/octet-stream\"\n\ndef main():\n\ - \ args = sys.argv[1:]\n brief, mime_only, files = False, False, []\n \ - \ for arg in args:\n if arg in (\"-b\", \"--brief\"):\n brief\ - \ = True\n elif arg in (\"-i\", \"--mime\", \"--mime-type\"):\n \ - \ mime_only = True\n elif arg.startswith(\"-\") and len(arg) > 1 and\ - \ not arg.startswith(\"--\"):\n if \"b\" in arg: brief = True\n \ - \ if \"i\" in arg: mime_only = True\n else:\n files.append(arg)\n\ - \ if not files:\n sys.stderr.write(\"Usage: file [-b] [-i|--mime-type]\ - \ ...\\n\")\n sys.exit(1)\n for f in files:\n desc, mime\ - \ = get_file_info(f, mime_only)\n res = mime if mime_only else desc\n \ - \ print(res if brief else f\"{f}: {res}\")\n\nif __name__ == \"__main__\"\ - :\n main()\n" - gateway-preflight.mjs: "import fs from \"node:fs\";\nimport path from \"node:path\"\ - ;\nimport { execSync, spawn } from \"node:child_process\";\n\nif (!process.env.__OPENCLAW_PREFLIGHT_DONE)\ - \ {\n process.env.__OPENCLAW_PREFLIGHT_DONE = \"1\";\n\n const isGateway = process.argv.some(arg\ - \ => arg.endsWith(\"openclaw.mjs\") || arg.endsWith(\"openclaw\")) && process.argv.includes(\"\ - gateway\");\n\n if (isGateway) {\n const t0 = Date.now();\n console.log(\"\ - [gateway-preflight] Initializing container self-healing preflight...\");\n\n \ - \ // 1. Clean up stale gateway lock files from ungraceful kills\n for (const\ - \ dir of [\"/tmp/openclaw-1000\", \"/tmp\"]) {\n if (fs.existsSync(dir))\ - \ {\n try {\n const files = fs.readdirSync(dir);\n for\ - \ (const file of files) {\n if (file.startsWith(\"gateway\") && file.endsWith(\"\ - .lock\")) {\n const fullPath = path.join(dir, file);\n \ - \ fs.unlinkSync(fullPath);\n console.log(`[gateway-preflight]\ - \ Removed stale lock: ${fullPath}`);\n }\n }\n } catch\ - \ (e) {\n console.warn(`[gateway-preflight] Lock cleanup warning in ${dir}:`,\ - \ e.message);\n }\n }\n }\n\n // 2. Clear false database quarantines\n\ - \ const qPath = \"/home/node/.openclaw/state/openclaw-quarantine.sqlite\";\n\ - \ if (fs.existsSync(qPath)) {\n try {\n execSync(`/home/node/.openclaw/.local/bin/python3\ - \ -c \"import sqlite3; c=sqlite3.connect(\\x27${qPath}\\x27); c.execute(\\x27DELETE\ - \ FROM quarantined_databases\\x27); c.commit(); c.close()\"`, { stdio: \"ignore\"\ - \ });\n console.log(\"[gateway-preflight] Cleared database quarantine.\"\ - );\n } catch (_) {}\n }\n\n // 3. Deduplicate subagent vector databases\ - \ (keeps non-main agent DBs < 15MB)\n try {\n execSync(\"/home/node/.openclaw/.local/bin/python3\ - \ /home/node/.openclaw/scripts/dedup_subagent_vectors.py\", { stdio: \"inherit\"\ - \ });\n } catch (e) {\n console.warn(\"[gateway-preflight] Vector deduplication\ - \ warning:\", e.message);\n }\n\n // 4. Pre-warm page cache sequentially\ - \ (transforms random 70 IOPS disk reads into sequential streaming)\n try {\n\ - \ const pw0 = Date.now();\n execSync(\"cat /home/node/.openclaw/state/openclaw.sqlite\ - \ /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 ||\ - \ true\");\n console.log(`[gateway-preflight] Pre-warmed SQLite databases\ - \ in ${Date.now() - pw0}ms`);\n } catch (_) {}\n\n // 5. Recover any wedged\ - \ running sessions left over from ungraceful container restart\n try {\n \ - \ execSync(\"/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/recover_sessions.py\"\ - , { stdio: \"inherit\" });\n } catch (e) {\n console.warn(\"[gateway-preflight]\ - \ Session recovery warning:\", e.message);\n }\n\n // 6. Spawn background\ - \ turn replay daemon if interrupted turns were detected\n try {\n if (fs.existsSync(\"\ - /tmp/openclaw-interrupted-turns.json\")) {\n console.log(\"[gateway-preflight]\ - \ Spawning background turn replay daemon...\");\n const child = spawn(\"\ - /home/node/.openclaw/.local/bin/python3\", [\"/home/node/.openclaw/scripts/replay_interrupted_turns.py\"\ - ], {\n detached: true,\n stdio: \"ignore\"\n });\n \ - \ child.unref();\n }\n } catch (e) {\n console.warn(\"[gateway-preflight]\ - \ Could not spawn turn replay daemon:\", e.message);\n }\n\n console.log(`[gateway-preflight]\ - \ Self-healing completed in ${Date.now() - t0}ms.`);\n }\n}\n" - openclaw-triage.py: "#!/usr/bin/env python3\n\"\"\"\nopenclaw-triage.py: Rapid zero-dependency\ - \ incident triage for OpenClaw multi-agent sessions.\nDirectly inspects ground-truth\ - \ SQLite session state and transcript events in < 1 second.\n\"\"\"\nimport argparse\n\ - import glob\nimport json\nimport os\nimport sqlite3\nimport sys\nimport time\n\ - \nTOPIC_TO_AGENT = {\n \"20\": \"fitness\",\n \"21\": \"koch\",\n \"\ - 22\": \"hebamme\",\n \"51\": \"travel\",\n \"115\": \"shopping\",\n \"\ - 116\": \"regensburg\",\n \"117\": \"recherche\",\n \"118\": \"doktor\",\n\ - \ \"119\": \"itguy\",\n \"164\": \"grabengasse_media\",\n \"2796\": \"\ - mila_tracker\",\n \"2797\": \"nutrition_tracker\",\n}\n\ndef parse_args():\n\ - \ parser = argparse.ArgumentParser(description=\"Rapid OpenClaw multi-agent\ - \ session triage.\")\n parser.add_argument(\"--topic\", \"-t\", help=\"Telegram\ - \ forum topic ID (e.g. 21, 22, 117)\")\n parser.add_argument(\"--agent\", \"\ - -a\", help=\"Agent identifier (e.g. koch, hebamme, recherche)\")\n parser.add_argument(\"\ - --limit\", \"-n\", type=int, default=10, help=\"Number of recent events to display\ - \ (default: 10)\")\n parser.add_argument(\"--db-root\", default=\"/home/node/.openclaw/agents\"\ - , help=\"Root path of agent SQLite databases\")\n return parser.parse_args()\n\ - \ndef triage_agent_session(agent, topic_id=None, limit=10, db_root=\"/home/node/.openclaw/agents\"\ - ):\n db_path = os.path.join(db_root, agent, \"agent\", \"openclaw-agent.sqlite\"\ - )\n if not os.path.exists(db_path):\n print(f\"[-] Database not found\ - \ for agent '{agent}': {db_path}\")\n return False\n\n try:\n \ - \ con = sqlite3.connect(db_path, timeout=5.0)\n except Exception as e:\n \ - \ print(f\"[-] Failed to connect to SQLite DB for agent '{agent}': {e}\"\ - )\n return False\n\n cur = con.cursor()\n # Find matching session\n\ - \ if topic_id:\n target_pattern = f\"%topic:{topic_id}%\"\n cur.execute(\n\ - \ \"SELECT session_key, current_session_id, status, updated_at, entry_json\ - \ FROM session_nodes WHERE session_key LIKE ? AND session_key NOT LIKE '%:heartbeat'\ - \ ORDER BY updated_at DESC LIMIT 1\",\n (target_pattern,)\n \ - \ )\n else:\n cur.execute(\n \"SELECT session_key, current_session_id,\ - \ status, updated_at, entry_json FROM session_nodes WHERE session_key NOT LIKE\ - \ '%:heartbeat' ORDER BY updated_at DESC LIMIT 1\"\n )\n\n row = cur.fetchone()\n\ - \ if not row:\n print(f\"[-] No session found in {agent} DB (filter:\ - \ topic={topic_id})\")\n con.close()\n return False\n\n skey,\ - \ sid, status, updated_at_ms, entry_json = row\n now_ms = time.time() * 1000\n\ - \ age_min = (now_ms - updated_at_ms) / 60000.0 if updated_at_ms else 0.0\n\n\ - \ entry = {}\n if entry_json:\n try:\n entry = json.loads(entry_json)\n\ - \ except Exception:\n pass\n\n tokens = entry.get(\"totalTokens\"\ - , \"?\")\n model = entry.get(\"model\", \"default\")\n last_run_id = entry.get(\"\ - activeWriterRunId\") or entry.get(\"activeRunId\", None)\n\n # Detect anomalies\n\ - \ badges = []\n if isinstance(tokens, int):\n if tokens > 60000:\n\ - \ badges.append(\"\U0001F6A8 [CRITICAL_CONTEXT_BLOAT: >60k tokens]\"\ - )\n elif tokens > 20000:\n badges.append(\"\u26A0\uFE0F [HIGH_CONTEXT_TOKENS:\ - \ >20k tokens]\")\n if status == \"running\" and age_min > 5.0:\n badges.append(\"\ - \U0001F512 [POSSIBLY_WEDGED_RUNNING: updated >5m ago]\")\n elif status == \"\ - processing\":\n badges.append(\"\u26A1 [ACTIVE_PROCESSING]\")\n\n #\ - \ Check ingress queue status\n state_db = \"/home/node/.openclaw/state/openclaw.sqlite\"\ - \n if topic_id and os.path.exists(state_db):\n try:\n s_con\ - \ = sqlite3.connect(state_db, timeout=2.0)\n lane_pat = f\"%topic:{topic_id}%\"\ - \n pending_cnt = s_con.execute(\n \"SELECT count(*)\ - \ FROM channel_ingress_events WHERE lane_key LIKE ? AND status IN ('pending',\ - \ 'claimed')\",\n (lane_pat,)\n ).fetchone()[0]\n \ - \ s_con.close()\n if pending_cnt > 0:\n badges.append(f\"\ - \U0001F4EC [INGRESS_STALLED: {pending_cnt} message(s) pending]\")\n except\ - \ Exception:\n pass\n\n print(\"=\" * 72)\n print(f\"\U0001F3AF\ - \ AGENT: {agent.upper():<12} | TOPIC: {topic_id or 'N/A':<6} | STATUS: {status.upper()}\"\ - )\n print(f\" SESSION KEY: {skey}\")\n print(f\" SESSION ID: {sid}\"\ - )\n print(f\" MODEL: {model}\")\n print(f\" TOKENS: {tokens}\ - \ | LAST UPDATED: {age_min:.1f}m ago\")\n if badges:\n print(f\" ANOMALIES:\ - \ {' '.join(badges)}\")\n print(\"=\" * 72)\n\n # Query transcript events\n\ - \ cur.execute(\n \"SELECT seq, created_at, event_json FROM transcript_events\ - \ WHERE session_id = ? ORDER BY seq DESC LIMIT ?\",\n (sid, limit)\n \ - \ )\n events = cur.fetchall()\n con.close()\n\n if not events:\n \ - \ print(\" (No transcript events found)\")\n return True\n\n events.reverse()\n\ - \ print(\"\\n\U0001F4DC RECENT TRANSCRIPT EVENTS (Chronological):\")\n\n \ - \ last_role = None\n for seq, ca_ms, ej_str in events:\n try:\n \ - \ ev = json.loads(ej_str)\n except Exception:\n print(f\"\ - [{seq}] \")\n continue\n\n etype = ev.get(\"\ - type\")\n if etype != \"message\":\n continue\n\n msg\ - \ = ev.get(\"message\", {})\n role = msg.get(\"role\")\n last_role\ - \ = role\n content = msg.get(\"content\", [])\n\n if role == \"\ - user\":\n txt = content if isinstance(content, str) else json.dumps(content)\n\ - \ # Check for audio transcript\n if isinstance(content,\ - \ str) and \"[Audio]\" in content:\n print(f\"\\n\U0001F449 [SEQ\ - \ {seq}] \U0001F3A4 USER AUDIO PROMPT:\")\n for line in content.splitlines()[:6]:\n\ - \ print(f\" {line}\")\n if len(content.splitlines())\ - \ > 6:\n print(\" ...\")\n else:\n \ - \ txt_preview = txt.replace(\"\\n\", \" \")[:240]\n print(f\"\ - \\n\U0001F449 [SEQ {seq}] \U0001F464 USER: {txt_preview}\")\n\n elif role\ - \ == \"assistant\":\n print(f\"\\n\U0001F916 [SEQ {seq}] ASSISTANT\ - \ TURN:\")\n if isinstance(content, list):\n for item\ - \ in content:\n if not isinstance(item, dict):\n \ - \ continue\n itype = item.get(\"type\")\n \ - \ if itype == \"thinking\":\n th = item.get(\"\ - thinking\", \"\").replace(\"\\n\", \" \")\n print(f\" \ - \ \U0001F9E0 THINKING: {th[:280]}...\")\n elif itype == \"\ - toolCall\":\n tname = item.get(\"name\", \"unknown\")\n\ - \ targs = json.dumps(item.get(\"arguments\", {}))\n \ - \ print(f\" \u2699\uFE0F TOOL CALL: {tname}({targs[:160]})\"\ - )\n elif itype == \"text\":\n tx = item.get(\"\ - text\", \"\")\n lines = tx.splitlines()\n \ - \ prev = lines[0][:150] if lines else \"\"\n \ - \ print(f\" \U0001F4AC TEXT OUTPUT: {prev}...\")\n elif isinstance(content,\ - \ str):\n print(f\" \U0001F4AC TEXT OUTPUT: {content[:200]}...\"\ - )\n\n elif role == \"toolResult\":\n is_err = ev.get(\"isError\"\ - , False)\n tname = ev.get(\"details\", {}).get(\"tool\") or msg.get(\"\ - toolName\", \"tool\")\n res_txt = json.dumps(content) if not isinstance(content,\ - \ str) else content\n if is_err or \"error\" in res_txt.lower() or\ - \ \"must have required property\" in res_txt:\n print(f\" \u274C\ - \ [SEQ {seq}] TOOL ERROR ({tname}): {res_txt[:260]}\")\n else:\n \ - \ print(f\" \u2705 [SEQ {seq}] TOOL RESULT ({tname}): {res_txt[:180]}\"\ - )\n\n if last_role == \"user\":\n print(\"\\n\u26A0\uFE0F WARNING: The\ - \ last event in the transcript is an UNANSWERED USER MESSAGE!\")\n print(\"\ - \ The agent turn either crashed, timed out, or was killed before producing a\ - \ response.\")\n\n print(\"\\n\" + \"=\" * 72)\n return True\n\ndef main():\n\ - \ args = parse_args()\n if not args.topic and not args.agent:\n print(\"\ - Usage: openclaw-triage.py (--topic | --agent ) [--limit ]\")\n \ - \ sys.exit(1)\n\n agent = args.agent\n topic = args.topic\n\n if\ - \ topic and not agent:\n agent = TOPIC_TO_AGENT.get(str(topic))\n \ - \ if not agent:\n # Search all DBs\n for a, t in TOPIC_TO_AGENT.items():\n\ - \ if t == topic:\n agent = a\n \ - \ break\n\n if not agent:\n # Scan all DBs for the topic\n \ - \ for db in sorted(glob.glob(os.path.join(args.db_root, \"*/agent/openclaw-agent.sqlite\"\ - ))):\n cand_agent = db.split(os.sep)[-3]\n try:\n \ - \ con = sqlite3.connect(db, timeout=2.0)\n cur = con.cursor()\n\ - \ cur.execute(\n \"SELECT session_key FROM session_nodes\ - \ WHERE session_key LIKE ? LIMIT 1\",\n (f\"%topic:{topic}%\"\ - ,)\n )\n if cur.fetchone():\n \ - \ agent = cand_agent\n con.close()\n break\n\ - \ con.close()\n except Exception:\n pass\n\ - \n if not agent:\n print(f\"[-] Could not map topic '{topic}' to any\ - \ known OpenClaw agent.\")\n sys.exit(1)\n\n success = triage_agent_session(agent,\ - \ topic_id=topic, limit=args.limit, db_root=args.db_root)\n sys.exit(0 if success\ - \ else 1)\n\nif __name__ == \"__main__\":\n main()\n" - patch_garmin_mcp.py: "import os\n\np = \"/home/node/.openclaw/mcp-venv/lib/python3.12/site-packages/garmin_mcp/workouts.py\"\ - \nif os.path.exists(p):\n with open(p, \"r\") as f:\n c = f.read()\n\ - \ if \"_normalize_sport_type\" not in c:\n patch = '''\nSPORT_TYPE_MAPPINGS\ - \ = {\n \"running\": {\"sportTypeId\": 1, \"sportTypeKey\": \"running\", \"\ - displayOrder\": 1},\n \"cycling\": {\"sportTypeId\": 2, \"sportTypeKey\": \"\ - cycling\", \"displayOrder\": 2},\n \"other\": {\"sportTypeId\": 3, \"sportTypeKey\"\ - : \"other\", \"displayOrder\": 3},\n \"lap_swimming\": {\"sportTypeId\": 4,\ - \ \"sportTypeKey\": \"lap_swimming\", \"displayOrder\": 4},\n \"swimming\"\ - : {\"sportTypeId\": 4, \"sportTypeKey\": \"lap_swimming\", \"displayOrder\": 4},\n\ - \ \"strength_training\": {\"sportTypeId\": 5, \"sportTypeKey\": \"strength_training\"\ - , \"displayOrder\": 5},\n \"cardio_training\": {\"sportTypeId\": 6, \"sportTypeKey\"\ - : \"cardio_training\", \"displayOrder\": 6},\n \"cardio\": {\"sportTypeId\"\ - : 6, \"sportTypeKey\": \"cardio_training\", \"displayOrder\": 6},\n \"yoga\"\ - : {\"sportTypeId\": 7, \"sportTypeKey\": \"yoga\", \"displayOrder\": 8},\n \ - \ \"pilates\": {\"sportTypeId\": 8, \"sportTypeKey\": \"pilates\", \"displayOrder\"\ - : 9},\n \"hiit\": {\"sportTypeId\": 9, \"sportTypeKey\": \"hiit\", \"displayOrder\"\ - : 7},\n \"mobility\": {\"sportTypeId\": 11, \"sportTypeKey\": \"mobility\"\ - , \"displayOrder\": 11},\n \"walking\": {\"sportTypeId\": 12, \"sportTypeKey\"\ - : \"walking\", \"displayOrder\": 10},\n \"rucking\": {\"sportTypeId\": 13,\ - \ \"sportTypeKey\": \"rucking\", \"displayOrder\": 12},\n}\nSPORT_ID_MAPPINGS\ - \ = {v[\"sportTypeId\"]: v for v in SPORT_TYPE_MAPPINGS.values()}\n\ndef _resolve_sport_block(st):\n\ - \ if isinstance(st, str):\n k = st.strip().lower()\n if k in\ - \ SPORT_TYPE_MAPPINGS:\n return dict(SPORT_TYPE_MAPPINGS[k])\n elif\ - \ isinstance(st, dict):\n key = st.get(\"sportTypeKey\")\n sid =\ - \ st.get(\"sportTypeId\")\n if key and str(key).strip().lower() in SPORT_TYPE_MAPPINGS:\n\ - \ base = dict(SPORT_TYPE_MAPPINGS[str(key).strip().lower()])\n \ - \ if sid is not None and sid != 0 and sid != base[\"sportTypeId\"]:\n \ - \ raise ValueError(f\"sportType mismatch: {key} vs {sid}\")\n \ - \ return base\n if sid is not None and sid in SPORT_ID_MAPPINGS:\n\ - \ return dict(SPORT_ID_MAPPINGS[sid])\n return None\n\ndef _normalize_sport_type(workout_data:\ - \ dict) -> None:\n resolved = _resolve_sport_block(workout_data.get(\"sportType\"\ - ))\n segments = workout_data.get(\"workoutSegments\") or []\n if not resolved\ - \ and segments:\n for seg in segments:\n resolved = _resolve_sport_block(seg.get(\"\ - sportType\"))\n if resolved:\n break\n if not resolved:\n\ - \ raise ValueError(\"Missing or invalid sportType. Both sportTypeId and\ - \ sportTypeKey are required.\")\n workout_data[\"sportType\"] = resolved\n\ - \ for seg in segments:\n st = _resolve_sport_block(seg.get(\"sportType\"\ - ))\n seg[\"sportType\"] = st if st else dict(resolved)\n\n'''\n \ - \ t = \"def _normalize_workout_steps(workout_data: dict) -> None:\"\n if\ - \ t in c:\n with open(p, \"w\") as f:\n f.write(c.replace(t,\ - \ patch + t + \"\\n _normalize_sport_type(workout_data)\", 1))\n \ - \ print(\"Successfully patched garmin_mcp workouts.py!\")\n" - patch_perplexity.mjs: "import fs from \"node:fs\";\nimport path from \"node:path\"\ - ;\nimport { globSync } from \"node:fs\";\n\nconst baseDir = \"/home/node/.openclaw/extensions/perplexity/dist\"\ - ;\nif (fs.existsSync(baseDir)) {\n const files = fs.readdirSync(baseDir).filter(f\ - \ => f.startsWith(\"perplexity-web-search-provider.runtime-\") && f.endsWith(\"\ - .js\"));\n for (const filename of files) {\n const file = path.join(baseDir,\ - \ filename);\n let code = fs.readFileSync(file, \"utf8\");\n if (!code.includes(\"\ - /* PATCHED_CHAT_COMPLETIONS */\")) {\n const target = \"const structured\ - \ = runtime.transport === \\\"search_api\\\";\";\n const fastPath = `const\ - \ structured = runtime.transport === \"search_api\"; /* PATCHED_CHAT_COMPLETIONS\ - \ */\n if (!structured) {\n const cacheKey = buildSearchCacheKey([\n\ - \ \"perplexity\",\n runtime.transport,\n runtime.baseUrl,\n\ - \ runtime.model,\n query,\n freshness\n \ - \ ]);\n const cacheTtlMs = resolveSearchCacheTtlMs(searchConfig);\n\ - \ const cached = readCachedSearchPayload(cacheKey, cacheTtlMs);\n \ - \ if (cached) return cached;\n const start = Date.now();\n \ - \ const timeoutSeconds = resolveSearchTimeoutSeconds(searchConfig);\n \ - \ const result = await runPerplexitySearch({\n query,\n \ - \ apiKey: runtime.apiKey,\n baseUrl: runtime.baseUrl,\n \ - \ model: runtime.model,\n timeoutSeconds,\n signal,\n\ - \ freshness\n });\n const payload = {\n \ - \ query,\n provider: \"perplexity\",\n model: runtime.model,\n\ - \ tookMs: Date.now() - start,\n externalContent: {\n \ - \ untrusted: true,\n source: \"web_search\",\n \ - \ provider: \"perplexity\",\n wrapped: true\n },\n\ - \ content: wrapWebContent(result.content, \"web_search\"),\n \ - \ citations: result.citations\n };\n writeCachedSearchPayload(cacheKey,\ - \ payload);\n return payload;\n }`;\n if (code.includes(target))\ - \ {\n code = code.replace(target, fastPath);\n fs.writeFileSync(file,\ - \ code, \"utf8\");\n console.log(\"Patched perplexity runtime for chat\ - \ completions:\", file);\n }\n }\n }\n}\n" - recover_sessions.py: "#!/usr/bin/env python3\nimport glob, sqlite3, json, os, sys,\ - \ time\n\nt0 = time.time()\nrecovered = 0\ninterrupted_turns = []\n\ndef extract_prompt(msg):\n\ - \ if not msg: return \"\"\n content = msg.get(\"content\")\n if isinstance(content,\ - \ str): return content\n if isinstance(content, list):\n parts = []\n\ - \ for c in content:\n if isinstance(c, dict): parts.append(c.get(\"\ - text\") or c.get(\"content\") or \"\")\n elif isinstance(c, str): parts.append(c)\n\ - \ return \"\\n\".join(filter(None, parts))\n return str(content or \"\ - \")\n\nfor db in sorted(glob.glob(\"/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite\"\ - )):\n try:\n conn = sqlite3.connect(db, timeout=10.0)\n cursor\ - \ = conn.cursor()\n while True:\n cursor.execute(\"PRAGMA foreign_key_check;\"\ - )\n violations = cursor.fetchall()\n if not violations:\ - \ break\n for table, rowid, parent, fkid in violations:\n \ - \ cursor.execute(f\"DELETE FROM {table} WHERE rowid = ?\", (rowid,))\n \ - \ cursor.execute(\"SELECT session_key, entry_json, current_session_id FROM\ - \ session_nodes WHERE status = \\\"running\\\"\")\n rows = cursor.fetchall()\n\ - \ agent_id = os.path.basename(os.path.dirname(os.path.dirname(db)))\n \ - \ for sk, ej, csid in rows:\n try:\n p = json.loads(ej)\ - \ if ej else {}\n sid = csid or p.get(\"sessionId\")\n \ - \ if sid:\n ev = cursor.execute(\"SELECT seq, event_json\ - \ FROM transcript_events WHERE session_id = ? ORDER BY seq DESC LIMIT 1\", (sid,)).fetchone()\n\ - \ if ev:\n ev_seq, ev_raw = ev\n \ - \ try:\n ev_data = json.loads(ev_raw)\n\ - \ msg = ev_data.get(\"message\") or ev_data\n \ - \ if msg.get(\"role\") == \"user\":\n \ - \ prompt_text = extract_prompt(msg)\n \ - \ ctx = p.get(\"restartRecoveryDeliveryContext\") or p.get(\"delivery\", {}).get(\"\ - context\") or p.get(\"delivery\", {}).get(\"route\", {}).get(\"context\") or {}\n\ - \ interrupted_turns.append({\n \ - \ \"agent\": agent_id,\n \ - \ \"session_key\": sk,\n \"session_id\": sid,\n\ - \ \"context\": ctx,\n \ - \ \"prompt\": prompt_text\n })\n\ - \ cursor.execute(\"PRAGMA foreign_keys = ON\")\n\ - \ cursor.execute(\"DELETE FROM transcript_events\ - \ WHERE session_id = ? AND seq = ?\", (sid, ev_seq))\n \ - \ except Exception as ex:\n print(f\"[recover_sessions]\ - \ Error parsing event: {ex}\", file=sys.stderr)\n p[\"status\"\ - ] = \"done\"\n p[\"activeWriterRunId\"] = None\n \ - \ p[\"activeClaimRunId\"] = None\n p[\"restartRecoveryDeliveryRunId\"\ - ] = None\n cursor.execute(\"UPDATE session_nodes SET status = \\\ - \"done\\\", entry_json = ? WHERE session_key = ?\", (json.dumps(p), sk))\n \ - \ cursor.execute(\"UPDATE session_nodes SET entry_valid = 1 WHERE\ - \ session_key = ?\", (sk,))\n recovered += 1\n except\ - \ Exception: pass\n cursor.execute(\"UPDATE session_windows SET status\ - \ = \\\"done\\\" WHERE status = \\\"running\\\"\")\n conn.commit()\n \ - \ conn.close()\n except Exception as e:\n print(f\"[recover_sessions]\ - \ Warning for {db}: {e}\", file=sys.stderr)\n\nif interrupted_turns:\n try:\n\ - \ with open(\"/tmp/openclaw-interrupted-turns.json\", \"w\") as f:\n \ - \ json.dump(interrupted_turns, f, indent=2)\n print(f\"[recover_sessions]\ - \ Detected {len(interrupted_turns)} interrupted user turn(s) saved to /tmp/openclaw-interrupted-turns.json\"\ - )\n except Exception as e:\n print(f\"[recover_sessions] Error writing\ - \ interrupted turns: {e}\", file=sys.stderr)\n\nprint(f\"[recover_sessions] Completed\ - \ in {time.time() - t0:.2f}s ({recovered} wedged sessions recovered)\")\n" - replay_interrupted_turns.py: "#!/usr/bin/env python3\nimport json, os, subprocess,\ - \ sys, time, urllib.request\n\nINTERRUPTED_FILE = \"/tmp/openclaw-interrupted-turns.json\"\ - \n\nif not os.path.exists(INTERRUPTED_FILE):\n sys.exit(0)\n\ntry:\n with\ - \ open(INTERRUPTED_FILE, \"r\") as f:\n turns = json.load(f)\nexcept Exception\ - \ as e:\n print(f\"[replay_interrupted_turns] Failed to read {INTERRUPTED_FILE}:\ - \ {e}\", file=sys.stderr)\n sys.exit(0)\n\nif not turns:\n sys.exit(0)\n\ - \nprint(f\"[replay_interrupted_turns] Found {len(turns)} interrupted turn(s).\ - \ Waiting for gateway /healthz...\")\n\ngateway_ready = False\nfor _ in range(60):\n\ - \ try:\n with urllib.request.urlopen(\"http://127.0.0.1:18790/healthz\"\ - , timeout=3) as resp:\n if resp.status == 200:\n gateway_ready\ - \ = True\n break\n except Exception:\n pass\n time.sleep(2)\n\ - \nif not gateway_ready:\n print(\"[replay_interrupted_turns] Gateway did not\ - \ become healthy in 120s. Aborting.\", file=sys.stderr)\n sys.exit(1)\n\n#\ - \ Allow 5s for Telegram ingress to bind\ntime.sleep(5)\n\nfor turn in turns:\n\ - \ sk = turn.get(\"session_key\")\n sid = turn.get(\"session_id\")\n agent_id\ - \ = turn.get(\"agent\")\n ctx = turn.get(\"context\") or {}\n prompt = turn.get(\"\ - prompt\") or \"\"\n\n channel = ctx.get(\"channel\") or \"telegram\"\n to_target\ - \ = ctx.get(\"to\") or \"\"\n thread_id = ctx.get(\"threadId\")\n\n target\ - \ = to_target\n if target.startswith(\"telegram:\"):\n parts = target.split(\"\ - :\")\n target = parts[1] if len(parts) > 1 else target\n\n print(f\"\ - [replay_interrupted_turns] Processing agent '{agent_id}' on session '{sk}' target='{target}'\ - \ thread='{thread_id}'\")\n\n # 1. Send status notification to Telegram if\ - \ target exists\n if target:\n notice = \"\u26A0\uFE0F OpenClaw wurde\ - \ w\xE4hrend der Bearbeitung neu gestartet \u2013 deine Anfrage wird jetzt nachgeholt...\"\ - \n notify_cmd = [\n \"openclaw\", \"message\", \"send\",\n \ - \ \"--channel\", channel,\n \"--target\", str(target),\n\ - \ \"-m\", notice\n ]\n if thread_id is not None:\n \ - \ notify_cmd.extend([\"--thread-id\", str(thread_id)])\n try:\n\ - \ subprocess.run(notify_cmd, check=False, timeout=30)\n \ - \ print(f\"[replay_interrupted_turns] Sent recovery notice to {target}\")\n \ - \ except Exception as e:\n print(f\"[replay_interrupted_turns]\ - \ Notice failed: {e}\", file=sys.stderr)\n\n # 2. Replay agent turn\n if\ - \ sk and prompt:\n p_short = (prompt[:60] + \"...\") if len(prompt) > 60\ - \ else prompt\n print(f\"[replay_interrupted_turns] Replaying agent turn:\ - \ {p_short}\")\n agent_cmd = [\n \"openclaw\", \"agent\",\n\ - \ \"--session-key\", sk,\n \"-m\", prompt,\n \ - \ \"--deliver\"\n ]\n try:\n res = subprocess.run(agent_cmd,\ - \ capture_output=True, text=True, timeout=300)\n if res.returncode\ - \ == 0:\n print(f\"[replay_interrupted_turns] Turn replay succeeded\ - \ for {sk}\")\n else:\n print(f\"[replay_interrupted_turns]\ - \ Turn replay exited with code {res.returncode}: {res.stderr[:200]}\", file=sys.stderr)\n\ - \ except Exception as e:\n print(f\"[replay_interrupted_turns]\ - \ Replay execution error: {e}\", file=sys.stderr)\n\ntry:\n os.remove(INTERRUPTED_FILE)\n\ - \ print(f\"[replay_interrupted_turns] Cleaned up {INTERRUPTED_FILE}\")\nexcept\ - \ Exception:\n pass\n" - sync_crons.py: "#!/usr/bin/env python3\n\"\"\"\nsync_crons.py\nIdempotently reconciles\ - \ declarative OpenClaw automations defined in crons.yaml\nwith the running OpenClaw\ - \ gateway state.\n\nCan run directly inside the container or remotely via kubectl\ - \ exec.\n\"\"\"\n\nimport json\nimport os\nimport subprocess\nimport sys\n\ntry:\n\ - \ import yaml\nexcept ImportError:\n venv_py = \"/home/node/.openclaw/mcp-venv/bin/python3\"\ - \n if os.path.exists(venv_py) and sys.executable != venv_py:\n os.execv(venv_py,\ - \ [venv_py] + sys.argv)\n raise\n\ndef find_crons_file():\n candidates =\ - \ [\n os.environ.get(\"CRONS_FILE\"),\n \"crons.yaml\",\n \ - \ \"k8s/openclaw/crons.yaml\",\n \"/home/node/.openclaw/scripts/crons.yaml\"\ - ,\n os.path.join(os.path.dirname(__file__), \"../crons.yaml\"),\n \ - \ os.path.join(os.path.dirname(__file__), \"crons.yaml\"),\n ]\n for c\ - \ in candidates:\n if c and os.path.exists(c):\n return os.path.abspath(c)\n\ - \ raise FileNotFoundError(\"Could not find crons.yaml. Pass via CRONS_FILE\ - \ environment variable.\")\n\ndef is_inside_container():\n return os.path.exists(\"\ - /home/node/.openclaw\") or os.environ.get(\"OPENCLAW_HOME\") is not None\n\ndef\ - \ run_openclaw_cmd(args):\n if is_inside_container():\n cmd = [\"openclaw\"\ - ] + args\n else:\n namespace = os.environ.get(\"NAMESPACE\", \"openclaw\"\ - )\n pod = os.environ.get(\"POD_NAME\", \"openclaw-0\")\n container\ - \ = os.environ.get(\"CONTAINER_NAME\", \"openclaw\")\n cmd = [\"kubectl\"\ - , \"exec\", \"-n\", namespace, pod, \"-c\", container, \"--\", \"openclaw\"] +\ - \ args\n\n res = subprocess.run(cmd, capture_output=True, text=True)\n if\ - \ res.returncode != 0:\n raise RuntimeError(f\"OpenClaw CLI command failed\ - \ ({' '.join(cmd)}):\\n{res.stderr}\\n{res.stdout}\")\n return res.stdout\n\ - \ndef list_existing_jobs():\n raw = run_openclaw_cmd([\"cron\", \"list\", \"\ - --json\"])\n try:\n data = json.loads(raw)\n return data.get(\"\ - jobs\", [])\n except Exception as e:\n print(f\"Warning: could not parse\ - \ cron list json: {e}\\nRaw output: {raw[:300]}\", file=sys.stderr)\n return\ - \ []\n\ndef reconcile_jobs():\n file_path = find_crons_file()\n print(f\"\ - === Syncing OpenClaw Crons-as-Code from {file_path} ===\")\n\n with open(file_path,\ - \ \"r\", encoding=\"utf-8\") as f:\n spec = yaml.safe_load(f)\n\n declarations\ - \ = spec.get(\"jobs\", [])\n existing_jobs = list_existing_jobs()\n\n print(f\"\ - Found {len(declarations)} declared jobs in YAML, {len(existing_jobs)} active jobs\ - \ on gateway.\")\n\n for job in declarations:\n name = job.get(\"name\"\ - )\n decl_key = job.get(\"declarationKey\") or name\n desc = job.get(\"\ - description\", \"\")\n sched = job.get(\"schedule\", {})\n\n # Match\ - \ existing job by declarationKey or exact name\n match = next((j for j\ - \ in existing_jobs if j.get(\"declarationKey\") == decl_key or j.get(\"name\"\ - ) == name), None)\n\n if match:\n existing_id = match.get(\"\ - id\")\n print(f\"Removing existing job '{name}' (ID {existing_id})\ - \ for idempotent recreation...\")\n try:\n run_openclaw_cmd([\"\ - cron\", \"rm\", existing_id])\n except Exception as e:\n \ - \ print(f\"Warning removing {existing_id}: {e}\")\n\n # Build add\ - \ command\n cmd = [\n \"cron\", \"add\",\n \"--name\"\ - , name,\n \"--declaration-key\", decl_key,\n \"--description\"\ - , desc,\n ]\n\n if sched.get(\"kind\") == \"every\":\n \ - \ cmd.extend([\"--every\", sched.get(\"every\", \"15m\")])\n elif sched.get(\"\ - kind\") == \"cron\":\n cmd.extend([\"--cron\", sched.get(\"expr\",\ - \ \"* * * * *\")])\n if sched.get(\"tz\"):\n cmd.extend([\"\ - --tz\", sched.get(\"tz\")])\n\n if job.get(\"session\"):\n cmd.extend([\"\ - --session\", job.get(\"session\")])\n\n if job.get(\"delivery\") == \"\ - none\":\n cmd.append(\"--no-deliver\")\n\n if job.get(\"command\"\ - ):\n cmd.extend([\"--command\", job.get(\"command\")])\n\n if\ - \ job.get(\"agent\"):\n cmd.extend([\"--agent\", job.get(\"agent\"\ - )])\n\n if job.get(\"model\"):\n cmd.extend([\"--model\", job.get(\"\ - model\")])\n\n if job.get(\"channel\"):\n cmd.extend([\"--channel\"\ - , job.get(\"channel\")])\n\n if job.get(\"to\"):\n cmd.extend([\"\ - --to\", job.get(\"to\")])\n\n if job.get(\"threadId\"):\n cmd.extend([\"\ - --thread-id\", str(job.get(\"threadId\"))])\n\n if job.get(\"announce\"\ - ):\n cmd.append(\"--announce\")\n\n if job.get(\"message\"):\n\ - \ cmd.extend([\"--message\", job.get(\"message\")])\n\n print(f\"\ - Registering job '{name}'...\")\n run_openclaw_cmd(cmd)\n print(f\"\ - Successfully registered '{name}'.\")\n\n print(\"=== OpenClaw Crons-as-Code\ - \ reconciliation complete! ===\")\n\nif __name__ == \"__main__\":\n reconcile_jobs()\n" - crons.yaml: "# Declarative Crons-as-Code for OpenClaw (Haumdaucher)\n# Reconciled\ - \ idempotently via k8s/openclaw/scripts/sync_crons.py\n\nversion: 1\njobs:\n \ - \ # 1. Maintenance & State Truncation\n - name: \"tracking-session-sliding-window\"\ - \n declarationKey: \"tracking-session-sliding-window\"\n description: \"\ - Lightweight sliding-window truncation (max 8 events) for mila_tracker and nutrition_tracker\"\ - \n schedule:\n kind: \"every\"\n every: \"15m\"\n session: \"\ - isolated\"\n delivery: \"none\"\n command: \"python3 /home/node/.openclaw/scripts/compact_tracking_sessions.py\"\ - \n\n - name: \"nightly-session-compaction\"\n declarationKey: \"nightly-session-compaction\"\ - \n description: \"Nightly multi-agent session compaction and SQLite WAL checkpointing\"\ - \n schedule:\n kind: \"cron\"\n expr: \"45 4 * * *\"\n tz: \"\ - Europe/Berlin\"\n session: \"isolated\"\n delivery: \"none\"\n command:\ - \ \"python3 /home/node/.openclaw/scripts/compact_sessions.py\"\n\n # 2. Family\ - \ Dashboards & Web Analytics\n - name: \"dashboards-periodic-publish\"\n declarationKey:\ - \ \"dashboards-periodic-publish\"\n description: \"Periodically regenerates\ - \ and uploads Mila & Moritz family HTML dashboards to S3\"\n schedule:\n \ - \ kind: \"every\"\n every: \"10m\"\n session: \"isolated\"\n delivery:\ - \ \"none\"\n command: \"sh -lc 'export PATH=/home/node/.openclaw/bin:$PATH;\ - \ cd /home/node/mop-skills/mop-skills/skills/datastore && uv run scripts/update_website.py\ - \ --data-dir /home/node/.openclaw/wiki/main/data && uv run scripts/update_moritz_website.py\ - \ --data-dir /home/node/.openclaw/wiki/main/data'\"\n\n # 3. Nightly Garmin Connect\ - \ Mirror\n - name: \"garmin-nightly-mirror\"\n declarationKey: \"garmin-nightly-mirror\"\ - \n description: \"N\xE4chtlicher deterministischer Spiegel der Garmin Connect\ - \ Daten (Aktivit\xE4ten, Daily Summary, Schlaf) in Git JSONL\"\n schedule:\n\ - \ kind: \"cron\"\n expr: \"30 3 * * *\"\n tz: \"Europe/Berlin\"\ - \n agent: \"fitness\"\n model: \"openrouter/deepseek/deepseek-v4.1-flash\"\ - \n session: \"isolated\"\n channel: \"telegram\"\n to: \"-1004326921510\"\ - \n threadId: \"20\"\n announce: true\n message: |\n F\xFChre den\ - \ n\xE4chtlichen Garmin Connect Sync durch.\n Vorgehen:\n 1. F\xFChre\ - \ den Garmin-Sync-Befehl aus:\n uv run /home/node/mop-skills/mop-skills/skills/datastore/scripts/garmin_sync.py\ - \ --days 2 --data-dir /home/node/.openclaw/wiki/main/data\n 2. Aktualisiere\ - \ die gerenderten Markdown-Ansichten:\n PYTHONPATH=/home/node/mop-skills/mop-skills/skills/datastore/scripts\ - \ uv run /home/node/mop-skills/mop-skills/skills/datastore/scripts/render_views.py\ - \ --data-dir /home/node/.openclaw/wiki/main/data --wiki-dir /home/node/.openclaw/wiki/main\n\ - \ 3. Aktualisiere das pers\xF6nliche Health & Nutrition Dashboard (moritz.html):\n\ - \ uv run /home/node/mop-skills/mop-skills/skills/datastore/scripts/update_moritz_website.py\ - \ --data-dir /home/node/.openclaw/wiki/main/data\n 4. Git Commit & Push im\ - \ Knowledge Wiki:\n cd /home/node/.openclaw/wiki/main && git add data/garmin/\ - \ entities/nutrition-log/ public/ && git diff --staged --quiet || (git commit\ - \ -m \"chore(garmin): automated nightly mirror [skip ci]\" && git push origin\ - \ main)\n 5. Finale Antwort:\n - Bei Erfolg: Gib exakt 'NO_REPLY'\ - \ aus (intentional silence, kein Telegram-Spam in der Nacht).\n - Nur\ - \ bei Authentifizierungsfehlern: Informiere kurz via Telegram (Thema #Sport),\ - \ dass ein Garmin-Re-Auth n\xF6tig ist, oder starte den Re-Auth-Flow.\n\n # 4.\ - \ Weekly Autonomous System & Error Audit\n - name: \"itguy-weekly-health-audit\"\ - \n declarationKey: \"itguy-weekly-health-audit\"\n description: \"W\xF6\ - chentlicher Self-Audit der OpenClaw-Logs, Tool-Failures und Session-Transkripte\ - \ mit Git-Commit-Korrelation\"\n schedule:\n kind: \"cron\"\n expr:\ - \ \"0 3 * * 0\"\n tz: \"Europe/Berlin\"\n agent: \"itguy\"\n model:\ - \ \"openrouter/deepseek/deepseek-v4.1-flash\"\n session: \"isolated\"\n \ - \ channel: \"telegram\"\n to: \"-1004326921510\"\n threadId: \"119\"\n \ - \ announce: true\n message: |\n F\xFChre den w\xF6chentlichen OpenClaw\ - \ System- und Log-Audit durch.\n Vorgehen:\n 1. F\xFChre den Audit-Befehl\ - \ aus:\n uv run /home/node/mop-skills/mop-skills/skills/openclaw-audit/scripts/audit_openclaw.py\ - \ --since 7d --write-report --add-task --git-push\n 2. Analysiere die Ausgabe\ - \ und korreliere eventuelle Fehler-Cluster mit den j\xFCngsten Commits in infrapuzzle,\ - \ mop-skills und knowledge-wiki.\n 3. Finale Antwort:\n - Wenn NEUE\ - \ Probleme gefunden oder Tasks in tasks/antigravity.md eingetragen wurden: Fasse\ - \ die Ergebnisse kurz, pr\xE4zise und l\xF6sungsorientiert zusammen (betroffene\ - \ Tools, vermutete Ursache, Link zur Task-Queue) und gib diese Zusammenfassung\ - \ als finale Antwort aus.\n - Wenn das System gesund ist und KEINE neuen\ - \ Tasks erzeugt wurden: Gib als finale Antwort exakt 'NO_REPLY' aus (intentional\ - \ silence, kein Telegram-Spam).\n\n # 5. Weekly AI Roundup (Recherche)\n - name:\ - \ \"weekly-ai-roundup\"\n declarationKey: \"weekly-ai-roundup:recherche\"\n\ - \ description: \"W\xF6chentlicher AI-Roundup und Tech-Scanning f\xFCr Topic\ - \ #Research\"\n schedule:\n kind: \"cron\"\n expr: \"30 6 * * 5\"\ - \n tz: \"Europe/Berlin\"\n agent: \"recherche\"\n model: \"openrouter/deepseek/deepseek-v4.1-flash\"\ - \n session: \"isolated\"\n channel: \"telegram\"\n to: \"-1004326921510\"\ - \n threadId: \"117\"\n announce: true\n message: |\n \U0001F5D3\uFE0F\ - \ W\xD6CHENTLICHER AI-ROUNDUP (Freitag). F\xFChre den in sources.md im Workspace\ - \ definierten Ablauf aus und poste das Ergebnis als EINE kompakte Telegram-Nachricht\ - \ in diesen Topic (#Research).\n\n 1) Hacker News: Algolia API \u2013 https://hn.algolia.com/api/v1/search_by_date?tags=story&numericFilters=points>150,created_at_i>{Unix-Timestamp\ - \ von vor 7 Tagen}&hitsPerPage=100 \u2192 daraus die KI-relevanten Top-Stories\ - \ nach Punkten ziehen.\n 2) Frontier-Labs pr\xFCfen: OpenAI, Anthropic, Google\ - \ DeepMind, Meta AI, Mistral, xAI.\n 3) China-Labs pr\xFCfen: GLM/Z.ai, Qwen,\ - \ DeepSeek, Moonshot/Kimi, MiniMax.\n 4) Tracker: Artificial Analysis Releases,\ - \ Hugging Face Trending, OpenRouter Rankings, LMArena.\n 5) Newsletter/Blogs:\ - \ The Batch, Import AI, Interconnects, Ahead of AI, Simon Willison, TLDR AI.\n\ - \ 6) GitHub: Trending (weekly) + neue Releases der Kern-Orgs (openai, anthropics,\ - \ google-deepmind, QwenLM, deepseek-ai, zai-org, meta-llama, mistralai).\n \ - \ 7) YouTube nur als Hype-Indikator (Titel/Snippets), KEINE Transkripte.\n\n\ - \ Ausgabe (Deutsch, Markdown, KEINE Tabellen):\n - Top-3-Highlights\ - \ der Woche (je 1 Satz + Link)\n - Danach Bullets je Kategorie: \u2460 Frontier-Modelle\ - \ \xB7 \u2461 China-Modelle \xB7 \u2462 Tools/Infra/Procedures \xB7 \u2463 Trends\ - \ (HN/GitHub/YouTube), je Eintrag genau 1 Quell-Link\n - Scope 60/40 (Modelle/Tools).\ - \ Wenn eine Kategorie nichts Neues hat: kurz 'nichts Neues'.\n - Am Ende\ - \ NUR falls n\xF6tig eine Zeile: welche Quelle in sources.md erg\xE4nzt/entfernt\ - \ werden sollte.\n\n Nutze web_search/web_fetch. Effort: medium.\n\n # 6.\ - \ Weekly Bridge Tracker (Holzgartensteg)\n - name: \"Recherche: Holzgartensteg\ - \ (w\xF6chentlich)\"\n declarationKey: \"recherche:holzgartensteg\"\n description:\ - \ \"W\xF6chentlicher Status-Check zum Holzgartensteg Regensburg via ALLRIS & Web\"\ - \n schedule:\n kind: \"cron\"\n expr: \"0 16 * * 5\"\n tz: \"\ - Europe/Berlin\"\n agent: \"main\"\n session: \"isolated\"\n channel:\ - \ \"telegram\"\n to: \"306373425\"\n announce: true\n message: |\n \ - \ W\xF6chentlicher Recherche-Task (Holzgartensteg, Stadt Regensburg).\n\n \ - \ AUFGABE: Pr\xFCfe, ob es NEUIGKEITEN zum geplanten Holzgartensteg (geplante\ - \ Fu\xDF-/Radbr\xFCcke zwischen Grieser Spitz und Maria-Beer-Platz / Weichs) gibt\ - \ \u2014 seit dem letzten Check bzw. den letzten ~7 Tagen.\n\n QUELLEN:\n\ - \ 1) ALLRIS via Skill 'regensburg-stadtrat': Script /home/node/mop-skills/mop-skills/skills/regensburg-stadtrat/scripts/regensburg_ris.py\ - \ \u2014 Befehle: search \"Holzgartensteg\" --limit 30 ; calendar --days 30 ;\ - \ vorlage ; session . Achte auf NEUE Vorlagen, Beschl\xFCsse, Anfragen\ - \ und Sitzungstermine (Ausschuss f\xFCr Stadtplanung, Verkehr und Wohnungsfragen\ - \ / Stadtrat).\n 2) Lokale Medien via web_search: regensburg-digital.de,\ - \ idowa.de, mittelbayerische.de, tvaktuell.com, regensburger-nachrichten.de.\n\ - \n FOKUS: neue Vorlagen/Beschl\xFCsse, Ausschuss-/Stadtrattermine, Mittel/Budget,\ - \ Planungsstand (Variante, Kosten, Termin), politische Positionen.\n\n BEKANNTER\ - \ STAND (nicht wiederholen, nur Abweichungen melden): 2020 aus Investitionsprogramm\ - \ gestrichen ('beerdigt'); Comeback ab 2025 (Gr\xFCne/Verb\xE4nde); Kommunalwahl\ - \ 2026 mehrere Parteien daf\xFCr; Haushalt 2026: ~100.000 EUR Planungsmittel im\ - \ Investitionsprogramm, Stadtrat-Beschluss 26.02.2026, Genehmigung Regierung der\ - \ Oberpfalz 06.03.2026; Koalitionsvereinbarung 2026 mit Wiederaufnahme der Planungen.\n\ - \n AUSGABE: Kurz auf Deutsch an Moritz. Falls echte Neuigkeiten: 3-6 Bullets\ - \ mit Quelle+Link. Falls NICHTS Neues: eine kurze Zeile, z.B. 'Holzgartensteg:\ - \ keine neuen Entwicklungen diese Woche.' Kein Hintergrund-Recycling.\n" diff --git a/k8s/openclaw/crons.yaml b/k8s/openclaw/crons.yaml index 497d02d..3088045 100644 --- a/k8s/openclaw/crons.yaml +++ b/k8s/openclaw/crons.yaml @@ -34,7 +34,7 @@ jobs: expr: "30 3 * * *" tz: "Europe/Berlin" agent: "fitness" - model: "openrouter/deepseek/deepseek-v4.1-flash" + model: "openrouter/deepseek/deepseek-v4.1-flash-1017" session: "isolated" channel: "telegram" to: "-1004326921510" @@ -58,7 +58,7 @@ jobs: expr: "0 3 * * 0" tz: "Europe/Berlin" agent: "itguy" - model: "openrouter/deepseek/deepseek-v4.1-flash" + model: "openrouter/deepseek/deepseek-v4.1-flash-1017" session: "isolated" channel: "telegram" to: "-1004326921510" @@ -83,7 +83,7 @@ jobs: expr: "30 6 * * 5" tz: "Europe/Berlin" agent: "recherche" - model: "openrouter/deepseek/deepseek-v4.1-flash" + model: "openrouter/deepseek/deepseek-v4.1-flash-1017" session: "isolated" channel: "telegram" to: "-1004326921510" diff --git a/k8s/openclaw/openclaw-instance.secret.yaml b/k8s/openclaw/openclaw-instance.secret.yaml index 2a64040..67ee73d 100644 Binary files a/k8s/openclaw/openclaw-instance.secret.yaml and b/k8s/openclaw/openclaw-instance.secret.yaml differ diff --git a/k8s/openclaw/scripts/compact_sessions.py b/k8s/openclaw/scripts/compact_sessions.py index fd50c46..b53e5d4 100644 --- a/k8s/openclaw/scripts/compact_sessions.py +++ b/k8s/openclaw/scripts/compact_sessions.py @@ -41,11 +41,12 @@ TIER_A_TARGETS = [ ] def prewarm_page_cache(): - print("=== Pre-warming SQLite databases into OS page cache ===") + print("=== Pre-warming SQLite databases and Knowledge Wiki into OS page cache ===") t0 = time.time() try: subprocess.run( - "cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 || true", + "cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 || true; " + "find /home/node/.openclaw/wiki/main /home/node/.openclaw/workspace* -maxdepth 3 -name '*.md' -exec cat {} + > /dev/null 2>&1 || true", shell=True, timeout=90.0 ) @@ -329,6 +330,7 @@ def main(): compact_active_sessions() checkpoint_wal_journals() vacuum_and_optimize() + prewarm_page_cache() if __name__ == "__main__": main() diff --git a/k8s/openclaw/scripts/dedup_subagent_vectors.py b/k8s/openclaw/scripts/dedup_subagent_vectors.py index b4752ac..5434578 100644 --- a/k8s/openclaw/scripts/dedup_subagent_vectors.py +++ b/k8s/openclaw/scripts/dedup_subagent_vectors.py @@ -1,34 +1,48 @@ - #!/usr/bin/env python3 - import glob, os, sqlite3, sys +#!/usr/bin/env python3 +import glob, json, os, sqlite3, sys - vec_path = "/app/node_modules/sqlite-vec-linux-x64/vec0.so" - for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")): - agent = os.path.basename(os.path.dirname(os.path.dirname(db))) - if agent == "main": - continue - try: - conn = sqlite3.connect(db, timeout=10.0) - if os.path.exists(vec_path): - conn.enable_load_extension(True) - conn.load_extension(vec_path) - cur = conn.cursor() - chk = cur.execute("SELECT count(*) FROM memory_index_chunks").fetchone()[0] - if chk > 0: - print(f"[dedup_subagents] Purging {chk} duplicated vector chunks from agent {agent}...") - for t in [ - "memory_index_sources", - "memory_index_chunks", - "memory_index_chunk_recall_metadata", - "memory_index_chunk_provenance", - "memory_entry_origins", - "memory_embedding_cache", - "memory_index_chunks_vec" - ]: - try: cur.execute(f"DELETE FROM {t}") - except Exception: pass - conn.commit() - cur.execute("VACUUM") - print(f"[dedup_subagents] Reclaimed disk space for agent {agent}") - conn.close() - except Exception as e: - print(f"[dedup_subagents] Notice for {agent}: {e}", file=sys.stderr) +cfg_path = "/home/node/.openclaw/openclaw.json" +if os.path.exists(cfg_path): + try: + with open(cfg_path, "r") as f: + cfg = json.load(f) + mem_search = cfg.get("memory", {}).get("search", {}) + if mem_search.get("extraPaths"): + mem_search["extraPaths"] = [] + with open(cfg_path, "w") as f: + json.dump(cfg, f, indent=2) + print("[dedup_subagents] Cleared top-level memory.search.extraPaths from openclaw.json (scoped to main only)") + except Exception as e: + print(f"[dedup_subagents] Warning checking openclaw.json: {e}", file=sys.stderr) + +vec_path = "/app/node_modules/sqlite-vec-linux-x64/vec0.so" +for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")): + agent = os.path.basename(os.path.dirname(os.path.dirname(db))) + if agent == "main": + continue + try: + conn = sqlite3.connect(db, timeout=10.0) + if os.path.exists(vec_path): + conn.enable_load_extension(True) + conn.load_extension(vec_path) + cur = conn.cursor() + chk = cur.execute("SELECT count(*) FROM memory_index_chunks").fetchone()[0] + if chk > 0: + print(f"[dedup_subagents] Purging {chk} duplicated vector chunks from agent {agent}...") + for t in [ + "memory_index_sources", + "memory_index_chunks", + "memory_index_chunk_recall_metadata", + "memory_index_chunk_provenance", + "memory_entry_origins", + "memory_embedding_cache", + "memory_index_chunks_vec" + ]: + try: cur.execute(f"DELETE FROM {t}") + except Exception: pass + conn.commit() + cur.execute("VACUUM") + print(f"[dedup_subagents] Reclaimed disk space for agent {agent}") + conn.close() + except Exception as e: + print(f"[dedup_subagents] Notice for {agent}: {e}", file=sys.stderr) diff --git a/k8s/openclaw/scripts/gateway-preflight.mjs b/k8s/openclaw/scripts/gateway-preflight.mjs index 0c6b2f9..a536d45 100644 --- a/k8s/openclaw/scripts/gateway-preflight.mjs +++ b/k8s/openclaw/scripts/gateway-preflight.mjs @@ -1,78 +1,78 @@ - import fs from "node:fs"; - import path from "node:path"; - import { execSync, spawn } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { execSync, spawn } from "node:child_process"; - if (!process.env.__OPENCLAW_PREFLIGHT_DONE) { - process.env.__OPENCLAW_PREFLIGHT_DONE = "1"; +if (!process.env.__OPENCLAW_PREFLIGHT_DONE) { + process.env.__OPENCLAW_PREFLIGHT_DONE = "1"; - const isGateway = process.argv.some(arg => arg.endsWith("openclaw.mjs") || arg.endsWith("openclaw")) && process.argv.includes("gateway"); + const isGateway = process.argv.some(arg => arg.endsWith("openclaw.mjs") || arg.endsWith("openclaw")) && process.argv.includes("gateway"); - if (isGateway) { - const t0 = Date.now(); - console.log("[gateway-preflight] Initializing container self-healing preflight..."); + if (isGateway) { + const t0 = Date.now(); + console.log("[gateway-preflight] Initializing container self-healing preflight..."); - // 1. Clean up stale gateway lock files from ungraceful kills - for (const dir of ["/tmp/openclaw-1000", "/tmp"]) { - if (fs.existsSync(dir)) { - try { - const files = fs.readdirSync(dir); - for (const file of files) { - if (file.startsWith("gateway") && file.endsWith(".lock")) { - const fullPath = path.join(dir, file); - fs.unlinkSync(fullPath); - console.log(`[gateway-preflight] Removed stale lock: ${fullPath}`); - } - } - } catch (e) { - console.warn(`[gateway-preflight] Lock cleanup warning in ${dir}:`, e.message); - } - } - } - - // 2. Clear false database quarantines - const qPath = "/home/node/.openclaw/state/openclaw-quarantine.sqlite"; - if (fs.existsSync(qPath)) { - try { - execSync(`/home/node/.openclaw/.local/bin/python3 -c "import sqlite3; c=sqlite3.connect(\x27${qPath}\x27); c.execute(\x27DELETE FROM quarantined_databases\x27); c.commit(); c.close()"`, { stdio: "ignore" }); - console.log("[gateway-preflight] Cleared database quarantine."); - } catch (_) {} - } - - // 3. Deduplicate subagent vector databases (keeps non-main agent DBs < 15MB) - try { - execSync("/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/dedup_subagent_vectors.py", { stdio: "inherit" }); - } catch (e) { - console.warn("[gateway-preflight] Vector deduplication warning:", e.message); - } - - // 4. Pre-warm page cache sequentially (transforms random 70 IOPS disk reads into sequential streaming) - try { - const pw0 = Date.now(); - execSync("cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 || true"); - console.log(`[gateway-preflight] Pre-warmed SQLite databases in ${Date.now() - pw0}ms`); - } catch (_) {} - - // 5. Recover any wedged running sessions left over from ungraceful container restart - try { - execSync("/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/recover_sessions.py", { stdio: "inherit" }); - } catch (e) { - console.warn("[gateway-preflight] Session recovery warning:", e.message); - } - - // 6. Spawn background turn replay daemon if interrupted turns were detected - try { - if (fs.existsSync("/tmp/openclaw-interrupted-turns.json")) { - console.log("[gateway-preflight] Spawning background turn replay daemon..."); - const child = spawn("/home/node/.openclaw/.local/bin/python3", ["/home/node/.openclaw/scripts/replay_interrupted_turns.py"], { - detached: true, - stdio: "ignore" - }); - child.unref(); - } - } catch (e) { - console.warn("[gateway-preflight] Could not spawn turn replay daemon:", e.message); - } - - console.log(`[gateway-preflight] Self-healing completed in ${Date.now() - t0}ms.`); + // 1. Clean up stale gateway lock files from ungraceful kills + for (const dir of ["/tmp/openclaw-1000", "/tmp"]) { + if (fs.existsSync(dir)) { + try { + const files = fs.readdirSync(dir); + for (const file of files) { + if (file.startsWith("gateway") && file.endsWith(".lock")) { + const fullPath = path.join(dir, file); + fs.unlinkSync(fullPath); + console.log(`[gateway-preflight] Removed stale lock: ${fullPath}`); } } + } catch (e) { + console.warn(`[gateway-preflight] Lock cleanup warning in ${dir}:`, e.message); + } + } + } + + // 2. Clear false database quarantines + const qPath = "/home/node/.openclaw/state/openclaw-quarantine.sqlite"; + if (fs.existsSync(qPath)) { + try { + execSync(`/home/node/.openclaw/.local/bin/python3 -c "import sqlite3; c=sqlite3.connect(\x27${qPath}\x27); c.execute(\x27DELETE FROM quarantined_databases\x27); c.commit(); c.close()"`, { stdio: "ignore" }); + console.log("[gateway-preflight] Cleared database quarantine."); + } catch (_) {} + } + + // 3. Deduplicate subagent vector databases (keeps non-main agent DBs < 15MB) + try { + execSync("/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/dedup_subagent_vectors.py", { stdio: "inherit" }); + } catch (e) { + console.warn("[gateway-preflight] Vector deduplication warning:", e.message); + } + + // 4. Pre-warm page cache sequentially (transforms random 70 IOPS disk reads into sequential streaming) + try { + const pw0 = Date.now(); + execSync("cat /home/node/.openclaw/state/openclaw.sqlite /home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite > /dev/null 2>&1 || true; find /home/node/.openclaw/wiki/main /home/node/.openclaw/workspace* -maxdepth 3 -name '*.md' -exec cat {} + > /dev/null 2>&1 || true"); + console.log(`[gateway-preflight] Pre-warmed SQLite databases and Wiki markdown in ${Date.now() - pw0}ms`); + } catch (_) {} + + // 5. Recover any wedged running sessions left over from ungraceful container restart + try { + execSync("/home/node/.openclaw/.local/bin/python3 /home/node/.openclaw/scripts/recover_sessions.py", { stdio: "inherit" }); + } catch (e) { + console.warn("[gateway-preflight] Session recovery warning:", e.message); + } + + // 6. Spawn background turn replay daemon if interrupted turns were detected + try { + if (fs.existsSync("/tmp/openclaw-interrupted-turns.json")) { + console.log("[gateway-preflight] Spawning background turn replay daemon..."); + const child = spawn("/home/node/.openclaw/.local/bin/python3", ["/home/node/.openclaw/scripts/replay_interrupted_turns.py"], { + detached: true, + stdio: "ignore" + }); + child.unref(); + } + } catch (e) { + console.warn("[gateway-preflight] Could not spawn turn replay daemon:", e.message); + } + + console.log(`[gateway-preflight] Self-healing completed in ${Date.now() - t0}ms.`); + } +} diff --git a/k8s/openclaw/scripts/openclaw-triage.py b/k8s/openclaw/scripts/openclaw-triage.py index 3c993d2..188c9b9 100644 --- a/k8s/openclaw/scripts/openclaw-triage.py +++ b/k8s/openclaw/scripts/openclaw-triage.py @@ -132,7 +132,7 @@ def triage_agent_session(agent, topic_id=None, limit=10, db_root="/home/node/.op events.reverse() print("\n📜 RECENT TRANSCRIPT EVENTS (Chronological):") - + last_role = None for seq, ca_ms, ej_str in events: try: diff --git a/k8s/openclaw/scripts/recover_sessions.py b/k8s/openclaw/scripts/recover_sessions.py index 8db7cc6..c8840c5 100644 --- a/k8s/openclaw/scripts/recover_sessions.py +++ b/k8s/openclaw/scripts/recover_sessions.py @@ -1,80 +1,96 @@ - #!/usr/bin/env python3 - import glob, sqlite3, json, os, sys, time +#!/usr/bin/env python3 +import glob, sqlite3, json, os, sys, time - t0 = time.time() - recovered = 0 - interrupted_turns = [] +t0 = time.time() +recovered = 0 +interrupted_turns = [] - def extract_prompt(msg): - if not msg: return "" - content = msg.get("content") - if isinstance(content, str): return content - if isinstance(content, list): - parts = [] - for c in content: - if isinstance(c, dict): parts.append(c.get("text") or c.get("content") or "") - elif isinstance(c, str): parts.append(c) - return "\n".join(filter(None, parts)) - return str(content or "") +def extract_prompt(msg): + if not msg: return "" + content = msg.get("content") + if isinstance(content, str): return content + if isinstance(content, list): + parts = [] + for c in content: + if isinstance(c, dict): parts.append(c.get("text") or c.get("content") or "") + elif isinstance(c, str): parts.append(c) + return "\n".join(filter(None, parts)) + return str(content or "") - for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")): - try: - conn = sqlite3.connect(db, timeout=10.0) - cursor = conn.cursor() - while True: - cursor.execute("PRAGMA foreign_key_check;") - violations = cursor.fetchall() - if not violations: break - for table, rowid, parent, fkid in violations: - cursor.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,)) - cursor.execute("SELECT session_key, entry_json, current_session_id FROM session_nodes WHERE status = \"running\"") - rows = cursor.fetchall() - agent_id = os.path.basename(os.path.dirname(os.path.dirname(db))) - for sk, ej, csid in rows: - try: - p = json.loads(ej) if ej else {} - sid = csid or p.get("sessionId") - if sid: - ev = cursor.execute("SELECT seq, event_json FROM transcript_events WHERE session_id = ? ORDER BY seq DESC LIMIT 1", (sid,)).fetchone() - if ev: - ev_seq, ev_raw = ev - try: - ev_data = json.loads(ev_raw) - msg = ev_data.get("message") or ev_data - if msg.get("role") == "user": - prompt_text = extract_prompt(msg) - ctx = p.get("restartRecoveryDeliveryContext") or p.get("delivery", {}).get("context") or p.get("delivery", {}).get("route", {}).get("context") or {} - interrupted_turns.append({ - "agent": agent_id, - "session_key": sk, - "session_id": sid, - "context": ctx, - "prompt": prompt_text - }) - cursor.execute("PRAGMA foreign_keys = ON") - cursor.execute("DELETE FROM transcript_events WHERE session_id = ? AND seq = ?", (sid, ev_seq)) - except Exception as ex: - print(f"[recover_sessions] Error parsing event: {ex}", file=sys.stderr) - p["status"] = "done" - p["activeWriterRunId"] = None - p["activeClaimRunId"] = None - p["restartRecoveryDeliveryRunId"] = None - cursor.execute("UPDATE session_nodes SET status = \"done\", entry_json = ? WHERE session_key = ?", (json.dumps(p), sk)) - cursor.execute("UPDATE session_nodes SET entry_valid = 1 WHERE session_key = ?", (sk,)) - recovered += 1 - except Exception: pass - cursor.execute("UPDATE session_windows SET status = \"done\" WHERE status = \"running\"") - conn.commit() - conn.close() - except Exception as e: - print(f"[recover_sessions] Warning for {db}: {e}", file=sys.stderr) +for db in sorted(glob.glob("/home/node/.openclaw/agents/*/agent/openclaw-agent.sqlite")): + try: + conn = sqlite3.connect(db, timeout=10.0) + cursor = conn.cursor() + while True: + cursor.execute("PRAGMA foreign_key_check;") + violations = cursor.fetchall() + if not violations: break + for table, rowid, parent, fkid in violations: + cursor.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,)) + cursor.execute("SELECT session_key, entry_json, current_session_id, status FROM session_nodes WHERE status IN (\"running\", \"failed\", \"error\")") + rows = cursor.fetchall() + agent_id = os.path.basename(os.path.dirname(os.path.dirname(db))) + for sk, ej, csid, st in rows: + try: + p = json.loads(ej) if ej else {} + sid = csid or p.get("sessionId") + if sid: + ev = cursor.execute("SELECT seq, event_json FROM transcript_events WHERE session_id = ? ORDER BY seq DESC LIMIT 1", (sid,)).fetchone() + if ev: + ev_seq, ev_raw = ev + try: + ev_data = json.loads(ev_raw) + msg = ev_data.get("message") or ev_data + if st == "running" and msg.get("role") == "user": + prompt_text = extract_prompt(msg) + ctx = p.get("restartRecoveryDeliveryContext") or p.get("delivery", {}).get("context") or p.get("delivery", {}).get("route", {}).get("context") or {} + interrupted_turns.append({ + "agent": agent_id, + "session_key": sk, + "session_id": sid, + "context": ctx, + "prompt": prompt_text + }) + cursor.execute("PRAGMA foreign_keys = ON") + cursor.execute("DELETE FROM transcript_events WHERE session_id = ? AND seq = ?", (sid, ev_seq)) + elif msg.get("role") == "assistant" and msg.get("stopReason") == "error": + cursor.execute("PRAGMA foreign_keys = ON") + cursor.execute("DELETE FROM transcript_events WHERE session_id = ? AND seq = ?", (sid, ev_seq)) + except Exception as ex: + print(f"[recover_sessions] Error parsing event: {ex}", file=sys.stderr) + p["status"] = "done" + p["activeWriterRunId"] = None + p["activeClaimRunId"] = None + p["restartRecoveryDeliveryRunId"] = None + cursor.execute("UPDATE session_nodes SET status = \"done\", entry_json = ? WHERE session_key = ?", (json.dumps(p), sk)) + cursor.execute("UPDATE session_nodes SET entry_valid = 1 WHERE session_key = ?", (sk,)) + recovered += 1 + except Exception: pass + cursor.execute("UPDATE session_nodes SET entry_valid = 1 WHERE entry_valid != 1") + cursor.execute("UPDATE session_windows SET status = \"done\" WHERE status IN (\"running\", \"failed\", \"error\")") + conn.commit() + conn.close() + except Exception as e: + print(f"[recover_sessions] Warning for {db}: {e}", file=sys.stderr) - if interrupted_turns: - try: - with open("/tmp/openclaw-interrupted-turns.json", "w") as f: - json.dump(interrupted_turns, f, indent=2) - print(f"[recover_sessions] Detected {len(interrupted_turns)} interrupted user turn(s) saved to /tmp/openclaw-interrupted-turns.json") - except Exception as e: - print(f"[recover_sessions] Error writing interrupted turns: {e}", file=sys.stderr) +state_db = "/home/node/.openclaw/state/openclaw.sqlite" +if os.path.exists(state_db): + try: + sconn = sqlite3.connect(state_db, timeout=10.0) + scur = sconn.cursor() + scur.execute("UPDATE channel_ingress_events SET attempts = 0, last_attempt_at = NULL, last_error = NULL WHERE status = 'pending' AND attempts > 0") + sconn.commit() + sconn.close() + except Exception as e: + print(f"[recover_sessions] Warning resetting ingress backoff: {e}", file=sys.stderr) + +if interrupted_turns: + try: + with open("/tmp/openclaw-interrupted-turns.json", "w") as f: + json.dump(interrupted_turns, f, indent=2) + print(f"[recover_sessions] Detected {len(interrupted_turns)} interrupted user turn(s) saved to /tmp/openclaw-interrupted-turns.json") + except Exception as e: + print(f"[recover_sessions] Error writing interrupted turns: {e}", file=sys.stderr) + +print(f"[recover_sessions] Completed in {time.time() - t0:.2f}s ({recovered} wedged sessions recovered)") - print(f"[recover_sessions] Completed in {time.time() - t0:.2f}s ({recovered} wedged sessions recovered)") diff --git a/k8s/openclaw/scripts/replay_interrupted_turns.py b/k8s/openclaw/scripts/replay_interrupted_turns.py index 47f5e0a..6774e1b 100644 --- a/k8s/openclaw/scripts/replay_interrupted_turns.py +++ b/k8s/openclaw/scripts/replay_interrupted_turns.py @@ -1,97 +1,97 @@ - #!/usr/bin/env python3 - import json, os, subprocess, sys, time, urllib.request +#!/usr/bin/env python3 +import json, os, subprocess, sys, time, urllib.request - INTERRUPTED_FILE = "/tmp/openclaw-interrupted-turns.json" +INTERRUPTED_FILE = "/tmp/openclaw-interrupted-turns.json" - if not os.path.exists(INTERRUPTED_FILE): - sys.exit(0) +if not os.path.exists(INTERRUPTED_FILE): + sys.exit(0) - try: - with open(INTERRUPTED_FILE, "r") as f: - turns = json.load(f) - except Exception as e: - print(f"[replay_interrupted_turns] Failed to read {INTERRUPTED_FILE}: {e}", file=sys.stderr) - sys.exit(0) +try: + with open(INTERRUPTED_FILE, "r") as f: + turns = json.load(f) +except Exception as e: + print(f"[replay_interrupted_turns] Failed to read {INTERRUPTED_FILE}: {e}", file=sys.stderr) + sys.exit(0) - if not turns: - sys.exit(0) +if not turns: + sys.exit(0) - print(f"[replay_interrupted_turns] Found {len(turns)} interrupted turn(s). Waiting for gateway /healthz...") +print(f"[replay_interrupted_turns] Found {len(turns)} interrupted turn(s). Waiting for gateway /healthz...") - gateway_ready = False - for _ in range(60): - try: - with urllib.request.urlopen("http://127.0.0.1:18790/healthz", timeout=3) as resp: - if resp.status == 200: - gateway_ready = True - break - except Exception: - pass - time.sleep(2) +gateway_ready = False +for _ in range(60): + try: + with urllib.request.urlopen("http://127.0.0.1:18790/healthz", timeout=3) as resp: + if resp.status == 200: + gateway_ready = True + break + except Exception: + pass + time.sleep(2) - if not gateway_ready: - print("[replay_interrupted_turns] Gateway did not become healthy in 120s. Aborting.", file=sys.stderr) - sys.exit(1) +if not gateway_ready: + print("[replay_interrupted_turns] Gateway did not become healthy in 120s. Aborting.", file=sys.stderr) + sys.exit(1) - # Allow 5s for Telegram ingress to bind - time.sleep(5) +# Allow 5s for Telegram ingress to bind +time.sleep(5) - for turn in turns: - sk = turn.get("session_key") - sid = turn.get("session_id") - agent_id = turn.get("agent") - ctx = turn.get("context") or {} - prompt = turn.get("prompt") or "" - - channel = ctx.get("channel") or "telegram" - to_target = ctx.get("to") or "" - thread_id = ctx.get("threadId") - - target = to_target - if target.startswith("telegram:"): - parts = target.split(":") - target = parts[1] if len(parts) > 1 else target +for turn in turns: + sk = turn.get("session_key") + sid = turn.get("session_id") + agent_id = turn.get("agent") + ctx = turn.get("context") or {} + prompt = turn.get("prompt") or "" - print(f"[replay_interrupted_turns] Processing agent '{agent_id}' on session '{sk}' target='{target}' thread='{thread_id}'") - - # 1. Send status notification to Telegram if target exists - if target: - notice = "⚠️ OpenClaw wurde während der Bearbeitung neu gestartet – deine Anfrage wird jetzt nachgeholt..." - notify_cmd = [ - "openclaw", "message", "send", - "--channel", channel, - "--target", str(target), - "-m", notice - ] - if thread_id is not None: - notify_cmd.extend(["--thread-id", str(thread_id)]) - try: - subprocess.run(notify_cmd, check=False, timeout=30) - print(f"[replay_interrupted_turns] Sent recovery notice to {target}") - except Exception as e: - print(f"[replay_interrupted_turns] Notice failed: {e}", file=sys.stderr) + channel = ctx.get("channel") or "telegram" + to_target = ctx.get("to") or "" + thread_id = ctx.get("threadId") - # 2. Replay agent turn - if sk and prompt: - p_short = (prompt[:60] + "...") if len(prompt) > 60 else prompt - print(f"[replay_interrupted_turns] Replaying agent turn: {p_short}") - agent_cmd = [ - "openclaw", "agent", - "--session-key", sk, - "-m", prompt, - "--deliver" - ] - try: - res = subprocess.run(agent_cmd, capture_output=True, text=True, timeout=300) - if res.returncode == 0: - print(f"[replay_interrupted_turns] Turn replay succeeded for {sk}") - else: - print(f"[replay_interrupted_turns] Turn replay exited with code {res.returncode}: {res.stderr[:200]}", file=sys.stderr) - except Exception as e: - print(f"[replay_interrupted_turns] Replay execution error: {e}", file=sys.stderr) + target = to_target + if target.startswith("telegram:"): + parts = target.split(":") + target = parts[1] if len(parts) > 1 else target - try: - os.remove(INTERRUPTED_FILE) - print(f"[replay_interrupted_turns] Cleaned up {INTERRUPTED_FILE}") - except Exception: - pass + print(f"[replay_interrupted_turns] Processing agent '{agent_id}' on session '{sk}' target='{target}' thread='{thread_id}'") + + # 1. Send status notification to Telegram if target exists + if target: + notice = "⚠️ OpenClaw wurde während der Bearbeitung neu gestartet – deine Anfrage wird jetzt nachgeholt..." + notify_cmd = [ + "openclaw", "message", "send", + "--channel", channel, + "--target", str(target), + "-m", notice + ] + if thread_id is not None: + notify_cmd.extend(["--thread-id", str(thread_id)]) + try: + subprocess.run(notify_cmd, check=False, timeout=30) + print(f"[replay_interrupted_turns] Sent recovery notice to {target}") + except Exception as e: + print(f"[replay_interrupted_turns] Notice failed: {e}", file=sys.stderr) + + # 2. Replay agent turn + if sk and prompt: + p_short = (prompt[:60] + "...") if len(prompt) > 60 else prompt + print(f"[replay_interrupted_turns] Replaying agent turn: {p_short}") + agent_cmd = [ + "openclaw", "agent", + "--session-key", sk, + "-m", prompt, + "--deliver" + ] + try: + res = subprocess.run(agent_cmd, capture_output=True, text=True, timeout=300) + if res.returncode == 0: + print(f"[replay_interrupted_turns] Turn replay succeeded for {sk}") + else: + print(f"[replay_interrupted_turns] Turn replay exited with code {res.returncode}: {res.stderr[:200]}", file=sys.stderr) + except Exception as e: + print(f"[replay_interrupted_turns] Replay execution error: {e}", file=sys.stderr) + +try: + os.remove(INTERRUPTED_FILE) + print(f"[replay_interrupted_turns] Cleaned up {INTERRUPTED_FILE}") +except Exception: + pass diff --git a/k8s/openclaw/setup-garmin-cron.sh b/k8s/openclaw/setup-garmin-cron.sh index 9ee2644..cc984b2 100755 --- a/k8s/openclaw/setup-garmin-cron.sh +++ b/k8s/openclaw/setup-garmin-cron.sh @@ -54,7 +54,7 @@ kubectl exec -n "$NAMESPACE" "$POD_NAME" -c "$CONTAINER_NAME" -- openclaw cron a --cron "$CRON_SCHEDULE" \ --tz "$TIMEZONE" \ --agent "fitness" \ - --model "openrouter/deepseek/deepseek-v4.1-flash" \ + --model "openrouter/deepseek/deepseek-v4.1-flash-1017" \ --session "isolated" \ --channel "telegram" \ --to "$TELEGRAM_GROUP" \ diff --git a/k8s/openclaw/setup-itguy-cron.sh b/k8s/openclaw/setup-itguy-cron.sh index fa7e272..f32c313 100755 --- a/k8s/openclaw/setup-itguy-cron.sh +++ b/k8s/openclaw/setup-itguy-cron.sh @@ -48,7 +48,7 @@ kubectl exec -n "$NAMESPACE" "$POD_NAME" -c "$CONTAINER_NAME" -- openclaw cron a --cron "$CRON_SCHEDULE" \ --tz "$TIMEZONE" \ --agent "itguy" \ - --model "openrouter/deepseek/deepseek-v4.1-flash" \ + --model "openrouter/deepseek/deepseek-v4.1-flash-1017" \ --session "isolated" \ --channel "telegram" \ --to "$TELEGRAM_GROUP" \ diff --git a/knowledge-wiki b/knowledge-wiki index 419b006..720c5cf 160000 --- a/knowledge-wiki +++ b/knowledge-wiki @@ -1 +1 @@ -Subproject commit 419b00657436708969a5f69fe00baee998fe9c48 +Subproject commit 720c5cf74bf9625c88411649ff9ca2724c18119a diff --git a/projects/duckdb-backend/app/engine.py b/projects/duckdb-backend/app/engine.py index 1e26478..a753fb4 100644 --- a/projects/duckdb-backend/app/engine.py +++ b/projects/duckdb-backend/app/engine.py @@ -734,44 +734,93 @@ class DuckDBEngine: today_wet_count = int(diaper_agg[1]) if diaper_agg and diaper_agg[1] is not None else 0 today_dirty_count = int(diaper_agg[2]) if diaper_agg and diaper_agg[2] is not None else 0 - # 5. 7d and 30d sleep averages + # 5. 7d and 30d sleep averages (use completed days < today_str when available) avg_7 = self.con.execute( - """ + f""" SELECT ROUND(AVG(total_sleep_hours), 2), ROUND(AVG(night_sleep_minutes)/60.0, 2), ROUND(AVG(nap_sleep_minutes)/60.0, 2), ROUND(AVG(nap_count), 1) - FROM (SELECT total_sleep_hours, night_sleep_minutes, nap_sleep_minutes, nap_count - FROM v_sleep_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7) + FROM ( + SELECT total_sleep_hours, night_sleep_minutes, nap_sleep_minutes, nap_count + FROM v_sleep_daily + WHERE subject = 'mila' AND log_date < '{today_str}' + ORDER BY log_date DESC LIMIT 7 + ) """ ).fetchone() + if not avg_7 or avg_7[0] is None: + avg_7 = self.con.execute( + """ + SELECT ROUND(AVG(total_sleep_hours), 2), ROUND(AVG(night_sleep_minutes)/60.0, 2), + ROUND(AVG(nap_sleep_minutes)/60.0, 2), ROUND(AVG(nap_count), 1) + FROM (SELECT total_sleep_hours, night_sleep_minutes, nap_sleep_minutes, nap_count + FROM v_sleep_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7) + """ + ).fetchone() avg_7_tot = f"{avg_7[0]:.2f} h" if avg_7 and avg_7[0] is not None else "—" avg_7_night = f"{avg_7[1]:.2f} h" if avg_7 and avg_7[1] is not None else "—" avg_7_naps = f"{avg_7[3]} Naps ({avg_7[2]:.2f} h)" if avg_7 and avg_7[3] is not None else "—" avg_30 = self.con.execute( - """ + f""" SELECT ROUND(AVG(total_sleep_hours), 2) - FROM (SELECT total_sleep_hours FROM v_sleep_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 30) + FROM ( + SELECT total_sleep_hours FROM v_sleep_daily + WHERE subject = 'mila' AND log_date < '{today_str}' + ORDER BY log_date DESC LIMIT 30 + ) """ ).fetchone() + if not avg_30 or avg_30[0] is None: + avg_30 = self.con.execute( + """ + SELECT ROUND(AVG(total_sleep_hours), 2) + FROM (SELECT total_sleep_hours FROM v_sleep_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 30) + """ + ).fetchone() avg_30_tot = f"{avg_30[0]:.2f} h" if avg_30 and avg_30[0] is not None else "—" feed_avg_7 = self.con.execute( - """ + f""" SELECT ROUND(AVG(total_consumed_ml), 0), ROUND(AVG(total_solid_g), 0), ROUND(AVG(total_kcal), 0), ROUND(AVG(bottle_count), 1) - FROM (SELECT total_consumed_ml, total_solid_g, total_kcal, bottle_count - FROM v_feeding_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7) + FROM ( + SELECT total_consumed_ml, total_solid_g, total_kcal, bottle_count + FROM v_feeding_daily + WHERE subject = 'mila' AND log_date < '{today_str}' + ORDER BY log_date DESC LIMIT 7 + ) """ ).fetchone() + if not feed_avg_7 or feed_avg_7[0] is None: + feed_avg_7 = self.con.execute( + """ + SELECT ROUND(AVG(total_consumed_ml), 0), ROUND(AVG(total_solid_g), 0), + ROUND(AVG(total_kcal), 0), ROUND(AVG(bottle_count), 1) + FROM (SELECT total_consumed_ml, total_solid_g, total_kcal, bottle_count + FROM v_feeding_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7) + """ + ).fetchone() diaper_avg_7 = self.con.execute( - """ + f""" SELECT ROUND(AVG(wet_count), 1), ROUND(AVG(total_changes), 1) - FROM (SELECT wet_count, total_changes - FROM v_diaper_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7) + FROM ( + SELECT wet_count, total_changes + FROM v_diaper_daily + WHERE subject = 'mila' AND log_date < '{today_str}' + ORDER BY log_date DESC LIMIT 7 + ) """ ).fetchone() + if not diaper_avg_7 or diaper_avg_7[0] is None: + diaper_avg_7 = self.con.execute( + """ + SELECT ROUND(AVG(wet_count), 1), ROUND(AVG(total_changes), 1) + FROM (SELECT wet_count, total_changes + FROM v_diaper_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7) + """ + ).fetchone() # 6. Past 30 Days History Table & sessions history_rows_data = self.con.execute( @@ -837,10 +886,18 @@ class DuckDBEngine: for r in history_rows_data: d_str = str(r[0]) d_obj = datetime.date.fromisoformat(d_str) - nh, nm = divmod(r[2] or 0, 60) - d_sessions = sessions_by_date.get(d_str, []) + d_sessions = list(sessions_by_date.get(d_str, [])) + row_night_min = int(r[2] or 0) + row_nap_min = int(r[4] or 0) + row_tot_hours = round(r[1], 2) if r[1] is not None else 0.0 + + if d_str == today_str and row_night_min == 0 and today_night_min > 0: + row_night_min = today_night_min + row_tot_hours = round((row_night_min + row_nap_min) / 60.0, 2) + + nh, nm = divmod(row_night_min, 60) night_durs = [s["duration_minutes"] for s in d_sessions if s["type"] == "night" and s["duration_minutes"]] - longest_night = max(night_durs) if night_durs else int(r[2] or 0) + longest_night = max(night_durs) if night_durs else row_night_min wake_windows = [] day_sessions = [s for s in d_sessions if s["type"] == "nap" and s.get("local_start") and s.get("local_end")] @@ -860,12 +917,12 @@ class DuckDBEngine: "weekday": WEEKDAYS_DE[d_obj.weekday()], "formatted_date": d_obj.strftime("%d.%m."), "is_today": (d_str == today_str), - "total_hours": round(r[1], 2) if r[1] is not None else 0.0, - "night_minutes": int(r[2] or 0), + "total_hours": row_tot_hours, + "night_minutes": row_night_min, "night_duration_str": f"{nh} h {nm:02d} min" if nh > 0 else f"{nm} min", - "nap_minutes": int(r[4] or 0), + "nap_minutes": row_nap_min, "nap_count": int(r[3] or 0), - "nap_str": f"{r[3]} ({r[4]} min)", + "nap_str": f"{r[3]} ({row_nap_min} min)", "longest_night_stretch_min": longest_night, "avg_wake_window_min": avg_wake, "feed_bottles_ml": int(r[5]), @@ -1457,6 +1514,37 @@ class DuckDBEngine: for r in records: f.write(json.dumps(r, ensure_ascii=False) + "\n") + def _parse_event_dt( + self, + raw: str, + now_dt: datetime.datetime, + actor: str = "", + base_date: Optional[str] = None, + ) -> datetime.datetime: + s = raw.strip() + m_time = re.fullmatch(r"^(\d{1,2}):(\d{2})(?::(\d{2}))?$", s) + if m_time: + d_part = base_date or now_dt.astimezone(BERLIN_TZ).strftime("%Y-%m-%d") + hh = int(m_time.group(1)) + mm = int(m_time.group(2)) + ss = int(m_time.group(3) or 0) + return datetime.datetime.fromisoformat(f"{d_part}T{hh:02d}:{mm:02d}:{ss:02d}").replace(tzinfo=BERLIN_TZ) + + dt = datetime.datetime.fromisoformat(s.replace("Z", "+00:00")) + if dt.tzinfo is None: + return dt.replace(tzinfo=BERLIN_TZ) + + if s.endswith("+00:00") or s.endswith("Z"): + dt_wall_berlin = dt.replace(tzinfo=BERLIN_TZ) + # Guard against LLM agents formatting local Europe/Berlin wall-clock times with +00:00 / Z + if ( + (dt > now_dt + datetime.timedelta(minutes=5) and dt_wall_berlin <= now_dt + datetime.timedelta(minutes=5)) + or ((actor or "").startswith("agent:") and dt.second == 0 and dt.microsecond == 0) + ): + return dt_wall_berlin + + return dt.astimezone(BERLIN_TZ) + def add_event(self, req: EventCreateRequest) -> Dict[str, Any]: with self._lock: event_id = str(ULID()) @@ -1476,9 +1564,7 @@ class DuckDBEngine: start_dt = (now_dt - delta).astimezone(BERLIN_TZ) start_iso = start_dt.isoformat() else: - start_dt = datetime.datetime.fromisoformat(req.start) - if start_dt.tzinfo is None: - start_dt = start_dt.replace(tzinfo=BERLIN_TZ) + start_dt = self._parse_event_dt(req.start, now_dt, actor=req.actor, base_date=req.date) start_iso = start_dt.isoformat() elif req.since: delta = parse_duration(req.since) @@ -1488,21 +1574,48 @@ class DuckDBEngine: start_dt = now_dt.astimezone(BERLIN_TZ) start_iso = start_dt.isoformat() else: - start_dt = datetime.datetime.fromisoformat(req.start) - if start_dt.tzinfo is None: - start_dt = start_dt.replace(tzinfo=BERLIN_TZ) + start_dt = self._parse_event_dt(req.start, now_dt, actor=req.actor) start_iso = start_dt.isoformat() end_iso = None end_dt = None if req.end: - end_dt = datetime.datetime.fromisoformat(req.end) - if end_dt.tzinfo is None: - end_dt = end_dt.replace(tzinfo=BERLIN_TZ) + end_dt = self._parse_event_dt( + req.end, + now_dt, + actor=req.actor, + base_date=start_dt.astimezone(BERLIN_TZ).strftime("%Y-%m-%d"), + ) if end_dt < start_dt: raise ValueError(f"End time {req.end} cannot be earlier than start time {req.start}") end_iso = end_dt.isoformat() + # Populate Mila standard recipe defaults when omitted by tracking agents + if req.subject == "mila": + if req.kind == "feeding.solid" or (req.kind == "nutrition.intake" and "brei" in str(req.payload.get("item") or req.payload.get("type") or "").lower()): + if req.kind == "nutrition.intake": + req.kind = "feeding.solid" + has_grams = any(req.payload.get(k) not in (None, 0) for k in ("amount_g", "consumed_g", "qty")) + if not has_grams: + item_l = str(req.payload.get("item") or req.meal or "").lower() + if any(w in item_l for w in ("abend", "nachmittag", "dinner")): + req.payload["amount_g"] = 200 + req.payload.setdefault("kcal", 110) + else: + req.payload["amount_g"] = 190 + req.payload.setdefault("kcal", 135) + elif req.kind == "feeding.bottle": + c_ml = req.payload.get("consumed_ml") + a_ml = req.payload.get("amount_ml") + l_ml = req.payload.get("left_ml") + if (c_ml is None or c_ml == 0) and (a_ml is None or a_ml == 0): + if l_ml is not None: + req.payload["amount_ml"] = 270 + req.payload["consumed_ml"] = max(0, 270 - int(l_ml)) + else: + req.payload["amount_ml"] = 270 + req.payload["consumed_ml"] = 270 + # 15-Minute Duplicate Guard if not req.allow_duplicate: if req.kind in ("nutrition.intake", "feeding.bottle", "feeding.solid"): @@ -1722,9 +1835,7 @@ class DuckDBEngine: delta = parse_duration(req.since) end_dt = (now_dt - delta).astimezone(BERLIN_TZ) elif req.end_time: - end_dt = datetime.datetime.fromisoformat(req.end_time) - if end_dt.tzinfo is None: - end_dt = end_dt.replace(tzinfo=BERLIN_TZ) + end_dt = self._parse_event_dt(req.end_time, now_dt, actor=req.actor) else: end_dt = now_dt.astimezone(BERLIN_TZ) diff --git a/projects/duckdb-backend/app/git_sync.py b/projects/duckdb-backend/app/git_sync.py index ebf2eb1..93d89cd 100644 --- a/projects/duckdb-backend/app/git_sync.py +++ b/projects/duckdb-backend/app/git_sync.py @@ -43,7 +43,7 @@ class GitSyncManager: try: subprocess.run(["git", "config", "--global", "http.sslVerify", "false"], check=False) subprocess.run(["git", "config", "--global", "--add", "safe.directory", "*"], check=False) - if (self.wiki_dir / ".git").is_dir(): + if (self.wiki_dir / ".git").exists(): subprocess.run(["git", "-C", str(self.wiki_dir), "config", "user.name", GIT_AUTHOR_NAME], check=False) subprocess.run(["git", "-C", str(self.wiki_dir), "config", "user.email", GIT_AUTHOR_EMAIL], check=False) except Exception as e: @@ -64,7 +64,7 @@ class GitSyncManager: """Ensures repository exists and is initialized.""" with self._lock: self._configure_git_client() - if (self.wiki_dir / ".git").is_dir(): + if (self.wiki_dir / ".git").exists(): try: res = subprocess.run( ["git", "-C", str(self.wiki_dir), "rev-parse", "--short", "HEAD"], @@ -144,7 +144,7 @@ class GitSyncManager: def pull_rebase(self) -> bool: """Pulls latest changes via rebase. Handles conflicts cleanly.""" - if not (self.wiki_dir / ".git").is_dir(): + if not (self.wiki_dir / ".git").exists(): return False with self._lock: @@ -208,7 +208,7 @@ class GitSyncManager: def commit_and_push(self, commit_msg: str) -> bool: """Stages data/, commits with descriptive message, rebases and pushes.""" - if not (self.wiki_dir / ".git").is_dir(): + if not (self.wiki_dir / ".git").exists(): return False with self._lock: diff --git a/projects/duckdb-backend/app/main.py b/projects/duckdb-backend/app/main.py index 56fc9e0..1c4a990 100644 --- a/projects/duckdb-backend/app/main.py +++ b/projects/duckdb-backend/app/main.py @@ -21,23 +21,26 @@ logger = logging.getLogger("duckdb_backend") @asynccontextmanager async def lifespan(app: FastAPI): logger.info("Initializing DuckDB Backend...") - git_sync = GitSyncManager( - wiki_dir=WIKI_DIR, - on_data_pulled=lambda: app.state.engine.reload_all() if hasattr(app.state, "engine") else None, - ) - git_sync.ensure_repo() - app.state.git_sync = git_sync + if getattr(app.state, "git_sync", None) is None or getattr(app.state, "engine", None) is None: + git_sync = GitSyncManager( + wiki_dir=WIKI_DIR, + on_data_pulled=lambda: app.state.engine.reload_all() if hasattr(app.state, "engine") else None, + ) + git_sync.ensure_repo() + app.state.git_sync = git_sync - engine = DuckDBEngine(data_dir=DATA_DIR, git_sync_manager=git_sync) - app.state.engine = engine + engine = DuckDBEngine(data_dir=DATA_DIR, git_sync_manager=git_sync) + app.state.engine = engine - await git_sync.start_periodic_sync() - logger.info("DuckDB Backend is ready to serve queries.") - - yield - - logger.info("Shutting down DuckDB Backend...") - await git_sync.stop_periodic_sync() + await git_sync.start_periodic_sync() + logger.info("DuckDB Backend is ready to serve queries.") + try: + yield + finally: + logger.info("Shutting down DuckDB Backend...") + await git_sync.stop_periodic_sync() + else: + yield app = FastAPI( diff --git a/projects/duckdb-backend/tests/conftest.py b/projects/duckdb-backend/tests/conftest.py index 5e76595..26cb453 100644 --- a/projects/duckdb-backend/tests/conftest.py +++ b/projects/duckdb-backend/tests/conftest.py @@ -13,13 +13,14 @@ from app.main import app @pytest.fixture(scope="session") def test_data_dir(): - # Use real wiki data dir if exists, else temporary copy + temp_root = Path(tempfile.mkdtemp()) + temp_data = temp_root / "data" if DATA_DIR.is_dir(): - yield DATA_DIR + shutil.copytree(DATA_DIR, temp_data) else: - temp_dir = Path(tempfile.mkdtemp()) - yield temp_dir - shutil.rmtree(temp_dir, ignore_errors=True) + temp_data.mkdir(parents=True, exist_ok=True) + yield temp_data + shutil.rmtree(temp_root, ignore_errors=True) @pytest.fixture diff --git a/projects/duckdb-backend/tests/test_api_events.py b/projects/duckdb-backend/tests/test_api_events.py index 1230215..e243b74 100644 --- a/projects/duckdb-backend/tests/test_api_events.py +++ b/projects/duckdb-backend/tests/test_api_events.py @@ -127,3 +127,50 @@ def test_duplicate_open_sleep_api_endpoint(client: TestClient): "subject": "test_baby", }) assert stop_res.status_code == 200 + + +def test_agent_utc_wallclock_guard_and_mila_recipe_defaults(tmp_path: Path): + engine = DuckDBEngine(data_dir=tmp_path) + + # 1. Agent passes +00:00 on local wall-clock times (e.g. 08:30 to 10:20) + nap_req = EventCreateRequest( + kind="sleep.session", + subject="mila", + actor="agent:mila_tracker", + start="2026-10-17T08:30:00+00:00", + end="2026-10-17T10:20:00+00:00", + payload={"type": "nap", "location": "bett"}, + ) + res_nap = engine.add_event(nap_req) + assert res_nap["status"] == "success" + assert res_nap["record"]["startIso"] == "2026-10-17T08:30:00+02:00" + assert res_nap["record"]["endIso"] == "2026-10-17T10:20:00+02:00" + assert res_nap["record"]["sleepTimeSeconds"] == 6600 + + # 2. Solid food without amount_g gets standard recipe defaults + brei_req = EventCreateRequest( + kind="feeding.solid", + subject="mila", + actor="agent:mila_tracker", + start="2026-10-17T12:20:00+00:00", + meal="lunch", + payload={"item": "mittagsbrei"}, + ) + res_brei = engine.add_event(brei_req) + assert res_brei["record"]["start"] == "2026-10-17T12:20:00+02:00" + assert res_brei["record"]["payload"]["amount_g"] == 190 + assert res_brei["record"]["payload"]["kcal"] == 135 + + abend_req = EventCreateRequest( + kind="feeding.solid", + subject="mila", + actor="agent:mila_tracker", + start="2026-10-17T16:40:00+00:00", + meal="dinner", + payload={"item": "abendbrei"}, + ) + res_abend = engine.add_event(abend_req) + assert res_abend["record"]["start"] == "2026-10-17T16:40:00+02:00" + assert res_abend["record"]["payload"]["amount_g"] == 200 + assert res_abend["record"]["payload"]["kcal"] == 110 + diff --git a/projects/mila/.dockerignore b/projects/mila/.dockerignore new file mode 100644 index 0000000..c6590c8 --- /dev/null +++ b/projects/mila/.dockerignore @@ -0,0 +1,4 @@ +node_modules +dist +.git +.DS_Store diff --git a/projects/mila/src/App.vue b/projects/mila/src/App.vue index 64800c3..16dda6b 100644 --- a/projects/mila/src/App.vue +++ b/projects/mila/src/App.vue @@ -178,7 +178,16 @@ const liveCounterStr = computed(() => { // Wake Window status evaluation (sweet spot 135 to 180 min, i.e. 2h15m to 3h00m) const wakeWindowStatus = computed(() => { if (!data.value || data.value.live_status.is_sleeping) return null - const mins = data.value.live_status.wake_window_minutes ?? 0 + const ls = data.value.live_status + let mins = ls.wake_window_minutes ?? 0 + if (ls.last_wake_iso) { + try { + const wakeTime = new Date(ls.last_wake_iso).getTime() + if (!isNaN(wakeTime)) { + mins = Math.max(0, Math.floor((nowEpoch.value - wakeTime) / 60000)) + } + } catch {} + } if (mins < 135) { return { text: 'Entspannt wach', @@ -255,8 +264,21 @@ const lastFeedDetail = computed(() => { const diaperStatus = computed(() => { if (!data.value) return null const ls = data.value.live_status - const elapsed = ls.last_diaper_since_minutes ?? (ls as any).last_diaper_elapsed_minutes - const elapsedStool = (ls as any).last_stool_elapsed_minutes + let elapsed = ls.last_diaper_since_minutes ?? (ls as any).last_diaper_elapsed_minutes + let elapsedStool = (ls as any).last_stool_elapsed_minutes + + if ((ls as any).last_diaper_iso) { + try { + const t = new Date((ls as any).last_diaper_iso).getTime() + if (!isNaN(t)) elapsed = Math.max(0, Math.floor((nowEpoch.value - t) / 60000)) + } catch {} + } + if ((ls as any).last_stool_iso) { + try { + const t = new Date((ls as any).last_stool_iso).getTime() + if (!isNaN(t)) elapsedStool = Math.max(0, Math.floor((nowEpoch.value - t) / 60000)) + } catch {} + } const formatElapsed = (mins: number | null | undefined) => { if (mins === null || mins === undefined) return null @@ -549,6 +571,50 @@ const currentAgeMonths = computed(() => {
+ +
+
+ 🌙 + Nachtschlaf ({{ data.today.nights.length }} erfasst) +
+
+
+
+
+ 🌙 +
+
+
+ {{ ns.start }} – {{ ns.end || 'schläft gerade' }} Uhr + +
+
+ ({{ ns.location }}) + · {{ ns.note }} +
+
+
+
+ + {{ ns.duration_minutes ? `${ns.duration_minutes} min` : 'in Ausführung' }} + + +
+
+
+
+
diff --git a/projects/mila/src/components/SleepChart.vue b/projects/mila/src/components/SleepChart.vue index 3aedad6..a49f6be 100644 --- a/projects/mila/src/components/SleepChart.vue +++ b/projects/mila/src/components/SleepChart.vue @@ -57,7 +57,10 @@ const bars = computed(() => { let nightHours = 0 let napHours = 0 - if (day.sessions && day.sessions.length > 0) { + if (typeof day.night_minutes === 'number' || typeof day.nap_minutes === 'number') { + nightHours = Math.max(0, (day.night_minutes || 0) / 60) + napHours = Math.max(0, (day.nap_minutes || 0) / 60) + } else if (day.sessions && day.sessions.length > 0) { for (const s of day.sessions) { const mins = s.duration_minutes || 0 if (s.type === 'night') { @@ -71,7 +74,7 @@ const bars = computed(() => { napHours = Math.max(0, day.total_hours - nightHours) } - const totalHours = Math.max(0, day.total_hours) + const totalHours = Math.max(0, day.total_hours, nightHours + napHours) const yTotal = getY(totalHours) const yNight = getY(nightHours) const nightHeight = Math.max(0, (paddingTop + innerHeight) - yNight) @@ -172,6 +175,59 @@ function getMinuteFromTime(timeStr: string): number { return parseInt(parts[0], 10) * 60 + parseInt(parts[1], 10) } +interface RasterBlock { + startMin: number + durationMin: number + type: 'night' | 'nap' + label: string + title: string +} + +function getDay24hBlocks(day: HistoryRow, dayIdx: number): RasterBlock[] { + const blocks: RasterBlock[] = [] + + // 1. Check previous calendar day (props.history[dayIdx + 1]) for overnight sleep spilling into 00:00..wake_time + const prevDay = props.history[dayIdx + 1] + if (prevDay?.sessions) { + for (const s of prevDay.sessions) { + if (!s.start) continue + const sMin = getMinuteFromTime(s.start) + const dur = s.duration_minutes || 60 + if (sMin + dur > 1440) { + const morningMin = Math.min(1440, (sMin + dur) - 1440) + if (morningMin > 0) { + blocks.push({ + startMin: 0, + durationMin: morningMin, + type: s.type === 'night' ? 'night' : 'nap', + label: `${morningMin}m`, + title: `Nachtschlaf (Vorabend ${s.start} – ${s.end || 'morgens'}): ${morningMin} min nach 00:00 Uhr (${s.duration_minutes} min gesamt)`, + }) + } + } + } + } + + // 2. Add sessions starting on this calendar day (clamped to 24:00) + if (day.sessions) { + for (const s of day.sessions) { + if (!s.start) continue + const sMin = getMinuteFromTime(s.start) + const dur = s.duration_minutes || 60 + const clampedDur = Math.max(1, Math.min(dur, 1440 - sMin)) + blocks.push({ + startMin: sMin, + durationMin: clampedDur, + type: s.type === 'night' ? 'night' : 'nap', + label: `${clampedDur}m`, + title: `${s.type === 'night' ? 'Nachtschlaf' : 'Nap'}: ${s.start}–${s.end || 'jetzt'} (${s.duration_minutes} min)`, + }) + } + } + + return blocks +} + function handlePointer(event: MouseEvent | TouchEvent) { const svg = (event.currentTarget as HTMLElement).getBoundingClientRect() const clientX = 'touches' in event ? event.touches[0].clientX : event.clientX @@ -474,7 +530,7 @@ function handlePointer(event: MouseEvent | TouchEvent) {
@@ -494,24 +550,22 @@ function handlePointer(event: MouseEvent | TouchEvent) {
-