feat(openclaw): upgrade fleet to DeepSeek V4.1 Flash 1017 and harden operations

- Upgrade OpenClaw primary model from deepseek-v4.1-flash to deepseek-v4.1-flash-1017 across all interactive agents, heartbeats, compaction, and cron jobs. - Configure hardened OpenRouter provider routing for 1017 (deepinfra, inference-net, parasail, cloudflare) with strict fp8/bf16 quantization and tool_choice verification. - Retain deepseek-v4.1-flash (0910), z-ai/glm-5.3-flash, and qwen3.8-flash in the automated fallback chain. - Update ADR 0002 with 1017 economics (30% price reduction on prompt/cache, +60% intelligence benchmark increase) and a 7-point provider qualification playbook. - Add operational scripts and tooling for session compaction, subagent vector deduplication, gateway preflight, and triage diagnostics in k8s/openclaw. - Update DuckDB backend engine, git-sync logic, and test suites. - Enhance Mila tracking UI components and add container build configuration. - Sync Terraform infrastructure state for Authentik.
2026-10-18 10:54:59 +02:00
parent 7ce1da18ab
commit dd60a095c6
26 changed files with 1786 additions and 1301 deletions
@@ -734,44 +734,93 @@ class DuckDBEngine:
            today_wet_count = int(diaper_agg[1]) if diaper_agg and diaper_agg[1] is not None else 0
            today_dirty_count = int(diaper_agg[2]) if diaper_agg and diaper_agg[2] is not None else 0

-            # 5. 7d and 30d sleep averages
+            # 5. 7d and 30d sleep averages (use completed days < today_str when available)
            avg_7 = self.con.execute(
-                """
+                f"""
                SELECT ROUND(AVG(total_sleep_hours), 2), ROUND(AVG(night_sleep_minutes)/60.0, 2),
                       ROUND(AVG(nap_sleep_minutes)/60.0, 2), ROUND(AVG(nap_count), 1)
-                FROM (SELECT total_sleep_hours, night_sleep_minutes, nap_sleep_minutes, nap_count
-                      FROM v_sleep_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7)
+                FROM (
+                    SELECT total_sleep_hours, night_sleep_minutes, nap_sleep_minutes, nap_count
+                    FROM v_sleep_daily
+                    WHERE subject = 'mila' AND log_date < '{today_str}'
+                    ORDER BY log_date DESC LIMIT 7
+                )
                """
            ).fetchone()
+            if not avg_7 or avg_7[0] is None:
+                avg_7 = self.con.execute(
+                    """
+                    SELECT ROUND(AVG(total_sleep_hours), 2), ROUND(AVG(night_sleep_minutes)/60.0, 2),
+                           ROUND(AVG(nap_sleep_minutes)/60.0, 2), ROUND(AVG(nap_count), 1)
+                    FROM (SELECT total_sleep_hours, night_sleep_minutes, nap_sleep_minutes, nap_count
+                          FROM v_sleep_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7)
+                    """
+                ).fetchone()

            avg_7_tot = f"{avg_7[0]:.2f} h" if avg_7 and avg_7[0] is not None else "—"
            avg_7_night = f"{avg_7[1]:.2f} h" if avg_7 and avg_7[1] is not None else "—"
            avg_7_naps = f"{avg_7[3]} Naps ({avg_7[2]:.2f} h)" if avg_7 and avg_7[3] is not None else "—"

            avg_30 = self.con.execute(
-                """
+                f"""
                SELECT ROUND(AVG(total_sleep_hours), 2)
-                FROM (SELECT total_sleep_hours FROM v_sleep_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 30)
+                FROM (
+                    SELECT total_sleep_hours FROM v_sleep_daily
+                    WHERE subject = 'mila' AND log_date < '{today_str}'
+                    ORDER BY log_date DESC LIMIT 30
+                )
                """
            ).fetchone()
+            if not avg_30 or avg_30[0] is None:
+                avg_30 = self.con.execute(
+                    """
+                    SELECT ROUND(AVG(total_sleep_hours), 2)
+                    FROM (SELECT total_sleep_hours FROM v_sleep_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 30)
+                    """
+                ).fetchone()
            avg_30_tot = f"{avg_30[0]:.2f} h" if avg_30 and avg_30[0] is not None else "—"

            feed_avg_7 = self.con.execute(
-                """
+                f"""
                SELECT ROUND(AVG(total_consumed_ml), 0), ROUND(AVG(total_solid_g), 0),
                       ROUND(AVG(total_kcal), 0), ROUND(AVG(bottle_count), 1)
-                FROM (SELECT total_consumed_ml, total_solid_g, total_kcal, bottle_count
-                      FROM v_feeding_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7)
+                FROM (
+                    SELECT total_consumed_ml, total_solid_g, total_kcal, bottle_count
+                    FROM v_feeding_daily
+                    WHERE subject = 'mila' AND log_date < '{today_str}'
+                    ORDER BY log_date DESC LIMIT 7
+                )
                """
            ).fetchone()
+            if not feed_avg_7 or feed_avg_7[0] is None:
+                feed_avg_7 = self.con.execute(
+                    """
+                    SELECT ROUND(AVG(total_consumed_ml), 0), ROUND(AVG(total_solid_g), 0),
+                           ROUND(AVG(total_kcal), 0), ROUND(AVG(bottle_count), 1)
+                    FROM (SELECT total_consumed_ml, total_solid_g, total_kcal, bottle_count
+                          FROM v_feeding_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7)
+                    """
+                ).fetchone()

            diaper_avg_7 = self.con.execute(
-                """
+                f"""
                SELECT ROUND(AVG(wet_count), 1), ROUND(AVG(total_changes), 1)
-                FROM (SELECT wet_count, total_changes
-                      FROM v_diaper_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7)
+                FROM (
+                    SELECT wet_count, total_changes
+                    FROM v_diaper_daily
+                    WHERE subject = 'mila' AND log_date < '{today_str}'
+                    ORDER BY log_date DESC LIMIT 7
+                )
                """
            ).fetchone()
+            if not diaper_avg_7 or diaper_avg_7[0] is None:
+                diaper_avg_7 = self.con.execute(
+                    """
+                    SELECT ROUND(AVG(wet_count), 1), ROUND(AVG(total_changes), 1)
+                    FROM (SELECT wet_count, total_changes
+                          FROM v_diaper_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7)
+                    """
+                ).fetchone()

            # 6. Past 30 Days History Table & sessions
            history_rows_data = self.con.execute(
@@ -837,10 +886,18 @@ class DuckDBEngine:
            for r in history_rows_data:
                d_str = str(r[0])
                d_obj = datetime.date.fromisoformat(d_str)
-                nh, nm = divmod(r[2] or 0, 60)
-                d_sessions = sessions_by_date.get(d_str, [])
+                d_sessions = list(sessions_by_date.get(d_str, []))
+                row_night_min = int(r[2] or 0)
+                row_nap_min = int(r[4] or 0)
+                row_tot_hours = round(r[1], 2) if r[1] is not None else 0.0
+
+                if d_str == today_str and row_night_min == 0 and today_night_min > 0:
+                    row_night_min = today_night_min
+                    row_tot_hours = round((row_night_min + row_nap_min) / 60.0, 2)
+
+                nh, nm = divmod(row_night_min, 60)
                night_durs = [s["duration_minutes"] for s in d_sessions if s["type"] == "night" and s["duration_minutes"]]
-                longest_night = max(night_durs) if night_durs else int(r[2] or 0)
+                longest_night = max(night_durs) if night_durs else row_night_min

                wake_windows = []
                day_sessions = [s for s in d_sessions if s["type"] == "nap" and s.get("local_start") and s.get("local_end")]
@@ -860,12 +917,12 @@ class DuckDBEngine:
                    "weekday": WEEKDAYS_DE[d_obj.weekday()],
                    "formatted_date": d_obj.strftime("%d.%m."),
                    "is_today": (d_str == today_str),
-                    "total_hours": round(r[1], 2) if r[1] is not None else 0.0,
-                    "night_minutes": int(r[2] or 0),
+                    "total_hours": row_tot_hours,
+                    "night_minutes": row_night_min,
                    "night_duration_str": f"{nh} h {nm:02d} min" if nh > 0 else f"{nm} min",
-                    "nap_minutes": int(r[4] or 0),
+                    "nap_minutes": row_nap_min,
                    "nap_count": int(r[3] or 0),
-                    "nap_str": f"{r[3]} ({r[4]} min)",
+                    "nap_str": f"{r[3]} ({row_nap_min} min)",
                    "longest_night_stretch_min": longest_night,
                    "avg_wake_window_min": avg_wake,
                    "feed_bottles_ml": int(r[5]),
@@ -1457,6 +1514,37 @@ class DuckDBEngine:
            for r in records:
                f.write(json.dumps(r, ensure_ascii=False) + "\n")

+    def _parse_event_dt(
+        self,
+        raw: str,
+        now_dt: datetime.datetime,
+        actor: str = "",
+        base_date: Optional[str] = None,
+    ) -> datetime.datetime:
+        s = raw.strip()
+        m_time = re.fullmatch(r"^(\d{1,2}):(\d{2})(?::(\d{2}))?$", s)
+        if m_time:
+            d_part = base_date or now_dt.astimezone(BERLIN_TZ).strftime("%Y-%m-%d")
+            hh = int(m_time.group(1))
+            mm = int(m_time.group(2))
+            ss = int(m_time.group(3) or 0)
+            return datetime.datetime.fromisoformat(f"{d_part}T{hh:02d}:{mm:02d}:{ss:02d}").replace(tzinfo=BERLIN_TZ)
+
+        dt = datetime.datetime.fromisoformat(s.replace("Z", "+00:00"))
+        if dt.tzinfo is None:
+            return dt.replace(tzinfo=BERLIN_TZ)
+
+        if s.endswith("+00:00") or s.endswith("Z"):
+            dt_wall_berlin = dt.replace(tzinfo=BERLIN_TZ)
+            # Guard against LLM agents formatting local Europe/Berlin wall-clock times with +00:00 / Z
+            if (
+                (dt > now_dt + datetime.timedelta(minutes=5) and dt_wall_berlin <= now_dt + datetime.timedelta(minutes=5))
+                or ((actor or "").startswith("agent:") and dt.second == 0 and dt.microsecond == 0)
+            ):
+                return dt_wall_berlin
+
+        return dt.astimezone(BERLIN_TZ)
+
    def add_event(self, req: EventCreateRequest) -> Dict[str, Any]:
        with self._lock:
            event_id = str(ULID())
@@ -1476,9 +1564,7 @@ class DuckDBEngine:
                    start_dt = (now_dt - delta).astimezone(BERLIN_TZ)
                    start_iso = start_dt.isoformat()
                else:
-                    start_dt = datetime.datetime.fromisoformat(req.start)
-                    if start_dt.tzinfo is None:
-                        start_dt = start_dt.replace(tzinfo=BERLIN_TZ)
+                    start_dt = self._parse_event_dt(req.start, now_dt, actor=req.actor, base_date=req.date)
                    start_iso = start_dt.isoformat()
            elif req.since:
                delta = parse_duration(req.since)
@@ -1488,21 +1574,48 @@ class DuckDBEngine:
                start_dt = now_dt.astimezone(BERLIN_TZ)
                start_iso = start_dt.isoformat()
            else:
-                start_dt = datetime.datetime.fromisoformat(req.start)
-                if start_dt.tzinfo is None:
-                    start_dt = start_dt.replace(tzinfo=BERLIN_TZ)
+                start_dt = self._parse_event_dt(req.start, now_dt, actor=req.actor)
                start_iso = start_dt.isoformat()

            end_iso = None
            end_dt = None
            if req.end:
-                end_dt = datetime.datetime.fromisoformat(req.end)
-                if end_dt.tzinfo is None:
-                    end_dt = end_dt.replace(tzinfo=BERLIN_TZ)
+                end_dt = self._parse_event_dt(
+                    req.end,
+                    now_dt,
+                    actor=req.actor,
+                    base_date=start_dt.astimezone(BERLIN_TZ).strftime("%Y-%m-%d"),
+                )
                if end_dt < start_dt:
                    raise ValueError(f"End time {req.end} cannot be earlier than start time {req.start}")
                end_iso = end_dt.isoformat()

+            # Populate Mila standard recipe defaults when omitted by tracking agents
+            if req.subject == "mila":
+                if req.kind == "feeding.solid" or (req.kind == "nutrition.intake" and "brei" in str(req.payload.get("item") or req.payload.get("type") or "").lower()):
+                    if req.kind == "nutrition.intake":
+                        req.kind = "feeding.solid"
+                    has_grams = any(req.payload.get(k) not in (None, 0) for k in ("amount_g", "consumed_g", "qty"))
+                    if not has_grams:
+                        item_l = str(req.payload.get("item") or req.meal or "").lower()
+                        if any(w in item_l for w in ("abend", "nachmittag", "dinner")):
+                            req.payload["amount_g"] = 200
+                            req.payload.setdefault("kcal", 110)
+                        else:
+                            req.payload["amount_g"] = 190
+                            req.payload.setdefault("kcal", 135)
+                elif req.kind == "feeding.bottle":
+                    c_ml = req.payload.get("consumed_ml")
+                    a_ml = req.payload.get("amount_ml")
+                    l_ml = req.payload.get("left_ml")
+                    if (c_ml is None or c_ml == 0) and (a_ml is None or a_ml == 0):
+                        if l_ml is not None:
+                            req.payload["amount_ml"] = 270
+                            req.payload["consumed_ml"] = max(0, 270 - int(l_ml))
+                        else:
+                            req.payload["amount_ml"] = 270
+                            req.payload["consumed_ml"] = 270
+
            # 15-Minute Duplicate Guard
            if not req.allow_duplicate:
                if req.kind in ("nutrition.intake", "feeding.bottle", "feeding.solid"):
@@ -1722,9 +1835,7 @@ class DuckDBEngine:
                delta = parse_duration(req.since)
                end_dt = (now_dt - delta).astimezone(BERLIN_TZ)
            elif req.end_time:
-                end_dt = datetime.datetime.fromisoformat(req.end_time)
-                if end_dt.tzinfo is None:
-                    end_dt = end_dt.replace(tzinfo=BERLIN_TZ)
+                end_dt = self._parse_event_dt(req.end_time, now_dt, actor=req.actor)
            else:
                end_dt = now_dt.astimezone(BERLIN_TZ)

@@ -43,7 +43,7 @@ class GitSyncManager:
        try:
            subprocess.run(["git", "config", "--global", "http.sslVerify", "false"], check=False)
            subprocess.run(["git", "config", "--global", "--add", "safe.directory", "*"], check=False)
-            if (self.wiki_dir / ".git").is_dir():
+            if (self.wiki_dir / ".git").exists():
                subprocess.run(["git", "-C", str(self.wiki_dir), "config", "user.name", GIT_AUTHOR_NAME], check=False)
                subprocess.run(["git", "-C", str(self.wiki_dir), "config", "user.email", GIT_AUTHOR_EMAIL], check=False)
        except Exception as e:
@@ -64,7 +64,7 @@ class GitSyncManager:
        """Ensures repository exists and is initialized."""
        with self._lock:
            self._configure_git_client()
-            if (self.wiki_dir / ".git").is_dir():
+            if (self.wiki_dir / ".git").exists():
                try:
                    res = subprocess.run(
                        ["git", "-C", str(self.wiki_dir), "rev-parse", "--short", "HEAD"],
@@ -144,7 +144,7 @@ class GitSyncManager:

    def pull_rebase(self) -> bool:
        """Pulls latest changes via rebase. Handles conflicts cleanly."""
-        if not (self.wiki_dir / ".git").is_dir():
+        if not (self.wiki_dir / ".git").exists():
            return False

        with self._lock:
@@ -208,7 +208,7 @@ class GitSyncManager:

    def commit_and_push(self, commit_msg: str) -> bool:
        """Stages data/, commits with descriptive message, rebases and pushes."""
-        if not (self.wiki_dir / ".git").is_dir():
+        if not (self.wiki_dir / ".git").exists():
            return False

        with self._lock:
@@ -21,23 +21,26 @@ logger = logging.getLogger("duckdb_backend")
@asynccontextmanager
 async def lifespan(app: FastAPI):
    logger.info("Initializing DuckDB Backend...")
-    git_sync = GitSyncManager(
-        wiki_dir=WIKI_DIR,
-        on_data_pulled=lambda: app.state.engine.reload_all() if hasattr(app.state, "engine") else None,
-    )
-    git_sync.ensure_repo()
-    app.state.git_sync = git_sync
+    if getattr(app.state, "git_sync", None) is None or getattr(app.state, "engine", None) is None:
+        git_sync = GitSyncManager(
+            wiki_dir=WIKI_DIR,
+            on_data_pulled=lambda: app.state.engine.reload_all() if hasattr(app.state, "engine") else None,
+        )
+        git_sync.ensure_repo()
+        app.state.git_sync = git_sync

-    engine = DuckDBEngine(data_dir=DATA_DIR, git_sync_manager=git_sync)
-    app.state.engine = engine
+        engine = DuckDBEngine(data_dir=DATA_DIR, git_sync_manager=git_sync)
+        app.state.engine = engine

-    await git_sync.start_periodic_sync()
-    logger.info("DuckDB Backend is ready to serve queries.")
-
-    yield
-
-    logger.info("Shutting down DuckDB Backend...")
-    await git_sync.stop_periodic_sync()
+        await git_sync.start_periodic_sync()
+        logger.info("DuckDB Backend is ready to serve queries.")
+        try:
+            yield
+        finally:
+            logger.info("Shutting down DuckDB Backend...")
+            await git_sync.stop_periodic_sync()
+    else:
+        yield


 app = FastAPI(
@@ -13,13 +13,14 @@ from app.main import app

@pytest.fixture(scope="session")
 def test_data_dir():
-    # Use real wiki data dir if exists, else temporary copy
+    temp_root = Path(tempfile.mkdtemp())
+    temp_data = temp_root / "data"
    if DATA_DIR.is_dir():
-        yield DATA_DIR
+        shutil.copytree(DATA_DIR, temp_data)
    else:
-        temp_dir = Path(tempfile.mkdtemp())
-        yield temp_dir
-        shutil.rmtree(temp_dir, ignore_errors=True)
+        temp_data.mkdir(parents=True, exist_ok=True)
+    yield temp_data
+    shutil.rmtree(temp_root, ignore_errors=True)


@pytest.fixture
@@ -127,3 +127,50 @@ def test_duplicate_open_sleep_api_endpoint(client: TestClient):
        "subject": "test_baby",
    })
    assert stop_res.status_code == 200
+
+
+def test_agent_utc_wallclock_guard_and_mila_recipe_defaults(tmp_path: Path):
+    engine = DuckDBEngine(data_dir=tmp_path)
+
+    # 1. Agent passes +00:00 on local wall-clock times (e.g. 08:30 to 10:20)
+    nap_req = EventCreateRequest(
+        kind="sleep.session",
+        subject="mila",
+        actor="agent:mila_tracker",
+        start="2026-10-17T08:30:00+00:00",
+        end="2026-10-17T10:20:00+00:00",
+        payload={"type": "nap", "location": "bett"},
+    )
+    res_nap = engine.add_event(nap_req)
+    assert res_nap["status"] == "success"
+    assert res_nap["record"]["startIso"] == "2026-10-17T08:30:00+02:00"
+    assert res_nap["record"]["endIso"] == "2026-10-17T10:20:00+02:00"
+    assert res_nap["record"]["sleepTimeSeconds"] == 6600
+
+    # 2. Solid food without amount_g gets standard recipe defaults
+    brei_req = EventCreateRequest(
+        kind="feeding.solid",
+        subject="mila",
+        actor="agent:mila_tracker",
+        start="2026-10-17T12:20:00+00:00",
+        meal="lunch",
+        payload={"item": "mittagsbrei"},
+    )
+    res_brei = engine.add_event(brei_req)
+    assert res_brei["record"]["start"] == "2026-10-17T12:20:00+02:00"
+    assert res_brei["record"]["payload"]["amount_g"] == 190
+    assert res_brei["record"]["payload"]["kcal"] == 135
+
+    abend_req = EventCreateRequest(
+        kind="feeding.solid",
+        subject="mila",
+        actor="agent:mila_tracker",
+        start="2026-10-17T16:40:00+00:00",
+        meal="dinner",
+        payload={"item": "abendbrei"},
+    )
+    res_abend = engine.add_event(abend_req)
+    assert res_abend["record"]["start"] == "2026-10-17T16:40:00+02:00"
+    assert res_abend["record"]["payload"]["amount_g"] == 200
+    assert res_abend["record"]["payload"]["kcal"] == 110
+