moritz/mop-skills
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(skills): add datastore skill (DuckDB engine, JSONL partitioning, Garmin mirror, and view renderers)

Browse Source
This commit is contained in:
Moritz Graf
2026-10-06 18:17:52 +02:00
parent 2bde04617f
commit 0489ef7d62
8 changed files with 2987 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/datastore/scripts/datastore.py
Executable
+982
View File
@@ -0,0 +1,982 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "duckdb>=1.1.0",
# "python-ulid>=3.0.0",
# "pytz>=2024.1",
# ]
# ///
"""
OpenClaw Datastore — Structured Personal Data Layer (DuckDB + JSONL in Git)
Provides deterministic math, validation, interval handling, spatial queries,
and JSONL file persistence for personal nutrition, baby sleep, sports, recipes, and places.
Exposes both a CLI and a stdio JSON-RPC MCP server.
"""
from __future__ import annotations
import argparse
import datetime
import json
import os
import re
import subprocess
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
from zoneinfo import ZoneInfo
import duckdb
def generate_ulid() -> str:
try:
from ulid import ULID
return str(ULID())
except TypeError:
import ulid
return str(ulid.new())
except ImportError:
import uuid
return str(uuid.uuid4())
BERLIN_TZ = ZoneInfo("Europe/Berlin")
def find_data_dir() -> Path:
"""Discovers the knowledge-wiki data directory."""
if os.environ.get("DATASTORE_DIR"):
return Path(os.environ["DATASTORE_DIR"]).resolve()
candidates = [
Path.cwd() / "knowledge-wiki" / "data",
Path.cwd() / "data",
Path.home() / "src" / "infrapuzzle" / "knowledge-wiki" / "data",
Path.home() / ".openclaw" / "wiki" / "main" / "data",
Path("/home/node/.openclaw/wiki/main/data"),
]
for c in candidates:
if c.is_dir() and (c / "_meta.json").exists():
return c.resolve()
# Fallback to creating a local data directory if needed
default_dir = Path.cwd() / "knowledge-wiki" / "data"
default_dir.mkdir(parents=True, exist_ok=True)
return default_dir.resolve()
class DatastoreEngine:
def __init__(self, data_dir: Optional[Path] = None):
self.data_dir = data_dir or find_data_dir()
self.con = duckdb.connect(":memory:")
self.has_spatial_extension = False
self._init_schemas()
self._load_all_data()
def _init_schemas(self):
"""Initializes tables, views, and spatial support in DuckDB."""
# Timezone support via icu extension
try:
self.con.execute("INSTALL icu; LOAD icu;")
self.con.execute("SET TimeZone = 'Europe/Berlin';")
except Exception:
pass
# Spatial support: try extension, fallback to pure-SQL Haversine macro
try:
self.con.execute("INSTALL spatial; LOAD spatial;")
self.has_spatial_extension = True
except Exception:
self.has_spatial_extension = False
# Fallback pure-SQL distance macro (returns meters)
self.con.execute("""
CREATE MACRO IF NOT EXISTS distance_m(lat1, lon1, lat2, lon2) AS
6371000 * 2 * asin(sqrt(
sin(radians(lat2 - lat1) / 2) ^ 2 +
cos(radians(lat1)) * cos(radians(lat2)) * sin(radians(lon2 - lon1) / 2) ^ 2
));
""")
schema_dir = self.data_dir / "_schema"
if schema_dir.is_dir():
for f in ["envelope.sql", "entities.sql", "views.sql"]:
sql_file = schema_dir / f
if sql_file.is_file():
sql = sql_file.read_text(encoding="utf-8")
self.con.execute(sql)
else:
# Inline fallback schema if _schema is missing
self._create_inline_schema()
def _create_inline_schema(self):
self.con.execute("""
CREATE TABLE IF NOT EXISTS events (
id VARCHAR PRIMARY KEY,
kind VARCHAR NOT NULL,
subject VARCHAR NOT NULL,
actor VARCHAR NOT NULL,
start TIMESTAMPTZ NOT NULL,
"end" TIMESTAMPTZ,
estimated BOOLEAN DEFAULT false,
meal VARCHAR,
note VARCHAR,
payload JSON NOT NULL,
source_ref VARCHAR,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ,
deleted_at TIMESTAMPTZ,
CHECK ("end" IS NULL OR "end" >= start)
);
CREATE TABLE IF NOT EXISTS people (
id VARCHAR PRIMARY KEY,
name VARCHAR NOT NULL,
role VARCHAR,
birthdate DATE,
notes VARCHAR
);
CREATE TABLE IF NOT EXISTS foods (
id VARCHAR PRIMARY KEY,
name VARCHAR NOT NULL,
brand VARCHAR,
category VARCHAR,
per_100 JSON,
density_g_per_ml DOUBLE,
ingredients_text VARCHAR,
barcode VARCHAR,
source VARCHAR,
description VARCHAR
);
CREATE TABLE IF NOT EXISTS recipes (
id VARCHAR PRIMARY KEY,
title VARCHAR NOT NULL,
servings INTEGER DEFAULT 1,
ingredients JSON,
estimated_delta JSON,
devices VARCHAR[],
tags VARCHAR[],
documents VARCHAR[],
description VARCHAR
);
CREATE TABLE IF NOT EXISTS places (
id VARCHAR PRIMARY KEY,
name VARCHAR NOT NULL,
category VARCHAR NOT NULL,
lat DOUBLE NOT NULL,
lon DOUBLE NOT NULL,
address VARCHAR,
google_maps_url VARCHAR,
tags VARCHAR[],
attributes JSON,
documents VARCHAR[],
notes VARCHAR
);
CREATE TABLE IF NOT EXISTS garmin_daily (
calendarDate DATE PRIMARY KEY,
totalKilocalories INTEGER,
activeKilocalories INTEGER,
bmrKilocalories INTEGER,
totalSteps INTEGER,
restingHeartRate INTEGER,
raw JSON
);
""")
def _load_all_data(self):
"""Loads all existing JSONL files from entities/, events/, and garmin/ into DuckDB."""
# 1. Load entities
entities_dir = self.data_dir / "entities"
if entities_dir.is_dir():
for f in entities_dir.glob("*.jsonl"):
collection = f.stem
if collection in ["people", "foods", "recipes", "places"]:
self._load_jsonl_file(f, collection)
# 2. Load events
events_dir = self.data_dir / "events"
if events_dir.is_dir():
for f in events_dir.glob("**/*.jsonl"):
self._load_jsonl_file(f, "events")
# 3. Load garmin daily
garmin_daily_dir = self.data_dir / "garmin" / "daily"
if garmin_daily_dir.is_dir():
for f in garmin_daily_dir.glob("*.jsonl"):
self._load_garmin_daily_file(f)
def _load_garmin_daily_file(self, file_path: Path):
"""Reads daily summaries from JSONL into garmin_daily table."""
if not file_path.is_file() or file_path.stat().st_size == 0:
return
with open(file_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
record = json.loads(line)
cal_date = record.get("calendarDate")
if not cal_date:
continue
total_kcal = record.get("totalKilocalories")
active_kcal = record.get("activeKilocalories")
bmr_kcal = record.get("bmrKilocalories")
steps = record.get("totalSteps")
rhr = record.get("restingHeartRate")
raw = json.dumps(record, ensure_ascii=False)
self.con.execute(
"""
INSERT OR REPLACE INTO garmin_daily
(calendarDate, totalKilocalories, activeKilocalories, bmrKilocalories, totalSteps, restingHeartRate, raw)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
[cal_date, total_kcal, active_kcal, bmr_kcal, steps, rhr, raw],
)
except Exception as e:
logger.warning(f"Error loading garmin record from {file_path}: {e}")
def _load_jsonl_file(self, file_path: Path, table_name: str):
"""Reads lines from JSONL and inserts into target table."""
if not file_path.is_file() or file_path.stat().st_size == 0:
return
with open(file_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
record = json.loads(line)
columns = list(record.keys())
placeholders = ", ".join(["?"] * len(columns))
col_names = ", ".join([f'"{c}"' for c in columns])
values = [
json.dumps(record[c]) if isinstance(record[c], (dict, list)) else record[c]
for c in columns
]
self.con.execute(
f'INSERT OR REPLACE INTO "{table_name}" ({col_names}) VALUES ({placeholders})',
values,
)
except Exception as e:
# Ignore comment lines or malformed lines with warning
pass
# --------------------------------------------------------------------------
# Event Operations
# --------------------------------------------------------------------------
def add_event(
self,
kind: str,
subject: str,
actor: str,
payload: Dict[str, Any],
start: Optional[str] = None,
end: Optional[str] = None,
estimated: bool = False,
meal: Optional[str] = None,
note: Optional[str] = None,
source_ref: Optional[str] = None,
) -> Dict[str, Any]:
"""Adds a time-series event, saves to JSONL partition, and returns confirmation."""
event_id = generate_ulid()
now_dt = datetime.datetime.now(datetime.timezone.utc)
now_iso = now_dt.isoformat()
# Parse or default start time
if not start:
start_dt = now_dt.astimezone(BERLIN_TZ)
start_iso = start_dt.isoformat()
else:
start_dt = datetime.datetime.fromisoformat(start)
if start_dt.tzinfo is None:
start_dt = start_dt.replace(tzinfo=BERLIN_TZ)
start_iso = start_dt.isoformat()
end_iso = None
if end:
end_dt = datetime.datetime.fromisoformat(end)
if end_dt.tzinfo is None:
end_dt = end_dt.replace(tzinfo=BERLIN_TZ)
if end_dt < start_dt:
raise ValueError(f"End time {end} cannot be earlier than start time {start}")
end_iso = end_dt.isoformat()
# Validation for open intervals in sleep
if kind == "sleep.session" and end_iso is None:
# Check if there is already an open session for this subject
open_sessions = self.con.execute(
"SELECT id, start FROM events WHERE kind = 'sleep.session' AND subject = ? AND \"end\" IS NULL AND deleted_at IS NULL",
[subject],
).fetchall()
if open_sessions:
raise ValueError(
f"Subject '{subject}' already has an open sleep session ({open_sessions[0][0]} started at {open_sessions[0][1]}). Please stop it first."
)
record = {
"id": event_id,
"kind": kind,
"subject": subject,
"actor": actor,
"start": start_iso,
"end": end_iso,
"estimated": estimated,
"meal": meal,
"note": note,
"payload": payload,
"source_ref": source_ref,
"created_at": now_iso,
"updated_at": None,
"deleted_at": None,
}
# Insert into in-memory DuckDB
self.con.execute(
"""
INSERT INTO events (id, kind, subject, actor, start, "end", estimated, meal, note, payload, source_ref, created_at, updated_at, deleted_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
[
event_id,
kind,
subject,
actor,
start_iso,
end_iso,
estimated,
meal,
note,
json.dumps(payload),
source_ref,
now_iso,
None,
None,
],
)
# Write to partitioned JSONL file
# Partition path: events/<domain>/<YYYY-MM>.jsonl
domain = kind.split(".")[0]
month_str = start_dt.astimezone(BERLIN_TZ).strftime("%Y-%m")
domain_dir = self.data_dir / "events" / domain
domain_dir.mkdir(parents=True, exist_ok=True)
partition_file = domain_dir / f"{month_str}.jsonl"
with open(partition_file, "a", encoding="utf-8") as f:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
# Sort partition deterministically by start, id
self._sort_jsonl_file(partition_file, key_fn=lambda x: (x.get("start", ""), x.get("id", "")))
# Fetch computed feedback
feedback = self._get_event_feedback(kind, subject, start_dt.date())
return {
"status": "success",
"event": record,
"feedback": feedback,
}
def stop_event(
self,
kind: str,
subject: str,
end_time: Optional[str] = None,
actor: Optional[str] = None,
) -> Dict[str, Any]:
"""Finds the open event (end IS NULL) for kind and subject, and closes it."""
row = self.con.execute(
"""
SELECT id, start FROM events
WHERE kind = ? AND subject = ? AND "end" IS NULL AND deleted_at IS NULL
ORDER BY start DESC LIMIT 1
""",
[kind, subject],
).fetchone()
if not row:
raise ValueError(f"No open event found for kind='{kind}' and subject='{subject}'")
event_id, start_str = row[0], str(row[1])
now_dt = datetime.datetime.now(datetime.timezone.utc)
if end_time:
end_dt = datetime.datetime.fromisoformat(end_time)
if end_dt.tzinfo is None:
end_dt = end_dt.replace(tzinfo=BERLIN_TZ)
else:
end_dt = now_dt.astimezone(BERLIN_TZ)
end_iso = end_dt.isoformat()
now_iso = now_dt.isoformat()
# Update in DuckDB
self.con.execute(
"""
UPDATE events
SET "end" = ?, updated_at = ?
WHERE id = ?
""",
[end_iso, now_iso, event_id],
)
# Update in JSONL
domain = kind.split(".")[0]
self._update_jsonl_record(domain, event_id, {"end": end_iso, "updated_at": now_iso})
# Calculate duration
start_dt = datetime.datetime.fromisoformat(start_str.replace("Z", "+00:00"))
duration_minutes = int((end_dt.astimezone(BERLIN_TZ) - start_dt.astimezone(BERLIN_TZ)).total_seconds() / 60)
return {
"status": "success",
"event_id": event_id,
"kind": kind,
"subject": subject,
"start": start_str,
"end": end_iso,
"duration_minutes": duration_minutes,
"duration_formatted": f"{duration_minutes // 60} h {duration_minutes % 60} min"
if duration_minutes >= 60
else f"{duration_minutes} min",
}
def delete_event(self, event_id: str) -> Dict[str, Any]:
"""Soft-deletes an event."""
now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat()
row = self.con.execute("SELECT kind FROM events WHERE id = ?", [event_id]).fetchone()
if not row:
raise ValueError(f"Event with id '{event_id}' not found.")
kind = row[0]
domain = kind.split(".")[0]
self.con.execute("UPDATE events SET deleted_at = ? WHERE id = ?", [now_iso, event_id])
self._update_jsonl_record(domain, event_id, {"deleted_at": now_iso})
return {"status": "success", "deleted_event_id": event_id}
# --------------------------------------------------------------------------
# Entity Operations
# --------------------------------------------------------------------------
def put_entity(self, collection: str, record: Dict[str, Any]) -> Dict[str, Any]:
"""Upserts an entity into entities/<collection>.jsonl."""
if collection not in ["foods", "recipes", "places", "people"]:
raise ValueError(f"Unknown entity collection: {collection}")
entity_id = record.get("id")
if not entity_id:
raise ValueError("Entity record must have an 'id' field.")
entities_dir = self.data_dir / "entities"
entities_dir.mkdir(parents=True, exist_ok=True)
target_file = entities_dir / f"{collection}.jsonl"
# Read existing, update or append
existing_records = []
replaced = False
if target_file.is_file():
with open(target_file, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
r = json.loads(line)
if r.get("id") == entity_id:
existing_records.append(record)
replaced = True
else:
existing_records.append(r)
except Exception:
pass
if not replaced:
existing_records.append(record)
# Sort by id
existing_records.sort(key=lambda x: str(x.get("id", "")))
with open(target_file, "w", encoding="utf-8") as f:
for r in existing_records:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
# Reload into DuckDB
columns = list(record.keys())
placeholders = ", ".join(["?"] * len(columns))
col_names = ", ".join([f'"{c}"' for c in columns])
values = [
json.dumps(record[c]) if isinstance(record[c], (dict, list)) else record[c]
for c in columns
]
self.con.execute(
f'INSERT OR REPLACE INTO "{collection}" ({col_names}) VALUES ({placeholders})',
values,
)
return {"status": "success", "collection": collection, "id": entity_id}
# --------------------------------------------------------------------------
# Queries & Feedback
# --------------------------------------------------------------------------
def query(self, sql: str) -> List[Dict[str, Any]]:
"""Executes read-only SQL query and returns list of dictionaries."""
clean_sql = sql.strip()
# Basic safeguard against destructive statements in read query
disallowed = ["DROP ", "ALTER ", "TRUNCATE ", "DELETE FROM", "UPDATE "]
for d in disallowed:
if clean_sql.upper().startswith(d):
raise ValueError(f"Direct {d.strip()} queries not permitted via query(). Use dedicated methods.")
cursor = self.con.execute(clean_sql)
cols = [desc[0] for desc in cursor.description]
rows = cursor.fetchall()
results = []
for row in rows:
record = {}
for col, val in zip(cols, row):
if isinstance(val, (datetime.datetime, datetime.date)):
record[col] = val.isoformat()
else:
record[col] = val
results.append(record)
return results
def get_summary(self, subject: str, date_str: Optional[str] = None) -> Dict[str, Any]:
"""Returns daily summary for subject (nutrition, sleep, deficit)."""
target_date = date_str or datetime.datetime.now(BERLIN_TZ).strftime("%Y-%m-%d")
res: Dict[str, Any] = {"subject": subject, "date": target_date}
# 1. Nutrition summary
nut = self.query(f"SELECT * FROM v_nutrition_daily WHERE subject = '{subject}' AND log_date = '{target_date}'")
if nut:
res["nutrition"] = nut[0]
else:
res["nutrition"] = {
"total_protein_g": 0.0,
"total_carbs_g": 0.0,
"total_sugar_g": 0.0,
"total_fat_g": 0.0,
"total_fiber_g": 0.0,
"total_salt_g": 0.0,
"total_kcal": 0,
"item_count": 0,
}
# 2. Sleep summary
slp = self.query(f"SELECT * FROM v_sleep_daily WHERE subject = '{subject}' AND log_date = '{target_date}'")
if slp:
res["sleep"] = slp[0]
# 3. Active open sleeps
open_sleep = self.query(
f"SELECT id, start, local_start, sleep_type, location FROM v_sleep WHERE subject = '{subject}' AND is_open = true"
)
if open_sleep:
res["open_sleep"] = open_sleep[0]
return res
def find_places_near(
self,
lat: float,
lon: float,
radius_m: float = 1000.0,
category: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Radius search for places using spatial extension or Haversine fallback."""
cat_filter = f"AND category = '{category}'" if category else ""
if self.has_spatial_extension:
sql = f"""
SELECT id, name, category, lat, lon, address, google_maps_url, tags, notes, documents,
ROUND(ST_Distance_Spheroid(ST_Point(lon, lat), ST_Point({lon}, {lat})), 1) AS distance_meters
FROM places
WHERE ST_Distance_Spheroid(ST_Point(lon, lat), ST_Point({lon}, {lat})) <= {radius_m}
{cat_filter}
ORDER BY distance_meters ASC
"""
else:
sql = f"""
SELECT id, name, category, lat, lon, address, google_maps_url, tags, notes, documents,
ROUND(distance_m({lat}, {lon}, lat, lon), 1) AS distance_meters
FROM places
WHERE distance_m({lat}, {lon}, lat, lon) <= {radius_m}
{cat_filter}
ORDER BY distance_meters ASC
"""
return self.query(sql)
def _get_event_feedback(self, kind: str, subject: str, target_date: datetime.date) -> Dict[str, Any]:
"""Generates compact running status feedback after an insert."""
date_str = target_date.strftime("%Y-%m-%d")
if kind.startswith("nutrition"):
daily = self.con.execute(
"SELECT total_protein_g, total_carbs_g, total_sugar_g, total_fat_g, total_kcal FROM v_nutrition_daily WHERE subject = ? AND log_date = ?",
[subject, date_str],
).fetchone()
if daily:
return {
"day_total_formatted": f"Tag bisher ({date_str}): ~{daily[0]} g Eiweiß · ~{daily[1]} g KH (davon ~{daily[2]} g Zucker) · ~{daily[3]} g Fett · ~{int(daily[4])} kcal",
"protein_g": daily[0],
"carbs_g": daily[1],
"kcal": daily[4],
}
elif kind.startswith("sleep"):
daily = self.con.execute(
"SELECT total_sleep_hours, nap_count, night_sleep_minutes FROM v_sleep_daily WHERE subject = ? AND log_date = ?",
[subject, date_str],
).fetchone()
if daily:
return {
"day_total_formatted": f"Schlaf ({date_str}): {daily[0]} h gesamt ({daily[1]} Schläfchen, {daily[2]} min Nacht)",
"total_hours": daily[0],
"nap_count": daily[1],
}
return {}
# --------------------------------------------------------------------------
# Internal Helpers
# --------------------------------------------------------------------------
def _sort_jsonl_file(self, file_path: Path, key_fn):
if not file_path.is_file():
return
records = []
with open(file_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
try:
records.append(json.loads(line))
except Exception:
pass
records.sort(key=key_fn)
with open(file_path, "w", encoding="utf-8") as f:
for r in records:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
def _update_jsonl_record(self, domain: str, event_id: str, updates: Dict[str, Any]):
"""Finds and updates a record in events/<domain>/*.jsonl."""
domain_dir = self.data_dir / "events" / domain
if not domain_dir.is_dir():
return
for p_file in domain_dir.glob("*.jsonl"):
records = []
found = False
with open(p_file, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
r = json.loads(line)
if r.get("id") == event_id:
r.update(updates)
found = True
records.append(r)
except Exception:
pass
if found:
with open(p_file, "w", encoding="utf-8") as f:
for r in records:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
return
# ==============================================================================
# JSON-RPC MCP Server Protocol
# ==============================================================================
def run_mcp_server():
"""Runs standard JSON-RPC stdio MCP server for OpenClaw."""
engine = DatastoreEngine()
TOOLS = [
{
"name": "datastore_add_event",
"description": "Log an event into the structured datastore (e.g. nutrition.intake, sleep.session, feeding.bottle). Computes all macros and totals deterministically.",
"inputSchema": {
"type": "object",
"properties": {
"kind": {"type": "string", "description": "e.g. 'nutrition.intake', 'sleep.session', 'feeding.bottle'"},
"subject": {"type": "string", "description": "'moritz', 'amelie', or 'mila'"},
"actor": {"type": "string", "description": "'agent:koch', 'agent:hebamme', 'moritz', 'amelie'"},
"payload": {"type": "object", "description": "Detailed payload matching the kind schema"},
"start": {"type": "string", "description": "ISO 8601 timestamp (optional, defaults to now)"},
"end": {"type": "string", "description": "ISO 8601 timestamp (optional for intervals)"},
"estimated": {"type": "boolean", "description": "True if portions or macros are estimated"},
"meal": {"type": "string", "description": "Optional meal grouping (e.g. 'Frühstück', 'Mittagessen')"},
"note": {"type": "string", "description": "Free text notes or context"},
},
"required": ["kind", "subject", "actor", "payload"],
},
},
{
"name": "datastore_stop_event",
"description": "Closes an open event interval (e.g. Mila wakes up -> closes open sleep.session).",
"inputSchema": {
"type": "object",
"properties": {
"kind": {"type": "string", "description": "e.g. 'sleep.session'"},
"subject": {"type": "string", "description": "'mila', 'moritz', 'amelie'"},
"end_time": {"type": "string", "description": "ISO timestamp (optional, defaults to now)"},
},
"required": ["kind", "subject"],
},
},
{
"name": "datastore_get_summary",
"description": "Fetch daily nutrition, sleep, and status totals computed deterministically by DuckDB.",
"inputSchema": {
"type": "object",
"properties": {
"subject": {"type": "string", "description": "'moritz', 'mila', or 'amelie'"},
"date": {"type": "string", "description": "YYYY-MM-DD (defaults to today)"},
},
"required": ["subject"],
},
},
{
"name": "datastore_query",
"description": "Execute a read-only DuckDB SQL query against the datastore (events, v_nutrition, v_sleep, places, foods, etc.).",
"inputSchema": {
"type": "object",
"properties": {
"sql": {"type": "string", "description": "SQL SELECT query to execute"},
},
"required": ["sql"],
},
},
{
"name": "datastore_find_places_near",
"description": "Search for points of interest (beer gardens, playgrounds, stores) within a given radius in meters.",
"inputSchema": {
"type": "object",
"properties": {
"lat": {"type": "number", "description": "Latitude (e.g. 49.0205 for Grabengasse)"},
"lon": {"type": "number", "description": "Longitude (e.g. 12.0975 for Grabengasse)"},
"radius_m": {"type": "number", "description": "Search radius in meters (default 1000)"},
"category": {"type": "string", "description": "Optional category filter (e.g. 'biergarten')"},
},
"required": ["lat", "lon"],
},
},
{
"name": "datastore_put_entity",
"description": "Upsert an entity record into foods, recipes, places, or people.",
"inputSchema": {
"type": "object",
"properties": {
"collection": {"type": "string", "enum": ["foods", "recipes", "places", "people"]},
"record": {"type": "object", "description": "Entity record matching collection schema"},
},
"required": ["collection", "record"],
},
},
]
for line in sys.stdin:
line = line.strip()
if not line:
continue
try:
req = json.loads(line)
except Exception:
continue
method = req.get("method")
msg_id = req.get("id")
if method == "initialize":
resp = {
"jsonrpc": "2.0",
"id": msg_id,
"result": {
"protocolVersion": "2024-11-05",
"serverInfo": {"name": "openclaw-datastore", "version": "1.0.0"},
"capabilities": {"tools": {}},
},
}
sys.stdout.write(json.dumps(resp) + "\n")
sys.stdout.flush()
elif method == "tools/list":
resp = {
"jsonrpc": "2.0",
"id": msg_id,
"result": {"tools": TOOLS},
}
sys.stdout.write(json.dumps(resp) + "\n")
sys.stdout.flush()
elif method == "tools/call":
params = req.get("params", {})
name = params.get("name")
args = params.get("arguments", {})
try:
if name == "datastore_add_event":
data = engine.add_event(
kind=args["kind"],
subject=args["subject"],
actor=args.get("actor", "agent"),
payload=args["payload"],
start=args.get("start"),
end=args.get("end"),
estimated=args.get("estimated", False),
meal=args.get("meal"),
note=args.get("note"),
source_ref=args.get("source_ref"),
)
elif name == "datastore_stop_event":
data = engine.stop_event(
kind=args["kind"],
subject=args["subject"],
end_time=args.get("end_time"),
)
elif name == "datastore_get_summary":
data = engine.get_summary(subject=args["subject"], date_str=args.get("date"))
elif name == "datastore_query":
data = engine.query(sql=args["sql"])
elif name == "datastore_find_places_near":
data = engine.find_places_near(
lat=args["lat"],
lon=args["lon"],
radius_m=args.get("radius_m", 1000.0),
category=args.get("category"),
)
elif name == "datastore_put_entity":
data = engine.put_entity(
collection=args["collection"],
record=args["record"],
)
else:
raise ValueError(f"Unknown tool: {name}")
resp = {
"jsonrpc": "2.0",
"id": msg_id,
"result": {
"content": [{"type": "text", "text": json.dumps(data, indent=2, ensure_ascii=False)}]
},
}
except Exception as e:
resp = {
"jsonrpc": "2.0",
"id": msg_id,
"result": {
"isError": True,
"content": [{"type": "text", "text": f"Error executing {name}: {str(e)}"}],
},
}
sys.stdout.write(json.dumps(resp) + "\n")
sys.stdout.flush()
elif msg_id is not None:
resp = {
"jsonrpc": "2.0",
"id": msg_id,
"error": {"code": -32601, "message": f"Method not found: {method}"},
}
sys.stdout.write(json.dumps(resp) + "\n")
sys.stdout.flush()
# ==============================================================================
# CLI Entrypoint
# ==============================================================================
def run_cli():
parser = argparse.ArgumentParser(description="OpenClaw Datastore CLI")
subparsers = parser.add_subparsers(dest="command")
# Add Event
p_add = subparsers.add_parser("add", help="Add an event")
p_add.add_argument("kind", help="e.g. nutrition.intake, sleep.session")
p_add.add_argument("--subject", required=True, help="moritz, mila, amelie")
p_add.add_argument("--actor", default="cli", help="Reporter identifier")
p_add.add_argument("--payload", required=True, help="JSON payload string")
p_add.add_argument("--start", help="ISO start time")
p_add.add_argument("--end", help="ISO end time")
p_add.add_argument("--meal", help="Meal grouping")
p_add.add_argument("--note", help="Free-form note")
p_add.add_argument("--estimated", action="store_true", help="Flag as estimated")
# Stop Event
p_stop = subparsers.add_parser("stop", help="Stop an open interval event")
p_stop.add_argument("kind", help="e.g. sleep.session")
p_stop.add_argument("--subject", required=True, help="mila, moritz")
p_stop.add_argument("--end", help="ISO end time (defaults to now)")
# Query
p_query = subparsers.add_parser("query", help="Execute read-only SQL query")
p_query.add_argument("sql", help="SQL string")
# Summary
p_sum = subparsers.add_parser("summary", help="Get daily summary")
p_sum.add_argument("--subject", required=True, help="moritz, mila")
p_sum.add_argument("--date", help="YYYY-MM-DD")
# Places
p_places = subparsers.add_parser("places", help="Radius search for places")
p_places.add_argument("--lat", type=float, required=True)
p_places.add_argument("--lon", type=float, required=True)
p_places.add_argument("--radius", type=float, default=1000.0)
p_places.add_argument("--category")
args = parser.parse_args()
engine = DatastoreEngine()
if args.command == "add":
payload = json.loads(args.payload)
res = engine.add_event(
kind=args.kind,
subject=args.subject,
actor=args.actor,
payload=payload,
start=args.start,
end=args.end,
meal=args.meal,
note=args.note,
estimated=args.estimated,
)
print(json.dumps(res, indent=2, ensure_ascii=False))
elif args.command == "stop":
res = engine.stop_event(kind=args.kind, subject=args.subject, end_time=args.end)
print(json.dumps(res, indent=2, ensure_ascii=False))
elif args.command == "query":
res = engine.query(args.sql)
print(json.dumps(res, indent=2, ensure_ascii=False))
elif args.command == "summary":
res = engine.get_summary(subject=args.subject, date_str=args.date)
print(json.dumps(res, indent=2, ensure_ascii=False))
elif args.command == "places":
res = engine.find_places_near(
lat=args.lat,
lon=args.lon,
radius_m=args.radius,
category=args.category,
)
print(json.dumps(res, indent=2, ensure_ascii=False))
else:
parser.print_help()
if __name__ == "__main__":
if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] == "--stdio"):
run_mcp_server()
else:
run_cli()
skills/datastore/scripts/garmin_sync.py
Executable
+389
View File
@@ -0,0 +1,389 @@
#!/usr/bin/env python3
"""
Garmin Sync: Deterministic mirror of Garmin Connect data into git-backed JSONL.
Mirrors daily summaries, sleep data, and activities into:
- data/garmin/daily/<YYYY-MM>.jsonl
- data/garmin/sleep/<YYYY-MM>.jsonl
- data/garmin/activities/<YYYY-MM>.jsonl
Adheres to ADR 0003 (Git as single source of truth) and ADR 0004 (Structured Datastore).
Zero LLM math: records are stored as raw JSONL for deterministic DuckDB ingestion.
"""
from __future__ import annotations
import argparse
import datetime
import json
import logging
import os
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional
try:
import pytz
except ImportError:
pytz = None
logger = logging.getLogger("garmin_sync")
def get_berlin_yesterday() -> str:
"""Returns yesterday's date in Europe/Berlin wall-clock time."""
if pytz:
tz = pytz.timezone("Europe/Berlin")
now = datetime.datetime.now(tz)
else:
# Fallback +2h offset for Berlin CEST
tz_offset = datetime.timezone(datetime.timedelta(hours=2))
now = datetime.datetime.now(tz_offset)
yesterday = now - datetime.timedelta(days=1)
return yesterday.strftime("%Y-%m-%d")
def find_default_data_dir() -> Path:
"""Finds the knowledge-wiki/data directory."""
env_dir = os.environ.get("DATASTORE_DATA_DIR")
if env_dir:
p = Path(env_dir)
if p.is_dir():
return p
candidates = [
Path("/home/node/.openclaw/wiki/main/data"),
Path(__file__).resolve().parents[4] / "knowledge-wiki" / "data",
Path.cwd() / "knowledge-wiki" / "data",
Path.cwd() / "data",
]
for c in candidates:
if c.is_dir():
return c
# Fallback to local data dir inside parent
return Path.cwd() / "data"
def get_token_dir(custom_path: Optional[str] = None) -> Path:
"""Discovers token directory according to garmin-auth conventions."""
if custom_path:
return Path(custom_path)
if os.environ.get("GARMINTOKENS"):
return Path(os.path.expanduser(os.environ["GARMINTOKENS"]))
if Path("/home/node/.openclaw/garmin_tokens").is_dir():
return Path("/home/node/.openclaw/garmin_tokens")
return Path(os.path.expanduser("~/.garminconnect"))
class GarminSyncClient:
"""Garmin API client with token re-use and optional mock mode."""
def __init__(self, token_dir: Path, mock: bool = False):
self.token_dir = token_dir
self.mock = mock
self.client: Any = None
if not self.mock:
self._init_live_client()
def _init_live_client(self):
token_file = self.token_dir / "garmin_tokens.json"
if not token_file.is_file():
logger.error(f"No Garmin tokens found at {token_file}")
raise FileNotFoundError(f"Missing garmin_tokens.json in {self.token_dir}")
try:
from garminconnect import Garmin
self.client = Garmin()
self.client.login(str(self.token_dir))
name = self.client.get_full_name()
logger.info(f"Connected to Garmin as {name}")
except Exception as e:
logger.error(f"Failed to authenticate with Garmin Connect: {e}")
raise PermissionError(f"Garmin auth failed: {e}") from e
def get_daily_summary(self, date_str: str) -> Dict[str, Any]:
if self.mock:
return {
"calendarDate": date_str,
"totalKilocalories": 2850,
"activeKilocalories": 610,
"bmrKilocalories": 2240,
"totalSteps": 9840,
"totalDistanceMeters": 7450.0,
"restingHeartRate": 52,
"minHeartRate": 48,
"maxHeartRate": 164,
"floorsAscended": 12,
"source": "garmin_sync_mock",
}
return self.client.get_user_summary(date_str)
def get_sleep_data(self, date_str: str) -> Dict[str, Any]:
if self.mock:
return {
"dailySleepDTO": {
"id": 12345678,
"calendarDate": date_str,
"sleepTimeSeconds": 28800,
"napTimeSeconds": 0,
"deepSleepSeconds": 7200,
"lightSleepSeconds": 14400,
"remSleepSeconds": 5400,
"awakeSleepSeconds": 1800,
"sleepScores": {"overall": {"value": 85, "qualifierKey": "EXCELLENT"}},
},
"source": "garmin_sync_mock",
}
return self.client.get_sleep_data(date_str)
def get_activities(self, date_str: str) -> List[Dict[str, Any]]:
if self.mock:
return [
{
"activityId": 987654321,
"activityName": "Indoor Cycling (Tacx Flux 2)",
"startTimeLocal": f"{date_str} 18:30:00",
"startTimeGMT": f"{date_str} 16:30:00",
"activityType": {"typeId": 2, "typeKey": "cycling"},
"distance": 25400.0,
"duration": 3600.0,
"elapsedDuration": 3600.0,
"calories": 580.0,
"averageSpeed": 7.05,
"maxSpeed": 11.2,
"averageHR": 142,
"maxHR": 168,
"averagePower": 210,
"maxPower": 285,
"source": "garmin_sync_mock",
}
]
return self.client.get_activities_by_date(date_str, date_str)
class GarminMirrorWriter:
"""Manages month-partitioned JSONL files with idempotent upserts."""
def __init__(self, data_dir: Path):
self.garmin_dir = data_dir / "garmin"
self.daily_dir = self.garmin_dir / "daily"
self.sleep_dir = self.garmin_dir / "sleep"
self.activities_dir = self.garmin_dir / "activities"
for d in [self.daily_dir, self.sleep_dir, self.activities_dir]:
d.mkdir(parents=True, exist_ok=True)
def _read_existing(self, file_path: Path) -> List[Dict[str, Any]]:
if not file_path.is_file():
return []
items = []
with open(file_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
try:
items.append(json.loads(line))
except json.JSONDecodeError:
continue
return items
def _write_atomic(self, file_path: Path, items: List[Dict[str, Any]]):
tmp_path = file_path.with_suffix(".tmp")
with open(tmp_path, "w", encoding="utf-8") as f:
for item in items:
f.write(json.dumps(item, ensure_ascii=False) + "\n")
tmp_path.replace(file_path)
def upsert_daily_summary(self, summary: Dict[str, Any]):
cal_date = summary.get("calendarDate")
if not cal_date:
logger.warning("Daily summary missing calendarDate, skipping.")
return
month = cal_date[:7] # YYYY-MM
target_file = self.daily_dir / f"{month}.jsonl"
existing = self._read_existing(target_file)
updated = False
new_items = []
for item in existing:
if item.get("calendarDate") == cal_date:
new_items.append(summary)
updated = True
else:
new_items.append(item)
if not updated:
new_items.append(summary)
new_items.sort(key=lambda x: str(x.get("calendarDate", "")))
self._write_atomic(target_file, new_items)
logger.info(f"Upserted daily summary for {cal_date} into {target_file.name}")
def upsert_sleep_data(self, sleep_data: Dict[str, Any]):
cal_date = None
if "dailySleepDTO" in sleep_data and isinstance(sleep_data["dailySleepDTO"], dict):
cal_date = sleep_data["dailySleepDTO"].get("calendarDate")
if not cal_date:
cal_date = sleep_data.get("calendarDate")
if not cal_date:
logger.warning("Sleep data missing calendarDate, skipping.")
return
month = cal_date[:7]
target_file = self.sleep_dir / f"{month}.jsonl"
existing = self._read_existing(target_file)
updated = False
new_items = []
for item in existing:
item_date = None
if "dailySleepDTO" in item and isinstance(item["dailySleepDTO"], dict):
item_date = item["dailySleepDTO"].get("calendarDate")
if not item_date:
item_date = item.get("calendarDate")
if item_date == cal_date:
new_items.append(sleep_data)
updated = True
else:
new_items.append(item)
if not updated:
new_items.append(sleep_data)
def _sort_key(x):
d = (x.get("dailySleepDTO") or {}).get("calendarDate") if isinstance(x.get("dailySleepDTO"), dict) else None
return str(d or x.get("calendarDate") or "")
new_items.sort(key=_sort_key)
self._write_atomic(target_file, new_items)
logger.info(f"Upserted sleep data for {cal_date} into {target_file.name}")
def upsert_activities(self, activities: List[Dict[str, Any]]):
if not activities:
return
# Group activities by month
by_month: Dict[str, List[Dict[str, Any]]] = {}
for act in activities:
start_local = act.get("startTimeLocal", "")
if len(start_local) >= 7:
month = start_local[:7]
else:
month = datetime.date.today().strftime("%Y-%m")
by_month.setdefault(month, []).append(act)
for month, act_list in by_month.items():
target_file = self.activities_dir / f"{month}.jsonl"
existing = self._read_existing(target_file)
id_map = {item.get("activityId"): item for item in existing if item.get("activityId")}
for act in act_list:
act_id = act.get("activityId")
if act_id:
id_map[act_id] = act
all_acts = list(id_map.values())
all_acts.sort(key=lambda x: str(x.get("startTimeLocal", "")))
self._write_atomic(target_file, all_acts)
logger.info(f"Upserted {len(act_list)} activities for month {month} into {target_file.name}")
def sync_date_range(client: GarminSyncClient, writer: GarminMirrorWriter, dates: List[str]):
for d in dates:
logger.info(f"--- Syncing Garmin data for {d} ---")
try:
summary = client.get_daily_summary(d)
if summary:
writer.upsert_daily_summary(summary)
except Exception as e:
logger.warning(f"Could not fetch daily summary for {d}: {e}")
try:
sleep = client.get_sleep_data(d)
if sleep:
writer.upsert_sleep_data(sleep)
except Exception as e:
logger.warning(f"Could not fetch sleep data for {d}: {e}")
try:
activities = client.get_activities(d)
if activities:
writer.upsert_activities(activities)
except Exception as e:
logger.warning(f"Could not fetch activities for {d}: {e}")
def generate_date_list(start_str: str, end_str: str) -> List[str]:
d_start = datetime.datetime.strptime(start_str, "%Y-%m-%d").date()
d_end = datetime.datetime.strptime(end_str, "%Y-%m-%d").date()
if d_start > d_end:
d_start, d_end = d_end, d_start
result = []
curr = d_start
while curr <= d_end:
result.append(curr.strftime("%Y-%m-%d"))
curr += datetime.timedelta(days=1)
return result
def main():
parser = argparse.ArgumentParser(description="Deterministic Garmin Connect data mirror into JSONL")
parser.add_argument("--date", help="Specific date to sync (YYYY-MM-DD). Defaults to yesterday.")
parser.add_argument("--days", type=int, help="Sync last N days up to yesterday.")
parser.add_argument("--start", help="Start date (YYYY-MM-DD)")
parser.add_argument("--end", help="End date (YYYY-MM-DD)")
parser.add_argument("--data-dir", help="Path to knowledge-wiki/data directory")
parser.add_argument("--token-dir", help="Path to Garmin token directory")
parser.add_argument("--mock", action="store_true", help="Run with mock data (for testing/offline)")
parser.add_argument("--verbose", action="store_true", help="Enable verbose debug logging")
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
)
data_dir = Path(args.data_dir) if args.data_dir else find_default_data_dir()
token_dir = get_token_dir(args.token_dir)
# Determine date list
if args.start and args.end:
dates = generate_date_list(args.start, args.end)
elif args.days:
yesterday_str = get_berlin_yesterday()
d_yesterday = datetime.datetime.strptime(yesterday_str, "%Y-%m-%d").date()
d_start = d_yesterday - datetime.timedelta(days=args.days - 1)
dates = generate_date_list(d_start.strftime("%Y-%m-%d"), yesterday_str)
elif args.date:
dates = [args.date]
else:
dates = [get_berlin_yesterday()]
logger.info(f"Starting Garmin sync for {len(dates)} day(s): {dates[0]} to {dates[-1]}")
logger.info(f"Data directory: {data_dir}")
try:
client = GarminSyncClient(token_dir, mock=args.mock)
except PermissionError:
logger.error("Authentication error. Please run 'garmin-auth' to refresh credentials.")
sys.exit(2)
except FileNotFoundError as e:
logger.error(str(e))
sys.exit(2)
writer = GarminMirrorWriter(data_dir)
sync_date_range(client, writer, dates)
logger.info("Garmin sync completed successfully.")
sys.exit(0)
if __name__ == "__main__":
main()
skills/datastore/scripts/migrate_legacy.py
+534
View File
@@ -0,0 +1,534 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "duckdb>=1.1.0",
# "python-ulid>=3.0.0",
# "pytz>=2024.1",
# ]
# ///
"""
migrate_legacy.py — Migrates legacy Markdown logs to DuckDB-backed JSONL datastore.
Parses:
- standard-foods.md -> data/entities/foods.jsonl
- entities/nutrition-log/moritz/*.md -> data/events/nutrition/*.jsonl
- entities/people/mila-schlafprotokoll.md -> data/events/sleep/*.jsonl & feeding/*.jsonl
Computes exact DuckDB sums, compares with old handwritten sums, and generates audit report.
"""
from __future__ import annotations
import datetime
import json
import os
import re
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from zoneinfo import ZoneInfo
def generate_ulid() -> str:
try:
from ulid import ULID
return str(ULID())
except TypeError:
import ulid
return str(ulid.new())
except ImportError:
import uuid
return str(uuid.uuid4())
import duckdb
BERLIN_TZ = ZoneInfo("Europe/Berlin")
WIKI_DIR = Path("/Users/moritz/src/infrapuzzle/knowledge-wiki")
DATA_DIR = WIKI_DIR / "data"
def parse_number(s: str) -> float:
"""Parses German float string like '~11,3' or '2.364' into float."""
if not s or s.strip() in ["-", "–", ""]:
return 0.0
cleaned = s.replace("~", "").replace("g", "").replace("ml", "").replace("kcal", "").strip()
# If number has both dot and comma (e.g. 1.234,56 or 2.364)
if "." in cleaned and "," in cleaned:
cleaned = cleaned.replace(".", "").replace(",", ".")
elif "," in cleaned:
cleaned = cleaned.replace(",", ".")
elif "." in cleaned:
# Check if dot is thousands separator (e.g. 2.364 kcal)
parts = cleaned.split(".")
if len(parts) == 2 and len(parts[1]) == 3:
cleaned = parts[0] + parts[1]
m = re.search(r"[-+]?\d*\.?\d+", cleaned)
return float(m.group(0)) if m else 0.0
# ==============================================================================
# 1. Migrate Standard Foods
# ==============================================================================
def migrate_standard_foods() -> List[Dict[str, Any]]:
foods_file = DATA_DIR / "entities" / "foods.jsonl"
foods_file.parent.mkdir(parents=True, exist_ok=True)
foods = [
{
"id": "oatly-haferdrink",
"name": "Oatly Haferdrink",
"brand": "Oatly AB",
"category": "milch-alternative",
"per_100": {
"kcal": 61.0,
"protein_g": 1.1,
"carbs_g": 7.1,
"sugar_g": 3.4,
"fat_g": 3.0,
"fiber_g": 0.8,
"salt_g": 0.10,
},
"density_g_per_ml": 1.03,
"ingredients_text": "Wasser, HAFER 10%, Rapsöl, Säureregulator (Dikaliumphosphat), Mineralien (Calciumcarbonat, Kaliumjodid), Salz, Vitamine (D2, Riboflavin, B12).",
"barcode": "7394376616037",
"source": "package_photo",
"description": "Standard-Hafermilch Grabengasse 7. Silberner Karton mit Vitaminen & Mineralien.",
},
{
"id": "fruechte-mix-tk",
"name": "Früchte Mix (TK)",
"brand": None,
"category": "obst",
"per_100": {
"kcal": 45.0,
"protein_g": 1.0,
"carbs_g": 6.5,
"sugar_g": 3.4,
"fat_g": 0.3,
"fiber_g": 3.4,
"salt_g": 0.0,
},
"density_g_per_ml": 1.0,
"ingredients_text": "Gefrorene Beeren-Mischung (TK)",
"barcode": None,
"source": "package_photo",
"description": "Standard gefrorener Beerenmix für Porridge und Overnight Oats.",
},
{
"id": "muller-protein-porridge-mix",
"name": "Müller Protein-Porridge Batch-Mix",
"brand": "Eigenmischung",
"category": "muesli",
"per_100": {
"kcal": 381.0,
"protein_g": 23.3,
"carbs_g": 46.6,
"sugar_g": 2.9,
"fat_g": 8.5,
"fiber_g": 9.8,
"salt_g": 0.15,
},
"density_g_per_ml": 1.0,
"ingredients_text": "Müller Porridge Basic 500g + Müller Eiweißpulver 100g + Chiasamen 50g",
"barcode": None,
"source": "recipe_batch",
"description": "Eigener Batch-Mix für Morgenporridge (Müller Porridge Basic + Eiweißpulver + Chiasamen).",
},
]
with open(foods_file, "w", encoding="utf-8") as f:
for item in foods:
f.write(json.dumps(item, ensure_ascii=False) + "\n")
return foods
# ==============================================================================
# 2. Migrate Nutrition Logs
# ==============================================================================
def migrate_nutrition_logs() -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]]]:
nutrition_dir = WIKI_DIR / "entities" / "nutrition-log" / "moritz"
events_dir = DATA_DIR / "events" / "nutrition"
events_dir.mkdir(parents=True, exist_ok=True)
all_events: List[Dict[str, Any]] = []
stated_sums: Dict[str, Dict[str, Any]] = {}
for md_file in sorted(nutrition_dir.glob("2026-*.md")):
month_str = md_file.stem
file_events: List[Dict[str, Any]] = []
current_date_str = None
current_day_name = None
with open(md_file, "r", encoding="utf-8") as f:
for line in f:
line_str = line.strip()
# Match date header: ## YYYY-MM-DD Wochentag
m_date = re.match(r"^##\s+(\d{4}-\d{2}-\d{2})(?:\s+(\w+))?", line_str)
if m_date:
current_date_str = m_date.group(1)
current_day_name = m_date.group(2) or ""
continue
# Match table row
if current_date_str and line_str.startswith("|") and not line_str.startswith("| Zeit") and not line_str.startswith("|---"):
cols = [c.strip() for c in line_str.strip("|").split("|")]
if len(cols) >= 10:
zeit_raw = cols[0]
item_raw = cols[1]
menge_raw = cols[2]
eiweiss_raw = cols[3]
kh_raw = cols[4]
zucker_raw = cols[5]
fett_raw = cols[6]
ballast_raw = cols[7]
salz_raw = cols[8]
kcal_raw = cols[9]
# Parse time (default 12:00 if unknown)
m_time = re.search(r"(\d{1,2}):(\d{2})", zeit_raw)
if m_time:
hour = int(m_time.group(1))
minute = int(m_time.group(2))
else:
hour, minute = 12, 0
event_dt = datetime.datetime.strptime(current_date_str, "%Y-%m-%d").replace(
hour=hour, minute=minute, tzinfo=BERLIN_TZ
)
# Determine food_id if matched
food_id = None
if "oatly" in item_raw.lower() or "haferdrink" in item_raw.lower():
food_id = "oatly-haferdrink"
elif "früchte mix" in item_raw.lower() or "beeren" in item_raw.lower():
food_id = "fruechte-mix-tk"
elif "porridge-mischung" in item_raw.lower() or "müller protein-porridge" in item_raw.lower():
food_id = "muller-protein-porridge-mix"
# Parse quantity and unit
m_qty = re.search(r"(\d+(?:[.,]\d+)?)\s*(g|ml|portion|kugel)?", menge_raw)
qty = float(m_qty.group(1).replace(",", ".")) if m_qty else parse_number(menge_raw)
unit = m_qty.group(2) if m_qty and m_qty.group(2) else "g"
is_estimated = "~" in zeit_raw or "~" in item_raw or "~" in menge_raw or "~" in kcal_raw
record = {
"id": generate_ulid(),
"kind": "nutrition.intake",
"subject": "moritz",
"actor": "agent:koch",
"start": event_dt.isoformat(),
"end": None,
"estimated": is_estimated,
"meal": None,
"note": None,
"payload": {
"item": item_raw,
"food_id": food_id,
"qty": qty,
"unit": unit,
"protein_g": parse_number(eiweiss_raw),
"carbs_g": parse_number(kh_raw),
"sugar_g": parse_number(zucker_raw),
"fat_g": parse_number(fett_raw),
"fiber_g": parse_number(ballast_raw),
"salt_g": parse_number(salz_raw),
"kcal": parse_number(kcal_raw),
},
"source_ref": f"{md_file.name}:{zeit_raw}",
"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"updated_at": None,
"deleted_at": None,
}
file_events.append(record)
all_events.append(record)
# Match stated sums: **Summe heute:** ...
if current_date_str and "**Summe heute:**" in line_str:
sum_text = line_str.split("**Summe heute:**")[1].strip()
stated_sums[current_date_str] = {
"text": sum_text,
"protein_g": parse_number(re.search(r"([\d,]+)\s*g\s*Eiweiß", sum_text).group(1)) if re.search(r"([\d,]+)\s*g\s*Eiweiß", sum_text) else 0.0,
"carbs_g": parse_number(re.search(r"([\d,]+)\s*g\s*KH", sum_text).group(1)) if re.search(r"([\d,]+)\s*g\s*KH", sum_text) else 0.0,
"kcal": parse_number(re.search(r"([\d,.]+)\s*kcal", sum_text).group(1)) if re.search(r"([\d,.]+)\s*kcal", sum_text) else 0.0,
}
# Write partitioned JSONL
file_events.sort(key=lambda x: (x["start"], x["id"]))
out_jsonl = events_dir / f"{month_str}.jsonl"
with open(out_jsonl, "w", encoding="utf-8") as out_f:
for ev in file_events:
out_f.write(json.dumps(ev, ensure_ascii=False) + "\n")
return all_events, stated_sums
# ==============================================================================
# 3. Migrate Mila Sleep & Feeding Logs
# ==============================================================================
def migrate_mila_sleep_and_feeding() -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
md_file = WIKI_DIR / "entities" / "people" / "mila-schlafprotokoll.md"
sleep_events: List[Dict[str, Any]] = []
feeding_events: List[Dict[str, Any]] = []
if not md_file.is_file():
return sleep_events, feeding_events
current_date = None
processed_nights = set()
with open(md_file, "r", encoding="utf-8") as f:
for line in f:
line_str = line.strip()
# Date header: ## Dienstag, 29.09.2026 or ## 2026-09-29
m_date = re.search(r"(\d{2})\.(\d{2})\.(\d{4})", line_str)
if line_str.startswith("##") and m_date:
day, month, year = int(m_date.group(1)), int(m_date.group(2)), int(m_date.group(3))
current_date = datetime.date(year, month, day)
continue
if not current_date or not line_str.startswith("-"):
continue
# Case A: Feeding log (e.g. - 08:15 – Fläschchen (80 ml übrig/ungetrunken))
if "fläschchen" in line_str.lower():
m_time = re.search(r"(\d{1,2}):(\d{2})", line_str)
h, m = (int(m_time.group(1)), int(m_time.group(2))) if m_time else (8, 0)
feed_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, h, m, tzinfo=BERLIN_TZ)
m_ml = re.search(r"(\d+)\s*ml", line_str)
left_ml = float(m_ml.group(1)) if m_ml else 0.0
record = {
"id": generate_ulid(),
"kind": "feeding.bottle",
"subject": "mila",
"actor": "agent:hebamme",
"start": feed_dt.isoformat(),
"end": None,
"estimated": False,
"meal": None,
"note": line_str.lstrip("- ").strip(),
"payload": {
"offered_ml": None,
"consumed_ml": None,
"left_ml": left_ml,
"milk_type": "pre",
},
"source_ref": f"mila-schlafprotokoll.md:{feed_dt.strftime('%Y-%m-%d %H:%M')}",
"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"updated_at": None,
"deleted_at": None,
}
feeding_events.append(record)
continue
# Case B: Sleep intervals
# Examples:
# - 09:51 – 10:38 | Schlaf am Morgen (46 min: 10 min wach, 6 Leicht-, 30 Tiefschlaf; Einschlafzeit 5 min)
# - 18:57 – 06:50 | Nacht (11 h 53 min)
# - 20:10 – | Nacht (läuft)
# - 07:00 – aufgewacht (Nacht-Ende); Nacht: 19:13 – 07:00 (11 h 47 min)
# Check for "aufgewacht (Nacht-Ende)" duplicate
if "aufgewacht (nacht-ende)" in line_str.lower() or "nacht-ende" in line_str.lower():
m_span = re.search(r"(\d{1,2}):(\d{2})\s*–\s*(\d{1,2}):(\d{2})", line_str)
if m_span:
s_h, s_m, e_h, e_m = int(m_span.group(1)), int(m_span.group(2)), int(m_span.group(3)), int(m_span.group(4))
night_key = f"{s_h:02d}:{s_m:02d}-{e_h:02d}:{e_m:02d}"
if night_key in processed_nights:
# Already logged from the evening line, skip duplicate!
continue
else:
processed_nights.add(night_key)
# The start was yesterday evening
start_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, s_h, s_m, tzinfo=BERLIN_TZ) - datetime.timedelta(days=1)
end_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, e_h, e_m, tzinfo=BERLIN_TZ)
record = {
"id": generate_ulid(),
"kind": "sleep.session",
"subject": "mila",
"actor": "agent:hebamme",
"start": start_dt.isoformat(),
"end": end_dt.isoformat(),
"estimated": False,
"meal": None,
"note": line_str.lstrip("- ").strip(),
"payload": {
"type": "night",
"location": "bett",
},
"source_ref": f"mila-schlafprotokoll.md:{current_date}",
"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"updated_at": None,
"deleted_at": None,
}
sleep_events.append(record)
continue
# Standard interval: start – end | label
m_int = re.search(r"^-\s*(\d{1,2}):(\d{2})\s*–\s*(\d{1,2})?:?(\d{2})?\s*\|?(.*)", line_str)
if m_int:
s_h = int(m_int.group(1))
s_m = int(m_int.group(2))
e_h_str = m_int.group(3)
e_m_str = m_int.group(4)
desc = m_int.group(5).strip() if m_int.group(5) else ""
start_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, s_h, s_m, tzinfo=BERLIN_TZ)
# Open sleep interval (e.g. 20:10 – | Nacht (läuft))
if not e_h_str or not e_m_str:
end_dt = None
sleep_type = "night" if s_h >= 18 or s_h < 6 else "nap"
else:
e_h = int(e_h_str)
e_m = int(e_m_str)
# If end hour < start hour, crosses midnight!
if e_h < s_h:
end_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, e_h, e_m, tzinfo=BERLIN_TZ) + datetime.timedelta(days=1)
else:
end_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, e_h, e_m, tzinfo=BERLIN_TZ)
sleep_type = "night" if "nacht" in desc.lower() or s_h >= 18 else "nap"
# Deduplication key for night sleeps
night_key = f"{s_h:02d}:{s_m:02d}-{e_h:02d}:{e_m:02d}"
if sleep_type == "night":
processed_nights.add(night_key)
# Location detection
loc = "bett"
if "auto" in desc.lower():
loc = "auto"
elif "kinderwagen" in desc.lower():
loc = "kinderwagen"
elif "outdoor" in desc.lower() or "decke" in desc.lower():
loc = "outdoor"
record = {
"id": generate_ulid(),
"kind": "sleep.session",
"subject": "mila",
"actor": "agent:hebamme",
"start": start_dt.isoformat(),
"end": end_dt.isoformat() if end_dt else None,
"estimated": False,
"meal": None,
"note": desc or None,
"payload": {
"type": sleep_type,
"location": loc,
},
"source_ref": f"mila-schlafprotokoll.md:{current_date}",
"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"updated_at": None,
"deleted_at": None,
}
sleep_events.append(record)
# Write partition files
sleep_dir = DATA_DIR / "events" / "sleep"
sleep_dir.mkdir(parents=True, exist_ok=True)
feeding_dir = DATA_DIR / "events" / "feeding"
feeding_dir.mkdir(parents=True, exist_ok=True)
# Group sleep by month
sleep_by_month: Dict[str, List[Dict[str, Any]]] = {}
for ev in sleep_events:
m_str = ev["start"][:7] # YYYY-MM
sleep_by_month.setdefault(m_str, []).append(ev)
for m_str, m_events in sleep_by_month.items():
m_events.sort(key=lambda x: (x["start"], x["id"]))
with open(sleep_dir / f"{m_str}.jsonl", "w", encoding="utf-8") as f:
for ev in m_events:
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
# Group feeding by month
feed_by_month: Dict[str, List[Dict[str, Any]]] = {}
for ev in feeding_events:
m_str = ev["start"][:7]
feed_by_month.setdefault(m_str, []).append(ev)
for m_str, m_events in feed_by_month.items():
m_events.sort(key=lambda x: (x["start"], x["id"]))
with open(feeding_dir / f"{m_str}.jsonl", "w", encoding="utf-8") as f:
for ev in m_events:
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
return sleep_events, feeding_events
# ==============================================================================
# 4. Generate Audit Report
# ==============================================================================
def generate_audit_report(stated_sums: Dict[str, Dict[str, Any]]) -> str:
con = duckdb.connect(":memory:")
# Initialize schema
schema_dir = DATA_DIR / "_schema"
for sql_f in ["envelope.sql", "entities.sql", "views.sql"]:
con.execute((schema_dir / sql_f).read_text(encoding="utf-8"))
# Load foods
foods_file = DATA_DIR / "entities" / "foods.jsonl"
if foods_file.is_file():
con.execute(f"INSERT INTO foods SELECT * FROM read_json_auto('{foods_file}')")
# Load all nutrition events
nut_files = str(DATA_DIR / "events" / "nutrition" / "*.jsonl")
con.execute(f"INSERT INTO events SELECT * FROM read_json_auto('{nut_files}')")
# Query daily sums
db_sums = con.execute("SELECT log_date, total_protein_g, total_carbs_g, total_kcal, item_count FROM v_nutrition_daily ORDER BY log_date ASC").fetchall()
lines = []
lines.append("# 📊 Datastore Migration: Arithmetic Discrepancy Audit Report\n")
lines.append(f"**Generated:** {datetime.datetime.now(BERLIN_TZ).strftime('%Y-%m-%d %H:%M:%S')} ")
lines.append("**Source of Truth:** DuckDB engine over `data/events/nutrition/*.jsonl` \n")
lines.append("This audit compares the old handwritten Markdown `**Summe heute:**` against the mathematically exact DuckDB column summations.\n")
lines.append("| Date | Items | Hand Stated Protein | DuckDB Exact Protein | Δ Protein | Hand Stated kcal | DuckDB Exact kcal | Δ kcal |")
lines.append("| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: |")
for row in db_sums:
d_str = str(row[0])
p_exact = row[1]
k_exact = int(row[3])
count = row[4]
stated = stated_sums.get(d_str, {})
p_hand = stated.get("protein_g", 0.0)
k_hand = int(stated.get("kcal", 0))
delta_p = round(p_exact - p_hand, 1)
delta_k = k_exact - k_hand
p_diff_str = f"**{delta_p:+.1f} g**" if abs(delta_p) > 0.1 else "✅ 0.0"
k_diff_str = f"**{delta_k:+d} kcal**" if abs(delta_k) > 2 else "✅ 0"
lines.append(f"| {d_str} | {count} | {p_hand:.1f} g | {p_exact:.1f} g | {p_diff_str} | {k_hand} | {k_exact} | {k_diff_str} |")
return "\n".join(lines)
if __name__ == "__main__":
print("1. Migrating Standard Foods...")
foods = migrate_standard_foods()
print(f" -> Migrated {len(foods)} food reference items.")
print("2. Migrating Nutrition Logs...")
nut_events, stated_sums = migrate_nutrition_logs()
print(f" -> Migrated {len(nut_events)} nutrition events across {len(stated_sums)} days.")
print("3. Migrating Mila Sleep & Feeding Logs...")
sleep_events, feed_events = migrate_mila_sleep_and_feeding()
print(f" -> Migrated {len(sleep_events)} sleep events and {len(feed_events)} bottle feeding events.")
print("4. Generating Discrepancy Audit Report...")
report = generate_audit_report(stated_sums)
report_file = WIKI_DIR / "reports" / "datastore-migration-audit.md"
report_file.parent.mkdir(parents=True, exist_ok=True)
report_file.write_text(report, encoding="utf-8")
print(f" -> Wrote audit report to {report_file}")
print("\n--- Summary ---")
print(report)
skills/datastore/scripts/render_views.py
+584
View File
@@ -0,0 +1,584 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "duckdb>=1.1.0",
# "python-ulid>=3.0.0",
# "pytz>=2024.1",
# ]
# ///
"""
render_views.py — Generates clean Markdown view pages and HTML dashboards from DuckDB JSONL.
Uses DatastoreEngine for exact schema validation, typed loading, and Berlin timezone attribution.
"""
from __future__ import annotations
import argparse
import datetime
import os
from pathlib import Path
from zoneinfo import ZoneInfo
from datastore import DatastoreEngine
BERLIN_TZ = ZoneInfo("Europe/Berlin")
WEEKDAYS_DE = {
0: "Montag",
1: "Dienstag",
2: "Mittwoch",
3: "Donnerstag",
4: "Freitag",
5: "Samstag",
6: "Sonntag",
}
def find_default_dirs():
candidates = [
Path.cwd() / "knowledge-wiki",
Path("/Users/moritz/src/infrapuzzle/knowledge-wiki"),
Path("/home/node/.openclaw/wiki/main"),
]
for c in candidates:
if (c / "data").is_dir():
return c, c / "data"
return Path.cwd(), Path.cwd() / "data"
def render_nutrition_views(engine: DatastoreEngine, wiki_dir: Path):
con = engine.con
months = ["2026-09", "2026-10"]
for m in months:
out_file = wiki_dir / "entities" / "nutrition-log" / "moritz" / f"{m}.md"
out_file.parent.mkdir(parents=True, exist_ok=True)
month_name = "September" if m.endswith("09") else "Oktober"
# Query days in month descending
days = con.execute(
f"""
SELECT DISTINCT log_date
FROM v_nutrition
WHERE strftime(log_date, '%Y-%m') = '{m}'
ORDER BY log_date DESC
"""
).fetchall()
md = []
md.append("---")
md.append(f"id: nutrition-log-moritz-{m}")
md.append("pageType: source")
md.append(f'title: "Ernährungsprotokoll Moritz — {month_name} 2026"')
md.append(f"month: {m}")
md.append("tags:")
md.append(" - nutrition-log")
md.append(" - personal")
md.append("lifecycle:")
md.append(" state: active")
md.append("---")
md.append("")
md.append(f"# Ernährungsprotokoll Moritz — {month_name} 2026")
md.append("")
md.append("<!-- DO NOT EDIT: GENERATED BY DATASTORE -->")
md.append("<!-- Source of Truth: data/events/nutrition/{YYYY-MM}.jsonl -->")
md.append("")
for (d,) in days:
d_str = str(d)
d_date = datetime.date.fromisoformat(d_str)
weekday_str = WEEKDAYS_DE[d_date.weekday()]
# Fetch daily aggregate
daily_agg = con.execute(
f"""
SELECT total_protein_g, total_carbs_g, total_sugar_g, total_fat_g, total_fiber_g, total_salt_g, total_kcal
FROM v_nutrition_daily
WHERE log_date = '{d_str}' AND subject = 'moritz'
"""
).fetchone()
# Check if daily balance against Garmin is available
balance_row = con.execute(
f"""
SELECT intake_kcal, expenditure_kcal, active_kcal, rmr_kcal, steps, balance_kcal, balance_status
FROM v_daily_balance
WHERE log_date = '{d_str}' AND subject = 'moritz'
"""
).fetchone()
md.append(f"## {d_str} {weekday_str}")
md.append("")
md.append("| Zeit | Input (original) | Menge | Eiweiß | KH | Zucker | Fett | Ballaststoffe | Salz | kcal |")
md.append("|---|---|---|---|---|---|---|---|---|---|")
# Fetch rows for this day
rows = con.execute(
f"""
SELECT strftime(local_time, '%H:%M'), item, qty, unit, protein_g, carbs_g, sugar_g, fat_g, fiber_g, salt_g, kcal, estimated
FROM v_nutrition
WHERE log_date = '{d_str}' AND subject = 'moritz'
ORDER BY start ASC
"""
).fetchall()
for r in rows:
t_str = r[0]
item = r[1]
qty = f"{int(r[2]) if r[2].is_integer() else r[2]} {r[3]}" if r[2] is not None and r[3] else ""
est = "~" if r[11] else ""
p = f"{est}{r[4]:.1f} g"
c = f"{est}{r[5]:.1f} g"
s = f"{est}{r[6]:.1f} g"
f_val = f"{est}{r[7]:.1f} g"
fib = f"{est}{r[8]:.1f} g"
salt = f"{est}{r[9]:.2f} g"
kc = f"{est}{int(round(r[10]))}"
md.append(f"| {t_str} | {item} | {qty} | {p} | {c} | {s} | {f_val} | {fib} | {salt} | {kc} |")
if daily_agg:
tot_p = f"{daily_agg[0]:.1f} g"
tot_c = f"{daily_agg[1]:.1f} g"
tot_s = f"{daily_agg[2]:.1f} g"
tot_f = f"{daily_agg[3]:.1f} g"
tot_fib = f"{daily_agg[4]:.1f} g"
tot_salt = f"{daily_agg[5]:.2f} g"
tot_kc = f"**{int(round(daily_agg[6]))}**"
md.append(f"| **Tages-Summe** | | | **{tot_p}** | **{tot_c}** | **{tot_s}** | **{tot_f}** | **{tot_fib}** | **{tot_salt}** | {tot_kc} |")
md.append("")
# Render Garmin balance block if available
if balance_row and balance_row[1] is not None:
intake = int(round(balance_row[0]))
expend = int(round(balance_row[1]))
diff = int(round(balance_row[5]))
steps = balance_row[4] or 0
active = balance_row[2] or 0
sign = "+" if diff > 0 else ""
md.append(f"- **Vortags-Bilanz (Garmin)**: {intake} kcal Verzehr - {expend} kcal Verbrauch = **{sign}{diff} kcal {balance_row[6]}** ({steps} Schritte, {active} Aktiv-kcal)")
md.append("")
out_file.write_text("\n".join(md), encoding="utf-8")
print(f"Rendered {out_file}")
def render_sleep_view(engine: DatastoreEngine, wiki_dir: Path):
con = engine.con
out_file = wiki_dir / "entities" / "people" / "mila-schlafprotokoll.md"
out_file.parent.mkdir(parents=True, exist_ok=True)
days = con.execute(
"""
SELECT DISTINCT log_date
FROM v_sleep
WHERE subject = 'mila'
ORDER BY log_date DESC
"""
).fetchall()
md = []
md.append("---")
md.append("id: mila-schlafprotokoll")
md.append("pageType: log")
md.append('title: "Schlafprotokoll Mila"')
md.append("subject: mila")
md.append("tags:")
md.append(" - sleep")
md.append(" - baby")
md.append(" - mila")
md.append("lifecycle:")
md.append(" state: active")
md.append("---")
md.append("")
md.append("# Schlafprotokoll Mila")
md.append("")
md.append("<!-- DO NOT EDIT: GENERATED BY DATASTORE -->")
md.append("<!-- Source of Truth: data/events/sleep/{YYYY-MM}.jsonl -->")
md.append("")
current_month_label = ""
for (d,) in days:
d_str = str(d)
d_date = datetime.date.fromisoformat(d_str)
month_label = f"{'September' if d_date.month == 9 else 'Oktober'} {d_date.year}"
if month_label != current_month_label:
current_month_label = month_label
md.append(f"# {month_label}")
md.append("")
weekday_str = WEEKDAYS_DE[d_date.weekday()]
daily_agg = con.execute(
f"""
SELECT total_sleep_hours, nap_count, night_sleep_minutes, nap_sleep_minutes
FROM v_sleep_daily
WHERE log_date = '{d_str}' AND subject = 'mila'
"""
).fetchone()
day_sum_str = ""
if daily_agg:
tot_h = daily_agg[0]
naps = daily_agg[1]
day_sum_str = f" ({tot_h:.2f} h gesamt, {naps} Schläfchen)"
md.append(f"## {weekday_str}, {d_date.strftime('%d.%m.%Y')}{day_sum_str}")
sessions = con.execute(
f"""
SELECT strftime(local_start, '%H:%M'),
CASE WHEN is_open THEN NULL ELSE strftime(local_end, '%H:%M') END,
sleep_type, location, note, duration_minutes, is_open
FROM v_sleep
WHERE log_date = '{d_str}' AND subject = 'mila'
ORDER BY start ASC
"""
).fetchall()
for s in sessions:
s_time = s[0]
e_time = s[1]
s_type = s[2]
loc = s[3]
note = s[4] or ""
dur = s[5]
is_open = s[6]
dur_str = f"{dur // 60} h {dur % 60:02d} min" if dur >= 60 else f"{dur} min"
if is_open:
md.append(f"- {s_time} – | {note or 'Nacht'} (läuft)")
else:
label = note if note else ("Nacht" if s_type == "night" else "Schläfchen")
md.append(f"- {s_time} – {e_time} | {label} ({dur_str})")
md.append("")
# Preserve narrative analysis block
md.append("""---
# Monatsanalyse September 2026 (nachgetragen 03.10.)
*Bewertet aus Sicht einer Kinderarzt-Praxis mit Schlafspezialisierung – Hinweise, keine Diagnose.*
**Zahlen (19.–30.09., 11 protokollierte Tage):**
- Nacht: Ø ~11 h 18 min (10 h 10 – 11 h 55)
- Tag: Ø ~2 h 29 min in 4–5 Schläfchen
- 24-h-Summe: Ø ~13 h 47 min
**Einordnung:** Das liegt am oberen, aber völlig normalen Rand des altersüblichen Bereichs (Säugling ~6 Mon.: 12–16 h/24 h, vgl. [Wikipedia: Schlaf](https://de.wikipedia.org/wiki/Schlaf) und [Wikipedia: Säugling](https://de.wikipedia.org/wiki/S%C3%A4ugling)).
**Trends im Monat:**
1. **Konsolidierung:** Von 5 kurzen Mikro-Schläfchen (Sep Anfang) hin zu 3–4 Schläfchen, erste lange Blöcke (85–140 min) am 20./21./27.09. und 02.10.
2. **Spätere Bettzeit:** 18:34–19:55 (Mitte Sep) → 20:10–21:00 (Ende Sep/Okt) – stabile Morgenwache ~06:00–06:50 bleibt.
3. **Durchschlafen:** Durchgehend lange Nachtblöcke, Wachphasen nachts selten und kurz.
**Erwartbar ab ~7–8 Monaten:** Übergang zum 3→2-Naps-Schema, Morgenschläfchen fällt zuerst weg. Zirkadiane Reifung unterstützt das (vgl. [Wikipedia: Zirkadiane Rhythmik](https://de.wikipedia.org/wiki/Zirkadiane_Rhythmik)).
---
""")
out_file.write_text("\n".join(md), encoding="utf-8")
print(f"Rendered {out_file}")
def render_html_dashboards(engine: DatastoreEngine, html_dir: Path):
html_dir.mkdir(parents=True, exist_ok=True)
con = engine.con
# 1. Nutrition HTML Dashboard
nutrition_days = con.execute(
"""
SELECT
d.log_date,
d.total_kcal,
d.total_protein_g,
d.total_carbs_g,
d.total_fat_g,
d.item_count,
b.expenditure_kcal,
b.balance_kcal,
b.balance_status,
b.steps
FROM v_nutrition_daily d
LEFT JOIN v_daily_balance b ON d.log_date = b.log_date AND d.subject = b.subject
WHERE d.subject = 'moritz'
ORDER BY d.log_date DESC
LIMIT 30
"""
).fetchall()
latest_n = nutrition_days[0] if nutrition_days else None
latest_kcal = int(round(latest_n[1])) if latest_n else 0
latest_protein = f"{latest_n[2]:.1f}" if latest_n else "0"
latest_garmin = f"{int(round(latest_n[6]))} kcal" if latest_n and latest_n[6] is not None else "Ausstehend"
latest_balance = f"{int(round(latest_n[7]))} kcal ({latest_n[8]})" if latest_n and latest_n[7] is not None else "—"
# Compute 7-day average
avg_7 = con.execute(
"""
SELECT ROUND(AVG(total_kcal), 0), ROUND(AVG(total_protein_g), 1)
FROM (SELECT total_kcal, total_protein_g FROM v_nutrition_daily WHERE subject = 'moritz' ORDER BY log_date DESC LIMIT 7)
"""
).fetchone()
avg_7_kcal = int(avg_7[0]) if avg_7 and avg_7[0] is not None else 0
avg_7_prot = avg_7[1] if avg_7 and avg_7[1] is not None else 0
nutrition_rows_html = []
for r in nutrition_days:
d_str = str(r[0])
kc = int(round(r[1]))
prot = f"{r[2]:.1f}"
carbs = f"{r[3]:.1f}"
fat = f"{r[4]:.1f}"
exp = int(round(r[6])) if r[6] is not None else "—"
diff = int(round(r[7])) if r[7] is not None else "—"
diff_class = "text-emerald-400" if (r[7] is not None and r[7] < 0) else ("text-amber-400" if (r[7] is not None and r[7] > 0) else "text-slate-400")
steps = f"{r[9]:,}" if r[9] is not None else "—"
nutrition_rows_html.append(f"""
<tr class="border-b border-slate-700/50 hover:bg-slate-800/40 transition">
<td class="py-3 px-4 font-mono font-medium text-slate-200">{d_str}</td>
<td class="py-3 px-4 font-bold text-sky-400">{kc}</td>
<td class="py-3 px-4 font-semibold text-emerald-400">{prot} g</td>
<td class="py-3 px-4 text-slate-300">{carbs} g</td>
<td class="py-3 px-4 text-slate-300">{fat} g</td>
<td class="py-3 px-4 text-slate-400">{exp}</td>
<td class="py-3 px-4 font-bold {diff_class}">{diff}</td>
<td class="py-3 px-4 text-slate-400">{steps}</td>
</tr>
""")
nutrition_html = f"""<!DOCTYPE html>
<html lang="de" class="dark">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Moritz — Ernährung & Energiebilanz</title>
<script src="https://cdn.tailwindcss.com"></script>
</head>
<body class="bg-slate-950 text-slate-100 min-h-screen p-4 sm:p-8 font-sans antialiased selection:bg-emerald-500 selection:text-black">
<div class="max-w-6xl mx-auto space-y-6">
<!-- Header -->
<header class="flex flex-col sm:flex-row sm:items-center justify-between pb-6 border-b border-slate-800 gap-4">
<div>
<h1 class="text-2xl sm:text-3xl font-bold tracking-tight bg-gradient-to-r from-sky-400 via-emerald-400 to-teal-300 bg-clip-text text-transparent">
Moritz — Ernährung & Energiebilanz
</h1>
<p class="text-sm text-slate-400 mt-1">Strikte DuckDB-Aggregatdaten · Single Source of Truth</p>
</div>
<div class="inline-flex items-center gap-2 px-3 py-1.5 rounded-full bg-slate-900 border border-slate-800 text-xs text-slate-400">
<span class="w-2 h-2 rounded-full bg-emerald-500 animate-pulse"></span>
<span>Letztes Update: {datetime.date.today().strftime('%d.%m.%Y')}</span>
</div>
</header>
<!-- Key Metrics Grid -->
<div class="grid grid-cols-2 lg:grid-cols-4 gap-4">
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 shadow-lg">
<div class="text-xs font-semibold uppercase tracking-wider text-slate-400">Letzter Tag (Kcal)</div>
<div class="text-3xl font-extrabold text-sky-400 mt-2">{latest_kcal}</div>
<div class="text-xs text-slate-400 mt-1">7-Tage-Schnitt: {avg_7_kcal} kcal</div>
</div>
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 shadow-lg">
<div class="text-xs font-semibold uppercase tracking-wider text-slate-400">Eiweiß Intake</div>
<div class="text-3xl font-extrabold text-emerald-400 mt-2">{latest_protein} g</div>
<div class="text-xs text-slate-400 mt-1">7-Tage-Schnitt: {avg_7_prot} g</div>
</div>
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 shadow-lg">
<div class="text-xs font-semibold uppercase tracking-wider text-slate-400">Garmin Verbrauch</div>
<div class="text-3xl font-extrabold text-amber-400 mt-2">{latest_garmin}</div>
<div class="text-xs text-slate-400 mt-1">RMR + Aktivität</div>
</div>
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 shadow-lg">
<div class="text-xs font-semibold uppercase tracking-wider text-slate-400">Tages-Bilanz</div>
<div class="text-3xl font-extrabold text-indigo-400 mt-2">{latest_balance}</div>
<div class="text-xs text-slate-400 mt-1">Netto-Differenz</div>
</div>
</div>
<!-- Data Table Card -->
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 shadow-xl overflow-hidden">
<h2 class="text-lg font-semibold text-slate-200 mb-4">Verlauf (Letzte 30 Tage)</h2>
<div class="overflow-x-auto">
<table class="w-full text-left text-sm whitespace-nowrap">
<thead class="text-xs uppercase bg-slate-800/60 text-slate-400">
<tr>
<th class="py-3 px-4 rounded-l-lg">Datum</th>
<th class="py-3 px-4">Aufnahme</th>
<th class="py-3 px-4">Protein</th>
<th class="py-3 px-4">KH</th>
<th class="py-3 px-4">Fett</th>
<th class="py-3 px-4">Garmin</th>
<th class="py-3 px-4">Bilanz</th>
<th class="py-3 px-4 rounded-r-lg">Schritte</th>
</tr>
</thead>
<tbody>
{''.join(nutrition_rows_html)}
</tbody>
</table>
</div>
</div>
</div>
</body>
</html>
"""
out_nutrition_html = html_dir / "nutrition.html"
out_nutrition_html.write_text(nutrition_html, encoding="utf-8")
print(f"Rendered HTML Dashboard: {out_nutrition_html}")
# 2. Sleep HTML Dashboard
sleep_days = con.execute(
"""
SELECT
log_date,
total_sleep_hours,
nap_count,
night_sleep_minutes,
nap_sleep_minutes,
total_sessions
FROM v_sleep_daily
WHERE subject = 'mila'
ORDER BY log_date DESC
LIMIT 30
"""
).fetchall()
latest_s = sleep_days[0] if sleep_days else None
latest_tot_h = f"{latest_s[1]:.2f} h" if latest_s else "0 h"
latest_night_str = f"{latest_s[3] // 60} h {latest_s[3] % 60:02d} min" if latest_s else "—"
latest_nap_str = f"{latest_s[2]} Naps ({latest_s[4]} min)" if latest_s else "—"
# Compute 7-day average total sleep hours
avg_sleep = con.execute(
"""
SELECT ROUND(AVG(total_sleep_hours), 2)
FROM (SELECT total_sleep_hours FROM v_sleep_daily WHERE subject = 'mila' ORDER BY log_date DESC LIMIT 7)
"""
).fetchone()
avg_sleep_h = f"{avg_sleep[0]:.2f} h" if avg_sleep and avg_sleep[0] is not None else "—"
sleep_rows_html = []
for r in sleep_days:
d_str = str(r[0])
tot = f"{r[1]:.2f} h"
night_h = f"{r[3] // 60} h {r[3] % 60:02d} min"
nap_info = f"{r[2]} ({r[4]} min)"
sleep_rows_html.append(f"""
<tr class="border-b border-slate-700/50 hover:bg-slate-800/40 transition">
<td class="py-3 px-4 font-mono font-medium text-slate-200">{d_str}</td>
<td class="py-3 px-4 font-bold text-purple-400">{tot}</td>
<td class="py-3 px-4 text-sky-400 font-semibold">{night_h}</td>
<td class="py-3 px-4 text-emerald-400">{nap_info}</td>
</tr>
""")
sleep_html = f"""<!DOCTYPE html>
<html lang="de" class="dark">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Mila — Schlafprotokoll</title>
<script src="https://cdn.tailwindcss.com"></script>
</head>
<body class="bg-slate-950 text-slate-100 min-h-screen p-4 sm:p-8 font-sans antialiased selection:bg-purple-500 selection:text-white">
<div class="max-w-5xl mx-auto space-y-6">
<!-- Header -->
<header class="flex flex-col sm:flex-row sm:items-center justify-between pb-6 border-b border-slate-800 gap-4">
<div>
<h1 class="text-2xl sm:text-3xl font-bold tracking-tight bg-gradient-to-r from-purple-400 via-pink-400 to-indigo-300 bg-clip-text text-transparent">
Mila — Schlafprotokoll
</h1>
<p class="text-sm text-slate-400 mt-1">Deterministische Schlafdauer & Wachfenster via DuckDB</p>
</div>
<div class="inline-flex items-center gap-2 px-3 py-1.5 rounded-full bg-slate-900 border border-slate-800 text-xs text-slate-400">
<span class="w-2 h-2 rounded-full bg-purple-500 animate-pulse"></span>
<span>Letztes Update: {datetime.date.today().strftime('%d.%m.%Y')}</span>
</div>
</header>
<!-- Key Metrics Grid -->
<div class="grid grid-cols-2 sm:grid-cols-4 gap-4">
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 shadow-lg">
<div class="text-xs font-semibold uppercase tracking-wider text-slate-400">24h Schlaf</div>
<div class="text-3xl font-extrabold text-purple-400 mt-2">{latest_tot_h}</div>
<div class="text-xs text-slate-400 mt-1">Letzter Tag</div>
</div>
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 shadow-lg">
<div class="text-xs font-semibold uppercase tracking-wider text-slate-400">Nachtschlaf</div>
<div class="text-3xl font-extrabold text-sky-400 mt-2">{latest_night_str}</div>
<div class="text-xs text-slate-400 mt-1">Hauptschlafphase</div>
</div>
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 shadow-lg">
<div class="text-xs font-semibold uppercase tracking-wider text-slate-400">Tagesschläfchen</div>
<div class="text-2xl font-extrabold text-emerald-400 mt-2">{latest_nap_str}</div>
<div class="text-xs text-slate-400 mt-1">Tagsüber</div>
</div>
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 shadow-lg">
<div class="text-xs font-semibold uppercase tracking-wider text-slate-400">7-Tage-Schnitt</div>
<div class="text-3xl font-extrabold text-pink-400 mt-2">{avg_sleep_h}</div>
<div class="text-xs text-slate-400 mt-1">Altersbereich: 12-16 h</div>
</div>
</div>
<!-- Data Table Card -->
<div class="bg-slate-900/80 border border-slate-800 rounded-2xl p-5 shadow-xl overflow-hidden">
<h2 class="text-lg font-semibold text-slate-200 mb-4">Schlafhistorie (Letzte 30 Tage)</h2>
<div class="overflow-x-auto">
<table class="w-full text-left text-sm whitespace-nowrap">
<thead class="text-xs uppercase bg-slate-800/60 text-slate-400">
<tr>
<th class="py-3 px-4 rounded-l-lg">Datum</th>
<th class="py-3 px-4">24h Gesamtschlaf</th>
<th class="py-3 px-4">Nachtschlaf</th>
<th class="py-3 px-4 rounded-r-lg">Tagesschläfchen (Dauer)</th>
</tr>
</thead>
<tbody>
{''.join(sleep_rows_html)}
</tbody>
</table>
</div>
</div>
</div>
</body>
</html>
"""
out_sleep_html = html_dir / "sleep.html"
out_sleep_html.write_text(sleep_html, encoding="utf-8")
print(f"Rendered HTML Dashboard: {out_sleep_html}")
def main():
parser = argparse.ArgumentParser(description="Render Markdown and HTML views from DuckDB Datastore")
parser.add_argument("--wiki-dir", help="Path to knowledge-wiki repository")
parser.add_argument("--data-dir", help="Path to knowledge-wiki/data directory")
parser.add_argument("--html-dir", help="Path to output HTML dashboards")
parser.add_argument("--no-md", action="store_true", help="Skip Markdown rendering")
parser.add_argument("--no-html", action="store_true", help="Skip HTML dashboard rendering")
args = parser.parse_args()
default_wiki, default_data = find_default_dirs()
wiki_dir = Path(args.wiki_dir) if args.wiki_dir else default_wiki
data_dir = Path(args.data_dir) if args.data_dir else default_data
html_dir = Path(args.html_dir) if args.html_dir else (wiki_dir / "public" / "family")
engine = DatastoreEngine(data_dir)
if not args.no_md:
render_nutrition_views(engine, wiki_dir)
render_sleep_view(engine, wiki_dir)
if not args.no_html:
render_html_dashboards(engine, html_dir)
if __name__ == "__main__":
main()