#!/usr/bin/env python3 # /// script # dependencies = [ # "duckdb>=1.1.0", # "python-ulid>=3.0.0", # "pytz>=2024.1", # ] # /// """ OpenClaw Datastore — Structured Personal Data Layer (DuckDB + JSONL in Git) Provides deterministic math, validation, interval handling, spatial queries, and JSONL file persistence for personal nutrition, baby sleep, sports, recipes, and places. Exposes both a CLI and a stdio JSON-RPC MCP server. """ from __future__ import annotations import argparse import datetime import json import os import re import subprocess import sys import logging from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union from zoneinfo import ZoneInfo import duckdb logger = logging.getLogger("datastore") def generate_ulid() -> str: try: from ulid import ULID return str(ULID()) except TypeError: import ulid return str(ulid.new()) except ImportError: import uuid return str(uuid.uuid4()) BERLIN_TZ = ZoneInfo("Europe/Berlin") def parse_duration(val: str) -> datetime.timedelta: """Parses duration strings like '15m', '15 min', '1h', '1h30m', '1.5h' into timedelta.""" s = val.strip().lower() if not s: raise ValueError("Empty duration string") # Decimal hours like "1.5h" or "1.5 stunden" m_dec_h = re.fullmatch(r"^(\d+(?:\.\d+)?)\s*(?:h|stunden?)$", s) if m_dec_h: return datetime.timedelta(hours=float(m_dec_h.group(1))) # Decimal minutes like "2.5m" or "15 min" m_dec_m = re.fullmatch(r"^(\d+(?:\.\d+)?)\s*(?:m|min|minuten?)$", s) if m_dec_m: return datetime.timedelta(minutes=float(m_dec_m.group(1))) # Composite like "1h 30m" or "1h30min" or "45s" m_comp = re.fullmatch( r"^(?:(\d+)\s*(?:h|stunden?))?\s*(?:(\d+)\s*(?:m|min|minuten?))?\s*(?:(\d+)\s*(?:s|sek|sekunden?))?$", s, ) if m_comp and any(m_comp.groups()): h = int(m_comp.group(1) or 0) m = int(m_comp.group(2) or 0) sec = int(m_comp.group(3) or 0) return datetime.timedelta(hours=h, minutes=m, seconds=sec) # Pure number without unit: default to minutes if s.isdigit(): return datetime.timedelta(minutes=int(s)) raise ValueError(f"Unrecognized duration format: '{val}'. Expected formats: '15m', '15 min', '1h', '1h30m'.") def find_data_dir() -> Path: """Discovers the knowledge-wiki data directory.""" for env_var in ["DATASTORE_DIR", "OPENCLAW_DATA_DIR"]: if os.environ.get(env_var): return Path(os.environ[env_var]).resolve() candidates = [ Path.cwd() / "knowledge-wiki" / "data", Path.cwd() / "data", Path.home() / "src" / "infrapuzzle" / "knowledge-wiki" / "data", Path.home() / ".openclaw" / "wiki" / "main" / "data", Path("/home/node/.openclaw/wiki/main/data"), ] for c in candidates: if c.is_dir() and (c / "_meta.json").exists(): return c.resolve() # Fallback to creating a local data directory if needed default_dir = Path.cwd() / "knowledge-wiki" / "data" default_dir.mkdir(parents=True, exist_ok=True) return default_dir.resolve() class DatastoreEngine: def __init__(self, data_dir: Optional[Path] = None): self.data_dir = data_dir or find_data_dir() self.con = duckdb.connect(":memory:") self.has_spatial_extension = False self._init_schemas() self._load_all_data() def _init_schemas(self): """Initializes tables, views, and spatial support in DuckDB.""" # Timezone support via icu extension try: self.con.execute("INSTALL icu; LOAD icu;") self.con.execute("SET TimeZone = 'Europe/Berlin';") except Exception: pass # Spatial support: try extension, fallback to pure-SQL Haversine macro try: self.con.execute("INSTALL spatial; LOAD spatial;") self.has_spatial_extension = True except Exception: self.has_spatial_extension = False # Fallback pure-SQL distance macro (returns meters) self.con.execute(""" CREATE MACRO IF NOT EXISTS distance_m(lat1, lon1, lat2, lon2) AS 6371000 * 2 * asin(sqrt( sin(radians(lat2 - lat1) / 2) ^ 2 + cos(radians(lat1)) * cos(radians(lat2)) * sin(radians(lon2 - lon1) / 2) ^ 2 )); """) schema_dir = self.data_dir / "_schema" if schema_dir.is_dir(): for f in ["envelope.sql", "entities.sql", "views.sql"]: sql_file = schema_dir / f if sql_file.is_file(): sql = sql_file.read_text(encoding="utf-8") self.con.execute(sql) else: # Inline fallback schema if _schema is missing self._create_inline_schema() def _create_inline_schema(self): self.con.execute(""" CREATE TABLE IF NOT EXISTS events ( id VARCHAR PRIMARY KEY, kind VARCHAR NOT NULL, subject VARCHAR NOT NULL, actor VARCHAR NOT NULL, start TIMESTAMPTZ NOT NULL, "end" TIMESTAMPTZ, estimated BOOLEAN DEFAULT false, meal VARCHAR, note VARCHAR, payload JSON NOT NULL, source_ref VARCHAR, created_at TIMESTAMPTZ NOT NULL, updated_at TIMESTAMPTZ, deleted_at TIMESTAMPTZ, CHECK ("end" IS NULL OR "end" >= start) ); CREATE TABLE IF NOT EXISTS people ( id VARCHAR PRIMARY KEY, name VARCHAR NOT NULL, role VARCHAR, birthdate DATE, notes VARCHAR ); CREATE TABLE IF NOT EXISTS foods ( id VARCHAR PRIMARY KEY, name VARCHAR NOT NULL, brand VARCHAR, category VARCHAR, per_100 JSON, density_g_per_ml DOUBLE, ingredients_text VARCHAR, barcode VARCHAR, source VARCHAR, description VARCHAR ); CREATE TABLE IF NOT EXISTS recipes ( id VARCHAR PRIMARY KEY, title VARCHAR NOT NULL, servings INTEGER DEFAULT 1, ingredients JSON, estimated_delta JSON, devices VARCHAR[], tags VARCHAR[], documents VARCHAR[], description VARCHAR ); CREATE TABLE IF NOT EXISTS places ( id VARCHAR PRIMARY KEY, name VARCHAR NOT NULL, category VARCHAR NOT NULL, lat DOUBLE NOT NULL, lon DOUBLE NOT NULL, address VARCHAR, google_maps_url VARCHAR, tags VARCHAR[], attributes JSON, documents VARCHAR[], notes VARCHAR ); CREATE TABLE IF NOT EXISTS garmin_daily ( calendarDate DATE PRIMARY KEY, totalKilocalories INTEGER, activeKilocalories INTEGER, bmrKilocalories INTEGER, totalSteps INTEGER, restingHeartRate INTEGER, raw JSON ); """) def _load_all_data(self): """Loads all existing JSONL files from entities/, events/, and garmin/ into DuckDB.""" # 1. Load entities entities_dir = self.data_dir / "entities" if entities_dir.is_dir(): for f in entities_dir.glob("*.jsonl"): collection = f.stem if collection in ["people", "foods", "recipes", "places"]: self._load_jsonl_file(f, collection) # 2. Load events events_dir = self.data_dir / "events" if events_dir.is_dir(): for f in events_dir.glob("**/*.jsonl"): self._load_jsonl_file(f, "events") # 3. Load garmin daily garmin_daily_dir = self.data_dir / "garmin" / "daily" if garmin_daily_dir.is_dir(): for f in garmin_daily_dir.glob("*.jsonl"): self._load_garmin_daily_file(f) def _load_garmin_daily_file(self, file_path: Path): """Reads daily summaries from JSONL into garmin_daily table.""" if not file_path.is_file() or file_path.stat().st_size == 0: return with open(file_path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue try: record = json.loads(line) cal_date = record.get("calendarDate") if not cal_date: continue total_kcal = record.get("totalKilocalories") active_kcal = record.get("activeKilocalories") bmr_kcal = record.get("bmrKilocalories") steps = record.get("totalSteps") rhr = record.get("restingHeartRate") raw = json.dumps(record, ensure_ascii=False) self.con.execute( """ INSERT OR REPLACE INTO garmin_daily (calendarDate, totalKilocalories, activeKilocalories, bmrKilocalories, totalSteps, restingHeartRate, raw) VALUES (?, ?, ?, ?, ?, ?, ?) """, [cal_date, total_kcal, active_kcal, bmr_kcal, steps, rhr, raw], ) except Exception as e: logger.warning(f"Error loading garmin record from {file_path}: {e}") def _load_jsonl_file(self, file_path: Path, table_name: str): """Reads lines from JSONL and inserts into target table.""" if not file_path.is_file() or file_path.stat().st_size == 0: return with open(file_path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue try: record = json.loads(line) columns = list(record.keys()) placeholders = ", ".join(["?"] * len(columns)) col_names = ", ".join([f'"{c}"' for c in columns]) values = [ json.dumps(record[c]) if isinstance(record[c], (dict, list)) else record[c] for c in columns ] self.con.execute( f'INSERT OR REPLACE INTO "{table_name}" ({col_names}) VALUES ({placeholders})', values, ) except Exception as e: # Ignore comment lines or malformed lines with warning pass def _matches_payload(self, kind: str, p1: Dict[str, Any], p2: Dict[str, Any]) -> bool: """Determines if two event payloads represent the same intake/session.""" if p1 == p2: return True if kind.startswith("nutrition."): food1 = p1.get("food_id") food2 = p2.get("food_id") item1 = str(p1.get("item", "")).strip().lower() item2 = str(p2.get("item", "")).strip().lower() same_item = ( (food1 and food2 and food1 == food2) or (item1 and item2 and item1 == item2) or (food1 and item2 and food1 == item2) or (food2 and item1 and food2 == item1) ) if same_item: qty1 = p1.get("qty") qty2 = p2.get("qty") if qty1 is not None and qty2 is not None: try: return abs(float(qty1) - float(qty2)) < 0.01 except (ValueError, TypeError): return qty1 == qty2 return True elif kind.startswith("feeding."): amt1 = p1.get("amount_ml") or p1.get("consumed_ml") amt2 = p2.get("amount_ml") or p2.get("consumed_ml") if amt1 is not None and amt2 is not None: return amt1 == amt2 elif kind.startswith("diaper."): if p1.get("pee") == p2.get("pee") and p1.get("poo") == p2.get("poo"): return True elif kind.startswith("sleep."): if p1.get("type") == p2.get("type"): return True return False def _is_duplicate_event( self, kind: str, subject: str, payload: Dict[str, Any], start_iso: str, tolerance_seconds: int = 900, ) -> Optional[Dict[str, Any]]: """Checks if a matching event already exists within tolerance_seconds (default 15m).""" rows = self.con.execute( """ SELECT id, kind, subject, actor, start, "end", estimated, meal, note, payload, source_ref, created_at, updated_at FROM events WHERE kind = ? AND subject = ? AND deleted_at IS NULL AND abs(epoch(start) - epoch(?::TIMESTAMPTZ)) <= ? ORDER BY start DESC """, [kind, subject, start_iso, tolerance_seconds], ).fetchall() for r in rows: existing_payload = json.loads(r[9]) if isinstance(r[9], str) else (r[9] or {}) if self._matches_payload(kind, payload, existing_payload): return { "id": r[0], "kind": r[1], "subject": r[2], "actor": r[3], "start": r[4].isoformat() if hasattr(r[4], "isoformat") else str(r[4]), "end": r[5].isoformat() if hasattr(r[5], "isoformat") and r[5] else (str(r[5]) if r[5] else None), "estimated": r[6], "meal": r[7], "note": r[8], "payload": existing_payload, "source_ref": r[10], "created_at": r[11].isoformat() if hasattr(r[11], "isoformat") else str(r[11]), "updated_at": r[12].isoformat() if hasattr(r[12], "isoformat") and r[12] else (str(r[12]) if r[12] else None), "deleted_at": None, } return None def _get_open_sleep_sessions(self, subject: str, exclude_id: Optional[str] = None) -> List[Dict[str, Any]]: """Finds open sleep sessions (end IS NULL) for subject, sorted chronologically.""" rows = self.con.execute( """ SELECT id, start, payload, note FROM events WHERE kind = 'sleep.session' AND subject = ? AND "end" IS NULL AND deleted_at IS NULL ORDER BY start ASC """, [subject], ).fetchall() results = [] for r in rows: if exclude_id and r[0] == exclude_id: continue results.append({ "id": r[0], "start": str(r[1]), "payload": json.loads(r[2]) if isinstance(r[2], str) else (r[2] or {}), "note": r[3], }) return results # -------------------------------------------------------------------------- # Event Operations # -------------------------------------------------------------------------- def add_event( self, kind: str, subject: str, actor: str, payload: Dict[str, Any], start: Optional[str] = None, end: Optional[str] = None, since: Optional[str] = None, estimated: bool = False, meal: Optional[str] = None, note: Optional[str] = None, source_ref: Optional[str] = None, allow_duplicate: bool = False, ) -> Dict[str, Any]: """Adds a time-series event, saves to JSONL partition, and returns confirmation.""" event_id = generate_ulid() now_dt = datetime.datetime.now(datetime.timezone.utc) now_iso = now_dt.isoformat() if since and start: raise ValueError("Cannot specify both 'since' and 'start'. Use 'since' for relative times or 'start' for absolute times.") # Parse or default start time if since: delta = parse_duration(since) start_dt = (now_dt - delta).astimezone(BERLIN_TZ) start_iso = start_dt.isoformat() elif not start: start_dt = now_dt.astimezone(BERLIN_TZ) start_iso = start_dt.isoformat() else: start_dt = datetime.datetime.fromisoformat(start) if start_dt.tzinfo is None: start_dt = start_dt.replace(tzinfo=BERLIN_TZ) start_iso = start_dt.isoformat() end_iso = None if end: end_dt = datetime.datetime.fromisoformat(end) if end_dt.tzinfo is None: end_dt = end_dt.replace(tzinfo=BERLIN_TZ) if end_dt < start_dt: raise ValueError(f"End time {end} cannot be earlier than start time {start}") end_iso = end_dt.isoformat() # Check for duplicate event within 15 minutes unless explicitly allowed if not allow_duplicate: existing_event = self._is_duplicate_event(kind, subject, payload, start_iso) if existing_event: feedback = self._get_event_feedback(kind, subject, start_dt.date()) return { "status": "success", "duplicate": True, "message": f"Duplicate event detected within 15 minutes (existing event ID: {existing_event['id']}). Event retained without duplicate insertion.", "event": existing_event, "feedback": feedback, } # Check open sleep sessions (notice on every log, never silently deckeln!) open_warning = None if kind == "sleep.session": open_sessions = self._get_open_sleep_sessions(subject) if open_sessions: first_open = open_sessions[0] o_start_dt = datetime.datetime.fromisoformat(str(first_open["start"]).replace("Z", "+00:00")).astimezone(BERLIN_TZ) open_warning = f"Vorheriges Schlaf-Event ({first_open['id']} ab {o_start_dt.strftime('%d.%m. %H:%M Uhr')}) ist noch offen! Bitte Aufwachzeit nachreichen." # If adding a new open interval (end is None) if end_iso is None: # If existing open session is recent (< 8 hours on same calendar day) and not allow_duplicate, prevent accidental duplicate live nap diff_hours = (start_dt - o_start_dt).total_seconds() / 3600.0 if diff_hours < 8.0 and start_dt.date() == o_start_dt.date() and not allow_duplicate: raise ValueError( f"Subject '{subject}' hat bereits ein aktives Schlaf-Event ({first_open['id']} gestartet um {o_start_dt.strftime('%H:%M Uhr')}). Bitte beende es zuerst." ) record = { "id": event_id, "kind": kind, "subject": subject, "actor": actor, "start": start_iso, "end": end_iso, "estimated": estimated, "meal": meal, "note": note, "payload": payload, "source_ref": source_ref, "created_at": now_iso, "updated_at": None, "deleted_at": None, } # Insert into in-memory DuckDB self.con.execute( """ INSERT INTO events (id, kind, subject, actor, start, "end", estimated, meal, note, payload, source_ref, created_at, updated_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ event_id, kind, subject, actor, start_iso, end_iso, estimated, meal, note, json.dumps(payload), source_ref, now_iso, None, None, ], ) # Write to partitioned JSONL file # Partition path: events//.jsonl domain = kind.split(".")[0] month_str = start_dt.astimezone(BERLIN_TZ).strftime("%Y-%m") domain_dir = self.data_dir / "events" / domain domain_dir.mkdir(parents=True, exist_ok=True) partition_file = domain_dir / f"{month_str}.jsonl" with open(partition_file, "a", encoding="utf-8") as f: f.write(json.dumps(record, ensure_ascii=False) + "\n") # Sort partition deterministically by start, id self._sort_jsonl_file(partition_file, key_fn=lambda x: (x.get("start", ""), x.get("id", ""))) # Fetch computed feedback feedback = self._get_event_feedback(kind, subject, start_dt.date()) if open_warning: feedback["open_session_warning"] = open_warning if "day_total_formatted" in feedback and open_warning not in feedback["day_total_formatted"]: feedback["day_total_formatted"] += f"\n{open_warning}" elif "day_total_formatted" not in feedback: feedback["day_total_formatted"] = open_warning return { "status": "success", "event": record, "feedback": feedback, } def stop_event( self, kind: str, subject: str, end_time: Optional[str] = None, event_id: Optional[str] = None, actor: Optional[str] = None, ) -> Dict[str, Any]: """Finds the open event (end IS NULL) for kind and subject, and closes it.""" if event_id: row = self.con.execute( """ SELECT id, start FROM events WHERE id = ? AND kind = ? AND subject = ? AND "end" IS NULL AND deleted_at IS NULL """, [event_id, kind, subject], ).fetchone() if not row: raise ValueError(f"No open event with id='{event_id}' found for kind='{kind}' and subject='{subject}'") else: row = self.con.execute( """ SELECT id, start FROM events WHERE kind = ? AND subject = ? AND "end" IS NULL AND deleted_at IS NULL ORDER BY start DESC LIMIT 1 """, [kind, subject], ).fetchone() if not row: raise ValueError(f"No open event found for kind='{kind}' and subject='{subject}'") resolved_event_id, start_str = row[0], str(row[1]) now_dt = datetime.datetime.now(datetime.timezone.utc) if end_time: end_dt = datetime.datetime.fromisoformat(end_time) if end_dt.tzinfo is None: end_dt = end_dt.replace(tzinfo=BERLIN_TZ) else: end_dt = now_dt.astimezone(BERLIN_TZ) end_iso = end_dt.isoformat() now_iso = now_dt.isoformat() # Update in DuckDB self.con.execute( """ UPDATE events SET "end" = ?, updated_at = ? WHERE id = ? """, [end_iso, now_iso, resolved_event_id], ) # Update in JSONL domain = kind.split(".")[0] self._update_jsonl_record(domain, resolved_event_id, {"end": end_iso, "updated_at": now_iso}) # Calculate duration start_dt = datetime.datetime.fromisoformat(start_str.replace("Z", "+00:00")) duration_minutes = int((end_dt.astimezone(BERLIN_TZ) - start_dt.astimezone(BERLIN_TZ)).total_seconds() / 60) return { "status": "success", "event_id": resolved_event_id, "kind": kind, "subject": subject, "start": start_str, "end": end_iso, "duration_minutes": duration_minutes, "duration_formatted": f"{duration_minutes // 60} h {duration_minutes % 60} min" if duration_minutes >= 60 else f"{duration_minutes} min", } def delete_event(self, event_id: str) -> Dict[str, Any]: """Soft-deletes an event.""" now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat() row = self.con.execute("SELECT kind FROM events WHERE id = ?", [event_id]).fetchone() if not row: raise ValueError(f"Event with id '{event_id}' not found.") kind = row[0] domain = kind.split(".")[0] self.con.execute("UPDATE events SET deleted_at = ? WHERE id = ?", [now_iso, event_id]) self._update_jsonl_record(domain, event_id, {"deleted_at": now_iso}) return {"status": "success", "deleted_event_id": event_id} # -------------------------------------------------------------------------- # Entity Operations # -------------------------------------------------------------------------- def put_entity(self, collection: str, record: Dict[str, Any]) -> Dict[str, Any]: """Upserts an entity into entities/.jsonl.""" if collection not in ["foods", "recipes", "places", "people"]: raise ValueError(f"Unknown entity collection: {collection}") entity_id = record.get("id") if not entity_id: raise ValueError("Entity record must have an 'id' field.") entities_dir = self.data_dir / "entities" entities_dir.mkdir(parents=True, exist_ok=True) target_file = entities_dir / f"{collection}.jsonl" # Read existing, update or append existing_records = [] replaced = False if target_file.is_file(): with open(target_file, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue try: r = json.loads(line) if r.get("id") == entity_id: existing_records.append(record) replaced = True else: existing_records.append(r) except Exception: pass if not replaced: existing_records.append(record) # Sort by id existing_records.sort(key=lambda x: str(x.get("id", ""))) with open(target_file, "w", encoding="utf-8") as f: for r in existing_records: f.write(json.dumps(r, ensure_ascii=False) + "\n") # Reload into DuckDB columns = list(record.keys()) placeholders = ", ".join(["?"] * len(columns)) col_names = ", ".join([f'"{c}"' for c in columns]) values = [ json.dumps(record[c]) if isinstance(record[c], (dict, list)) else record[c] for c in columns ] self.con.execute( f'INSERT OR REPLACE INTO "{collection}" ({col_names}) VALUES ({placeholders})', values, ) return {"status": "success", "collection": collection, "id": entity_id} # -------------------------------------------------------------------------- # Queries & Feedback # -------------------------------------------------------------------------- def query(self, sql: str) -> List[Dict[str, Any]]: """Executes read-only SQL query and returns list of dictionaries.""" clean_sql = sql.strip() # Basic safeguard against destructive statements in read query disallowed = ["DROP ", "ALTER ", "TRUNCATE ", "DELETE FROM", "UPDATE "] for d in disallowed: if clean_sql.upper().startswith(d): raise ValueError(f"Direct {d.strip()} queries not permitted via query(). Use dedicated methods.") cursor = self.con.execute(clean_sql) cols = [desc[0] for desc in cursor.description] rows = cursor.fetchall() results = [] for row in rows: record = {} for col, val in zip(cols, row): if isinstance(val, (datetime.datetime, datetime.date)): record[col] = val.isoformat() else: record[col] = val results.append(record) return results def get_summary(self, subject: str, date_str: Optional[str] = None) -> Dict[str, Any]: """Returns daily summary for subject (nutrition, sleep, deficit).""" target_date = date_str or datetime.datetime.now(BERLIN_TZ).strftime("%Y-%m-%d") res: Dict[str, Any] = {"subject": subject, "date": target_date} # 1. Nutrition summary nut = self.query(f"SELECT * FROM v_nutrition_daily WHERE subject = '{subject}' AND log_date = '{target_date}'") if nut: res["nutrition"] = nut[0] else: res["nutrition"] = { "total_protein_g": 0.0, "total_carbs_g": 0.0, "total_sugar_g": 0.0, "total_fat_g": 0.0, "total_fiber_g": 0.0, "total_salt_g": 0.0, "total_kcal": 0, "item_count": 0, } # 2. Sleep summary slp = self.query(f"SELECT * FROM v_sleep_daily WHERE subject = '{subject}' AND log_date = '{target_date}'") if slp: res["sleep"] = slp[0] # 3. Active open sleeps open_sleep = self.query( f"SELECT id, start, local_start, sleep_type, location FROM v_sleep WHERE subject = '{subject}' AND is_open = true" ) if open_sleep: res["open_sleep"] = open_sleep[0] return res def find_places_near( self, lat: float, lon: float, radius_m: float = 1000.0, category: Optional[str] = None, ) -> List[Dict[str, Any]]: """Radius search for places using spatial extension or Haversine fallback.""" cat_filter = f"AND category = '{category}'" if category else "" if self.has_spatial_extension: sql = f""" SELECT id, name, category, lat, lon, address, google_maps_url, tags, notes, documents, ROUND(ST_Distance_Spheroid(ST_Point(lon, lat), ST_Point({lon}, {lat})), 1) AS distance_meters FROM places WHERE ST_Distance_Spheroid(ST_Point(lon, lat), ST_Point({lon}, {lat})) <= {radius_m} {cat_filter} ORDER BY distance_meters ASC """ else: sql = f""" SELECT id, name, category, lat, lon, address, google_maps_url, tags, notes, documents, ROUND(distance_m({lat}, {lon}, lat, lon), 1) AS distance_meters FROM places WHERE distance_m({lat}, {lon}, lat, lon) <= {radius_m} {cat_filter} ORDER BY distance_meters ASC """ return self.query(sql) def _get_event_feedback(self, kind: str, subject: str, target_date: datetime.date) -> Dict[str, Any]: """Generates compact running status feedback after an insert.""" date_str = target_date.strftime("%Y-%m-%d") if kind.startswith("nutrition"): daily = self.con.execute( "SELECT total_protein_g, total_carbs_g, total_sugar_g, total_fat_g, total_kcal FROM v_nutrition_daily WHERE subject = ? AND log_date = ?", [subject, date_str], ).fetchone() if daily: return { "day_total_formatted": f"Tag bisher ({date_str}): ~{daily[0]} g Eiweiß · ~{daily[1]} g KH (davon ~{daily[2]} g Zucker) · ~{daily[3]} g Fett · ~{int(daily[4])} kcal", "protein_g": daily[0], "carbs_g": daily[1], "kcal": daily[4], } elif kind.startswith("sleep"): daily = self.con.execute( "SELECT total_sleep_hours, nap_count, night_sleep_minutes FROM v_sleep_daily WHERE subject = ? AND log_date = ?", [subject, date_str], ).fetchone() if daily: return { "day_total_formatted": f"Schlaf ({date_str}): {daily[0]} h gesamt ({daily[1]} Schläfchen, {daily[2]} min Nacht)", "total_hours": daily[0], "nap_count": daily[1], } return {} # -------------------------------------------------------------------------- # Internal Helpers # -------------------------------------------------------------------------- def _sort_jsonl_file(self, file_path: Path, key_fn): if not file_path.is_file(): return records = [] with open(file_path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if line: try: records.append(json.loads(line)) except Exception: pass records.sort(key=key_fn) with open(file_path, "w", encoding="utf-8") as f: for r in records: f.write(json.dumps(r, ensure_ascii=False) + "\n") def _update_jsonl_record(self, domain: str, event_id: str, updates: Dict[str, Any]): """Finds and updates a record in events//*.jsonl.""" domain_dir = self.data_dir / "events" / domain if not domain_dir.is_dir(): return for p_file in domain_dir.glob("*.jsonl"): records = [] found = False with open(p_file, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue try: r = json.loads(line) if r.get("id") == event_id: r.update(updates) found = True records.append(r) except Exception: pass if found: with open(p_file, "w", encoding="utf-8") as f: for r in records: f.write(json.dumps(r, ensure_ascii=False) + "\n") return # ============================================================================== # JSON-RPC MCP Server Protocol # ============================================================================== def run_mcp_server(): """Runs standard JSON-RPC stdio MCP server for OpenClaw.""" engine = DatastoreEngine() TOOLS = [ { "name": "datastore_add_event", "description": "Log an event into the structured datastore (e.g. nutrition.intake, sleep.session, feeding.bottle). Computes all macros and totals deterministically.", "inputSchema": { "type": "object", "properties": { "kind": {"type": "string", "description": "e.g. 'nutrition.intake', 'sleep.session', 'feeding.bottle'"}, "subject": {"type": "string", "description": "'moritz', 'amelie', or 'mila'"}, "actor": {"type": "string", "description": "'agent:koch', 'agent:hebamme', 'moritz', 'amelie'"}, "payload": {"type": "object", "description": "Detailed payload matching the kind schema"}, "start": {"type": "string", "description": "ISO 8601 timestamp (optional, defaults to now)"}, "end": {"type": "string", "description": "ISO 8601 timestamp (optional for intervals)"}, "since": {"type": "string", "description": "Relative start offset backwards from now (e.g. '5m', '15m', '1h'). Use when logging events that started X minutes ago without computing absolute timestamps."}, "estimated": {"type": "boolean", "description": "True if portions or macros are estimated"}, "meal": {"type": "string", "description": "Optional meal grouping (e.g. 'Frühstück', 'Mittagessen')"}, "note": {"type": "string", "description": "Free text notes or context"}, "allow_duplicate": {"type": "boolean", "description": "Set to true to force adding duplicate event within 15m"}, }, "required": ["kind", "subject", "actor", "payload"], }, }, { "name": "datastore_stop_event", "description": "Closes an open event interval (e.g. Mila wakes up -> closes open sleep.session).", "inputSchema": { "type": "object", "properties": { "kind": {"type": "string", "description": "e.g. 'sleep.session'"}, "subject": {"type": "string", "description": "'mila', 'moritz', 'amelie'"}, "end_time": {"type": "string", "description": "ISO timestamp (optional, defaults to now)"}, }, "required": ["kind", "subject"], }, }, { "name": "datastore_get_summary", "description": "Fetch daily nutrition, sleep, and status totals computed deterministically by DuckDB.", "inputSchema": { "type": "object", "properties": { "subject": {"type": "string", "description": "'moritz', 'mila', or 'amelie'"}, "date": {"type": "string", "description": "YYYY-MM-DD (defaults to today)"}, }, "required": ["subject"], }, }, { "name": "datastore_query", "description": "Execute a read-only DuckDB SQL query against the datastore (events, v_nutrition, v_sleep, places, foods, etc.).", "inputSchema": { "type": "object", "properties": { "sql": {"type": "string", "description": "SQL SELECT query to execute"}, }, "required": ["sql"], }, }, { "name": "datastore_find_places_near", "description": "Search for points of interest (beer gardens, playgrounds, stores) within a given radius in meters.", "inputSchema": { "type": "object", "properties": { "lat": {"type": "number", "description": "Latitude (e.g. 49.0205 for Grabengasse)"}, "lon": {"type": "number", "description": "Longitude (e.g. 12.0975 for Grabengasse)"}, "radius_m": {"type": "number", "description": "Search radius in meters (default 1000)"}, "category": {"type": "string", "description": "Optional category filter (e.g. 'biergarten')"}, }, "required": ["lat", "lon"], }, }, { "name": "datastore_put_entity", "description": "Upsert an entity record into foods, recipes, places, or people.", "inputSchema": { "type": "object", "properties": { "collection": {"type": "string", "enum": ["foods", "recipes", "places", "people"]}, "record": {"type": "object", "description": "Entity record matching collection schema"}, }, "required": ["collection", "record"], }, }, ] for line in sys.stdin: line = line.strip() if not line: continue try: req = json.loads(line) except Exception: continue method = req.get("method") msg_id = req.get("id") if method == "initialize": resp = { "jsonrpc": "2.0", "id": msg_id, "result": { "protocolVersion": "2024-11-05", "serverInfo": {"name": "openclaw-datastore", "version": "1.0.0"}, "capabilities": {"tools": {}}, }, } sys.stdout.write(json.dumps(resp) + "\n") sys.stdout.flush() elif method == "tools/list": resp = { "jsonrpc": "2.0", "id": msg_id, "result": {"tools": TOOLS}, } sys.stdout.write(json.dumps(resp) + "\n") sys.stdout.flush() elif method == "tools/call": params = req.get("params", {}) name = params.get("name") args = params.get("arguments", {}) try: if name == "datastore_add_event": data = engine.add_event( kind=args["kind"], subject=args["subject"], actor=args.get("actor", "agent"), payload=args["payload"], start=args.get("start"), end=args.get("end"), since=args.get("since"), estimated=args.get("estimated", False), meal=args.get("meal"), note=args.get("note"), source_ref=args.get("source_ref"), allow_duplicate=args.get("allow_duplicate", False), ) elif name == "datastore_stop_event": data = engine.stop_event( kind=args["kind"], subject=args["subject"], end_time=args.get("end_time"), ) elif name == "datastore_get_summary": data = engine.get_summary(subject=args["subject"], date_str=args.get("date")) elif name == "datastore_query": data = engine.query(sql=args["sql"]) elif name == "datastore_find_places_near": data = engine.find_places_near( lat=args["lat"], lon=args["lon"], radius_m=args.get("radius_m", 1000.0), category=args.get("category"), ) elif name == "datastore_put_entity": data = engine.put_entity( collection=args["collection"], record=args["record"], ) else: raise ValueError(f"Unknown tool: {name}") resp = { "jsonrpc": "2.0", "id": msg_id, "result": { "content": [{"type": "text", "text": json.dumps(data, indent=2, ensure_ascii=False)}] }, } except Exception as e: resp = { "jsonrpc": "2.0", "id": msg_id, "result": { "isError": True, "content": [{"type": "text", "text": f"Error executing {name}: {str(e)}"}], }, } sys.stdout.write(json.dumps(resp) + "\n") sys.stdout.flush() elif msg_id is not None: resp = { "jsonrpc": "2.0", "id": msg_id, "error": {"code": -32601, "message": f"Method not found: {method}"}, } sys.stdout.write(json.dumps(resp) + "\n") sys.stdout.flush() # ============================================================================== # CLI Entrypoint # ============================================================================== def run_cli(): parser = argparse.ArgumentParser(description="OpenClaw Datastore CLI") subparsers = parser.add_subparsers(dest="command") # Add Event p_add = subparsers.add_parser("add", help="Add an event") p_add.add_argument("kind", nargs="?", default=None, help="e.g. nutrition.intake, sleep.session") p_add.add_argument("--kind", dest="kind_flag", help="e.g. nutrition.intake, sleep.session") p_add.add_argument("--subject", required=True, help="moritz, mila, amelie") p_add.add_argument("--actor", default="cli", help="Reporter identifier") p_add.add_argument("--payload", required=True, help="JSON payload string") p_add.add_argument("--start", help="ISO start time") p_add.add_argument("--end", help="ISO end time") p_add.add_argument("--since", help="Relative start offset backwards from now (e.g. '5m', '15m', '1h')") p_add.add_argument("--meal", help="Meal grouping") p_add.add_argument("--note", help="Free-form note") p_add.add_argument("--estimated", action="store_true", help="Flag as estimated") p_add.add_argument("--allow-duplicate", action="store_true", help="Allow duplicate event within 15 minutes") # Stop Event p_stop = subparsers.add_parser("stop", help="Stop an open interval event") p_stop.add_argument("kind", nargs="?", default=None, help="e.g. sleep.session") p_stop.add_argument("--kind", dest="kind_flag", help="e.g. sleep.session") p_stop.add_argument("--subject", required=True, help="mila, moritz") p_stop.add_argument("--id", dest="event_id", help="Specific event ID to close") p_stop.add_argument("--end", help="ISO end time (defaults to now)") # Query p_query = subparsers.add_parser("query", help="Execute read-only SQL query") p_query.add_argument("sql", help="SQL string") # Summary p_sum = subparsers.add_parser("summary", help="Get daily summary") p_sum.add_argument("--subject", required=True, help="moritz, mila") p_sum.add_argument("--date", help="YYYY-MM-DD") # Places p_places = subparsers.add_parser("places", help="Radius search for places") p_places.add_argument("--lat", type=float, required=True) p_places.add_argument("--lon", type=float, required=True) p_places.add_argument("--radius", type=float, default=1000.0) p_places.add_argument("--category") args = parser.parse_args() engine = DatastoreEngine() if args.command == "add": kind = args.kind or args.kind_flag if not kind: parser.error("the 'add' command requires 'kind' (positional or via --kind)") if kind.lower() in ("true", "false", "yes", "no"): parser.error(f"Invalid kind '{kind}'. Did you mean '--estimated' without an argument?") if "." not in kind: parser.error(f"Invalid kind '{kind}'. Expected domain.event (e.g. 'nutrition.intake', 'sleep.session', 'feeding.bottle').") payload = json.loads(args.payload) res = engine.add_event( kind=kind, subject=args.subject, actor=args.actor, payload=payload, start=args.start, end=args.end, since=args.since, meal=args.meal, note=args.note, estimated=args.estimated, allow_duplicate=args.allow_duplicate, ) print(json.dumps(res, indent=2, ensure_ascii=False)) elif args.command == "stop": kind = args.kind or args.kind_flag if not kind: parser.error("the 'stop' command requires 'kind' (positional or via --kind)") res = engine.stop_event(kind=kind, subject=args.subject, end_time=args.end, event_id=getattr(args, "event_id", None)) print(json.dumps(res, indent=2, ensure_ascii=False)) elif args.command == "query": res = engine.query(args.sql) print(json.dumps(res, indent=2, ensure_ascii=False)) elif args.command == "summary": res = engine.get_summary(subject=args.subject, date_str=args.date) print(json.dumps(res, indent=2, ensure_ascii=False)) elif args.command == "places": res = engine.find_places_near( lat=args.lat, lon=args.lon, radius_m=args.radius, category=args.category, ) print(json.dumps(res, indent=2, ensure_ascii=False)) else: parser.print_help() if __name__ == "__main__": if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] == "--stdio"): run_mcp_server() else: run_cli()