1136 lines
45 KiB
Python
Executable File
1136 lines
45 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# /// script
|
|
# dependencies = [
|
|
# "duckdb>=1.1.0",
|
|
# "python-ulid>=3.0.0",
|
|
# "pytz>=2024.1",
|
|
# ]
|
|
# ///
|
|
"""
|
|
OpenClaw Datastore — Structured Personal Data Layer (DuckDB + JSONL in Git)
|
|
Provides deterministic math, validation, interval handling, spatial queries,
|
|
and JSONL file persistence for personal nutrition, baby sleep, sports, recipes, and places.
|
|
|
|
Exposes both a CLI and a stdio JSON-RPC MCP server.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
from zoneinfo import ZoneInfo
|
|
|
|
import duckdb
|
|
|
|
def generate_ulid() -> str:
|
|
try:
|
|
from ulid import ULID
|
|
return str(ULID())
|
|
except TypeError:
|
|
import ulid
|
|
return str(ulid.new())
|
|
except ImportError:
|
|
import uuid
|
|
return str(uuid.uuid4())
|
|
|
|
|
|
BERLIN_TZ = ZoneInfo("Europe/Berlin")
|
|
|
|
|
|
def parse_duration(val: str) -> datetime.timedelta:
|
|
"""Parses duration strings like '15m', '15 min', '1h', '1h30m', '1.5h' into timedelta."""
|
|
s = val.strip().lower()
|
|
if not s:
|
|
raise ValueError("Empty duration string")
|
|
|
|
# Decimal hours like "1.5h" or "1.5 stunden"
|
|
m_dec_h = re.fullmatch(r"^(\d+(?:\.\d+)?)\s*(?:h|stunden?)$", s)
|
|
if m_dec_h:
|
|
return datetime.timedelta(hours=float(m_dec_h.group(1)))
|
|
|
|
# Decimal minutes like "2.5m" or "15 min"
|
|
m_dec_m = re.fullmatch(r"^(\d+(?:\.\d+)?)\s*(?:m|min|minuten?)$", s)
|
|
if m_dec_m:
|
|
return datetime.timedelta(minutes=float(m_dec_m.group(1)))
|
|
|
|
# Composite like "1h 30m" or "1h30min" or "45s"
|
|
m_comp = re.fullmatch(
|
|
r"^(?:(\d+)\s*(?:h|stunden?))?\s*(?:(\d+)\s*(?:m|min|minuten?))?\s*(?:(\d+)\s*(?:s|sek|sekunden?))?$",
|
|
s,
|
|
)
|
|
if m_comp and any(m_comp.groups()):
|
|
h = int(m_comp.group(1) or 0)
|
|
m = int(m_comp.group(2) or 0)
|
|
sec = int(m_comp.group(3) or 0)
|
|
return datetime.timedelta(hours=h, minutes=m, seconds=sec)
|
|
|
|
# Pure number without unit: default to minutes
|
|
if s.isdigit():
|
|
return datetime.timedelta(minutes=int(s))
|
|
|
|
raise ValueError(f"Unrecognized duration format: '{val}'. Expected formats: '15m', '15 min', '1h', '1h30m'.")
|
|
|
|
|
|
|
|
def find_data_dir() -> Path:
|
|
"""Discovers the knowledge-wiki data directory."""
|
|
for env_var in ["DATASTORE_DIR", "OPENCLAW_DATA_DIR"]:
|
|
if os.environ.get(env_var):
|
|
return Path(os.environ[env_var]).resolve()
|
|
|
|
candidates = [
|
|
Path.cwd() / "knowledge-wiki" / "data",
|
|
Path.cwd() / "data",
|
|
Path.home() / "src" / "infrapuzzle" / "knowledge-wiki" / "data",
|
|
Path.home() / ".openclaw" / "wiki" / "main" / "data",
|
|
Path("/home/node/.openclaw/wiki/main/data"),
|
|
]
|
|
for c in candidates:
|
|
if c.is_dir() and (c / "_meta.json").exists():
|
|
return c.resolve()
|
|
|
|
# Fallback to creating a local data directory if needed
|
|
default_dir = Path.cwd() / "knowledge-wiki" / "data"
|
|
default_dir.mkdir(parents=True, exist_ok=True)
|
|
return default_dir.resolve()
|
|
|
|
|
|
class DatastoreEngine:
|
|
def __init__(self, data_dir: Optional[Path] = None):
|
|
self.data_dir = data_dir or find_data_dir()
|
|
self.con = duckdb.connect(":memory:")
|
|
self.has_spatial_extension = False
|
|
self._init_schemas()
|
|
self._load_all_data()
|
|
|
|
def _init_schemas(self):
|
|
"""Initializes tables, views, and spatial support in DuckDB."""
|
|
# Timezone support via icu extension
|
|
try:
|
|
self.con.execute("INSTALL icu; LOAD icu;")
|
|
self.con.execute("SET TimeZone = 'Europe/Berlin';")
|
|
except Exception:
|
|
pass
|
|
|
|
# Spatial support: try extension, fallback to pure-SQL Haversine macro
|
|
try:
|
|
self.con.execute("INSTALL spatial; LOAD spatial;")
|
|
self.has_spatial_extension = True
|
|
except Exception:
|
|
self.has_spatial_extension = False
|
|
|
|
# Fallback pure-SQL distance macro (returns meters)
|
|
self.con.execute("""
|
|
CREATE MACRO IF NOT EXISTS distance_m(lat1, lon1, lat2, lon2) AS
|
|
6371000 * 2 * asin(sqrt(
|
|
sin(radians(lat2 - lat1) / 2) ^ 2 +
|
|
cos(radians(lat1)) * cos(radians(lat2)) * sin(radians(lon2 - lon1) / 2) ^ 2
|
|
));
|
|
""")
|
|
|
|
schema_dir = self.data_dir / "_schema"
|
|
if schema_dir.is_dir():
|
|
for f in ["envelope.sql", "entities.sql", "views.sql"]:
|
|
sql_file = schema_dir / f
|
|
if sql_file.is_file():
|
|
sql = sql_file.read_text(encoding="utf-8")
|
|
self.con.execute(sql)
|
|
else:
|
|
# Inline fallback schema if _schema is missing
|
|
self._create_inline_schema()
|
|
|
|
def _create_inline_schema(self):
|
|
self.con.execute("""
|
|
CREATE TABLE IF NOT EXISTS events (
|
|
id VARCHAR PRIMARY KEY,
|
|
kind VARCHAR NOT NULL,
|
|
subject VARCHAR NOT NULL,
|
|
actor VARCHAR NOT NULL,
|
|
start TIMESTAMPTZ NOT NULL,
|
|
"end" TIMESTAMPTZ,
|
|
estimated BOOLEAN DEFAULT false,
|
|
meal VARCHAR,
|
|
note VARCHAR,
|
|
payload JSON NOT NULL,
|
|
source_ref VARCHAR,
|
|
created_at TIMESTAMPTZ NOT NULL,
|
|
updated_at TIMESTAMPTZ,
|
|
deleted_at TIMESTAMPTZ,
|
|
CHECK ("end" IS NULL OR "end" >= start)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS people (
|
|
id VARCHAR PRIMARY KEY,
|
|
name VARCHAR NOT NULL,
|
|
role VARCHAR,
|
|
birthdate DATE,
|
|
notes VARCHAR
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS foods (
|
|
id VARCHAR PRIMARY KEY,
|
|
name VARCHAR NOT NULL,
|
|
brand VARCHAR,
|
|
category VARCHAR,
|
|
per_100 JSON,
|
|
density_g_per_ml DOUBLE,
|
|
ingredients_text VARCHAR,
|
|
barcode VARCHAR,
|
|
source VARCHAR,
|
|
description VARCHAR
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS recipes (
|
|
id VARCHAR PRIMARY KEY,
|
|
title VARCHAR NOT NULL,
|
|
servings INTEGER DEFAULT 1,
|
|
ingredients JSON,
|
|
estimated_delta JSON,
|
|
devices VARCHAR[],
|
|
tags VARCHAR[],
|
|
documents VARCHAR[],
|
|
description VARCHAR
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS places (
|
|
id VARCHAR PRIMARY KEY,
|
|
name VARCHAR NOT NULL,
|
|
category VARCHAR NOT NULL,
|
|
lat DOUBLE NOT NULL,
|
|
lon DOUBLE NOT NULL,
|
|
address VARCHAR,
|
|
google_maps_url VARCHAR,
|
|
tags VARCHAR[],
|
|
attributes JSON,
|
|
documents VARCHAR[],
|
|
notes VARCHAR
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS garmin_daily (
|
|
calendarDate DATE PRIMARY KEY,
|
|
totalKilocalories INTEGER,
|
|
activeKilocalories INTEGER,
|
|
bmrKilocalories INTEGER,
|
|
totalSteps INTEGER,
|
|
restingHeartRate INTEGER,
|
|
raw JSON
|
|
);
|
|
""")
|
|
|
|
def _load_all_data(self):
|
|
"""Loads all existing JSONL files from entities/, events/, and garmin/ into DuckDB."""
|
|
# 1. Load entities
|
|
entities_dir = self.data_dir / "entities"
|
|
if entities_dir.is_dir():
|
|
for f in entities_dir.glob("*.jsonl"):
|
|
collection = f.stem
|
|
if collection in ["people", "foods", "recipes", "places"]:
|
|
self._load_jsonl_file(f, collection)
|
|
|
|
# 2. Load events
|
|
events_dir = self.data_dir / "events"
|
|
if events_dir.is_dir():
|
|
for f in events_dir.glob("**/*.jsonl"):
|
|
self._load_jsonl_file(f, "events")
|
|
|
|
# 3. Load garmin daily
|
|
garmin_daily_dir = self.data_dir / "garmin" / "daily"
|
|
if garmin_daily_dir.is_dir():
|
|
for f in garmin_daily_dir.glob("*.jsonl"):
|
|
self._load_garmin_daily_file(f)
|
|
|
|
def _load_garmin_daily_file(self, file_path: Path):
|
|
"""Reads daily summaries from JSONL into garmin_daily table."""
|
|
if not file_path.is_file() or file_path.stat().st_size == 0:
|
|
return
|
|
with open(file_path, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
record = json.loads(line)
|
|
cal_date = record.get("calendarDate")
|
|
if not cal_date:
|
|
continue
|
|
total_kcal = record.get("totalKilocalories")
|
|
active_kcal = record.get("activeKilocalories")
|
|
bmr_kcal = record.get("bmrKilocalories")
|
|
steps = record.get("totalSteps")
|
|
rhr = record.get("restingHeartRate")
|
|
raw = json.dumps(record, ensure_ascii=False)
|
|
self.con.execute(
|
|
"""
|
|
INSERT OR REPLACE INTO garmin_daily
|
|
(calendarDate, totalKilocalories, activeKilocalories, bmrKilocalories, totalSteps, restingHeartRate, raw)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
[cal_date, total_kcal, active_kcal, bmr_kcal, steps, rhr, raw],
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Error loading garmin record from {file_path}: {e}")
|
|
|
|
def _load_jsonl_file(self, file_path: Path, table_name: str):
|
|
"""Reads lines from JSONL and inserts into target table."""
|
|
if not file_path.is_file() or file_path.stat().st_size == 0:
|
|
return
|
|
|
|
with open(file_path, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
record = json.loads(line)
|
|
columns = list(record.keys())
|
|
placeholders = ", ".join(["?"] * len(columns))
|
|
col_names = ", ".join([f'"{c}"' for c in columns])
|
|
values = [
|
|
json.dumps(record[c]) if isinstance(record[c], (dict, list)) else record[c]
|
|
for c in columns
|
|
]
|
|
self.con.execute(
|
|
f'INSERT OR REPLACE INTO "{table_name}" ({col_names}) VALUES ({placeholders})',
|
|
values,
|
|
)
|
|
except Exception as e:
|
|
# Ignore comment lines or malformed lines with warning
|
|
pass
|
|
|
|
def _matches_payload(self, kind: str, p1: Dict[str, Any], p2: Dict[str, Any]) -> bool:
|
|
"""Determines if two event payloads represent the same intake/session."""
|
|
if p1 == p2:
|
|
return True
|
|
if kind.startswith("nutrition."):
|
|
food1 = p1.get("food_id")
|
|
food2 = p2.get("food_id")
|
|
item1 = str(p1.get("item", "")).strip().lower()
|
|
item2 = str(p2.get("item", "")).strip().lower()
|
|
same_item = (
|
|
(food1 and food2 and food1 == food2)
|
|
or (item1 and item2 and item1 == item2)
|
|
or (food1 and item2 and food1 == item2)
|
|
or (food2 and item1 and food2 == item1)
|
|
)
|
|
if same_item:
|
|
qty1 = p1.get("qty")
|
|
qty2 = p2.get("qty")
|
|
if qty1 is not None and qty2 is not None:
|
|
try:
|
|
return abs(float(qty1) - float(qty2)) < 0.01
|
|
except (ValueError, TypeError):
|
|
return qty1 == qty2
|
|
return True
|
|
elif kind.startswith("feeding."):
|
|
amt1 = p1.get("amount_ml") or p1.get("consumed_ml")
|
|
amt2 = p2.get("amount_ml") or p2.get("consumed_ml")
|
|
if amt1 is not None and amt2 is not None:
|
|
return amt1 == amt2
|
|
elif kind.startswith("diaper."):
|
|
if p1.get("pee") == p2.get("pee") and p1.get("poo") == p2.get("poo"):
|
|
return True
|
|
elif kind.startswith("sleep."):
|
|
if p1.get("type") == p2.get("type"):
|
|
return True
|
|
return False
|
|
|
|
def _is_duplicate_event(
|
|
self,
|
|
kind: str,
|
|
subject: str,
|
|
payload: Dict[str, Any],
|
|
start_iso: str,
|
|
tolerance_seconds: int = 900,
|
|
) -> Optional[Dict[str, Any]]:
|
|
"""Checks if a matching event already exists within tolerance_seconds (default 15m)."""
|
|
rows = self.con.execute(
|
|
"""
|
|
SELECT id, kind, subject, actor, start, "end", estimated, meal, note, payload, source_ref, created_at, updated_at
|
|
FROM events
|
|
WHERE kind = ?
|
|
AND subject = ?
|
|
AND deleted_at IS NULL
|
|
AND abs(epoch(start) - epoch(?::TIMESTAMPTZ)) <= ?
|
|
ORDER BY start DESC
|
|
""",
|
|
[kind, subject, start_iso, tolerance_seconds],
|
|
).fetchall()
|
|
|
|
for r in rows:
|
|
existing_payload = json.loads(r[9]) if isinstance(r[9], str) else (r[9] or {})
|
|
if self._matches_payload(kind, payload, existing_payload):
|
|
return {
|
|
"id": r[0],
|
|
"kind": r[1],
|
|
"subject": r[2],
|
|
"actor": r[3],
|
|
"start": r[4].isoformat() if hasattr(r[4], "isoformat") else str(r[4]),
|
|
"end": r[5].isoformat() if hasattr(r[5], "isoformat") and r[5] else (str(r[5]) if r[5] else None),
|
|
"estimated": r[6],
|
|
"meal": r[7],
|
|
"note": r[8],
|
|
"payload": existing_payload,
|
|
"source_ref": r[10],
|
|
"created_at": r[11].isoformat() if hasattr(r[11], "isoformat") else str(r[11]),
|
|
"updated_at": r[12].isoformat() if hasattr(r[12], "isoformat") and r[12] else (str(r[12]) if r[12] else None),
|
|
"deleted_at": None,
|
|
}
|
|
return None
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Event Operations
|
|
# --------------------------------------------------------------------------
|
|
def add_event(
|
|
self,
|
|
kind: str,
|
|
subject: str,
|
|
actor: str,
|
|
payload: Dict[str, Any],
|
|
start: Optional[str] = None,
|
|
end: Optional[str] = None,
|
|
since: Optional[str] = None,
|
|
estimated: bool = False,
|
|
meal: Optional[str] = None,
|
|
note: Optional[str] = None,
|
|
source_ref: Optional[str] = None,
|
|
allow_duplicate: bool = False,
|
|
) -> Dict[str, Any]:
|
|
"""Adds a time-series event, saves to JSONL partition, and returns confirmation."""
|
|
event_id = generate_ulid()
|
|
now_dt = datetime.datetime.now(datetime.timezone.utc)
|
|
now_iso = now_dt.isoformat()
|
|
|
|
if since and start:
|
|
raise ValueError("Cannot specify both 'since' and 'start'. Use 'since' for relative times or 'start' for absolute times.")
|
|
|
|
# Parse or default start time
|
|
if since:
|
|
delta = parse_duration(since)
|
|
start_dt = (now_dt - delta).astimezone(BERLIN_TZ)
|
|
start_iso = start_dt.isoformat()
|
|
elif not start:
|
|
start_dt = now_dt.astimezone(BERLIN_TZ)
|
|
start_iso = start_dt.isoformat()
|
|
else:
|
|
start_dt = datetime.datetime.fromisoformat(start)
|
|
if start_dt.tzinfo is None:
|
|
start_dt = start_dt.replace(tzinfo=BERLIN_TZ)
|
|
start_iso = start_dt.isoformat()
|
|
|
|
end_iso = None
|
|
if end:
|
|
end_dt = datetime.datetime.fromisoformat(end)
|
|
if end_dt.tzinfo is None:
|
|
end_dt = end_dt.replace(tzinfo=BERLIN_TZ)
|
|
if end_dt < start_dt:
|
|
raise ValueError(f"End time {end} cannot be earlier than start time {start}")
|
|
end_iso = end_dt.isoformat()
|
|
|
|
# Check for duplicate event within 15 minutes unless explicitly allowed
|
|
if not allow_duplicate:
|
|
existing_event = self._is_duplicate_event(kind, subject, payload, start_iso)
|
|
if existing_event:
|
|
feedback = self._get_event_feedback(kind, subject, start_dt.date())
|
|
return {
|
|
"status": "success",
|
|
"duplicate": True,
|
|
"message": f"Duplicate event detected within 15 minutes (existing event ID: {existing_event['id']}). Event retained without duplicate insertion.",
|
|
"event": existing_event,
|
|
"feedback": feedback,
|
|
}
|
|
|
|
# Validation for open intervals in sleep
|
|
if kind == "sleep.session" and end_iso is None:
|
|
# Check if there is already an open session for this subject
|
|
open_sessions = self.con.execute(
|
|
"SELECT id, start FROM events WHERE kind = 'sleep.session' AND subject = ? AND \"end\" IS NULL AND deleted_at IS NULL",
|
|
[subject],
|
|
).fetchall()
|
|
if open_sessions:
|
|
raise ValueError(
|
|
f"Subject '{subject}' already has an open sleep session ({open_sessions[0][0]} started at {open_sessions[0][1]}). Please stop it first."
|
|
)
|
|
|
|
record = {
|
|
"id": event_id,
|
|
"kind": kind,
|
|
"subject": subject,
|
|
"actor": actor,
|
|
"start": start_iso,
|
|
"end": end_iso,
|
|
"estimated": estimated,
|
|
"meal": meal,
|
|
"note": note,
|
|
"payload": payload,
|
|
"source_ref": source_ref,
|
|
"created_at": now_iso,
|
|
"updated_at": None,
|
|
"deleted_at": None,
|
|
}
|
|
|
|
# Insert into in-memory DuckDB
|
|
self.con.execute(
|
|
"""
|
|
INSERT INTO events (id, kind, subject, actor, start, "end", estimated, meal, note, payload, source_ref, created_at, updated_at, deleted_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
[
|
|
event_id,
|
|
kind,
|
|
subject,
|
|
actor,
|
|
start_iso,
|
|
end_iso,
|
|
estimated,
|
|
meal,
|
|
note,
|
|
json.dumps(payload),
|
|
source_ref,
|
|
now_iso,
|
|
None,
|
|
None,
|
|
],
|
|
)
|
|
|
|
# Write to partitioned JSONL file
|
|
# Partition path: events/<domain>/<YYYY-MM>.jsonl
|
|
domain = kind.split(".")[0]
|
|
month_str = start_dt.astimezone(BERLIN_TZ).strftime("%Y-%m")
|
|
domain_dir = self.data_dir / "events" / domain
|
|
domain_dir.mkdir(parents=True, exist_ok=True)
|
|
partition_file = domain_dir / f"{month_str}.jsonl"
|
|
|
|
with open(partition_file, "a", encoding="utf-8") as f:
|
|
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
|
|
# Sort partition deterministically by start, id
|
|
self._sort_jsonl_file(partition_file, key_fn=lambda x: (x.get("start", ""), x.get("id", "")))
|
|
|
|
# Fetch computed feedback
|
|
feedback = self._get_event_feedback(kind, subject, start_dt.date())
|
|
|
|
return {
|
|
"status": "success",
|
|
"event": record,
|
|
"feedback": feedback,
|
|
}
|
|
|
|
def stop_event(
|
|
self,
|
|
kind: str,
|
|
subject: str,
|
|
end_time: Optional[str] = None,
|
|
actor: Optional[str] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Finds the open event (end IS NULL) for kind and subject, and closes it."""
|
|
row = self.con.execute(
|
|
"""
|
|
SELECT id, start FROM events
|
|
WHERE kind = ? AND subject = ? AND "end" IS NULL AND deleted_at IS NULL
|
|
ORDER BY start DESC LIMIT 1
|
|
""",
|
|
[kind, subject],
|
|
).fetchone()
|
|
|
|
if not row:
|
|
raise ValueError(f"No open event found for kind='{kind}' and subject='{subject}'")
|
|
|
|
event_id, start_str = row[0], str(row[1])
|
|
now_dt = datetime.datetime.now(datetime.timezone.utc)
|
|
if end_time:
|
|
end_dt = datetime.datetime.fromisoformat(end_time)
|
|
if end_dt.tzinfo is None:
|
|
end_dt = end_dt.replace(tzinfo=BERLIN_TZ)
|
|
else:
|
|
end_dt = now_dt.astimezone(BERLIN_TZ)
|
|
|
|
end_iso = end_dt.isoformat()
|
|
now_iso = now_dt.isoformat()
|
|
|
|
# Update in DuckDB
|
|
self.con.execute(
|
|
"""
|
|
UPDATE events
|
|
SET "end" = ?, updated_at = ?
|
|
WHERE id = ?
|
|
""",
|
|
[end_iso, now_iso, event_id],
|
|
)
|
|
|
|
# Update in JSONL
|
|
domain = kind.split(".")[0]
|
|
self._update_jsonl_record(domain, event_id, {"end": end_iso, "updated_at": now_iso})
|
|
|
|
# Calculate duration
|
|
start_dt = datetime.datetime.fromisoformat(start_str.replace("Z", "+00:00"))
|
|
duration_minutes = int((end_dt.astimezone(BERLIN_TZ) - start_dt.astimezone(BERLIN_TZ)).total_seconds() / 60)
|
|
|
|
return {
|
|
"status": "success",
|
|
"event_id": event_id,
|
|
"kind": kind,
|
|
"subject": subject,
|
|
"start": start_str,
|
|
"end": end_iso,
|
|
"duration_minutes": duration_minutes,
|
|
"duration_formatted": f"{duration_minutes // 60} h {duration_minutes % 60} min"
|
|
if duration_minutes >= 60
|
|
else f"{duration_minutes} min",
|
|
}
|
|
|
|
def delete_event(self, event_id: str) -> Dict[str, Any]:
|
|
"""Soft-deletes an event."""
|
|
now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
row = self.con.execute("SELECT kind FROM events WHERE id = ?", [event_id]).fetchone()
|
|
if not row:
|
|
raise ValueError(f"Event with id '{event_id}' not found.")
|
|
|
|
kind = row[0]
|
|
domain = kind.split(".")[0]
|
|
|
|
self.con.execute("UPDATE events SET deleted_at = ? WHERE id = ?", [now_iso, event_id])
|
|
self._update_jsonl_record(domain, event_id, {"deleted_at": now_iso})
|
|
|
|
return {"status": "success", "deleted_event_id": event_id}
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Entity Operations
|
|
# --------------------------------------------------------------------------
|
|
def put_entity(self, collection: str, record: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Upserts an entity into entities/<collection>.jsonl."""
|
|
if collection not in ["foods", "recipes", "places", "people"]:
|
|
raise ValueError(f"Unknown entity collection: {collection}")
|
|
|
|
entity_id = record.get("id")
|
|
if not entity_id:
|
|
raise ValueError("Entity record must have an 'id' field.")
|
|
|
|
entities_dir = self.data_dir / "entities"
|
|
entities_dir.mkdir(parents=True, exist_ok=True)
|
|
target_file = entities_dir / f"{collection}.jsonl"
|
|
|
|
# Read existing, update or append
|
|
existing_records = []
|
|
replaced = False
|
|
if target_file.is_file():
|
|
with open(target_file, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
r = json.loads(line)
|
|
if r.get("id") == entity_id:
|
|
existing_records.append(record)
|
|
replaced = True
|
|
else:
|
|
existing_records.append(r)
|
|
except Exception:
|
|
pass
|
|
|
|
if not replaced:
|
|
existing_records.append(record)
|
|
|
|
# Sort by id
|
|
existing_records.sort(key=lambda x: str(x.get("id", "")))
|
|
|
|
with open(target_file, "w", encoding="utf-8") as f:
|
|
for r in existing_records:
|
|
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
|
|
# Reload into DuckDB
|
|
columns = list(record.keys())
|
|
placeholders = ", ".join(["?"] * len(columns))
|
|
col_names = ", ".join([f'"{c}"' for c in columns])
|
|
values = [
|
|
json.dumps(record[c]) if isinstance(record[c], (dict, list)) else record[c]
|
|
for c in columns
|
|
]
|
|
self.con.execute(
|
|
f'INSERT OR REPLACE INTO "{collection}" ({col_names}) VALUES ({placeholders})',
|
|
values,
|
|
)
|
|
|
|
return {"status": "success", "collection": collection, "id": entity_id}
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Queries & Feedback
|
|
# --------------------------------------------------------------------------
|
|
def query(self, sql: str) -> List[Dict[str, Any]]:
|
|
"""Executes read-only SQL query and returns list of dictionaries."""
|
|
clean_sql = sql.strip()
|
|
# Basic safeguard against destructive statements in read query
|
|
disallowed = ["DROP ", "ALTER ", "TRUNCATE ", "DELETE FROM", "UPDATE "]
|
|
for d in disallowed:
|
|
if clean_sql.upper().startswith(d):
|
|
raise ValueError(f"Direct {d.strip()} queries not permitted via query(). Use dedicated methods.")
|
|
|
|
cursor = self.con.execute(clean_sql)
|
|
cols = [desc[0] for desc in cursor.description]
|
|
rows = cursor.fetchall()
|
|
|
|
results = []
|
|
for row in rows:
|
|
record = {}
|
|
for col, val in zip(cols, row):
|
|
if isinstance(val, (datetime.datetime, datetime.date)):
|
|
record[col] = val.isoformat()
|
|
else:
|
|
record[col] = val
|
|
results.append(record)
|
|
return results
|
|
|
|
def get_summary(self, subject: str, date_str: Optional[str] = None) -> Dict[str, Any]:
|
|
"""Returns daily summary for subject (nutrition, sleep, deficit)."""
|
|
target_date = date_str or datetime.datetime.now(BERLIN_TZ).strftime("%Y-%m-%d")
|
|
|
|
res: Dict[str, Any] = {"subject": subject, "date": target_date}
|
|
|
|
# 1. Nutrition summary
|
|
nut = self.query(f"SELECT * FROM v_nutrition_daily WHERE subject = '{subject}' AND log_date = '{target_date}'")
|
|
if nut:
|
|
res["nutrition"] = nut[0]
|
|
else:
|
|
res["nutrition"] = {
|
|
"total_protein_g": 0.0,
|
|
"total_carbs_g": 0.0,
|
|
"total_sugar_g": 0.0,
|
|
"total_fat_g": 0.0,
|
|
"total_fiber_g": 0.0,
|
|
"total_salt_g": 0.0,
|
|
"total_kcal": 0,
|
|
"item_count": 0,
|
|
}
|
|
|
|
# 2. Sleep summary
|
|
slp = self.query(f"SELECT * FROM v_sleep_daily WHERE subject = '{subject}' AND log_date = '{target_date}'")
|
|
if slp:
|
|
res["sleep"] = slp[0]
|
|
|
|
# 3. Active open sleeps
|
|
open_sleep = self.query(
|
|
f"SELECT id, start, local_start, sleep_type, location FROM v_sleep WHERE subject = '{subject}' AND is_open = true"
|
|
)
|
|
if open_sleep:
|
|
res["open_sleep"] = open_sleep[0]
|
|
|
|
return res
|
|
|
|
def find_places_near(
|
|
self,
|
|
lat: float,
|
|
lon: float,
|
|
radius_m: float = 1000.0,
|
|
category: Optional[str] = None,
|
|
) -> List[Dict[str, Any]]:
|
|
"""Radius search for places using spatial extension or Haversine fallback."""
|
|
cat_filter = f"AND category = '{category}'" if category else ""
|
|
|
|
if self.has_spatial_extension:
|
|
sql = f"""
|
|
SELECT id, name, category, lat, lon, address, google_maps_url, tags, notes, documents,
|
|
ROUND(ST_Distance_Spheroid(ST_Point(lon, lat), ST_Point({lon}, {lat})), 1) AS distance_meters
|
|
FROM places
|
|
WHERE ST_Distance_Spheroid(ST_Point(lon, lat), ST_Point({lon}, {lat})) <= {radius_m}
|
|
{cat_filter}
|
|
ORDER BY distance_meters ASC
|
|
"""
|
|
else:
|
|
sql = f"""
|
|
SELECT id, name, category, lat, lon, address, google_maps_url, tags, notes, documents,
|
|
ROUND(distance_m({lat}, {lon}, lat, lon), 1) AS distance_meters
|
|
FROM places
|
|
WHERE distance_m({lat}, {lon}, lat, lon) <= {radius_m}
|
|
{cat_filter}
|
|
ORDER BY distance_meters ASC
|
|
"""
|
|
|
|
return self.query(sql)
|
|
|
|
def _get_event_feedback(self, kind: str, subject: str, target_date: datetime.date) -> Dict[str, Any]:
|
|
"""Generates compact running status feedback after an insert."""
|
|
date_str = target_date.strftime("%Y-%m-%d")
|
|
if kind.startswith("nutrition"):
|
|
daily = self.con.execute(
|
|
"SELECT total_protein_g, total_carbs_g, total_sugar_g, total_fat_g, total_kcal FROM v_nutrition_daily WHERE subject = ? AND log_date = ?",
|
|
[subject, date_str],
|
|
).fetchone()
|
|
if daily:
|
|
return {
|
|
"day_total_formatted": f"Tag bisher ({date_str}): ~{daily[0]} g Eiweiß · ~{daily[1]} g KH (davon ~{daily[2]} g Zucker) · ~{daily[3]} g Fett · ~{int(daily[4])} kcal",
|
|
"protein_g": daily[0],
|
|
"carbs_g": daily[1],
|
|
"kcal": daily[4],
|
|
}
|
|
elif kind.startswith("sleep"):
|
|
daily = self.con.execute(
|
|
"SELECT total_sleep_hours, nap_count, night_sleep_minutes FROM v_sleep_daily WHERE subject = ? AND log_date = ?",
|
|
[subject, date_str],
|
|
).fetchone()
|
|
if daily:
|
|
return {
|
|
"day_total_formatted": f"Schlaf ({date_str}): {daily[0]} h gesamt ({daily[1]} Schläfchen, {daily[2]} min Nacht)",
|
|
"total_hours": daily[0],
|
|
"nap_count": daily[1],
|
|
}
|
|
return {}
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Internal Helpers
|
|
# --------------------------------------------------------------------------
|
|
def _sort_jsonl_file(self, file_path: Path, key_fn):
|
|
if not file_path.is_file():
|
|
return
|
|
records = []
|
|
with open(file_path, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
try:
|
|
records.append(json.loads(line))
|
|
except Exception:
|
|
pass
|
|
records.sort(key=key_fn)
|
|
with open(file_path, "w", encoding="utf-8") as f:
|
|
for r in records:
|
|
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
|
|
def _update_jsonl_record(self, domain: str, event_id: str, updates: Dict[str, Any]):
|
|
"""Finds and updates a record in events/<domain>/*.jsonl."""
|
|
domain_dir = self.data_dir / "events" / domain
|
|
if not domain_dir.is_dir():
|
|
return
|
|
|
|
for p_file in domain_dir.glob("*.jsonl"):
|
|
records = []
|
|
found = False
|
|
with open(p_file, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
r = json.loads(line)
|
|
if r.get("id") == event_id:
|
|
r.update(updates)
|
|
found = True
|
|
records.append(r)
|
|
except Exception:
|
|
pass
|
|
if found:
|
|
with open(p_file, "w", encoding="utf-8") as f:
|
|
for r in records:
|
|
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
return
|
|
|
|
|
|
# ==============================================================================
|
|
# JSON-RPC MCP Server Protocol
|
|
# ==============================================================================
|
|
def run_mcp_server():
|
|
"""Runs standard JSON-RPC stdio MCP server for OpenClaw."""
|
|
engine = DatastoreEngine()
|
|
|
|
TOOLS = [
|
|
{
|
|
"name": "datastore_add_event",
|
|
"description": "Log an event into the structured datastore (e.g. nutrition.intake, sleep.session, feeding.bottle). Computes all macros and totals deterministically.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"kind": {"type": "string", "description": "e.g. 'nutrition.intake', 'sleep.session', 'feeding.bottle'"},
|
|
"subject": {"type": "string", "description": "'moritz', 'amelie', or 'mila'"},
|
|
"actor": {"type": "string", "description": "'agent:koch', 'agent:hebamme', 'moritz', 'amelie'"},
|
|
"payload": {"type": "object", "description": "Detailed payload matching the kind schema"},
|
|
"start": {"type": "string", "description": "ISO 8601 timestamp (optional, defaults to now)"},
|
|
"end": {"type": "string", "description": "ISO 8601 timestamp (optional for intervals)"},
|
|
"since": {"type": "string", "description": "Relative start offset backwards from now (e.g. '5m', '15m', '1h'). Use when logging events that started X minutes ago without computing absolute timestamps."},
|
|
"estimated": {"type": "boolean", "description": "True if portions or macros are estimated"},
|
|
"meal": {"type": "string", "description": "Optional meal grouping (e.g. 'Frühstück', 'Mittagessen')"},
|
|
"note": {"type": "string", "description": "Free text notes or context"},
|
|
"allow_duplicate": {"type": "boolean", "description": "Set to true to force adding duplicate event within 15m"},
|
|
},
|
|
"required": ["kind", "subject", "actor", "payload"],
|
|
},
|
|
},
|
|
{
|
|
"name": "datastore_stop_event",
|
|
"description": "Closes an open event interval (e.g. Mila wakes up -> closes open sleep.session).",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"kind": {"type": "string", "description": "e.g. 'sleep.session'"},
|
|
"subject": {"type": "string", "description": "'mila', 'moritz', 'amelie'"},
|
|
"end_time": {"type": "string", "description": "ISO timestamp (optional, defaults to now)"},
|
|
},
|
|
"required": ["kind", "subject"],
|
|
},
|
|
},
|
|
{
|
|
"name": "datastore_get_summary",
|
|
"description": "Fetch daily nutrition, sleep, and status totals computed deterministically by DuckDB.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"subject": {"type": "string", "description": "'moritz', 'mila', or 'amelie'"},
|
|
"date": {"type": "string", "description": "YYYY-MM-DD (defaults to today)"},
|
|
},
|
|
"required": ["subject"],
|
|
},
|
|
},
|
|
{
|
|
"name": "datastore_query",
|
|
"description": "Execute a read-only DuckDB SQL query against the datastore (events, v_nutrition, v_sleep, places, foods, etc.).",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"sql": {"type": "string", "description": "SQL SELECT query to execute"},
|
|
},
|
|
"required": ["sql"],
|
|
},
|
|
},
|
|
{
|
|
"name": "datastore_find_places_near",
|
|
"description": "Search for points of interest (beer gardens, playgrounds, stores) within a given radius in meters.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"lat": {"type": "number", "description": "Latitude (e.g. 49.0205 for Grabengasse)"},
|
|
"lon": {"type": "number", "description": "Longitude (e.g. 12.0975 for Grabengasse)"},
|
|
"radius_m": {"type": "number", "description": "Search radius in meters (default 1000)"},
|
|
"category": {"type": "string", "description": "Optional category filter (e.g. 'biergarten')"},
|
|
},
|
|
"required": ["lat", "lon"],
|
|
},
|
|
},
|
|
{
|
|
"name": "datastore_put_entity",
|
|
"description": "Upsert an entity record into foods, recipes, places, or people.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"collection": {"type": "string", "enum": ["foods", "recipes", "places", "people"]},
|
|
"record": {"type": "object", "description": "Entity record matching collection schema"},
|
|
},
|
|
"required": ["collection", "record"],
|
|
},
|
|
},
|
|
]
|
|
|
|
for line in sys.stdin:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
req = json.loads(line)
|
|
except Exception:
|
|
continue
|
|
|
|
method = req.get("method")
|
|
msg_id = req.get("id")
|
|
|
|
if method == "initialize":
|
|
resp = {
|
|
"jsonrpc": "2.0",
|
|
"id": msg_id,
|
|
"result": {
|
|
"protocolVersion": "2024-11-05",
|
|
"serverInfo": {"name": "openclaw-datastore", "version": "1.0.0"},
|
|
"capabilities": {"tools": {}},
|
|
},
|
|
}
|
|
sys.stdout.write(json.dumps(resp) + "\n")
|
|
sys.stdout.flush()
|
|
|
|
elif method == "tools/list":
|
|
resp = {
|
|
"jsonrpc": "2.0",
|
|
"id": msg_id,
|
|
"result": {"tools": TOOLS},
|
|
}
|
|
sys.stdout.write(json.dumps(resp) + "\n")
|
|
sys.stdout.flush()
|
|
|
|
elif method == "tools/call":
|
|
params = req.get("params", {})
|
|
name = params.get("name")
|
|
args = params.get("arguments", {})
|
|
|
|
try:
|
|
if name == "datastore_add_event":
|
|
data = engine.add_event(
|
|
kind=args["kind"],
|
|
subject=args["subject"],
|
|
actor=args.get("actor", "agent"),
|
|
payload=args["payload"],
|
|
start=args.get("start"),
|
|
end=args.get("end"),
|
|
since=args.get("since"),
|
|
estimated=args.get("estimated", False),
|
|
meal=args.get("meal"),
|
|
note=args.get("note"),
|
|
source_ref=args.get("source_ref"),
|
|
allow_duplicate=args.get("allow_duplicate", False),
|
|
)
|
|
elif name == "datastore_stop_event":
|
|
data = engine.stop_event(
|
|
kind=args["kind"],
|
|
subject=args["subject"],
|
|
end_time=args.get("end_time"),
|
|
)
|
|
elif name == "datastore_get_summary":
|
|
data = engine.get_summary(subject=args["subject"], date_str=args.get("date"))
|
|
elif name == "datastore_query":
|
|
data = engine.query(sql=args["sql"])
|
|
elif name == "datastore_find_places_near":
|
|
data = engine.find_places_near(
|
|
lat=args["lat"],
|
|
lon=args["lon"],
|
|
radius_m=args.get("radius_m", 1000.0),
|
|
category=args.get("category"),
|
|
)
|
|
elif name == "datastore_put_entity":
|
|
data = engine.put_entity(
|
|
collection=args["collection"],
|
|
record=args["record"],
|
|
)
|
|
else:
|
|
raise ValueError(f"Unknown tool: {name}")
|
|
|
|
resp = {
|
|
"jsonrpc": "2.0",
|
|
"id": msg_id,
|
|
"result": {
|
|
"content": [{"type": "text", "text": json.dumps(data, indent=2, ensure_ascii=False)}]
|
|
},
|
|
}
|
|
except Exception as e:
|
|
resp = {
|
|
"jsonrpc": "2.0",
|
|
"id": msg_id,
|
|
"result": {
|
|
"isError": True,
|
|
"content": [{"type": "text", "text": f"Error executing {name}: {str(e)}"}],
|
|
},
|
|
}
|
|
sys.stdout.write(json.dumps(resp) + "\n")
|
|
sys.stdout.flush()
|
|
|
|
elif msg_id is not None:
|
|
resp = {
|
|
"jsonrpc": "2.0",
|
|
"id": msg_id,
|
|
"error": {"code": -32601, "message": f"Method not found: {method}"},
|
|
}
|
|
sys.stdout.write(json.dumps(resp) + "\n")
|
|
sys.stdout.flush()
|
|
|
|
|
|
# ==============================================================================
|
|
# CLI Entrypoint
|
|
# ==============================================================================
|
|
def run_cli():
|
|
parser = argparse.ArgumentParser(description="OpenClaw Datastore CLI")
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
# Add Event
|
|
p_add = subparsers.add_parser("add", help="Add an event")
|
|
p_add.add_argument("kind", nargs="?", default=None, help="e.g. nutrition.intake, sleep.session")
|
|
p_add.add_argument("--kind", dest="kind_flag", help="e.g. nutrition.intake, sleep.session")
|
|
p_add.add_argument("--subject", required=True, help="moritz, mila, amelie")
|
|
p_add.add_argument("--actor", default="cli", help="Reporter identifier")
|
|
p_add.add_argument("--payload", required=True, help="JSON payload string")
|
|
p_add.add_argument("--start", help="ISO start time")
|
|
p_add.add_argument("--end", help="ISO end time")
|
|
p_add.add_argument("--since", help="Relative start offset backwards from now (e.g. '5m', '15m', '1h')")
|
|
p_add.add_argument("--meal", help="Meal grouping")
|
|
p_add.add_argument("--note", help="Free-form note")
|
|
p_add.add_argument("--estimated", action="store_true", help="Flag as estimated")
|
|
p_add.add_argument("--allow-duplicate", action="store_true", help="Allow duplicate event within 15 minutes")
|
|
|
|
# Stop Event
|
|
p_stop = subparsers.add_parser("stop", help="Stop an open interval event")
|
|
p_stop.add_argument("kind", nargs="?", default=None, help="e.g. sleep.session")
|
|
p_stop.add_argument("--kind", dest="kind_flag", help="e.g. sleep.session")
|
|
p_stop.add_argument("--subject", required=True, help="mila, moritz")
|
|
p_stop.add_argument("--end", help="ISO end time (defaults to now)")
|
|
|
|
# Query
|
|
p_query = subparsers.add_parser("query", help="Execute read-only SQL query")
|
|
p_query.add_argument("sql", help="SQL string")
|
|
|
|
# Summary
|
|
p_sum = subparsers.add_parser("summary", help="Get daily summary")
|
|
p_sum.add_argument("--subject", required=True, help="moritz, mila")
|
|
p_sum.add_argument("--date", help="YYYY-MM-DD")
|
|
|
|
# Places
|
|
p_places = subparsers.add_parser("places", help="Radius search for places")
|
|
p_places.add_argument("--lat", type=float, required=True)
|
|
p_places.add_argument("--lon", type=float, required=True)
|
|
p_places.add_argument("--radius", type=float, default=1000.0)
|
|
p_places.add_argument("--category")
|
|
|
|
args = parser.parse_args()
|
|
engine = DatastoreEngine()
|
|
|
|
if args.command == "add":
|
|
kind = args.kind or args.kind_flag
|
|
if not kind:
|
|
parser.error("the 'add' command requires 'kind' (positional or via --kind)")
|
|
payload = json.loads(args.payload)
|
|
res = engine.add_event(
|
|
kind=kind,
|
|
subject=args.subject,
|
|
actor=args.actor,
|
|
payload=payload,
|
|
start=args.start,
|
|
end=args.end,
|
|
since=args.since,
|
|
meal=args.meal,
|
|
note=args.note,
|
|
estimated=args.estimated,
|
|
allow_duplicate=args.allow_duplicate,
|
|
)
|
|
print(json.dumps(res, indent=2, ensure_ascii=False))
|
|
|
|
elif args.command == "stop":
|
|
kind = args.kind or args.kind_flag
|
|
if not kind:
|
|
parser.error("the 'stop' command requires 'kind' (positional or via --kind)")
|
|
res = engine.stop_event(kind=kind, subject=args.subject, end_time=args.end)
|
|
print(json.dumps(res, indent=2, ensure_ascii=False))
|
|
elif args.command == "query":
|
|
res = engine.query(args.sql)
|
|
print(json.dumps(res, indent=2, ensure_ascii=False))
|
|
|
|
elif args.command == "summary":
|
|
res = engine.get_summary(subject=args.subject, date_str=args.date)
|
|
print(json.dumps(res, indent=2, ensure_ascii=False))
|
|
|
|
elif args.command == "places":
|
|
res = engine.find_places_near(
|
|
lat=args.lat,
|
|
lon=args.lon,
|
|
radius_m=args.radius,
|
|
category=args.category,
|
|
)
|
|
print(json.dumps(res, indent=2, ensure_ascii=False))
|
|
|
|
else:
|
|
parser.print_help()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] == "--stdio"):
|
|
run_mcp_server()
|
|
else:
|
|
run_cli()
|