moritz/mop-skills
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(datastore): support --kind flag and add 15m duplicate suppression

Browse Source
This commit is contained in:
Moritz Graf
2026-10-09 10:28:11 +02:00
parent 77a0e56d96
commit 6a93539a97
2 changed files with 303 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/datastore/scripts/datastore.py
+113 -7
View File
@@ -81,8 +81,9 @@ def parse_duration(val: str) -> datetime.timedelta:
def find_data_dir() -> Path:
"""Discovers the knowledge-wiki data directory."""
if os.environ.get("DATASTORE_DIR"):
return Path(os.environ["DATASTORE_DIR"]).resolve()
for env_var in ["DATASTORE_DIR", "OPENCLAW_DATA_DIR"]:
if os.environ.get(env_var):
return Path(os.environ[env_var]).resolve()
candidates = [
Path.cwd() / "knowledge-wiki" / "data",
@@ -303,6 +304,86 @@ class DatastoreEngine:
# Ignore comment lines or malformed lines with warning
pass
def _matches_payload(self, kind: str, p1: Dict[str, Any], p2: Dict[str, Any]) -> bool:
"""Determines if two event payloads represent the same intake/session."""
if p1 == p2:
return True
if kind.startswith("nutrition."):
food1 = p1.get("food_id")
food2 = p2.get("food_id")
item1 = str(p1.get("item", "")).strip().lower()
item2 = str(p2.get("item", "")).strip().lower()
same_item = (
(food1 and food2 and food1 == food2)
or (item1 and item2 and item1 == item2)
or (food1 and item2 and food1 == item2)
or (food2 and item1 and food2 == item1)
)
if same_item:
qty1 = p1.get("qty")
qty2 = p2.get("qty")
if qty1 is not None and qty2 is not None:
try:
return abs(float(qty1) - float(qty2)) < 0.01
except (ValueError, TypeError):
return qty1 == qty2
return True
elif kind.startswith("feeding."):
amt1 = p1.get("amount_ml") or p1.get("consumed_ml")
amt2 = p2.get("amount_ml") or p2.get("consumed_ml")
if amt1 is not None and amt2 is not None:
return amt1 == amt2
elif kind.startswith("diaper."):
if p1.get("pee") == p2.get("pee") and p1.get("poo") == p2.get("poo"):
return True
elif kind.startswith("sleep."):
if p1.get("type") == p2.get("type"):
return True
return False
def _is_duplicate_event(
self,
kind: str,
subject: str,
payload: Dict[str, Any],
start_iso: str,
tolerance_seconds: int = 900,
) -> Optional[Dict[str, Any]]:
"""Checks if a matching event already exists within tolerance_seconds (default 15m)."""
rows = self.con.execute(
"""
SELECT id, kind, subject, actor, start, "end", estimated, meal, note, payload, source_ref, created_at, updated_at
FROM events
WHERE kind = ?
AND subject = ?
AND deleted_at IS NULL
AND abs(epoch(start) - epoch(?::TIMESTAMPTZ)) <= ?
ORDER BY start DESC
""",
[kind, subject, start_iso, tolerance_seconds],
).fetchall()
for r in rows:
existing_payload = json.loads(r[9]) if isinstance(r[9], str) else (r[9] or {})
if self._matches_payload(kind, payload, existing_payload):
return {
"id": r[0],
"kind": r[1],
"subject": r[2],
"actor": r[3],
"start": r[4].isoformat() if hasattr(r[4], "isoformat") else str(r[4]),
"end": r[5].isoformat() if hasattr(r[5], "isoformat") and r[5] else (str(r[5]) if r[5] else None),
"estimated": r[6],
"meal": r[7],
"note": r[8],
"payload": existing_payload,
"source_ref": r[10],
"created_at": r[11].isoformat() if hasattr(r[11], "isoformat") else str(r[11]),
"updated_at": r[12].isoformat() if hasattr(r[12], "isoformat") and r[12] else (str(r[12]) if r[12] else None),
"deleted_at": None,
}
return None
# --------------------------------------------------------------------------
# Event Operations
# --------------------------------------------------------------------------
@@ -319,6 +400,7 @@ class DatastoreEngine:
meal: Optional[str] = None,
note: Optional[str] = None,
source_ref: Optional[str] = None,
allow_duplicate: bool = False,
) -> Dict[str, Any]:
"""Adds a time-series event, saves to JSONL partition, and returns confirmation."""
event_id = generate_ulid()
@@ -351,6 +433,19 @@ class DatastoreEngine:
raise ValueError(f"End time {end} cannot be earlier than start time {start}")
end_iso = end_dt.isoformat()
# Check for duplicate event within 15 minutes unless explicitly allowed
if not allow_duplicate:
existing_event = self._is_duplicate_event(kind, subject, payload, start_iso)
if existing_event:
feedback = self._get_event_feedback(kind, subject, start_dt.date())
return {
"status": "success",
"duplicate": True,
"message": f"Duplicate event detected within 15 minutes (existing event ID: {existing_event['id']}). Event retained without duplicate insertion.",
"event": existing_event,
"feedback": feedback,
}
# Validation for open intervals in sleep
if kind == "sleep.session" and end_iso is None:
# Check if there is already an open session for this subject
@@ -760,6 +855,7 @@ def run_mcp_server():
"estimated": {"type": "boolean", "description": "True if portions or macros are estimated"},
"meal": {"type": "string", "description": "Optional meal grouping (e.g. 'Frühstück', 'Mittagessen')"},
"note": {"type": "string", "description": "Free text notes or context"},
"allow_duplicate": {"type": "boolean", "description": "Set to true to force adding duplicate event within 15m"},
},
"required": ["kind", "subject", "actor", "payload"],
},
@@ -881,6 +977,7 @@ def run_mcp_server():
meal=args.get("meal"),
note=args.get("note"),
source_ref=args.get("source_ref"),
allow_duplicate=args.get("allow_duplicate", False),
)
elif name == "datastore_stop_event":
data = engine.stop_event(
@@ -945,7 +1042,8 @@ def run_cli():
# Add Event
p_add = subparsers.add_parser("add", help="Add an event")
p_add.add_argument("kind", help="e.g. nutrition.intake, sleep.session")
p_add.add_argument("kind", nargs="?", default=None, help="e.g. nutrition.intake, sleep.session")
p_add.add_argument("--kind", dest="kind_flag", help="e.g. nutrition.intake, sleep.session")
p_add.add_argument("--subject", required=True, help="moritz, mila, amelie")
p_add.add_argument("--actor", default="cli", help="Reporter identifier")
p_add.add_argument("--payload", required=True, help="JSON payload string")
@@ -955,10 +1053,12 @@ def run_cli():
p_add.add_argument("--meal", help="Meal grouping")
p_add.add_argument("--note", help="Free-form note")
p_add.add_argument("--estimated", action="store_true", help="Flag as estimated")
p_add.add_argument("--allow-duplicate", action="store_true", help="Allow duplicate event within 15 minutes")
# Stop Event
p_stop = subparsers.add_parser("stop", help="Stop an open interval event")
p_stop.add_argument("kind", help="e.g. sleep.session")
p_stop.add_argument("kind", nargs="?", default=None, help="e.g. sleep.session")
p_stop.add_argument("--kind", dest="kind_flag", help="e.g. sleep.session")
p_stop.add_argument("--subject", required=True, help="mila, moritz")
p_stop.add_argument("--end", help="ISO end time (defaults to now)")
@@ -982,9 +1082,12 @@ def run_cli():
engine = DatastoreEngine()
if args.command == "add":
kind = args.kind or args.kind_flag
if not kind:
parser.error("the 'add' command requires 'kind' (positional or via --kind)")
payload = json.loads(args.payload)
res = engine.add_event(
kind=args.kind,
kind=kind,
subject=args.subject,
actor=args.actor,
payload=payload,
@@ -994,13 +1097,16 @@ def run_cli():
meal=args.meal,
note=args.note,
estimated=args.estimated,
allow_duplicate=args.allow_duplicate,
)
print(json.dumps(res, indent=2, ensure_ascii=False))
elif args.command == "stop":
res = engine.stop_event(kind=args.kind, subject=args.subject, end_time=args.end)
kind = args.kind or args.kind_flag
if not kind:
parser.error("the 'stop' command requires 'kind' (positional or via --kind)")
res = engine.stop_event(kind=kind, subject=args.subject, end_time=args.end)
print(json.dumps(res, indent=2, ensure_ascii=False))
elif args.command == "query":
res = engine.query(args.sql)
print(json.dumps(res, indent=2, ensure_ascii=False))