Files
mop-skills/skills/datastore/scripts/migrate_legacy.py
T

535 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
# /// script
# dependencies = [
# "duckdb>=1.1.0",
# "python-ulid>=3.0.0",
# "pytz>=2024.1",
# ]
# ///
"""
migrate_legacy.py — Migrates legacy Markdown logs to DuckDB-backed JSONL datastore.
Parses:
- standard-foods.md -> data/entities/foods.jsonl
- entities/nutrition-log/moritz/*.md -> data/events/nutrition/*.jsonl
- entities/people/mila-schlafprotokoll.md -> data/events/sleep/*.jsonl & feeding/*.jsonl
Computes exact DuckDB sums, compares with old handwritten sums, and generates audit report.
"""
from __future__ import annotations
import datetime
import json
import os
import re
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from zoneinfo import ZoneInfo
def generate_ulid() -> str:
try:
from ulid import ULID
return str(ULID())
except TypeError:
import ulid
return str(ulid.new())
except ImportError:
import uuid
return str(uuid.uuid4())
import duckdb
BERLIN_TZ = ZoneInfo("Europe/Berlin")
WIKI_DIR = Path("/Users/moritz/src/infrapuzzle/knowledge-wiki")
DATA_DIR = WIKI_DIR / "data"
def parse_number(s: str) -> float:
"""Parses German float string like '~11,3' or '2.364' into float."""
if not s or s.strip() in ["-", "–", ""]:
return 0.0
cleaned = s.replace("~", "").replace("g", "").replace("ml", "").replace("kcal", "").strip()
# If number has both dot and comma (e.g. 1.234,56 or 2.364)
if "." in cleaned and "," in cleaned:
cleaned = cleaned.replace(".", "").replace(",", ".")
elif "," in cleaned:
cleaned = cleaned.replace(",", ".")
elif "." in cleaned:
# Check if dot is thousands separator (e.g. 2.364 kcal)
parts = cleaned.split(".")
if len(parts) == 2 and len(parts[1]) == 3:
cleaned = parts[0] + parts[1]
m = re.search(r"[-+]?\d*\.?\d+", cleaned)
return float(m.group(0)) if m else 0.0
# ==============================================================================
# 1. Migrate Standard Foods
# ==============================================================================
def migrate_standard_foods() -> List[Dict[str, Any]]:
foods_file = DATA_DIR / "entities" / "foods.jsonl"
foods_file.parent.mkdir(parents=True, exist_ok=True)
foods = [
{
"id": "oatly-haferdrink",
"name": "Oatly Haferdrink",
"brand": "Oatly AB",
"category": "milch-alternative",
"per_100": {
"kcal": 61.0,
"protein_g": 1.1,
"carbs_g": 7.1,
"sugar_g": 3.4,
"fat_g": 3.0,
"fiber_g": 0.8,
"salt_g": 0.10,
},
"density_g_per_ml": 1.03,
"ingredients_text": "Wasser, HAFER 10%, Rapsöl, Säureregulator (Dikaliumphosphat), Mineralien (Calciumcarbonat, Kaliumjodid), Salz, Vitamine (D2, Riboflavin, B12).",
"barcode": "7394376616037",
"source": "package_photo",
"description": "Standard-Hafermilch Grabengasse 7. Silberner Karton mit Vitaminen & Mineralien.",
},
{
"id": "fruechte-mix-tk",
"name": "Früchte Mix (TK)",
"brand": None,
"category": "obst",
"per_100": {
"kcal": 45.0,
"protein_g": 1.0,
"carbs_g": 6.5,
"sugar_g": 3.4,
"fat_g": 0.3,
"fiber_g": 3.4,
"salt_g": 0.0,
},
"density_g_per_ml": 1.0,
"ingredients_text": "Gefrorene Beeren-Mischung (TK)",
"barcode": None,
"source": "package_photo",
"description": "Standard gefrorener Beerenmix für Porridge und Overnight Oats.",
},
{
"id": "muller-protein-porridge-mix",
"name": "Müller Protein-Porridge Batch-Mix",
"brand": "Eigenmischung",
"category": "muesli",
"per_100": {
"kcal": 381.0,
"protein_g": 23.3,
"carbs_g": 46.6,
"sugar_g": 2.9,
"fat_g": 8.5,
"fiber_g": 9.8,
"salt_g": 0.15,
},
"density_g_per_ml": 1.0,
"ingredients_text": "Müller Porridge Basic 500g + Müller Eiweißpulver 100g + Chiasamen 50g",
"barcode": None,
"source": "recipe_batch",
"description": "Eigener Batch-Mix für Morgenporridge (Müller Porridge Basic + Eiweißpulver + Chiasamen).",
},
]
with open(foods_file, "w", encoding="utf-8") as f:
for item in foods:
f.write(json.dumps(item, ensure_ascii=False) + "\n")
return foods
# ==============================================================================
# 2. Migrate Nutrition Logs
# ==============================================================================
def migrate_nutrition_logs() -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]]]:
nutrition_dir = WIKI_DIR / "entities" / "nutrition-log" / "moritz"
events_dir = DATA_DIR / "events" / "nutrition"
events_dir.mkdir(parents=True, exist_ok=True)
all_events: List[Dict[str, Any]] = []
stated_sums: Dict[str, Dict[str, Any]] = {}
for md_file in sorted(nutrition_dir.glob("2026-*.md")):
month_str = md_file.stem
file_events: List[Dict[str, Any]] = []
current_date_str = None
current_day_name = None
with open(md_file, "r", encoding="utf-8") as f:
for line in f:
line_str = line.strip()
# Match date header: ## YYYY-MM-DD Wochentag
m_date = re.match(r"^##\s+(\d{4}-\d{2}-\d{2})(?:\s+(\w+))?", line_str)
if m_date:
current_date_str = m_date.group(1)
current_day_name = m_date.group(2) or ""
continue
# Match table row
if current_date_str and line_str.startswith("|") and not line_str.startswith("| Zeit") and not line_str.startswith("|---"):
cols = [c.strip() for c in line_str.strip("|").split("|")]
if len(cols) >= 10:
zeit_raw = cols[0]
item_raw = cols[1]
menge_raw = cols[2]
eiweiss_raw = cols[3]
kh_raw = cols[4]
zucker_raw = cols[5]
fett_raw = cols[6]
ballast_raw = cols[7]
salz_raw = cols[8]
kcal_raw = cols[9]
# Parse time (default 12:00 if unknown)
m_time = re.search(r"(\d{1,2}):(\d{2})", zeit_raw)
if m_time:
hour = int(m_time.group(1))
minute = int(m_time.group(2))
else:
hour, minute = 12, 0
event_dt = datetime.datetime.strptime(current_date_str, "%Y-%m-%d").replace(
hour=hour, minute=minute, tzinfo=BERLIN_TZ
)
# Determine food_id if matched
food_id = None
if "oatly" in item_raw.lower() or "haferdrink" in item_raw.lower():
food_id = "oatly-haferdrink"
elif "früchte mix" in item_raw.lower() or "beeren" in item_raw.lower():
food_id = "fruechte-mix-tk"
elif "porridge-mischung" in item_raw.lower() or "müller protein-porridge" in item_raw.lower():
food_id = "muller-protein-porridge-mix"
# Parse quantity and unit
m_qty = re.search(r"(\d+(?:[.,]\d+)?)\s*(g|ml|portion|kugel)?", menge_raw)
qty = float(m_qty.group(1).replace(",", ".")) if m_qty else parse_number(menge_raw)
unit = m_qty.group(2) if m_qty and m_qty.group(2) else "g"
is_estimated = "~" in zeit_raw or "~" in item_raw or "~" in menge_raw or "~" in kcal_raw
record = {
"id": generate_ulid(),
"kind": "nutrition.intake",
"subject": "moritz",
"actor": "agent:koch",
"start": event_dt.isoformat(),
"end": None,
"estimated": is_estimated,
"meal": None,
"note": None,
"payload": {
"item": item_raw,
"food_id": food_id,
"qty": qty,
"unit": unit,
"protein_g": parse_number(eiweiss_raw),
"carbs_g": parse_number(kh_raw),
"sugar_g": parse_number(zucker_raw),
"fat_g": parse_number(fett_raw),
"fiber_g": parse_number(ballast_raw),
"salt_g": parse_number(salz_raw),
"kcal": parse_number(kcal_raw),
},
"source_ref": f"{md_file.name}:{zeit_raw}",
"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"updated_at": None,
"deleted_at": None,
}
file_events.append(record)
all_events.append(record)
# Match stated sums: **Summe heute:** ...
if current_date_str and "**Summe heute:**" in line_str:
sum_text = line_str.split("**Summe heute:**")[1].strip()
stated_sums[current_date_str] = {
"text": sum_text,
"protein_g": parse_number(re.search(r"([\d,]+)\s*g\s*Eiweiß", sum_text).group(1)) if re.search(r"([\d,]+)\s*g\s*Eiweiß", sum_text) else 0.0,
"carbs_g": parse_number(re.search(r"([\d,]+)\s*g\s*KH", sum_text).group(1)) if re.search(r"([\d,]+)\s*g\s*KH", sum_text) else 0.0,
"kcal": parse_number(re.search(r"([\d,.]+)\s*kcal", sum_text).group(1)) if re.search(r"([\d,.]+)\s*kcal", sum_text) else 0.0,
}
# Write partitioned JSONL
file_events.sort(key=lambda x: (x["start"], x["id"]))
out_jsonl = events_dir / f"{month_str}.jsonl"
with open(out_jsonl, "w", encoding="utf-8") as out_f:
for ev in file_events:
out_f.write(json.dumps(ev, ensure_ascii=False) + "\n")
return all_events, stated_sums
# ==============================================================================
# 3. Migrate Mila Sleep & Feeding Logs
# ==============================================================================
def migrate_mila_sleep_and_feeding() -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
md_file = WIKI_DIR / "entities" / "people" / "mila-schlafprotokoll.md"
sleep_events: List[Dict[str, Any]] = []
feeding_events: List[Dict[str, Any]] = []
if not md_file.is_file():
return sleep_events, feeding_events
current_date = None
processed_nights = set()
with open(md_file, "r", encoding="utf-8") as f:
for line in f:
line_str = line.strip()
# Date header: ## Dienstag, 29.09.2026 or ## 2026-09-29
m_date = re.search(r"(\d{2})\.(\d{2})\.(\d{4})", line_str)
if line_str.startswith("##") and m_date:
day, month, year = int(m_date.group(1)), int(m_date.group(2)), int(m_date.group(3))
current_date = datetime.date(year, month, day)
continue
if not current_date or not line_str.startswith("-"):
continue
# Case A: Feeding log (e.g. - 08:15 – Fläschchen (80 ml übrig/ungetrunken))
if "fläschchen" in line_str.lower():
m_time = re.search(r"(\d{1,2}):(\d{2})", line_str)
h, m = (int(m_time.group(1)), int(m_time.group(2))) if m_time else (8, 0)
feed_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, h, m, tzinfo=BERLIN_TZ)
m_ml = re.search(r"(\d+)\s*ml", line_str)
left_ml = float(m_ml.group(1)) if m_ml else 0.0
record = {
"id": generate_ulid(),
"kind": "feeding.bottle",
"subject": "mila",
"actor": "agent:hebamme",
"start": feed_dt.isoformat(),
"end": None,
"estimated": False,
"meal": None,
"note": line_str.lstrip("- ").strip(),
"payload": {
"offered_ml": None,
"consumed_ml": None,
"left_ml": left_ml,
"milk_type": "pre",
},
"source_ref": f"mila-schlafprotokoll.md:{feed_dt.strftime('%Y-%m-%d %H:%M')}",
"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"updated_at": None,
"deleted_at": None,
}
feeding_events.append(record)
continue
# Case B: Sleep intervals
# Examples:
# - 09:51 – 10:38 | Schlaf am Morgen (46 min: 10 min wach, 6 Leicht-, 30 Tiefschlaf; Einschlafzeit 5 min)
# - 18:57 – 06:50 | Nacht (11 h 53 min)
# - 20:10 – | Nacht (läuft)
# - 07:00 – aufgewacht (Nacht-Ende); Nacht: 19:13 – 07:00 (11 h 47 min)
# Check for "aufgewacht (Nacht-Ende)" duplicate
if "aufgewacht (nacht-ende)" in line_str.lower() or "nacht-ende" in line_str.lower():
m_span = re.search(r"(\d{1,2}):(\d{2})\s*–\s*(\d{1,2}):(\d{2})", line_str)
if m_span:
s_h, s_m, e_h, e_m = int(m_span.group(1)), int(m_span.group(2)), int(m_span.group(3)), int(m_span.group(4))
night_key = f"{s_h:02d}:{s_m:02d}-{e_h:02d}:{e_m:02d}"
if night_key in processed_nights:
# Already logged from the evening line, skip duplicate!
continue
else:
processed_nights.add(night_key)
# The start was yesterday evening
start_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, s_h, s_m, tzinfo=BERLIN_TZ) - datetime.timedelta(days=1)
end_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, e_h, e_m, tzinfo=BERLIN_TZ)
record = {
"id": generate_ulid(),
"kind": "sleep.session",
"subject": "mila",
"actor": "agent:hebamme",
"start": start_dt.isoformat(),
"end": end_dt.isoformat(),
"estimated": False,
"meal": None,
"note": line_str.lstrip("- ").strip(),
"payload": {
"type": "night",
"location": "bett",
},
"source_ref": f"mila-schlafprotokoll.md:{current_date}",
"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"updated_at": None,
"deleted_at": None,
}
sleep_events.append(record)
continue
# Standard interval: start – end | label
m_int = re.search(r"^-\s*(\d{1,2}):(\d{2})\s*–\s*(\d{1,2})?:?(\d{2})?\s*\|?(.*)", line_str)
if m_int:
s_h = int(m_int.group(1))
s_m = int(m_int.group(2))
e_h_str = m_int.group(3)
e_m_str = m_int.group(4)
desc = m_int.group(5).strip() if m_int.group(5) else ""
start_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, s_h, s_m, tzinfo=BERLIN_TZ)
# Open sleep interval (e.g. 20:10 – | Nacht (läuft))
if not e_h_str or not e_m_str:
end_dt = None
sleep_type = "night" if s_h >= 18 or s_h < 6 else "nap"
else:
e_h = int(e_h_str)
e_m = int(e_m_str)
# If end hour < start hour, crosses midnight!
if e_h < s_h:
end_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, e_h, e_m, tzinfo=BERLIN_TZ) + datetime.timedelta(days=1)
else:
end_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, e_h, e_m, tzinfo=BERLIN_TZ)
sleep_type = "night" if "nacht" in desc.lower() or s_h >= 18 else "nap"
# Deduplication key for night sleeps
night_key = f"{s_h:02d}:{s_m:02d}-{e_h:02d}:{e_m:02d}"
if sleep_type == "night":
processed_nights.add(night_key)
# Location detection
loc = "bett"
if "auto" in desc.lower():
loc = "auto"
elif "kinderwagen" in desc.lower():
loc = "kinderwagen"
elif "outdoor" in desc.lower() or "decke" in desc.lower():
loc = "outdoor"
record = {
"id": generate_ulid(),
"kind": "sleep.session",
"subject": "mila",
"actor": "agent:hebamme",
"start": start_dt.isoformat(),
"end": end_dt.isoformat() if end_dt else None,
"estimated": False,
"meal": None,
"note": desc or None,
"payload": {
"type": sleep_type,
"location": loc,
},
"source_ref": f"mila-schlafprotokoll.md:{current_date}",
"created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"updated_at": None,
"deleted_at": None,
}
sleep_events.append(record)
# Write partition files
sleep_dir = DATA_DIR / "events" / "sleep"
sleep_dir.mkdir(parents=True, exist_ok=True)
feeding_dir = DATA_DIR / "events" / "feeding"
feeding_dir.mkdir(parents=True, exist_ok=True)
# Group sleep by month
sleep_by_month: Dict[str, List[Dict[str, Any]]] = {}
for ev in sleep_events:
m_str = ev["start"][:7] # YYYY-MM
sleep_by_month.setdefault(m_str, []).append(ev)
for m_str, m_events in sleep_by_month.items():
m_events.sort(key=lambda x: (x["start"], x["id"]))
with open(sleep_dir / f"{m_str}.jsonl", "w", encoding="utf-8") as f:
for ev in m_events:
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
# Group feeding by month
feed_by_month: Dict[str, List[Dict[str, Any]]] = {}
for ev in feeding_events:
m_str = ev["start"][:7]
feed_by_month.setdefault(m_str, []).append(ev)
for m_str, m_events in feed_by_month.items():
m_events.sort(key=lambda x: (x["start"], x["id"]))
with open(feeding_dir / f"{m_str}.jsonl", "w", encoding="utf-8") as f:
for ev in m_events:
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
return sleep_events, feeding_events
# ==============================================================================
# 4. Generate Audit Report
# ==============================================================================
def generate_audit_report(stated_sums: Dict[str, Dict[str, Any]]) -> str:
con = duckdb.connect(":memory:")
# Initialize schema
schema_dir = DATA_DIR / "_schema"
for sql_f in ["envelope.sql", "entities.sql", "views.sql"]:
con.execute((schema_dir / sql_f).read_text(encoding="utf-8"))
# Load foods
foods_file = DATA_DIR / "entities" / "foods.jsonl"
if foods_file.is_file():
con.execute(f"INSERT INTO foods SELECT * FROM read_json_auto('{foods_file}')")
# Load all nutrition events
nut_files = str(DATA_DIR / "events" / "nutrition" / "*.jsonl")
con.execute(f"INSERT INTO events SELECT * FROM read_json_auto('{nut_files}')")
# Query daily sums
db_sums = con.execute("SELECT log_date, total_protein_g, total_carbs_g, total_kcal, item_count FROM v_nutrition_daily ORDER BY log_date ASC").fetchall()
lines = []
lines.append("# 📊 Datastore Migration: Arithmetic Discrepancy Audit Report\n")
lines.append(f"**Generated:** {datetime.datetime.now(BERLIN_TZ).strftime('%Y-%m-%d %H:%M:%S')} ")
lines.append("**Source of Truth:** DuckDB engine over `data/events/nutrition/*.jsonl` \n")
lines.append("This audit compares the old handwritten Markdown `**Summe heute:**` against the mathematically exact DuckDB column summations.\n")
lines.append("| Date | Items | Hand Stated Protein | DuckDB Exact Protein | Δ Protein | Hand Stated kcal | DuckDB Exact kcal | Δ kcal |")
lines.append("| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: |")
for row in db_sums:
d_str = str(row[0])
p_exact = row[1]
k_exact = int(row[3])
count = row[4]
stated = stated_sums.get(d_str, {})
p_hand = stated.get("protein_g", 0.0)
k_hand = int(stated.get("kcal", 0))
delta_p = round(p_exact - p_hand, 1)
delta_k = k_exact - k_hand
p_diff_str = f"**{delta_p:+.1f} g**" if abs(delta_p) > 0.1 else "✅ 0.0"
k_diff_str = f"**{delta_k:+d} kcal**" if abs(delta_k) > 2 else "✅ 0"
lines.append(f"| {d_str} | {count} | {p_hand:.1f} g | {p_exact:.1f} g | {p_diff_str} | {k_hand} | {k_exact} | {k_diff_str} |")
return "\n".join(lines)
if __name__ == "__main__":
print("1. Migrating Standard Foods...")
foods = migrate_standard_foods()
print(f" -> Migrated {len(foods)} food reference items.")
print("2. Migrating Nutrition Logs...")
nut_events, stated_sums = migrate_nutrition_logs()
print(f" -> Migrated {len(nut_events)} nutrition events across {len(stated_sums)} days.")
print("3. Migrating Mila Sleep & Feeding Logs...")
sleep_events, feed_events = migrate_mila_sleep_and_feeding()
print(f" -> Migrated {len(sleep_events)} sleep events and {len(feed_events)} bottle feeding events.")
print("4. Generating Discrepancy Audit Report...")
report = generate_audit_report(stated_sums)
report_file = WIKI_DIR / "reports" / "datastore-migration-audit.md"
report_file.parent.mkdir(parents=True, exist_ok=True)
report_file.write_text(report, encoding="utf-8")
print(f" -> Wrote audit report to {report_file}")
print("\n--- Summary ---")
print(report)