#!/usr/bin/env python3 # /// script # dependencies = [ # "duckdb>=1.1.0", # "python-ulid>=3.0.0", # "pytz>=2024.1", # ] # /// """ migrate_legacy.py — Migrates legacy Markdown logs to DuckDB-backed JSONL datastore. Parses: - standard-foods.md -> data/entities/foods.jsonl - entities/nutrition-log/moritz/*.md -> data/events/nutrition/*.jsonl - entities/people/mila-schlafprotokoll.md -> data/events/sleep/*.jsonl & feeding/*.jsonl Computes exact DuckDB sums, compares with old handwritten sums, and generates audit report. """ from __future__ import annotations import datetime import json import os import re from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from zoneinfo import ZoneInfo def generate_ulid() -> str: try: from ulid import ULID return str(ULID()) except TypeError: import ulid return str(ulid.new()) except ImportError: import uuid return str(uuid.uuid4()) import duckdb BERLIN_TZ = ZoneInfo("Europe/Berlin") WIKI_DIR = Path("/Users/moritz/src/infrapuzzle/knowledge-wiki") DATA_DIR = WIKI_DIR / "data" def parse_number(s: str) -> float: """Parses German float string like '~11,3' or '2.364' into float.""" if not s or s.strip() in ["-", "–", ""]: return 0.0 cleaned = s.replace("~", "").replace("g", "").replace("ml", "").replace("kcal", "").strip() # If number has both dot and comma (e.g. 1.234,56 or 2.364) if "." in cleaned and "," in cleaned: cleaned = cleaned.replace(".", "").replace(",", ".") elif "," in cleaned: cleaned = cleaned.replace(",", ".") elif "." in cleaned: # Check if dot is thousands separator (e.g. 2.364 kcal) parts = cleaned.split(".") if len(parts) == 2 and len(parts[1]) == 3: cleaned = parts[0] + parts[1] m = re.search(r"[-+]?\d*\.?\d+", cleaned) return float(m.group(0)) if m else 0.0 # ============================================================================== # 1. Migrate Standard Foods # ============================================================================== def migrate_standard_foods() -> List[Dict[str, Any]]: foods_file = DATA_DIR / "entities" / "foods.jsonl" foods_file.parent.mkdir(parents=True, exist_ok=True) foods = [ { "id": "oatly-haferdrink", "name": "Oatly Haferdrink", "brand": "Oatly AB", "category": "milch-alternative", "per_100": { "kcal": 61.0, "protein_g": 1.1, "carbs_g": 7.1, "sugar_g": 3.4, "fat_g": 3.0, "fiber_g": 0.8, "salt_g": 0.10, }, "density_g_per_ml": 1.03, "ingredients_text": "Wasser, HAFER 10%, Rapsöl, Säureregulator (Dikaliumphosphat), Mineralien (Calciumcarbonat, Kaliumjodid), Salz, Vitamine (D2, Riboflavin, B12).", "barcode": "7394376616037", "source": "package_photo", "description": "Standard-Hafermilch Grabengasse 7. Silberner Karton mit Vitaminen & Mineralien.", }, { "id": "fruechte-mix-tk", "name": "Früchte Mix (TK)", "brand": None, "category": "obst", "per_100": { "kcal": 45.0, "protein_g": 1.0, "carbs_g": 6.5, "sugar_g": 3.4, "fat_g": 0.3, "fiber_g": 3.4, "salt_g": 0.0, }, "density_g_per_ml": 1.0, "ingredients_text": "Gefrorene Beeren-Mischung (TK)", "barcode": None, "source": "package_photo", "description": "Standard gefrorener Beerenmix für Porridge und Overnight Oats.", }, { "id": "muller-protein-porridge-mix", "name": "Müller Protein-Porridge Batch-Mix", "brand": "Eigenmischung", "category": "muesli", "per_100": { "kcal": 381.0, "protein_g": 23.3, "carbs_g": 46.6, "sugar_g": 2.9, "fat_g": 8.5, "fiber_g": 9.8, "salt_g": 0.15, }, "density_g_per_ml": 1.0, "ingredients_text": "Müller Porridge Basic 500g + Müller Eiweißpulver 100g + Chiasamen 50g", "barcode": None, "source": "recipe_batch", "description": "Eigener Batch-Mix für Morgenporridge (Müller Porridge Basic + Eiweißpulver + Chiasamen).", }, ] with open(foods_file, "w", encoding="utf-8") as f: for item in foods: f.write(json.dumps(item, ensure_ascii=False) + "\n") return foods # ============================================================================== # 2. Migrate Nutrition Logs # ============================================================================== def migrate_nutrition_logs() -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]]]: nutrition_dir = WIKI_DIR / "entities" / "nutrition-log" / "moritz" events_dir = DATA_DIR / "events" / "nutrition" events_dir.mkdir(parents=True, exist_ok=True) all_events: List[Dict[str, Any]] = [] stated_sums: Dict[str, Dict[str, Any]] = {} for md_file in sorted(nutrition_dir.glob("2026-*.md")): month_str = md_file.stem file_events: List[Dict[str, Any]] = [] current_date_str = None current_day_name = None with open(md_file, "r", encoding="utf-8") as f: for line in f: line_str = line.strip() # Match date header: ## YYYY-MM-DD Wochentag m_date = re.match(r"^##\s+(\d{4}-\d{2}-\d{2})(?:\s+(\w+))?", line_str) if m_date: current_date_str = m_date.group(1) current_day_name = m_date.group(2) or "" continue # Match table row if current_date_str and line_str.startswith("|") and not line_str.startswith("| Zeit") and not line_str.startswith("|---"): cols = [c.strip() for c in line_str.strip("|").split("|")] if len(cols) >= 10: zeit_raw = cols[0] item_raw = cols[1] menge_raw = cols[2] eiweiss_raw = cols[3] kh_raw = cols[4] zucker_raw = cols[5] fett_raw = cols[6] ballast_raw = cols[7] salz_raw = cols[8] kcal_raw = cols[9] # Parse time (default 12:00 if unknown) m_time = re.search(r"(\d{1,2}):(\d{2})", zeit_raw) if m_time: hour = int(m_time.group(1)) minute = int(m_time.group(2)) else: hour, minute = 12, 0 event_dt = datetime.datetime.strptime(current_date_str, "%Y-%m-%d").replace( hour=hour, minute=minute, tzinfo=BERLIN_TZ ) # Determine food_id if matched food_id = None if "oatly" in item_raw.lower() or "haferdrink" in item_raw.lower(): food_id = "oatly-haferdrink" elif "früchte mix" in item_raw.lower() or "beeren" in item_raw.lower(): food_id = "fruechte-mix-tk" elif "porridge-mischung" in item_raw.lower() or "müller protein-porridge" in item_raw.lower(): food_id = "muller-protein-porridge-mix" # Parse quantity and unit m_qty = re.search(r"(\d+(?:[.,]\d+)?)\s*(g|ml|portion|kugel)?", menge_raw) qty = float(m_qty.group(1).replace(",", ".")) if m_qty else parse_number(menge_raw) unit = m_qty.group(2) if m_qty and m_qty.group(2) else "g" is_estimated = "~" in zeit_raw or "~" in item_raw or "~" in menge_raw or "~" in kcal_raw record = { "id": generate_ulid(), "kind": "nutrition.intake", "subject": "moritz", "actor": "agent:koch", "start": event_dt.isoformat(), "end": None, "estimated": is_estimated, "meal": None, "note": None, "payload": { "item": item_raw, "food_id": food_id, "qty": qty, "unit": unit, "protein_g": parse_number(eiweiss_raw), "carbs_g": parse_number(kh_raw), "sugar_g": parse_number(zucker_raw), "fat_g": parse_number(fett_raw), "fiber_g": parse_number(ballast_raw), "salt_g": parse_number(salz_raw), "kcal": parse_number(kcal_raw), }, "source_ref": f"{md_file.name}:{zeit_raw}", "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "updated_at": None, "deleted_at": None, } file_events.append(record) all_events.append(record) # Match stated sums: **Summe heute:** ... if current_date_str and "**Summe heute:**" in line_str: sum_text = line_str.split("**Summe heute:**")[1].strip() stated_sums[current_date_str] = { "text": sum_text, "protein_g": parse_number(re.search(r"([\d,]+)\s*g\s*Eiweiß", sum_text).group(1)) if re.search(r"([\d,]+)\s*g\s*Eiweiß", sum_text) else 0.0, "carbs_g": parse_number(re.search(r"([\d,]+)\s*g\s*KH", sum_text).group(1)) if re.search(r"([\d,]+)\s*g\s*KH", sum_text) else 0.0, "kcal": parse_number(re.search(r"([\d,.]+)\s*kcal", sum_text).group(1)) if re.search(r"([\d,.]+)\s*kcal", sum_text) else 0.0, } # Write partitioned JSONL file_events.sort(key=lambda x: (x["start"], x["id"])) out_jsonl = events_dir / f"{month_str}.jsonl" with open(out_jsonl, "w", encoding="utf-8") as out_f: for ev in file_events: out_f.write(json.dumps(ev, ensure_ascii=False) + "\n") return all_events, stated_sums # ============================================================================== # 3. Migrate Mila Sleep & Feeding Logs # ============================================================================== def migrate_mila_sleep_and_feeding() -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: md_file = WIKI_DIR / "entities" / "people" / "mila-schlafprotokoll.md" sleep_events: List[Dict[str, Any]] = [] feeding_events: List[Dict[str, Any]] = [] if not md_file.is_file(): return sleep_events, feeding_events current_date = None processed_nights = set() with open(md_file, "r", encoding="utf-8") as f: for line in f: line_str = line.strip() # Date header: ## Dienstag, 29.09.2026 or ## 2026-09-29 m_date = re.search(r"(\d{2})\.(\d{2})\.(\d{4})", line_str) if line_str.startswith("##") and m_date: day, month, year = int(m_date.group(1)), int(m_date.group(2)), int(m_date.group(3)) current_date = datetime.date(year, month, day) continue if not current_date or not line_str.startswith("-"): continue # Case A: Feeding log (e.g. - 08:15 – Fläschchen (80 ml übrig/ungetrunken)) if "fläschchen" in line_str.lower(): m_time = re.search(r"(\d{1,2}):(\d{2})", line_str) h, m = (int(m_time.group(1)), int(m_time.group(2))) if m_time else (8, 0) feed_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, h, m, tzinfo=BERLIN_TZ) m_ml = re.search(r"(\d+)\s*ml", line_str) left_ml = float(m_ml.group(1)) if m_ml else 0.0 record = { "id": generate_ulid(), "kind": "feeding.bottle", "subject": "mila", "actor": "agent:hebamme", "start": feed_dt.isoformat(), "end": None, "estimated": False, "meal": None, "note": line_str.lstrip("- ").strip(), "payload": { "offered_ml": None, "consumed_ml": None, "left_ml": left_ml, "milk_type": "pre", }, "source_ref": f"mila-schlafprotokoll.md:{feed_dt.strftime('%Y-%m-%d %H:%M')}", "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "updated_at": None, "deleted_at": None, } feeding_events.append(record) continue # Case B: Sleep intervals # Examples: # - 09:51 – 10:38 | Schlaf am Morgen (46 min: 10 min wach, 6 Leicht-, 30 Tiefschlaf; Einschlafzeit 5 min) # - 18:57 – 06:50 | Nacht (11 h 53 min) # - 20:10 – | Nacht (läuft) # - 07:00 – aufgewacht (Nacht-Ende); Nacht: 19:13 – 07:00 (11 h 47 min) # Check for "aufgewacht (Nacht-Ende)" duplicate if "aufgewacht (nacht-ende)" in line_str.lower() or "nacht-ende" in line_str.lower(): m_span = re.search(r"(\d{1,2}):(\d{2})\s*–\s*(\d{1,2}):(\d{2})", line_str) if m_span: s_h, s_m, e_h, e_m = int(m_span.group(1)), int(m_span.group(2)), int(m_span.group(3)), int(m_span.group(4)) night_key = f"{s_h:02d}:{s_m:02d}-{e_h:02d}:{e_m:02d}" if night_key in processed_nights: # Already logged from the evening line, skip duplicate! continue else: processed_nights.add(night_key) # The start was yesterday evening start_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, s_h, s_m, tzinfo=BERLIN_TZ) - datetime.timedelta(days=1) end_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, e_h, e_m, tzinfo=BERLIN_TZ) record = { "id": generate_ulid(), "kind": "sleep.session", "subject": "mila", "actor": "agent:hebamme", "start": start_dt.isoformat(), "end": end_dt.isoformat(), "estimated": False, "meal": None, "note": line_str.lstrip("- ").strip(), "payload": { "type": "night", "location": "bett", }, "source_ref": f"mila-schlafprotokoll.md:{current_date}", "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "updated_at": None, "deleted_at": None, } sleep_events.append(record) continue # Standard interval: start – end | label m_int = re.search(r"^-\s*(\d{1,2}):(\d{2})\s*–\s*(\d{1,2})?:?(\d{2})?\s*\|?(.*)", line_str) if m_int: s_h = int(m_int.group(1)) s_m = int(m_int.group(2)) e_h_str = m_int.group(3) e_m_str = m_int.group(4) desc = m_int.group(5).strip() if m_int.group(5) else "" start_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, s_h, s_m, tzinfo=BERLIN_TZ) # Open sleep interval (e.g. 20:10 – | Nacht (läuft)) if not e_h_str or not e_m_str: end_dt = None sleep_type = "night" if s_h >= 18 or s_h < 6 else "nap" else: e_h = int(e_h_str) e_m = int(e_m_str) # If end hour < start hour, crosses midnight! if e_h < s_h: end_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, e_h, e_m, tzinfo=BERLIN_TZ) + datetime.timedelta(days=1) else: end_dt = datetime.datetime(current_date.year, current_date.month, current_date.day, e_h, e_m, tzinfo=BERLIN_TZ) sleep_type = "night" if "nacht" in desc.lower() or s_h >= 18 else "nap" # Deduplication key for night sleeps night_key = f"{s_h:02d}:{s_m:02d}-{e_h:02d}:{e_m:02d}" if sleep_type == "night": processed_nights.add(night_key) # Location detection loc = "bett" if "auto" in desc.lower(): loc = "auto" elif "kinderwagen" in desc.lower(): loc = "kinderwagen" elif "outdoor" in desc.lower() or "decke" in desc.lower(): loc = "outdoor" record = { "id": generate_ulid(), "kind": "sleep.session", "subject": "mila", "actor": "agent:hebamme", "start": start_dt.isoformat(), "end": end_dt.isoformat() if end_dt else None, "estimated": False, "meal": None, "note": desc or None, "payload": { "type": sleep_type, "location": loc, }, "source_ref": f"mila-schlafprotokoll.md:{current_date}", "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "updated_at": None, "deleted_at": None, } sleep_events.append(record) # Write partition files sleep_dir = DATA_DIR / "events" / "sleep" sleep_dir.mkdir(parents=True, exist_ok=True) feeding_dir = DATA_DIR / "events" / "feeding" feeding_dir.mkdir(parents=True, exist_ok=True) # Group sleep by month sleep_by_month: Dict[str, List[Dict[str, Any]]] = {} for ev in sleep_events: m_str = ev["start"][:7] # YYYY-MM sleep_by_month.setdefault(m_str, []).append(ev) for m_str, m_events in sleep_by_month.items(): m_events.sort(key=lambda x: (x["start"], x["id"])) with open(sleep_dir / f"{m_str}.jsonl", "w", encoding="utf-8") as f: for ev in m_events: f.write(json.dumps(ev, ensure_ascii=False) + "\n") # Group feeding by month feed_by_month: Dict[str, List[Dict[str, Any]]] = {} for ev in feeding_events: m_str = ev["start"][:7] feed_by_month.setdefault(m_str, []).append(ev) for m_str, m_events in feed_by_month.items(): m_events.sort(key=lambda x: (x["start"], x["id"])) with open(feeding_dir / f"{m_str}.jsonl", "w", encoding="utf-8") as f: for ev in m_events: f.write(json.dumps(ev, ensure_ascii=False) + "\n") return sleep_events, feeding_events # ============================================================================== # 4. Generate Audit Report # ============================================================================== def generate_audit_report(stated_sums: Dict[str, Dict[str, Any]]) -> str: con = duckdb.connect(":memory:") # Initialize schema schema_dir = DATA_DIR / "_schema" for sql_f in ["envelope.sql", "entities.sql", "views.sql"]: con.execute((schema_dir / sql_f).read_text(encoding="utf-8")) # Load foods foods_file = DATA_DIR / "entities" / "foods.jsonl" if foods_file.is_file(): con.execute(f"INSERT INTO foods SELECT * FROM read_json_auto('{foods_file}')") # Load all nutrition events nut_files = str(DATA_DIR / "events" / "nutrition" / "*.jsonl") con.execute(f"INSERT INTO events SELECT * FROM read_json_auto('{nut_files}')") # Query daily sums db_sums = con.execute("SELECT log_date, total_protein_g, total_carbs_g, total_kcal, item_count FROM v_nutrition_daily ORDER BY log_date ASC").fetchall() lines = [] lines.append("# 📊 Datastore Migration: Arithmetic Discrepancy Audit Report\n") lines.append(f"**Generated:** {datetime.datetime.now(BERLIN_TZ).strftime('%Y-%m-%d %H:%M:%S')} ") lines.append("**Source of Truth:** DuckDB engine over `data/events/nutrition/*.jsonl` \n") lines.append("This audit compares the old handwritten Markdown `**Summe heute:**` against the mathematically exact DuckDB column summations.\n") lines.append("| Date | Items | Hand Stated Protein | DuckDB Exact Protein | Δ Protein | Hand Stated kcal | DuckDB Exact kcal | Δ kcal |") lines.append("| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: |") for row in db_sums: d_str = str(row[0]) p_exact = row[1] k_exact = int(row[3]) count = row[4] stated = stated_sums.get(d_str, {}) p_hand = stated.get("protein_g", 0.0) k_hand = int(stated.get("kcal", 0)) delta_p = round(p_exact - p_hand, 1) delta_k = k_exact - k_hand p_diff_str = f"**{delta_p:+.1f} g**" if abs(delta_p) > 0.1 else "✅ 0.0" k_diff_str = f"**{delta_k:+d} kcal**" if abs(delta_k) > 2 else "✅ 0" lines.append(f"| {d_str} | {count} | {p_hand:.1f} g | {p_exact:.1f} g | {p_diff_str} | {k_hand} | {k_exact} | {k_diff_str} |") return "\n".join(lines) if __name__ == "__main__": print("1. Migrating Standard Foods...") foods = migrate_standard_foods() print(f" -> Migrated {len(foods)} food reference items.") print("2. Migrating Nutrition Logs...") nut_events, stated_sums = migrate_nutrition_logs() print(f" -> Migrated {len(nut_events)} nutrition events across {len(stated_sums)} days.") print("3. Migrating Mila Sleep & Feeding Logs...") sleep_events, feed_events = migrate_mila_sleep_and_feeding() print(f" -> Migrated {len(sleep_events)} sleep events and {len(feed_events)} bottle feeding events.") print("4. Generating Discrepancy Audit Report...") report = generate_audit_report(stated_sums) report_file = WIKI_DIR / "reports" / "datastore-migration-audit.md" report_file.parent.mkdir(parents=True, exist_ok=True) report_file.write_text(report, encoding="utf-8") print(f" -> Wrote audit report to {report_file}") print("\n--- Summary ---") print(report)