moritz/mop-skills
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(skills): add regensburg_ris_mcp.py and universal_product_extractor.py

Browse Source
This commit is contained in:
Moritz Graf
2026-09-16 08:22:40 +02:00
parent 7e82424ca2
commit 5376ddbc20
2 changed files with 393 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/regensburg-stadtrat/scripts/regensburg_ris_mcp.py
Executable
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "beautifulsoup4>=4.12.0",
# "requests>=2.31.0",
# ]
# ///
"""
Regensburg Stadtrat (ALLRIS) MCP Server
Exposes municipal council documents, upcoming meetings, agenda items (TOPs),
and binding resolutions as Model Context Protocol (MCP) tools over stdio.
"""
import os
import sys
import json
import traceback
# Ensure current directory is on sys.path to import AllrisClient
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from regensburg_ris import AllrisClient
TOOLS = [
{
"name": "stadtrat_search",
"description": "Search Regensburg city council documents, resolutions (Beschlüsse), and motions (Vorlagen) by keyword (e.g. 'Kaufhof', 'Protzenweiherbrücke', 'Stadtbahn', 'Haushalt').",
"inputSchema": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Keyword to search in ALLRIS net (e.g. 'Brücke', 'Kaufhof', 'Verkehr')."
},
"limit": {
"type": "integer",
"description": "Maximum number of results to return (default: 10).",
"default": 10
}
},
"required": ["query"]
}
},
{
"name": "stadtrat_calendar",
"description": "Fetch upcoming municipal council and committee meetings (Stadtrat, Ausschüsse) in Regensburg.",
"inputSchema": {
"type": "object",
"properties": {
"days": {
"type": "integer",
"description": "Number of days in the future to search (default: 30).",
"default": 30
},
"from_date": {
"type": "string",
"description": "Start date in DD.MM.YYYY format (default: today)."
},
"to_date": {
"type": "string",
"description": "End date in DD.MM.YYYY format."
}
}
}
},
{
"name": "stadtrat_session",
"description": "Fetch detailed meeting agenda items (Tagesordnungspunkte / TOPs) and linked Vorlagen IDs for a meeting ID (SILFDNR).",
"inputSchema": {
"type": "object",
"properties": {
"session_id": {
"type": "string",
"description": "ALLRIS session ID (SILFDNR, e.g. '2665')."
}
},
"required": ["session_id"]
}
},
{
"name": "stadtrat_vorlage",
"description": "Inspect a specific council proposal (Vorlage) by VOLFDNR ID: extracts voting outcomes, responsible department, and verbatim resolution text.",
"inputSchema": {
"type": "object",
"properties": {
"vorlage_id": {
"type": "string",
"description": "ALLRIS proposal ID (VOLFDNR, e.g. '21362')."
},
"convert_attachments": {
"type": "boolean",
"description": "Whether to convert linked PDF attachments to Markdown (default: false).",
"default": False
}
},
"required": ["vorlage_id"]
}
},
{
"name": "stadtrat_compile_topic",
"description": "Compile a multi-document synthesis of a topic across ALLRIS into the Obsidian knowledge-wiki.",
"inputSchema": {
"type": "object",
"properties": {
"topic": {
"type": "string",
"description": "Topic keyword (e.g. 'Protzenweiherbrücke', 'Kaufhof')."
},
"vault": {
"type": "string",
"description": "Path to the Obsidian knowledge-wiki vault.",
"default": "/home/node/.openclaw/wiki/main"
}
},
"required": ["topic"]
}
}
]
def run_mcp_server():
client = AllrisClient()
for line in sys.stdin:
line = line.strip()
if not line:
continue
try:
req = json.loads(line)
except json.JSONDecodeError:
continue
method = req.get("method")
msg_id = req.get("id")
params = req.get("params", {})
if method == "initialize":
resp = {
"jsonrpc": "2.0",
"id": msg_id,
"result": {
"protocolVersion": "2024-11-05",
"serverInfo": {
"name": "regensburg-stadtrat-mcp",
"version": "1.0.0"
},
"capabilities": {
"tools": {}
}
}
}
sys.stdout.write(json.dumps(resp) + "\n")
sys.stdout.flush()
elif method == "notifications/initialized":
pass
elif method == "tools/list":
resp = {
"jsonrpc": "2.0",
"id": msg_id,
"result": {
"tools": TOOLS
}
}
sys.stdout.write(json.dumps(resp) + "\n")
sys.stdout.flush()
elif method == "tools/call":
tool_name = params.get("name")
args = params.get("arguments", {})
try:
if tool_name == "stadtrat_search":
data = client.search(
query=args.get("query", ""),
limit=int(args.get("limit", 10))
)
elif tool_name == "stadtrat_calendar":
data = client.get_calendar(
days=int(args.get("days", 30)),
from_date=args.get("from_date"),
to_date=args.get("to_date")
)
elif tool_name == "stadtrat_session":
data = client.get_session(
silfdnr=str(args.get("session_id", ""))
)
elif tool_name == "stadtrat_vorlage":
data = client.get_vorlage(
volfdnr=str(args.get("vorlage_id", "")),
convert_attachments=bool(args.get("convert_attachments", False))
)
elif tool_name == "stadtrat_compile_topic":
vault_path = args.get("vault", "/home/node/.openclaw/wiki/main")
data = client.compile_topic(
topic=args.get("topic", ""),
vault=vault_path
)
else:
raise ValueError(f"Unknown tool: {tool_name}")
text_result = json.dumps(data, indent=2, ensure_ascii=False)
resp = {
"jsonrpc": "2.0",
"id": msg_id,
"result": {
"content": [{"type": "text", "text": text_result}]
}
}
except Exception as e:
resp = {
"jsonrpc": "2.0",
"id": msg_id,
"result": {
"isError": True,
"content": [{"type": "text", "text": f"Error executing {tool_name}: {str(e)}\n{traceback.format_exc()}"}]
}
}
sys.stdout.write(json.dumps(resp) + "\n")
sys.stdout.flush()
else:
if msg_id is not None:
resp = {
"jsonrpc": "2.0",
"id": msg_id,
"error": {"code": -32601, "message": f"Method not found: {method}"}
}
sys.stdout.write(json.dumps(resp) + "\n")
sys.stdout.flush()
if __name__ == "__main__":
run_mcp_server()
skills/shopping/scripts/universal_product_extractor.py
Executable
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env python3
# /// script
# dependencies = [
# "beautifulsoup4>=4.12.0",
# "requests>=2.31.0",
# ]
# ///
"""
Universal E-Commerce Product Extractor (Tier 2 JSON-LD + OpenGraph + Microdata)
Extracts structured product data (title, brand, price, currency, availability, sku)
from arbitrary online stores in < 200ms using Schema.org metadata, bypassing heavy browser
overhead and bot detection. Falls back to Chromium CDP if JS rendering is strictly required.
"""
import os
import sys
import json
import re
import urllib.request
import urllib.parse
from typing import Dict, Any, Optional, List
DEFAULT_HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7",
}
class UniversalProductExtractor:
def __init__(self, cdp_url: Optional[str] = None):
self.cdp_url = cdp_url or os.environ.get("OPENCLAW_CHROMIUM_CDP", "http://127.0.0.1:9222")
def fetch_html(self, url: str, timeout: int = 15) -> str:
req = urllib.request.Request(url, headers=DEFAULT_HEADERS)
with urllib.request.urlopen(req, timeout=timeout) as response:
encoding = response.headers.get_content_charset() or "utf-8"
return response.read().decode(encoding, errors="replace")
def extract_from_jsonld(self, html: str) -> List[Dict[str, Any]]:
"""Extract all Schema.org Product objects from application/ld+json scripts."""
products = []
pattern = re.compile(r'<script[^>]*type=[\'"]application/ld\+json[\'"][^>]*>(.*?)</script>', re.DOTALL | re.IGNORECASE)
for match in pattern.finditer(html):
raw_json = match.group(1).strip()
if not raw_json:
continue
try:
data = json.loads(raw_json)
except Exception:
continue
items = data if isinstance(data, list) else [data]
for item in items:
if not isinstance(item, dict):
continue
# Support @graph structures
if "@graph" in item and isinstance(item["@graph"], list):
for sub in item["@graph"]:
if isinstance(sub, dict) and sub.get("@type") in ("Product", "IndividualProduct"):
products.append(self._normalize_schema_product(sub))
elif item.get("@type") in ("Product", "IndividualProduct"):
products.append(self._normalize_schema_product(item))
return products
def _normalize_schema_product(self, raw: dict) -> dict:
brand = raw.get("brand")
if isinstance(brand, dict):
brand = brand.get("name", "")
offers = raw.get("offers", {})
if isinstance(offers, list) and offers:
offers = offers[0]
price = None
currency = "EUR"
availability = "Unknown"
if isinstance(offers, dict):
price = offers.get("price") or offers.get("lowPrice")
currency = offers.get("priceCurrency", "EUR")
raw_avail = str(offers.get("availability", ""))
if "InStock" in raw_avail:
availability = "InStock"
elif "OutOfStock" in raw_avail:
availability = "OutOfStock"
elif "PreOrder" in raw_avail:
availability = "PreOrder"
return {
"title": raw.get("name", "").strip(),
"brand": str(brand or "").strip(),
"price": float(price) if price and str(price).replace(".", "", 1).isdigit() else price,
"currency": currency,
"availability": availability,
"sku": str(raw.get("sku") or raw.get("gtin13") or raw.get("gtin") or "").strip(),
"image": raw.get("image") if isinstance(raw.get("image"), str) else (raw.get("image", [None])[0] if isinstance(raw.get("image"), list) else None),
"description": raw.get("description", "").strip()[:500] if raw.get("description") else None,
"extraction_tier": "json-ld"
}
def extract_from_opengraph(self, html: str) -> Dict[str, Any]:
"""Fallback extraction from OpenGraph / Twitter Cards / Meta tags."""
title_m = re.search(r'<meta[^>]*property=[\'"]og:title[\'"][^>]*content=[\'"](.*?)[\'"]', html, re.IGNORECASE)
price_m = re.search(r'<meta[^>]*property=[\'"]product:price:amount[\'"][^>]*content=[\'"](.*?)[\'"]', html, re.IGNORECASE) or \
re.search(r'<meta[^>]*property=[\'"]og:price:amount[\'"][^>]*content=[\'"](.*?)[\'"]', html, re.IGNORECASE)
curr_m = re.search(r'<meta[^>]*property=[\'"]product:price:currency[\'"][^>]*content=[\'"](.*?)[\'"]', html, re.IGNORECASE) or \
re.search(r'<meta[^>]*property=[\'"]og:price:currency[\'"][^>]*content=[\'"](.*?)[\'"]', html, re.IGNORECASE)
img_m = re.search(r'<meta[^>]*property=[\'"]og:image[\'"][^>]*content=[\'"](.*?)[\'"]', html, re.IGNORECASE)
title = title_m.group(1).strip() if title_m else ""
price = price_m.group(1).strip() if price_m else None
currency = curr_m.group(1).strip() if curr_m else "EUR"
image = img_m.group(1).strip() if img_m else None
if title or price:
return {
"title": title,
"price": float(price) if price and price.replace(".", "", 1).isdigit() else price,
"currency": currency,
"image": image,
"extraction_tier": "opengraph"
}
return {}
def extract(self, url: str) -> Dict[str, Any]:
"""Main entry point: fetches URL, extracts JSON-LD, falls back to OpenGraph."""
html = self.fetch_html(url)
jsonld_products = self.extract_from_jsonld(html)
if jsonld_products:
res = jsonld_products[0]
res["source_url"] = url
return res
og_product = self.extract_from_opengraph(html)
if og_product:
og_product["source_url"] = url
return og_product
return {
"error": "No structured product metadata found (JSON-LD or OpenGraph)",
"source_url": url,
"extraction_tier": "failed"
}
def main():
if len(sys.argv) < 2:
print("Usage: universal_product_extractor.py <product_url>")
sys.exit(1)
url = sys.argv[1]
extractor = UniversalProductExtractor()
result = extractor.extract(url)
print(json.dumps(result, indent=2, ensure_ascii=False))
if __name__ == "__main__":
main()