feat(skills): add markitdown document converter and regensburg-stadtrat civic intelligence skills
This commit is contained in:
Moritz Graf
@@ -0,0 +1,45 @@
|
||||
---
|
||||
name: markitdown
|
||||
description: Universal document and media converter powered by Microsoft MarkItDown. Converts PDF, Word (docx), Excel (xlsx), PowerPoint (pptx), HTML, CSV, audio, and images to clean, structured Markdown. Use whenever you need to read, inspect, or extract text from PDFs, spreadsheets, office documents, or web files.
|
||||
---
|
||||
|
||||
# 📄 Microsoft MarkItDown Converter
|
||||
|
||||
Convert arbitrary documents and files into clean, structured Markdown for analysis, summarization, or archiving.
|
||||
|
||||
## 🚀 Usage
|
||||
|
||||
Use `scripts/convert_to_markdown.py`. The script runs automatically via `uv` with all document dependencies bundled.
|
||||
|
||||
### 1. Convert a Remote URL (PDF, Word, etc.)
|
||||
Download and convert in one step:
|
||||
```bash
|
||||
python3 scripts/convert_to_markdown.py --url "https://example.com/document.pdf"
|
||||
```
|
||||
|
||||
To save directly to a file with frontmatter metadata:
|
||||
```bash
|
||||
python3 scripts/convert_to_markdown.py --url "https://example.com/document.pdf" --frontmatter -o /path/to/output.md
|
||||
```
|
||||
|
||||
### 2. Convert a Local File
|
||||
Convert an existing local document or an uploaded attachment:
|
||||
```bash
|
||||
python3 scripts/convert_to_markdown.py /path/to/invoice.pdf
|
||||
python3 scripts/convert_to_markdown.py /path/to/spreadsheet.xlsx
|
||||
python3 scripts/convert_to_markdown.py /path/to/presentation.pptx
|
||||
```
|
||||
|
||||
## 🛠️ Supported Formats
|
||||
* **PDF (`.pdf`)**: Extracts text, columns, formatting, and tables.
|
||||
* **Microsoft Word (`.docx`)**: Preserves headers, bold/italic, lists, and tables.
|
||||
* **Microsoft Excel (`.xlsx`)**: Converts sheets and tables into Markdown pipe tables.
|
||||
* **Microsoft PowerPoint (`.pptx`)**: Extracts slide text, outlines, and speaker notes.
|
||||
* **HTML / Web pages**: Cleans boilerplate and outputs article markdown.
|
||||
* **CSV / JSON / XML**: Formats data into readable markdown tables or code blocks.
|
||||
* **Audio (`.wav`, `.mp3`)**: Transcribes speech (when speech models are active).
|
||||
* **Images**: Extracts metadata, EXIF details, and embedded text descriptions.
|
||||
|
||||
## 💡 Best Practices for OpenClaw
|
||||
- When the user sends a PDF or document link in Telegram, call `convert_to_markdown.py --url <URL>` to read its contents before answering.
|
||||
- For large documents (>50 pages), save the converted Markdown to a temporary file and search/filter for relevant sections.
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
# /// script
|
||||
# dependencies = [
|
||||
# "markitdown[all]>=0.0.1a4",
|
||||
# "requests>=2.31.0",
|
||||
# ]
|
||||
# ///
|
||||
"""
|
||||
Universal Document to Markdown Converter
|
||||
Powered by Microsoft MarkItDown.
|
||||
|
||||
Supports PDF, DOCX, XLSX, PPTX, HTML, CSV, TXT, JSON, XML, images, and audio.
|
||||
Can take a local file path, a remote URL, or stdin.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
|
||||
def ensure_dependencies_and_run():
|
||||
"""If markitdown is not installed in the current Python env, re-exec via uv run if available."""
|
||||
try:
|
||||
import markitdown # noqa: F401
|
||||
import requests # noqa: F401
|
||||
except ImportError:
|
||||
uv_path = shutil.which("uv")
|
||||
if uv_path and os.environ.get("_MARKITDOWN_REEXEC") != "1":
|
||||
env = os.environ.copy()
|
||||
env["_MARKITDOWN_REEXEC"] = "1"
|
||||
cmd = [uv_path, "run", os.path.abspath(__file__), *sys.argv[1:]]
|
||||
try:
|
||||
res = subprocess.run(cmd, env=env)
|
||||
sys.exit(res.returncode)
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"Failed to re-execute via uv: {e}\n")
|
||||
sys.stderr.write("Error: 'markitdown[all]' or 'requests' not found. Please install with:\n")
|
||||
sys.stderr.write(" uv pip install 'markitdown[all]' requests\n")
|
||||
sys.exit(1)
|
||||
|
||||
ensure_dependencies_and_run()
|
||||
|
||||
from markitdown import MarkItDown
|
||||
import requests
|
||||
|
||||
def download_url(url: str, output_dir: str) -> str:
|
||||
"""Download a remote URL to a temporary local file, guessing the filename from URL or headers."""
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
resp = requests.get(url, headers=headers, stream=True, timeout=60)
|
||||
resp.raise_for_status()
|
||||
|
||||
# Determine filename
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
basename = os.path.basename(parsed.path) or "downloaded_file"
|
||||
if not os.path.splitext(basename)[1]:
|
||||
ct = resp.headers.get("Content-Type", "")
|
||||
if "pdf" in ct:
|
||||
basename += ".pdf"
|
||||
elif "word" in ct or "docx" in ct:
|
||||
basename += ".docx"
|
||||
elif "sheet" in ct or "excel" in ct or "xlsx" in ct:
|
||||
basename += ".xlsx"
|
||||
elif "html" in ct:
|
||||
basename += ".html"
|
||||
|
||||
local_path = os.path.join(output_dir, basename)
|
||||
with open(local_path, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=65536):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
return local_path
|
||||
|
||||
def convert_file(path: str) -> str:
|
||||
"""Convert local file to Markdown text using MarkItDown."""
|
||||
md = MarkItDown()
|
||||
result = md.convert(path)
|
||||
return result.text_content
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Convert documents (PDF, Office, HTML, images) to Markdown via Microsoft MarkItDown"
|
||||
)
|
||||
parser.add_argument("file", nargs="?", help="Local path to the document file")
|
||||
parser.add_argument("--url", "-u", help="Remote URL of the document to download and convert")
|
||||
parser.add_argument("--output", "-o", help="Optional output file path (defaults to stdout)")
|
||||
parser.add_argument("--frontmatter", action="store_true", help="Add YAML frontmatter with file metadata")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.file and not args.url:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
temp_dir = None
|
||||
target_file = args.file
|
||||
|
||||
try:
|
||||
if args.url:
|
||||
temp_dir = tempfile.mkdtemp(prefix="markitdown_")
|
||||
target_file = download_url(args.url, temp_dir)
|
||||
|
||||
if not target_file or not os.path.exists(target_file):
|
||||
sys.stderr.write(f"Error: File not found: {target_file}\n")
|
||||
sys.exit(1)
|
||||
|
||||
content = convert_file(target_file)
|
||||
|
||||
if args.frontmatter:
|
||||
meta = [
|
||||
"---",
|
||||
f"source_file: {os.path.basename(target_file)}",
|
||||
f"original_url: {args.url or ''}",
|
||||
f"size_bytes: {os.path.getsize(target_file)}",
|
||||
"converted_by: microsoft-markitdown",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
content = "\n".join(meta) + content
|
||||
|
||||
if args.output:
|
||||
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
print(f"Successfully converted to {args.output}")
|
||||
else:
|
||||
print(content)
|
||||
|
||||
finally:
|
||||
if temp_dir and os.path.exists(temp_dir):
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user