218 lines
8.5 KiB
Python
Executable File
218 lines
8.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Google Drive to Google Photos Migration Verification Tool
|
|
|
|
This script analyzes the migration status of pictures from Google Drive to Google Photos.
|
|
It compares Google Drive folder contents with Pod logs from Kubernetes (and/or Google Photos API)
|
|
to determine exact file counts, total sizes, and completion status per subfolder.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from collections import defaultdict
|
|
|
|
DEFAULT_SECRET_FILE = os.path.join(os.path.dirname(__file__), "rclone.secret.yml")
|
|
DEFAULT_DRIVE_ROOT_ID = "1f43AtWvfb7qd-eGr9nw-8vHraA5k9mVl"
|
|
DEFAULT_PHOTOS_ALBUM = "All pictures of the last years"
|
|
|
|
def extract_rclone_conf(secret_file_path):
|
|
"""Extract rclone.conf from Kubernetes ConfigMap secret YAML file."""
|
|
if not os.path.exists(secret_file_path):
|
|
print(f"Error: Secret file not found at {secret_file_path}")
|
|
sys.exit(1)
|
|
|
|
with open(secret_file_path, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
|
|
match = re.search(r"rclone\.conf:\s*\|\n(.*)", content, re.DOTALL)
|
|
if not match:
|
|
print("Error: Could not find rclone.conf data in secret file.")
|
|
sys.exit(1)
|
|
|
|
conf_data = match.group(1)
|
|
# Remove leading 4-space indentation if present
|
|
cleaned_lines = []
|
|
for line in conf_data.split("\n"):
|
|
if line.startswith(" "):
|
|
cleaned_lines.append(line[4:])
|
|
else:
|
|
cleaned_lines.append(line)
|
|
|
|
tmp_conf = tempfile.NamedTemporaryFile(mode="w", suffix=".conf", delete=False)
|
|
tmp_conf.write("\n".join(cleaned_lines))
|
|
tmp_conf.close()
|
|
return tmp_conf.name
|
|
|
|
def get_k8s_job_logs(namespace="backup", job_name="gdrive-to-gphotos"):
|
|
"""Fetch logs from all pods associated with the Kubernetes job."""
|
|
print("Fetching execution logs from Kubernetes pods...")
|
|
try:
|
|
res = subprocess.run(
|
|
["kubectl", "get", "pods", "-n", namespace, "-l", f"job-name={job_name}", "-o", "jsonpath={.items[*].metadata.name}"],
|
|
capture_output=True, text=True, check=True
|
|
)
|
|
pod_names = res.stdout.strip().split()
|
|
if not pod_names:
|
|
print(f"Warning: No pods found for job '{job_name}' in namespace '{namespace}'.")
|
|
return ""
|
|
|
|
all_logs = []
|
|
for pod in pod_names:
|
|
print(f" Retrieving logs for pod: {pod}")
|
|
log_res = subprocess.run(
|
|
["kubectl", "logs", "-n", namespace, pod],
|
|
capture_output=True, text=True
|
|
)
|
|
all_logs.append(log_res.stdout)
|
|
return "\n".join(all_logs)
|
|
except Exception as e:
|
|
print(f"Warning: Could not fetch k8s logs directly ({e}).")
|
|
return ""
|
|
|
|
def parse_copied_files_from_logs(log_text):
|
|
"""Parse log text for confirmed uploaded files per subfolder."""
|
|
copied_files = defaultdict(set)
|
|
for line in log_text.splitlines():
|
|
if "Copied (new)" in line or "Copied (replaced)" in line:
|
|
m = re.search(r"INFO\s+:\s+(.*?)/(.*?):\s+Copied", line)
|
|
if m:
|
|
folder, filename = m.group(1), m.group(2)
|
|
copied_files[folder].add(filename)
|
|
return copied_files
|
|
|
|
def get_gdrive_subfolders(config_path, root_id):
|
|
"""List subdirectories in the source Google Drive folder."""
|
|
print("Querying subfolders from Google Drive...")
|
|
res = subprocess.run(
|
|
["rclone", "--config", config_path, "lsd", f"--drive-root-folder-id={root_id}", "gdrive:"],
|
|
capture_output=True, text=True
|
|
)
|
|
if res.returncode != 0:
|
|
print(f"Error listing Google Drive folders: {res.stderr}")
|
|
return []
|
|
|
|
subfolders = []
|
|
for line in res.stdout.strip().splitlines():
|
|
if line:
|
|
parts = line.split(maxsplit=4)
|
|
if len(parts) >= 5:
|
|
subfolders.append(parts[4])
|
|
return sorted(subfolders)
|
|
|
|
def get_gdrive_folder_stats(config_path, root_id, subfolder):
|
|
"""Get file count and size for a Google Drive subfolder using rclone size."""
|
|
res = subprocess.run(
|
|
["rclone", "--config", config_path, "size", f"--drive-root-folder-id={root_id}", f"gdrive:{subfolder}"],
|
|
capture_output=True, text=True
|
|
)
|
|
if res.returncode != 0:
|
|
return {"count": 0, "size_str": "Unknown", "bytes": 0}
|
|
|
|
output = res.stdout.strip()
|
|
# Handle formats like "Total objects: 2.563k (2563)" or "Total objects: 524"
|
|
count_match = re.search(r"Total objects:\s*(?:[\d\.\w]+\s*\()?(\d+)\)?", output)
|
|
size_match = re.search(r"Total size:\s*([\d\.]+\s*[KMGT]?i?B)\s*\((.*?)\s*Bytes\)", output)
|
|
|
|
count = int(count_match.group(1)) if count_match else 0
|
|
size_str = size_match.group(1) if size_match else "0 B"
|
|
size_bytes = int(size_match.group(2)) if size_match else 0
|
|
|
|
return {"count": count, "size_str": size_str, "bytes": size_bytes}
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Verify Google Drive to Google Photos migration status.")
|
|
parser.add_argument("--secret-file", default=DEFAULT_SECRET_FILE, help="Path to rclone.secret.yml")
|
|
parser.add_argument("--root-id", default=DEFAULT_DRIVE_ROOT_ID, help="Google Drive root folder ID")
|
|
parser.add_argument("--local-log", help="Path to local pod log file (optional override)")
|
|
parser.add_argument("--csv", help="Path to save CSV report output (optional)")
|
|
args = parser.parse_args()
|
|
|
|
print("==========================================================")
|
|
print(" Google Drive to Google Photos Migration Verification Tool")
|
|
print("==========================================================")
|
|
|
|
# 1. Extract Config
|
|
config_path = extract_rclone_conf(args.secret_file)
|
|
|
|
try:
|
|
# 2. Get Logs
|
|
if args.local_log and os.path.exists(args.local_log):
|
|
print(f"Reading logs from local file: {args.local_log}")
|
|
with open(args.local_log, "r", errors="ignore") as f:
|
|
log_text = f.read()
|
|
else:
|
|
log_text = get_k8s_job_logs()
|
|
|
|
copied_map = parse_copied_files_from_logs(log_text)
|
|
|
|
# 3. Get Google Drive Subfolders
|
|
subfolders = get_gdrive_subfolders(config_path, args.root_id)
|
|
if not subfolders:
|
|
print("No subfolders found or unable to connect to Google Drive.")
|
|
sys.exit(1)
|
|
|
|
print(f"\nAnalyzing {len(subfolders)} subfolders...\n")
|
|
|
|
results = []
|
|
total_drive_files = 0
|
|
total_drive_bytes = 0
|
|
total_copied_files = 0
|
|
|
|
hdr = f"{'Subfolder Name':<35} | {'GDrive Files':<12} | {'GDrive Size':<12} | {'Uploaded':<10} | {'Status':<15}"
|
|
print(hdr)
|
|
print("-" * len(hdr))
|
|
|
|
for folder in subfolders:
|
|
stats = get_gdrive_folder_stats(config_path, args.root_id, folder)
|
|
copied_count = len(copied_map.get(folder, set()))
|
|
|
|
drive_count = stats["count"]
|
|
drive_size = stats["size_str"]
|
|
total_drive_files += drive_count
|
|
total_drive_bytes += stats["bytes"]
|
|
total_copied_files += copied_count
|
|
|
|
if drive_count > 0 and copied_count >= drive_count:
|
|
status = "COMPLETE"
|
|
elif copied_count > 0:
|
|
pct = (copied_count / drive_count * 100) if drive_count > 0 else 0
|
|
status = f"IN PROGRESS ({pct:.1f}%)"
|
|
else:
|
|
status = "QUEUED / WAITING"
|
|
|
|
print(f"{folder:<35} | {drive_count:<12} | {drive_size:<12} | {copied_count:<10} | {status:<15}")
|
|
|
|
results.append({
|
|
"folder": folder,
|
|
"drive_files": drive_count,
|
|
"drive_size": drive_size,
|
|
"uploaded_files": copied_count,
|
|
"status": status
|
|
})
|
|
|
|
print("-" * len(hdr))
|
|
total_gb = total_drive_bytes / (1024 ** 3)
|
|
summary_str = f"TOTAL: {total_drive_files} files ({total_gb:.2f} GiB) | Uploaded: {total_copied_files} files"
|
|
print(f"\n{summary_str}")
|
|
|
|
# Save CSV if requested
|
|
if args.csv:
|
|
import csv
|
|
with open(args.csv, "w", newline="", encoding="utf-8") as csvfile:
|
|
writer = csv.DictWriter(csvfile, fieldnames=["folder", "drive_files", "drive_size", "uploaded_files", "status"])
|
|
writer.writeheader()
|
|
writer.writerows(results)
|
|
print(f"Saved CSV report to: {args.csv}")
|
|
|
|
finally:
|
|
if os.path.exists(config_path):
|
|
os.remove(config_path)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|