#!/usr/bin/env python3 """ Google Drive to Google Photos Migration Verification Tool This script analyzes the migration status of pictures from Google Drive to Google Photos. It compares Google Drive folder contents with Pod logs from Kubernetes (and/or Google Photos API) to determine exact file counts, total sizes, and completion status per subfolder. """ import argparse import json import os import re import subprocess import sys import tempfile from collections import defaultdict DEFAULT_SECRET_FILE = os.path.join(os.path.dirname(__file__), "rclone.secret.yml") DEFAULT_DRIVE_ROOT_ID = "1f43AtWvfb7qd-eGr9nw-8vHraA5k9mVl" DEFAULT_PHOTOS_ALBUM = "All pictures of the last years" def extract_rclone_conf(secret_file_path): """Extract rclone.conf from Kubernetes ConfigMap secret YAML file.""" if not os.path.exists(secret_file_path): print(f"Error: Secret file not found at {secret_file_path}") sys.exit(1) with open(secret_file_path, "r", encoding="utf-8") as f: content = f.read() match = re.search(r"rclone\.conf:\s*\|\n(.*)", content, re.DOTALL) if not match: print("Error: Could not find rclone.conf data in secret file.") sys.exit(1) conf_data = match.group(1) # Remove leading 4-space indentation if present cleaned_lines = [] for line in conf_data.split("\n"): if line.startswith(" "): cleaned_lines.append(line[4:]) else: cleaned_lines.append(line) tmp_conf = tempfile.NamedTemporaryFile(mode="w", suffix=".conf", delete=False) tmp_conf.write("\n".join(cleaned_lines)) tmp_conf.close() return tmp_conf.name def get_k8s_job_logs(namespace="backup", job_name="gdrive-to-gphotos"): """Fetch logs from all pods associated with the Kubernetes job.""" print("Fetching execution logs from Kubernetes pods...") try: res = subprocess.run( ["kubectl", "get", "pods", "-n", namespace, "-l", f"job-name={job_name}", "-o", "jsonpath={.items[*].metadata.name}"], capture_output=True, text=True, check=True ) pod_names = res.stdout.strip().split() if not pod_names: print(f"Warning: No pods found for job '{job_name}' in namespace '{namespace}'.") return "" all_logs = [] for pod in pod_names: print(f" Retrieving logs for pod: {pod}") log_res = subprocess.run( ["kubectl", "logs", "-n", namespace, pod], capture_output=True, text=True ) all_logs.append(log_res.stdout) return "\n".join(all_logs) except Exception as e: print(f"Warning: Could not fetch k8s logs directly ({e}).") return "" def parse_copied_files_from_logs(log_text): """Parse log text for confirmed uploaded files per subfolder.""" copied_files = defaultdict(set) for line in log_text.splitlines(): if "Copied (new)" in line or "Copied (replaced)" in line: m = re.search(r"INFO\s+:\s+(.*?)/(.*?):\s+Copied", line) if m: folder, filename = m.group(1), m.group(2) copied_files[folder].add(filename) return copied_files def get_gdrive_subfolders(config_path, root_id): """List subdirectories in the source Google Drive folder.""" print("Querying subfolders from Google Drive...") res = subprocess.run( ["rclone", "--config", config_path, "lsd", f"--drive-root-folder-id={root_id}", "gdrive:"], capture_output=True, text=True ) if res.returncode != 0: print(f"Error listing Google Drive folders: {res.stderr}") return [] subfolders = [] for line in res.stdout.strip().splitlines(): if line: parts = line.split(maxsplit=4) if len(parts) >= 5: subfolders.append(parts[4]) return sorted(subfolders) def get_gdrive_folder_stats(config_path, root_id, subfolder): """Get file count and size for a Google Drive subfolder using rclone size.""" res = subprocess.run( ["rclone", "--config", config_path, "size", f"--drive-root-folder-id={root_id}", f"gdrive:{subfolder}"], capture_output=True, text=True ) if res.returncode != 0: return {"count": 0, "size_str": "Unknown", "bytes": 0} output = res.stdout.strip() # Handle formats like "Total objects: 2.563k (2563)" or "Total objects: 524" count_match = re.search(r"Total objects:\s*(?:[\d\.\w]+\s*\()?(\d+)\)?", output) size_match = re.search(r"Total size:\s*([\d\.]+\s*[KMGT]?i?B)\s*\((.*?)\s*Bytes\)", output) count = int(count_match.group(1)) if count_match else 0 size_str = size_match.group(1) if size_match else "0 B" size_bytes = int(size_match.group(2)) if size_match else 0 return {"count": count, "size_str": size_str, "bytes": size_bytes} def main(): parser = argparse.ArgumentParser(description="Verify Google Drive to Google Photos migration status.") parser.add_argument("--secret-file", default=DEFAULT_SECRET_FILE, help="Path to rclone.secret.yml") parser.add_argument("--root-id", default=DEFAULT_DRIVE_ROOT_ID, help="Google Drive root folder ID") parser.add_argument("--local-log", help="Path to local pod log file (optional override)") parser.add_argument("--csv", help="Path to save CSV report output (optional)") args = parser.parse_args() print("==========================================================") print(" Google Drive to Google Photos Migration Verification Tool") print("==========================================================") # 1. Extract Config config_path = extract_rclone_conf(args.secret_file) try: # 2. Get Logs if args.local_log and os.path.exists(args.local_log): print(f"Reading logs from local file: {args.local_log}") with open(args.local_log, "r", errors="ignore") as f: log_text = f.read() else: log_text = get_k8s_job_logs() copied_map = parse_copied_files_from_logs(log_text) # 3. Get Google Drive Subfolders subfolders = get_gdrive_subfolders(config_path, args.root_id) if not subfolders: print("No subfolders found or unable to connect to Google Drive.") sys.exit(1) print(f"\nAnalyzing {len(subfolders)} subfolders...\n") results = [] total_drive_files = 0 total_drive_bytes = 0 total_copied_files = 0 hdr = f"{'Subfolder Name':<35} | {'GDrive Files':<12} | {'GDrive Size':<12} | {'Uploaded':<10} | {'Status':<15}" print(hdr) print("-" * len(hdr)) for folder in subfolders: stats = get_gdrive_folder_stats(config_path, args.root_id, folder) copied_count = len(copied_map.get(folder, set())) drive_count = stats["count"] drive_size = stats["size_str"] total_drive_files += drive_count total_drive_bytes += stats["bytes"] total_copied_files += copied_count if drive_count > 0 and copied_count >= drive_count: status = "COMPLETE" elif copied_count > 0: pct = (copied_count / drive_count * 100) if drive_count > 0 else 0 status = f"IN PROGRESS ({pct:.1f}%)" else: status = "QUEUED / WAITING" print(f"{folder:<35} | {drive_count:<12} | {drive_size:<12} | {copied_count:<10} | {status:<15}") results.append({ "folder": folder, "drive_files": drive_count, "drive_size": drive_size, "uploaded_files": copied_count, "status": status }) print("-" * len(hdr)) total_gb = total_drive_bytes / (1024 ** 3) summary_str = f"TOTAL: {total_drive_files} files ({total_gb:.2f} GiB) | Uploaded: {total_copied_files} files" print(f"\n{summary_str}") # Save CSV if requested if args.csv: import csv with open(args.csv, "w", newline="", encoding="utf-8") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=["folder", "drive_files", "drive_size", "uploaded_files", "status"]) writer.writeheader() writer.writerows(results) print(f"Saved CSV report to: {args.csv}") finally: if os.path.exists(config_path): os.remove(config_path) if __name__ == "__main__": main()