#!/usr/bin/env python3 import os import sys import time import shutil import hashlib import subprocess from concurrent.futures import ProcessPoolExecutor, as_completed from PIL import Image, ImageOps BUCKET_PATH = "/var/lib/docker/volumes/sss6crntvrcs54qbhvhy5728_appwrite-uploads/_data/app-69b84df5003bd8957c5c/media_bucket" CACHE_PATH = "/var/lib/docker/volumes/sss6crntvrcs54qbhvhy5728_appwrite-cache/_data/app-69b84df5003bd8957c5c" STAGING_DIR = "/tmp/img_opt_staging" MARIADB_CONTAINER = "appwrite-mariadb-sss6crntvrcs54qbhvhy5728" DB_USER = "sYoHWwl0DGvEyt3s" DB_PASS = "sn9YkZqoLV06CQxAy2qsKi36S7POVShT" DB_NAME = "appwrite" TABLE_NAME = "_1_bucket_1" MAX_DIMENSION = 1920 JPEG_QUALITY = 82 def optimize_single_file(filename): """ Worker function to optimize a single image file. Returns: (status, filename, file_id, orig_size, new_size, new_md5) """ orig_path = os.path.join(BUCKET_PATH, filename) file_id, ext = os.path.splitext(filename) ext_lower = ext.lower() if ext_lower not in ('.jpg', '.jpeg', '.png', '.webp'): return ('SKIPPED_NOT_IMAGE', filename, file_id, 0, 0, '') try: orig_size = os.path.getsize(orig_path) except Exception: return ('ERROR_STAT', filename, file_id, 0, 0, '') if orig_size < 1024: # Tiny icon, less than 1KB return ('SKIPPED_TOO_SMALL', filename, file_id, orig_size, orig_size, '') staging_path = os.path.join(STAGING_DIR, f"{filename}.tmp") try: with Image.open(orig_path) as img: # Transpose orientation according to EXIF try: img = ImageOps.exif_transpose(img) except Exception: pass w, h = img.size needs_resize = max(w, h) > MAX_DIMENSION if needs_resize: if w > h: new_w = MAX_DIMENSION new_h = int(h * (MAX_DIMENSION / w)) else: new_h = MAX_DIMENSION new_w = int(w * (MAX_DIMENSION / h)) img = img.resize((new_w, new_h), Image.Resampling.LANCZOS) # Save optimized if ext_lower in ('.jpg', '.jpeg'): if img.mode in ("RGBA", "P"): img = img.convert("RGB") img.save( staging_path, format="JPEG", quality=JPEG_QUALITY, optimize=True, progressive=True ) elif ext_lower == '.png': img.save( staging_path, format="PNG", optimize=True ) elif ext_lower == '.webp': img.save( staging_path, format="WEBP", quality=JPEG_QUALITY, method=6 ) new_size = os.path.getsize(staging_path) # Only replace if we actually saved space if new_size < orig_size: # Calculate new MD5 hasher = hashlib.md5() with open(staging_path, 'rb') as f: while chunk := f.read(65536): hasher.update(chunk) new_md5 = hasher.hexdigest() # Replace original atomically shutil.move(staging_path, orig_path) os.chmod(orig_path, 0o644) shutil.chown(orig_path, user="root", group="root") return ('OPTIMIZED', filename, file_id, orig_size, new_size, new_md5) else: if os.path.exists(staging_path): os.remove(staging_path) return ('SKIPPED_NO_SAVINGS', filename, file_id, orig_size, orig_size, '') except Exception as e: if os.path.exists(staging_path): try: os.remove(staging_path) except Exception: pass return ('ERROR_PROCESS', filename, file_id, orig_size, orig_size, str(e)) def apply_db_updates(updates): """ Applies updates in batches to MariaDB table _1_bucket_1. updates is a list of (file_id, new_size, new_md5) """ if not updates: return 0 print(f"\n[DB] Aplicando {len(updates)} actualizaciones en la base de datos MariaDB...") batch_size = 500 total_updated = 0 for i in range(0, len(updates), batch_size): chunk = updates[i:i + batch_size] sql_lines = ["START TRANSACTION;"] for file_id, new_size, new_md5 in chunk: sql_lines.append( f"UPDATE {TABLE_NAME} SET sizeOriginal = {new_size}, sizeActual = {new_size}, signature = '{new_md5}' WHERE _uid = '{file_id}';" ) sql_lines.append("COMMIT;") sql_script = "\n".join(sql_lines) tmp_sql_file = f"/tmp/update_batch_{i}.sql" with open(tmp_sql_file, "w", encoding="utf-8") as f: f.write(sql_script) # Pipe sql into docker exec cat_cmd = f"cat {tmp_sql_file} | docker exec -i {MARIADB_CONTAINER} mysql -u {DB_USER} -p{DB_PASS} {DB_NAME}" res = subprocess.run(cat_cmd, shell=True, capture_output=True, text=True) if os.path.exists(tmp_sql_file): os.remove(tmp_sql_file) if res.returncode != 0: print(f" [ERROR] Lote {i}-{i+len(chunk)} falló: {res.stderr}") else: total_updated += len(chunk) print(f" ✓ Lote actualizado: {total_updated} / {len(updates)} registros") return total_updated def main(): print("=" * 60) print("MOTOR DE OPTIMIZACIÓN EN MASA DE ALMACENAMIENTO (APPWRITE)") print("=" * 60) if not os.path.exists(BUCKET_PATH): print(f"[ERROR] No se encontró el bucket en: {BUCKET_PATH}") sys.exit(1) os.makedirs(STAGING_DIR, exist_ok=True) files = sorted(os.listdir(BUCKET_PATH)) total_files = len(files) print(f"[INFO] Archivos encontrados en el almacenamiento: {total_files}") start_time = time.time() workers = min(10, os.cpu_count() or 4) print(f"[INFO] Iniciando procesamiento paralelo con {workers} hilos de CPU...\n") optimized_count = 0 skipped_no_savings = 0 skipped_other = 0 error_count = 0 total_orig_bytes = 0 total_new_bytes = 0 db_updates = [] # Process in parallel with ProcessPoolExecutor(max_workers=workers) as executor: futures = {executor.submit(optimize_single_file, f): f for f in files} done_count = 0 for future in as_completed(futures): done_count += 1 status, filename, file_id, orig_sz, new_sz, new_md5 = future.result() if status == 'OPTIMIZED': optimized_count += 1 total_orig_bytes += orig_sz total_new_bytes += new_sz db_updates.append((file_id, new_sz, new_md5)) elif status == 'SKIPPED_NO_SAVINGS': skipped_no_savings += 1 total_orig_bytes += orig_sz total_new_bytes += orig_sz elif status.startswith('SKIPPED'): skipped_other += 1 total_orig_bytes += orig_sz total_new_bytes += orig_sz else: error_count += 1 total_orig_bytes += orig_sz total_new_bytes += orig_sz if done_count % 500 == 0 or done_count == total_files: elapsed = time.time() - start_time rate = done_count / elapsed if elapsed > 0 else 0 saved_mb = (total_orig_bytes - total_new_bytes) / (1024 * 1024) print(f"[{done_count}/{total_files}] ({done_count*100/total_files:.1f}%) | " f"Optimizadas: {optimized_count} | Ahorro actual: {saved_mb:.1f} MB | {rate:.1f} imgs/seg") # Clean staging dir shutil.rmtree(STAGING_DIR, ignore_errors=True) # Apply database updates if db_updates: apply_db_updates(db_updates) # Clear Appwrite preview cache so new previews reflect optimized images print("\n[CACHE] Limpiando caché de previsualizaciones de Appwrite...") if os.path.exists(CACHE_PATH): try: cache_count = len(os.listdir(CACHE_PATH)) for item in os.listdir(CACHE_PATH): item_path = os.path.join(CACHE_PATH, item) if os.path.isfile(item_path) or os.path.islink(item_path): os.unlink(item_path) elif os.path.isdir(item_path): shutil.rmtree(item_path) print(f" ✓ {cache_count} archivos de caché eliminados exitosamente.") except Exception as e: print(f" [AVISO] No se pudo vaciar la caché por completo: {e}") # Summary elapsed_total = time.time() - start_time total_orig_mb = total_orig_bytes / (1024 * 1024) total_new_mb = total_new_bytes / (1024 * 1024) total_saved_mb = total_orig_mb - total_new_mb pct_savings = (total_saved_mb / total_orig_mb * 100) if total_orig_mb > 0 else 0 print("\n" + "=" * 60) print("RESUMEN DE OPTIMIZACIÓN COMPLETADO") print("=" * 60) print(f"Total archivos escaneados: {total_files}") print(f"Archivos optimizados: {optimized_count}") print(f"Archivos sin ganancia (skip): {skipped_no_savings}") print(f"Otros archivos (videos/etc): {skipped_other}") print(f"Errores / omitidos: {error_count}") print(f"Tiempo total: {elapsed_total:.1f} segundos ({elapsed_total/60:.2f} minutos)") print("-" * 60) print(f"Tamaño original total: {total_orig_mb:.2f} MB ({total_orig_mb/1024:.2f} GB)") print(f"Tamaño final optimizado: {total_new_mb:.2f} MB ({total_new_mb/1024:.2f} GB)") print(f"Ahorro neto de espacio: {total_saved_mb:.2f} MB ({total_saved_mb/1024:.2f} GB)") print(f"Porcentaje de reducción: {pct_savings:.2f}%") print("=" * 60) if __name__ == '__main__': main()