273 lines
9.8 KiB
Python
273 lines
9.8 KiB
Python
#!/usr/bin/env python3
|
|
import os
|
|
import sys
|
|
import time
|
|
import shutil
|
|
import hashlib
|
|
import subprocess
|
|
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
from PIL import Image, ImageOps
|
|
|
|
BUCKET_PATH = "/var/lib/docker/volumes/sss6crntvrcs54qbhvhy5728_appwrite-uploads/_data/app-69b84df5003bd8957c5c/media_bucket"
|
|
CACHE_PATH = "/var/lib/docker/volumes/sss6crntvrcs54qbhvhy5728_appwrite-cache/_data/app-69b84df5003bd8957c5c"
|
|
STAGING_DIR = "/tmp/img_opt_staging"
|
|
MARIADB_CONTAINER = "appwrite-mariadb-sss6crntvrcs54qbhvhy5728"
|
|
DB_USER = "sYoHWwl0DGvEyt3s"
|
|
DB_PASS = "sn9YkZqoLV06CQxAy2qsKi36S7POVShT"
|
|
DB_NAME = "appwrite"
|
|
TABLE_NAME = "_1_bucket_1"
|
|
|
|
MAX_DIMENSION = 1920
|
|
JPEG_QUALITY = 82
|
|
|
|
def optimize_single_file(filename):
|
|
"""
|
|
Worker function to optimize a single image file.
|
|
Returns: (status, filename, file_id, orig_size, new_size, new_md5)
|
|
"""
|
|
orig_path = os.path.join(BUCKET_PATH, filename)
|
|
file_id, ext = os.path.splitext(filename)
|
|
ext_lower = ext.lower()
|
|
|
|
if ext_lower not in ('.jpg', '.jpeg', '.png', '.webp'):
|
|
return ('SKIPPED_NOT_IMAGE', filename, file_id, 0, 0, '')
|
|
|
|
try:
|
|
orig_size = os.path.getsize(orig_path)
|
|
except Exception:
|
|
return ('ERROR_STAT', filename, file_id, 0, 0, '')
|
|
|
|
if orig_size < 1024: # Tiny icon, less than 1KB
|
|
return ('SKIPPED_TOO_SMALL', filename, file_id, orig_size, orig_size, '')
|
|
|
|
staging_path = os.path.join(STAGING_DIR, f"{filename}.tmp")
|
|
|
|
try:
|
|
with Image.open(orig_path) as img:
|
|
# Transpose orientation according to EXIF
|
|
try:
|
|
img = ImageOps.exif_transpose(img)
|
|
except Exception:
|
|
pass
|
|
|
|
w, h = img.size
|
|
needs_resize = max(w, h) > MAX_DIMENSION
|
|
|
|
if needs_resize:
|
|
if w > h:
|
|
new_w = MAX_DIMENSION
|
|
new_h = int(h * (MAX_DIMENSION / w))
|
|
else:
|
|
new_h = MAX_DIMENSION
|
|
new_w = int(w * (MAX_DIMENSION / h))
|
|
img = img.resize((new_w, new_h), Image.Resampling.LANCZOS)
|
|
|
|
# Save optimized
|
|
if ext_lower in ('.jpg', '.jpeg'):
|
|
if img.mode in ("RGBA", "P"):
|
|
img = img.convert("RGB")
|
|
img.save(
|
|
staging_path,
|
|
format="JPEG",
|
|
quality=JPEG_QUALITY,
|
|
optimize=True,
|
|
progressive=True
|
|
)
|
|
elif ext_lower == '.png':
|
|
img.save(
|
|
staging_path,
|
|
format="PNG",
|
|
optimize=True
|
|
)
|
|
elif ext_lower == '.webp':
|
|
img.save(
|
|
staging_path,
|
|
format="WEBP",
|
|
quality=JPEG_QUALITY,
|
|
method=6
|
|
)
|
|
|
|
new_size = os.path.getsize(staging_path)
|
|
|
|
# Only replace if we actually saved space
|
|
if new_size < orig_size:
|
|
# Calculate new MD5
|
|
hasher = hashlib.md5()
|
|
with open(staging_path, 'rb') as f:
|
|
while chunk := f.read(65536):
|
|
hasher.update(chunk)
|
|
new_md5 = hasher.hexdigest()
|
|
|
|
# Replace original atomically
|
|
shutil.move(staging_path, orig_path)
|
|
os.chmod(orig_path, 0o644)
|
|
shutil.chown(orig_path, user="root", group="root")
|
|
|
|
return ('OPTIMIZED', filename, file_id, orig_size, new_size, new_md5)
|
|
else:
|
|
if os.path.exists(staging_path):
|
|
os.remove(staging_path)
|
|
return ('SKIPPED_NO_SAVINGS', filename, file_id, orig_size, orig_size, '')
|
|
|
|
except Exception as e:
|
|
if os.path.exists(staging_path):
|
|
try:
|
|
os.remove(staging_path)
|
|
except Exception:
|
|
pass
|
|
return ('ERROR_PROCESS', filename, file_id, orig_size, orig_size, str(e))
|
|
|
|
|
|
def apply_db_updates(updates):
|
|
"""
|
|
Applies updates in batches to MariaDB table _1_bucket_1.
|
|
updates is a list of (file_id, new_size, new_md5)
|
|
"""
|
|
if not updates:
|
|
return 0
|
|
|
|
print(f"\n[DB] Aplicando {len(updates)} actualizaciones en la base de datos MariaDB...")
|
|
batch_size = 500
|
|
total_updated = 0
|
|
|
|
for i in range(0, len(updates), batch_size):
|
|
chunk = updates[i:i + batch_size]
|
|
sql_lines = ["START TRANSACTION;"]
|
|
for file_id, new_size, new_md5 in chunk:
|
|
sql_lines.append(
|
|
f"UPDATE {TABLE_NAME} SET sizeOriginal = {new_size}, sizeActual = {new_size}, signature = '{new_md5}' WHERE _uid = '{file_id}';"
|
|
)
|
|
sql_lines.append("COMMIT;")
|
|
|
|
sql_script = "\n".join(sql_lines)
|
|
tmp_sql_file = f"/tmp/update_batch_{i}.sql"
|
|
with open(tmp_sql_file, "w", encoding="utf-8") as f:
|
|
f.write(sql_script)
|
|
|
|
# Pipe sql into docker exec
|
|
cat_cmd = f"cat {tmp_sql_file} | docker exec -i {MARIADB_CONTAINER} mysql -u {DB_USER} -p{DB_PASS} {DB_NAME}"
|
|
res = subprocess.run(cat_cmd, shell=True, capture_output=True, text=True)
|
|
|
|
if os.path.exists(tmp_sql_file):
|
|
os.remove(tmp_sql_file)
|
|
|
|
if res.returncode != 0:
|
|
print(f" [ERROR] Lote {i}-{i+len(chunk)} falló: {res.stderr}")
|
|
else:
|
|
total_updated += len(chunk)
|
|
print(f" ✓ Lote actualizado: {total_updated} / {len(updates)} registros")
|
|
|
|
return total_updated
|
|
|
|
|
|
def main():
|
|
print("=" * 60)
|
|
print("MOTOR DE OPTIMIZACIÓN EN MASA DE ALMACENAMIENTO (APPWRITE)")
|
|
print("=" * 60)
|
|
|
|
if not os.path.exists(BUCKET_PATH):
|
|
print(f"[ERROR] No se encontró el bucket en: {BUCKET_PATH}")
|
|
sys.exit(1)
|
|
|
|
os.makedirs(STAGING_DIR, exist_ok=True)
|
|
|
|
files = sorted(os.listdir(BUCKET_PATH))
|
|
total_files = len(files)
|
|
print(f"[INFO] Archivos encontrados en el almacenamiento: {total_files}")
|
|
|
|
start_time = time.time()
|
|
workers = min(10, os.cpu_count() or 4)
|
|
print(f"[INFO] Iniciando procesamiento paralelo con {workers} hilos de CPU...\n")
|
|
|
|
optimized_count = 0
|
|
skipped_no_savings = 0
|
|
skipped_other = 0
|
|
error_count = 0
|
|
|
|
total_orig_bytes = 0
|
|
total_new_bytes = 0
|
|
db_updates = []
|
|
|
|
# Process in parallel
|
|
with ProcessPoolExecutor(max_workers=workers) as executor:
|
|
futures = {executor.submit(optimize_single_file, f): f for f in files}
|
|
|
|
done_count = 0
|
|
for future in as_completed(futures):
|
|
done_count += 1
|
|
status, filename, file_id, orig_sz, new_sz, new_md5 = future.result()
|
|
|
|
if status == 'OPTIMIZED':
|
|
optimized_count += 1
|
|
total_orig_bytes += orig_sz
|
|
total_new_bytes += new_sz
|
|
db_updates.append((file_id, new_sz, new_md5))
|
|
elif status == 'SKIPPED_NO_SAVINGS':
|
|
skipped_no_savings += 1
|
|
total_orig_bytes += orig_sz
|
|
total_new_bytes += orig_sz
|
|
elif status.startswith('SKIPPED'):
|
|
skipped_other += 1
|
|
total_orig_bytes += orig_sz
|
|
total_new_bytes += orig_sz
|
|
else:
|
|
error_count += 1
|
|
total_orig_bytes += orig_sz
|
|
total_new_bytes += orig_sz
|
|
|
|
if done_count % 500 == 0 or done_count == total_files:
|
|
elapsed = time.time() - start_time
|
|
rate = done_count / elapsed if elapsed > 0 else 0
|
|
saved_mb = (total_orig_bytes - total_new_bytes) / (1024 * 1024)
|
|
print(f"[{done_count}/{total_files}] ({done_count*100/total_files:.1f}%) | "
|
|
f"Optimizadas: {optimized_count} | Ahorro actual: {saved_mb:.1f} MB | {rate:.1f} imgs/seg")
|
|
|
|
# Clean staging dir
|
|
shutil.rmtree(STAGING_DIR, ignore_errors=True)
|
|
|
|
# Apply database updates
|
|
if db_updates:
|
|
apply_db_updates(db_updates)
|
|
|
|
# Clear Appwrite preview cache so new previews reflect optimized images
|
|
print("\n[CACHE] Limpiando caché de previsualizaciones de Appwrite...")
|
|
if os.path.exists(CACHE_PATH):
|
|
try:
|
|
cache_count = len(os.listdir(CACHE_PATH))
|
|
for item in os.listdir(CACHE_PATH):
|
|
item_path = os.path.join(CACHE_PATH, item)
|
|
if os.path.isfile(item_path) or os.path.islink(item_path):
|
|
os.unlink(item_path)
|
|
elif os.path.isdir(item_path):
|
|
shutil.rmtree(item_path)
|
|
print(f" ✓ {cache_count} archivos de caché eliminados exitosamente.")
|
|
except Exception as e:
|
|
print(f" [AVISO] No se pudo vaciar la caché por completo: {e}")
|
|
|
|
# Summary
|
|
elapsed_total = time.time() - start_time
|
|
total_orig_mb = total_orig_bytes / (1024 * 1024)
|
|
total_new_mb = total_new_bytes / (1024 * 1024)
|
|
total_saved_mb = total_orig_mb - total_new_mb
|
|
pct_savings = (total_saved_mb / total_orig_mb * 100) if total_orig_mb > 0 else 0
|
|
|
|
print("\n" + "=" * 60)
|
|
print("RESUMEN DE OPTIMIZACIÓN COMPLETADO")
|
|
print("=" * 60)
|
|
print(f"Total archivos escaneados: {total_files}")
|
|
print(f"Archivos optimizados: {optimized_count}")
|
|
print(f"Archivos sin ganancia (skip): {skipped_no_savings}")
|
|
print(f"Otros archivos (videos/etc): {skipped_other}")
|
|
print(f"Errores / omitidos: {error_count}")
|
|
print(f"Tiempo total: {elapsed_total:.1f} segundos ({elapsed_total/60:.2f} minutos)")
|
|
print("-" * 60)
|
|
print(f"Tamaño original total: {total_orig_mb:.2f} MB ({total_orig_mb/1024:.2f} GB)")
|
|
print(f"Tamaño final optimizado: {total_new_mb:.2f} MB ({total_new_mb/1024:.2f} GB)")
|
|
print(f"Ahorro neto de espacio: {total_saved_mb:.2f} MB ({total_saved_mb/1024:.2f} GB)")
|
|
print(f"Porcentaje de reducción: {pct_savings:.2f}%")
|
|
print("=" * 60)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|