compression added

generate_manifest.py

@@ -1,5 +1,7 @@
 import os
 import json
+import gzip
+import shutil
 import hashlib
 import argparse
 
@@ -9,6 +11,13 @@ import argparse
 
 MANIFEST_FILENAME = "manifest.json"
 
+# Suffix used for the precompressed copy of each file, stored alongside the
+# original (e.g. "texture.dds" -> "texture.dds.gz").
+COMPRESSED_SUFFIX = ".gz"
+
+# gzip compression level, 1 (fastest, larger) to 9 (slowest, smaller).
+COMPRESSION_LEVEL = 6
+
 # Files the updater must never touch (launcher itself, local settings, etc.)
 IGNORE_FILES = {
     MANIFEST_FILENAME,
@@ -34,8 +43,13 @@ IGNORE_EXTENSIONS = {
 # ============================================================
 
 
+def get_md5_bytes(data):
+    """Computes the MD5 hash of an in-memory bytes object."""
+    return hashlib.md5(data).hexdigest()
+
+
 def get_md5(filepath):
-    """Computes the MD5 hash of a file."""
+    """Computes the MD5 hash of a file on disk."""
     hash_md5 = hashlib.md5()
     try:
         with open(filepath, "rb") as f:
@@ -46,6 +60,41 @@ def get_md5(filepath):
         return None
 
 
+def compress_file(filepath, compressed_path):
+    """
+    Compresses filepath into compressed_path with gzip. Every file is
+    compressed unconditionally - if the result isn't actually smaller
+    (common for already-compressed formats like .jpg/.mp3/.zip), the
+    compressed copy is discarded and the manifest entry omits "compressed",
+    so the server/updater fall back to serving the original file untouched.
+
+    Uses mtime=0 and a GzipFile with no filename so the compressed output
+    is byte-identical across runs when the input hasn't changed - otherwise
+    gzip embeds the source mtime in its header and compressed_hash would
+    change on every manifest regeneration even with no real content change.
+
+    Returns (compressed_size, compressed_hash) or (None, None) if the
+    compressed copy wasn't kept.
+    """
+    original_size = os.path.getsize(filepath)
+
+    with open(filepath, "rb") as f_in:
+        data = f_in.read()
+
+    with open(compressed_path, "wb") as raw_out:
+        with gzip.GzipFile(fileobj=raw_out, mode="wb", compresslevel=COMPRESSION_LEVEL, mtime=0) as f_out:
+            f_out.write(data)
+
+    compressed_size = os.path.getsize(compressed_path)
+
+    if compressed_size >= original_size:
+        os.remove(compressed_path)
+        return None, None
+
+    compressed_hash = get_md5(compressed_path)
+    return compressed_size, compressed_hash
+
+
 def generate(client_dir):
     if not os.path.exists(client_dir) or not os.path.isdir(client_dir):
         print(f"Error: directory '{client_dir}' not found!")
@@ -56,6 +105,9 @@ def generate(client_dir):
 
     print(f"Scanning client folder: {client_dir}")
 
+    total_original = 0
+    total_saved = 0
+
     for root, dirs, files in os.walk(client_dir):
         # Filter out ignored folders (case-insensitive)
         dirs[:] = [d for d in dirs if d.lower() not in IGNORE_FOLDERS]
@@ -65,24 +117,48 @@ def generate(client_dir):
                 continue
             if any(file.lower().endswith(ext) for ext in IGNORE_EXTENSIONS):
                 continue
+            # Skip stray leftover compressed copies from a previous run so
+            # we never treat "texture.dds.gz" as a distributable file itself.
+            if file.endswith(COMPRESSED_SUFFIX):
+                continue
 
             filepath = os.path.join(root, file)
 
             # Path relative to the client folder (this is what the updater expects)
             rel_path = os.path.relpath(filepath, client_dir)
-
-            # Convert Windows-style separators to web-style
             rel_path_web = rel_path.replace("\\", "/")
 
             file_hash = get_md5(filepath)
-            if file_hash:
-                manifest[rel_path_web] = file_hash
+            if not file_hash:
+                continue
+
+            entry = {
+                "hash": file_hash,
+                "size": os.path.getsize(filepath),
+            }
+
+            compressed_path = filepath + COMPRESSED_SUFFIX
+            compressed_size, compressed_hash = compress_file(filepath, compressed_path)
+
+            if compressed_hash:
+                entry["compressed"] = True
+                entry["compressed_size"] = compressed_size
+                entry["compressed_hash"] = compressed_hash
+                total_saved += entry["size"] - compressed_size
+            else:
+                entry["compressed"] = False
+
+            total_original += entry["size"]
+            manifest[rel_path_web] = entry
 
     try:
         with open(manifest_path, "w", encoding="utf-8") as f:
             json.dump(manifest, f, indent=4)
         print(f"Done! Manifest saved to: {manifest_path}")
         print(f"Total files in manifest: {len(manifest)}")
+        if total_original:
+            pct = (total_saved / total_original) * 100
+            print(f"Compression saved {total_saved:,} bytes of {total_original:,} ({pct:.1f}%)")
     except IOError as e:
         print(f"Error saving manifest: {e}")