webapp import: checksum-aware sync (skip unchanged, update only changed)

Image import previously full-copied everything and rmtree'd existing target dirs on every run. Replaced the shutil copy/move/copytree/rmtree with deploy.sync_tree(), which shells out to rsync -a --checksum: files whose content already matches the target are skipped, only new or changed files are written, and existing target files not in the source are left untouched (merge, not mirror). move=True uses --remove-source-files (frees the SMB upload dir) and prunes emptied source dirs. Applies to the Deploy import, the _shared redirections, and the root-level items. Big re-imports now only rewrite what actually changed.

webapp/app.py

@@ -113,9 +113,9 @@ def images_import():
             src_items = os.listdir(source)
 
             # Move files from network upload to save disk space; copy from USB.
+            # deploy.sync_tree uses rsync --checksum: unchanged files are
+            # skipped, only new/changed files are written (no full replace).
             use_move = source == config.UPLOAD_DIR or source.startswith(config.UPLOAD_DIR + "/")
-            _transfer = shutil.move if use_move else shutil.copy2
-            _transfer_tree = shutil.move if use_move else shutil.copytree
 
             top_dirs = {d for d in src_items if os.path.isdir(os.path.join(source, d))}
             full_layout = "Deploy" in top_dirs
@@ -134,8 +134,7 @@ def images_import():
                     elif os.path.isdir(src_item) and item in shared_root:
                         prefix_key = target.split("-")[0] + "-"
                         shared_dest = os.path.join(config.SHARED_DIR, f"{prefix_key}{item}")
-                        os.makedirs(shared_dest, exist_ok=True)
-                        deploy._merge_tree(src_item, shared_dest, move=use_move)
+                        deploy.sync_tree(src_item, shared_dest, move=use_move)
                         dst_item = os.path.join(root, item)
                         if os.path.islink(dst_item):
                             os.remove(dst_item)
@@ -143,12 +142,9 @@ def images_import():
                             shutil.rmtree(dst_item)
                         os.symlink(shared_dest, dst_item)
                     elif os.path.isdir(src_item):
-                        dst_item = os.path.join(root, item)
-                        if os.path.exists(dst_item):
-                            shutil.rmtree(dst_item)
-                        _transfer_tree(src_item, dst_item)
+                        deploy.sync_tree(src_item, os.path.join(root, item), move=use_move)
                     else:
-                        _transfer(src_item, os.path.join(root, item))
+                        deploy.sync_tree(src_item, os.path.join(root, item), move=use_move)
             else:
                 deploy.import_deploy(source, dest, target, move=use_move)
 

webapp/services/deploy.py

@@ -7,6 +7,7 @@ re-use the same multi-GB driver tree without doubling disk usage.
 
 import os
 import shutil
+import subprocess
 
 import config
 from services.system import find_usb_mounts
@@ -21,27 +22,43 @@ def _replace_with_symlink(link_path, target_path):
     os.symlink(target_path, link_path)
 
 
-def _merge_tree(src, dst, move=False):
-    """Recursively merge src tree into dst, overwriting existing files.
+def sync_tree(src, dst, move=False, checksum=True):
+    """Copy src -> dst with rsync, SKIPPING files whose content already
+    matches and transferring only new or changed files.
 
-    When move=True, files are moved instead of copied (saves disk space
-    on imports from the local upload-dir).
-    """
-    _transfer = shutil.move if move else shutil.copy2
-    _transfer_tree = shutil.move if move else shutil.copytree
-    for item in os.listdir(src):
-        s = os.path.join(src, item)
-        d = os.path.join(dst, item)
-        if os.path.isdir(s):
-            if os.path.isdir(d):
-                _merge_tree(s, d, move=move)
-            else:
-                if os.path.exists(d):
-                    os.remove(d)
-                _transfer_tree(s, d)
-        else:
-            os.makedirs(os.path.dirname(d), exist_ok=True)
-            _transfer(s, d)
+    src may be a file or a directory (its contents merge into dst). With
+    checksum=True rsync compares by hash (--checksum), so a re-import only
+    rewrites files that actually differ instead of replacing everything;
+    set checksum=False for the faster size+mtime comparison. move=True
+    removes source files after a successful transfer (frees the upload dir).
+    Existing dst files not present in src are left untouched (merge, not
+    mirror - no --delete)."""
+    flags = ["-a"]                        # recurse, preserve perms/times/symlinks
+    if checksum:
+        flags.append("--checksum")        # compare by content hash, not size+mtime
+    if move:
+        flags.append("--remove-source-files")
+    if os.path.isdir(src):
+        os.makedirs(dst, exist_ok=True)
+        src_arg = src.rstrip("/") + "/"   # trailing slash = merge CONTENTS into dst
+    else:
+        parent = os.path.dirname(dst)
+        if parent:
+            os.makedirs(parent, exist_ok=True)
+        src_arg = src
+    subprocess.run(["rsync", *flags, src_arg, dst], check=True)
+    if move and os.path.isdir(src):
+        # --remove-source-files empties files but leaves the dir skeleton; prune it
+        for root, _dirs, _files in os.walk(src, topdown=False):
+            try:
+                os.rmdir(root)
+            except OSError:
+                pass
+
+
+def _merge_tree(src, dst, move=False):
+    """Backwards-compatible alias: checksum-aware merge of src into dst."""
+    sync_tree(src, dst, move=move)
 
 
 def import_deploy(src_deploy, dst_deploy, target="", move=False):