diff --git a/webapp/app.py b/webapp/app.py index ad010ea..dbf5477 100644 --- a/webapp/app.py +++ b/webapp/app.py @@ -113,9 +113,9 @@ def images_import(): src_items = os.listdir(source) # Move files from network upload to save disk space; copy from USB. + # deploy.sync_tree uses rsync --checksum: unchanged files are + # skipped, only new/changed files are written (no full replace). use_move = source == config.UPLOAD_DIR or source.startswith(config.UPLOAD_DIR + "/") - _transfer = shutil.move if use_move else shutil.copy2 - _transfer_tree = shutil.move if use_move else shutil.copytree top_dirs = {d for d in src_items if os.path.isdir(os.path.join(source, d))} full_layout = "Deploy" in top_dirs @@ -134,8 +134,7 @@ def images_import(): elif os.path.isdir(src_item) and item in shared_root: prefix_key = target.split("-")[0] + "-" shared_dest = os.path.join(config.SHARED_DIR, f"{prefix_key}{item}") - os.makedirs(shared_dest, exist_ok=True) - deploy._merge_tree(src_item, shared_dest, move=use_move) + deploy.sync_tree(src_item, shared_dest, move=use_move) dst_item = os.path.join(root, item) if os.path.islink(dst_item): os.remove(dst_item) @@ -143,12 +142,9 @@ def images_import(): shutil.rmtree(dst_item) os.symlink(shared_dest, dst_item) elif os.path.isdir(src_item): - dst_item = os.path.join(root, item) - if os.path.exists(dst_item): - shutil.rmtree(dst_item) - _transfer_tree(src_item, dst_item) + deploy.sync_tree(src_item, os.path.join(root, item), move=use_move) else: - _transfer(src_item, os.path.join(root, item)) + deploy.sync_tree(src_item, os.path.join(root, item), move=use_move) else: deploy.import_deploy(source, dest, target, move=use_move) diff --git a/webapp/services/deploy.py b/webapp/services/deploy.py index 629758f..d7166de 100644 --- a/webapp/services/deploy.py +++ b/webapp/services/deploy.py @@ -7,6 +7,7 @@ re-use the same multi-GB driver tree without doubling disk usage. import os import shutil +import subprocess import config from services.system import find_usb_mounts @@ -21,27 +22,43 @@ def _replace_with_symlink(link_path, target_path): os.symlink(target_path, link_path) -def _merge_tree(src, dst, move=False): - """Recursively merge src tree into dst, overwriting existing files. +def sync_tree(src, dst, move=False, checksum=True): + """Copy src -> dst with rsync, SKIPPING files whose content already + matches and transferring only new or changed files. - When move=True, files are moved instead of copied (saves disk space - on imports from the local upload-dir). - """ - _transfer = shutil.move if move else shutil.copy2 - _transfer_tree = shutil.move if move else shutil.copytree - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isdir(s): - if os.path.isdir(d): - _merge_tree(s, d, move=move) - else: - if os.path.exists(d): - os.remove(d) - _transfer_tree(s, d) - else: - os.makedirs(os.path.dirname(d), exist_ok=True) - _transfer(s, d) + src may be a file or a directory (its contents merge into dst). With + checksum=True rsync compares by hash (--checksum), so a re-import only + rewrites files that actually differ instead of replacing everything; + set checksum=False for the faster size+mtime comparison. move=True + removes source files after a successful transfer (frees the upload dir). + Existing dst files not present in src are left untouched (merge, not + mirror - no --delete).""" + flags = ["-a"] # recurse, preserve perms/times/symlinks + if checksum: + flags.append("--checksum") # compare by content hash, not size+mtime + if move: + flags.append("--remove-source-files") + if os.path.isdir(src): + os.makedirs(dst, exist_ok=True) + src_arg = src.rstrip("/") + "/" # trailing slash = merge CONTENTS into dst + else: + parent = os.path.dirname(dst) + if parent: + os.makedirs(parent, exist_ok=True) + src_arg = src + subprocess.run(["rsync", *flags, src_arg, dst], check=True) + if move and os.path.isdir(src): + # --remove-source-files empties files but leaves the dir skeleton; prune it + for root, _dirs, _files in os.walk(src, topdown=False): + try: + os.rmdir(root) + except OSError: + pass + + +def _merge_tree(src, dst, move=False): + """Backwards-compatible alias: checksum-aware merge of src into dst.""" + sync_tree(src, dst, move=move) def import_deploy(src_deploy, dst_deploy, target="", move=False):