From 0cb6b26c272ae5c97fd1478b38d6907e8fdf1ce7 Mon Sep 17 00:00:00 2001 From: cproudlock Date: Thu, 23 Jul 2026 10:45:58 -0400 Subject: [PATCH] webapp import: checksum-aware sync (skip unchanged, update only changed) Image import previously full-copied everything and rmtree'd existing target dirs on every run. Replaced the shutil copy/move/copytree/rmtree with deploy.sync_tree(), which shells out to rsync -a --checksum: files whose content already matches the target are skipped, only new or changed files are written, and existing target files not in the source are left untouched (merge, not mirror). move=True uses --remove-source-files (frees the SMB upload dir) and prunes emptied source dirs. Applies to the Deploy import, the _shared redirections, and the root-level items. Big re-imports now only rewrite what actually changed. --- webapp/app.py | 14 ++++------ webapp/services/deploy.py | 57 +++++++++++++++++++++++++-------------- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/webapp/app.py b/webapp/app.py index ad010ea..dbf5477 100644 --- a/webapp/app.py +++ b/webapp/app.py @@ -113,9 +113,9 @@ def images_import(): src_items = os.listdir(source) # Move files from network upload to save disk space; copy from USB. + # deploy.sync_tree uses rsync --checksum: unchanged files are + # skipped, only new/changed files are written (no full replace). use_move = source == config.UPLOAD_DIR or source.startswith(config.UPLOAD_DIR + "/") - _transfer = shutil.move if use_move else shutil.copy2 - _transfer_tree = shutil.move if use_move else shutil.copytree top_dirs = {d for d in src_items if os.path.isdir(os.path.join(source, d))} full_layout = "Deploy" in top_dirs @@ -134,8 +134,7 @@ def images_import(): elif os.path.isdir(src_item) and item in shared_root: prefix_key = target.split("-")[0] + "-" shared_dest = os.path.join(config.SHARED_DIR, f"{prefix_key}{item}") - os.makedirs(shared_dest, exist_ok=True) - deploy._merge_tree(src_item, shared_dest, move=use_move) + deploy.sync_tree(src_item, shared_dest, move=use_move) dst_item = os.path.join(root, item) if os.path.islink(dst_item): os.remove(dst_item) @@ -143,12 +142,9 @@ def images_import(): shutil.rmtree(dst_item) os.symlink(shared_dest, dst_item) elif os.path.isdir(src_item): - dst_item = os.path.join(root, item) - if os.path.exists(dst_item): - shutil.rmtree(dst_item) - _transfer_tree(src_item, dst_item) + deploy.sync_tree(src_item, os.path.join(root, item), move=use_move) else: - _transfer(src_item, os.path.join(root, item)) + deploy.sync_tree(src_item, os.path.join(root, item), move=use_move) else: deploy.import_deploy(source, dest, target, move=use_move) diff --git a/webapp/services/deploy.py b/webapp/services/deploy.py index 629758f..d7166de 100644 --- a/webapp/services/deploy.py +++ b/webapp/services/deploy.py @@ -7,6 +7,7 @@ re-use the same multi-GB driver tree without doubling disk usage. import os import shutil +import subprocess import config from services.system import find_usb_mounts @@ -21,27 +22,43 @@ def _replace_with_symlink(link_path, target_path): os.symlink(target_path, link_path) -def _merge_tree(src, dst, move=False): - """Recursively merge src tree into dst, overwriting existing files. +def sync_tree(src, dst, move=False, checksum=True): + """Copy src -> dst with rsync, SKIPPING files whose content already + matches and transferring only new or changed files. - When move=True, files are moved instead of copied (saves disk space - on imports from the local upload-dir). - """ - _transfer = shutil.move if move else shutil.copy2 - _transfer_tree = shutil.move if move else shutil.copytree - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isdir(s): - if os.path.isdir(d): - _merge_tree(s, d, move=move) - else: - if os.path.exists(d): - os.remove(d) - _transfer_tree(s, d) - else: - os.makedirs(os.path.dirname(d), exist_ok=True) - _transfer(s, d) + src may be a file or a directory (its contents merge into dst). With + checksum=True rsync compares by hash (--checksum), so a re-import only + rewrites files that actually differ instead of replacing everything; + set checksum=False for the faster size+mtime comparison. move=True + removes source files after a successful transfer (frees the upload dir). + Existing dst files not present in src are left untouched (merge, not + mirror - no --delete).""" + flags = ["-a"] # recurse, preserve perms/times/symlinks + if checksum: + flags.append("--checksum") # compare by content hash, not size+mtime + if move: + flags.append("--remove-source-files") + if os.path.isdir(src): + os.makedirs(dst, exist_ok=True) + src_arg = src.rstrip("/") + "/" # trailing slash = merge CONTENTS into dst + else: + parent = os.path.dirname(dst) + if parent: + os.makedirs(parent, exist_ok=True) + src_arg = src + subprocess.run(["rsync", *flags, src_arg, dst], check=True) + if move and os.path.isdir(src): + # --remove-source-files empties files but leaves the dir skeleton; prune it + for root, _dirs, _files in os.walk(src, topdown=False): + try: + os.rmdir(root) + except OSError: + pass + + +def _merge_tree(src, dst, move=False): + """Backwards-compatible alias: checksum-aware merge of src into dst.""" + sync_tree(src, dst, move=move) def import_deploy(src_deploy, dst_deploy, target="", move=False):