Verified MCL uploads: hash every file, repair over HTTP, gate the import
Small Control JSONs have arrived empty or wrong more than once, and the old check compared sizes only and skipped missing files. A copy of Deploy/Control on the dev PC still holds two 0-byte JSONs. Upload-Image.ps1 (tested on Windows PowerShell 5.1 in the win11 VM): - refuses an unfinished MCL cache: exactly one <OS>-Production.tag (MCL writes it last), no Control file newer than it, selected OS matches, every Control/Tools JSON non-empty and valid - checks payload against the SHA-256 MCL's own Control JSONs publish. MCL resumes downloads onto old files (FileMode.Append) and then trusts any file of the right size: the September junk-tail corruption. Offers to delete the bad cache files so MCL downloads them again - hashes every file into a manifest, robocopies, POSTs the manifest - re-sends bad small files over HTTP (no proxy), large ones by deleting the bad copy and robocopy /J (robocopy skips a same-size, same-time file as "modified" even with /IS /IT) Webapp: - services/upload_verify.py hashes the upload on the server's own disk in a background thread; reports missing, differing, bad-in-MCL-cache and stray files; PUT /api/upload/file writes temp, checks hash, renames - import of the upload folder only after a passed verification, with every file unchanged (size, mtime, ctime, inode), no stray files, and the target the OS allows (LTSC: Shop Floor only; else Standard or Engineer; unknown OS: none). Import, verify and PUT share one lock - paths matched case-insensitively against the disk, as Samba does - import page shows the upload, MCL run, verification and stray files with a remove button; record of each import kept on the image Reviewed adversarially; all 11 confirmed findings fixed with tests. 42 new unit tests. VM test: clean upload; same-size same-mtime damage on 3 files repaired in one round; stray file blocks import until removed; Shop Floor refused for a Pro upload; import records and clears the folder; missing tag, 0-byte JSON, junk-head WIM and late Control file each refused with nothing uploaded.
This commit is contained in:
cproudlock
@@ -21,6 +21,12 @@
|
||||
.imp-actions { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; }
|
||||
.imp-empty { display: flex; flex-direction: column; gap: 10px; align-items: flex-start; font-size: 14px; color: var(--text-2); line-height: 1.6; }
|
||||
.imp-empty p { margin: 0; }
|
||||
.upv-rows { display: flex; flex-direction: column; gap: 6px; font-size: 14px; }
|
||||
.upv-row { display: grid; grid-template-columns: 92px minmax(0, 1fr); gap: 10px; align-items: center; }
|
||||
.upv-row .k { color: var(--muted); font-size: 13px; }
|
||||
.upv-bad summary { cursor: pointer; font-size: 13px; font-weight: 600; }
|
||||
.upv-bad ul { margin: 8px 0 0; padding-left: 18px; display: flex; flex-direction: column; gap: 3px; font-size: 13px; }
|
||||
.upv-bad li { min-width: 0; }
|
||||
|
||||
.ig { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.3fr) 92px 112px auto; align-items: center; gap: 0 14px;
|
||||
padding: 11px 24px; font-size: 14px; border-top: 1px solid var(--line); }
|
||||
@@ -50,40 +56,81 @@
|
||||
<section class="panel imp-panel" aria-labelledby="imp-up-h">
|
||||
<div>
|
||||
<h2 id="imp-up-h">From the network upload share</h2>
|
||||
<p class="lead">Files copied over SMB to <span class="mono">\\172.16.9.1\image-upload</span>. They are moved, not copied.</p>
|
||||
<p class="lead">Uploaded by <span class="mono">Upload-Image.ps1</span> to <span class="mono">\\172.16.9.1\image-upload</span> and checked file by file. They are moved, not copied.</p>
|
||||
</div>
|
||||
{% if upload_sources %}
|
||||
{% set man = upload_info.manifest %}
|
||||
{% set st = upload_info.state %}
|
||||
{% if man %}
|
||||
<div class="upv-rows">
|
||||
<div class="upv-row"><span class="k">Upload</span><span class="tip" data-tip="{{ man.upload_id }}{% if man.created %} created {{ man.created }}{% endif %}"><span class="trunc mono">{{ man.upload_id }}</span></span></div>
|
||||
{% if man.mcl %}
|
||||
<div class="upv-row"><span class="k">MCL cache</span><span class="tip" data-tip="{{ man.mcl.tag or '' }}"><span class="trunc">{{ man.mcl.os_name or 'Unknown OS' }}{% if man.mcl.os_version %} {{ man.mcl.os_version }}{% endif %}, finished {{ (man.mcl.finished or '?')|replace('T', ' ') }}</span></span></div>
|
||||
{% endif %}
|
||||
<div class="upv-row"><span class="k">Files</span><span>{{ man.files }} ({{ man.bytes|filesizeformat(true) }})</span></div>
|
||||
<div class="upv-row"><span class="k">Verified</span><span>
|
||||
{% if st.state == 'running' %}{{ ui.chip('Checking ' ~ st.done_files ~ '/' ~ st.total_files, 'warn') }}
|
||||
{% elif st.state == 'done' and st.ok %}{{ ui.chip('Every file matches', 'ok') }} <span class="sub">{{ (st.finished or '')|replace('T', ' ') }}</span>
|
||||
{% elif st.state == 'done' %}{{ ui.chip((st.missing|length + st.mismatch|length + st.bad_in_cache|length) ~ ' bad file(s)', 'err') }}
|
||||
{% elif st.state == 'error' %}{{ ui.chip('Check failed', 'err') }} <span class="sub">{{ st.error }}</span>
|
||||
{% else %}{{ ui.chip('Not verified', 'off') }}{% endif %}
|
||||
</span></div>
|
||||
</div>
|
||||
{% if st.state == 'done' and not st.ok %}
|
||||
<details class="upv-bad"><summary>Bad files</summary>
|
||||
<ul>
|
||||
{% for f in st.missing %}<li class="tip" data-tip="{{ f }}"><span class="trunc mono">Missing: {{ f }}</span></li>{% endfor %}
|
||||
{% for f in st.mismatch %}<li class="tip" data-tip="{{ f.path }}: {{ f.reason }}"><span class="trunc mono">Differs: {{ f.path }}</span></li>{% endfor %}
|
||||
{% for f in st.bad_in_cache %}<li class="tip" data-tip="{{ f.path }}: {{ f.reason }}"><span class="trunc mono">Bad in MCL cache: {{ f.path }}</span></li>{% endfor %}
|
||||
</ul>
|
||||
<p class="sub">Run Upload-Image.ps1 again: it re-sends the files that differ. Files bad in the MCL cache must be downloaded again by MCL.</p>
|
||||
</details>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
{% if upload_info.extra_count %}
|
||||
<div class="notice"><span><strong>{{ upload_info.extra_count }} file(s) in the upload folder are not part of this upload</strong>, left from an earlier one. They would be imported with it.</span></div>
|
||||
<details class="upv-bad"><summary>Show them</summary><ul>
|
||||
{% for f in upload_info.extra %}<li class="tip" data-tip="{{ f }}"><span class="trunc mono">{{ f }}</span></li>{% endfor %}
|
||||
{% if upload_info.extra_count > upload_info.extra|length %}<li class="sub">and {{ upload_info.extra_count - upload_info.extra|length }} more</li>{% endif %}
|
||||
</ul></details>
|
||||
<form method="POST" action="{{ url_for('images_import_remove_extras') }}" onsubmit="return confirm('Delete the {{ upload_info.extra_count }} file(s) that are not in this upload\'s manifest?');">
|
||||
<input type="hidden" name="_csrf_token" value="{{ csrf_token() }}">
|
||||
<button type="submit" class="ui-btn">Remove files not in this upload</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
|
||||
{% if upload_ok %}
|
||||
<form method="POST" id="uploadImportForm">
|
||||
<input type="hidden" name="_csrf_token" value="{{ csrf_token() }}">
|
||||
<div class="imp-field">
|
||||
<label for="uploadSource">Source</label>
|
||||
<select class="field" name="source" id="uploadSource" required>
|
||||
<option value="">Select an upload folder</option>
|
||||
{% for src in upload_sources %}
|
||||
<option value="{{ src }}">{{ src }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
<input type="hidden" name="source" value="{{ upload_dir }}">
|
||||
<div class="imp-field">
|
||||
<label for="uploadTarget">Target image</label>
|
||||
{% set allowed = upload_info.allowed_targets %}
|
||||
<select class="field" name="target" id="uploadTarget" required>
|
||||
<option value="">Select the image to import into</option>
|
||||
{% for it in image_types %}
|
||||
<option value="{{ it }}">{{ friendly_names[it] }} ({{ it }})</option>
|
||||
{% if allowed|length > 1 %}<option value="">Select the image to import into</option>{% endif %}
|
||||
{% for it in allowed %}
|
||||
{% set li = last_imports.get(it) %}
|
||||
{% set older = li and li.mcl and man.mcl and li.mcl.finished and man.mcl.finished and man.mcl.finished < li.mcl.finished %}
|
||||
<option value="{{ it }}" data-older="{{ '1' if older else '' }}" data-last="{{ (li.mcl.finished if li and li.mcl else '')|replace('T', ' ') }}">{{ friendly_names[it] }} ({{ it }})</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<span class="sub">Content goes into the image's Deploy folder. Out-of-box Drivers are pooled once in a shared folder and linked from each image to save disk space.</span>
|
||||
<span class="sub">{% if upload_info.os %}This upload carries {{ upload_info.os.name }}{% if upload_info.os.version %} {{ upload_info.os.version }}{% endif %}, so it can only go to the image(s) listed.{% endif %}
|
||||
Out-of-box Drivers are pooled once in a shared folder and linked from each image.</span>
|
||||
</div>
|
||||
<div class="notice" id="olderWarn" hidden><span><strong>This MCL cache is older than the one last imported into that image</strong> (<span id="olderWhen"></span>).</span></div>
|
||||
<div class="notice"><span><strong>The files are moved out of the upload folder</strong>, so the source is used up.
|
||||
Files with the same name in the image are overwritten. Large images take several minutes.</span></div>
|
||||
<div class="imp-actions">
|
||||
<button type="submit" class="ui-btn pri" id="uploadImportBtn">Start import</button>
|
||||
</div>
|
||||
</form>
|
||||
{% elif upload_sources %}
|
||||
<div class="notice"><span><strong>Import blocked:</strong> {{ upload_reason }}</span></div>
|
||||
<div class="imp-actions"><button type="button" class="ui-btn" onclick="location.reload()">Refresh</button></div>
|
||||
{% else %}
|
||||
<div class="imp-empty">
|
||||
{{ ui.chip('Nothing uploaded', 'off') }}
|
||||
<p>Map <span class="mono">\\172.16.9.1\image-upload</span> on your Windows PC and copy the Deploy folder contents there, then refresh.</p>
|
||||
<p>On the PC with the Media Creator Lite cache, run <span class="mono">scripts\Upload-Image.ps1</span>. It copies the cache here and checks every file.</p>
|
||||
<button type="button" class="ui-btn" onclick="location.reload()">Refresh</button>
|
||||
</div>
|
||||
{% endif %}
|
||||
@@ -152,6 +199,15 @@
|
||||
{% block extra_scripts %}
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
var tsel = document.getElementById('uploadTarget');
|
||||
function olderCheck() {
|
||||
var w = document.getElementById('olderWarn');
|
||||
if (!tsel || !w) return;
|
||||
var o = tsel.options[tsel.selectedIndex];
|
||||
w.hidden = !(o && o.dataset.older);
|
||||
document.getElementById('olderWhen').textContent = o && o.dataset.last ? 'last import finished ' + o.dataset.last : '';
|
||||
}
|
||||
if (tsel) { tsel.addEventListener('change', olderCheck); olderCheck(); }
|
||||
['importForm', 'uploadImportForm'].forEach(function(formId) {
|
||||
var form = document.getElementById(formId);
|
||||
if (form) {
|
||||
|
||||
Reference in New Issue
Block a user