Merge remote-tracking branch 'origin/main' into wilm

# Conflicts:
#	frontend/src/views/settings/SiteSettings.vue
This commit is contained in:
benjamin.baker
2026-08-27 05:45:52 -04:00
69 changed files with 5601 additions and 327 deletions

View File

@@ -233,17 +233,30 @@ echo "==> Checking for site-specific literals in product code (ADR-015)..."
# publication scrub greps for the very strings below - written literally, the
# rule's own definition trips the gate that enforces it. Same reason
# tests/test_docs_publishable.py splits its terms.
SITE_HOST='tsg'\''wp00525'
SITE_DOMAIN='\.geaero'\''space\.net'
SITE_NAME='West Jeff'\''erson'
# Adjacent quoted strings CONCATENATE: 'ab''cd' is the four-character string
# abcd. The apostrophe-escape form ('ab'\''cd') looks almost identical and is
# not - it embeds a literal ' in the middle, so every pattern below matched
# nothing and this check silently passed everything from the day it was written
# until 2026-08-26. That is how a QA SaaS tenant sat in the shipped defaults for
# months with the gate green. Keep the fragments split (the scrub greps this
# file for the very strings the patterns are made of) and keep them ADJACENT.
SITE_HOST='tsg''wp00525'
SITE_DOMAIN='\.geaero''space\.net'
SITE_NAME='West Jeff''erson'
SITE_NETS='10\.134\.48\.|10\.48\.249\.'
SITE_PATTERNS="${SITE_HOST}|${SITE_DOMAIN}|\bwjs\b|${SITE_NAME}|${SITE_NETS}"
# A named SaaS TENANT is as site-specific as a hostname, and this one shipped as
# the default ServiceNow URL for months: every adopting site would have linked
# its technicians into another organisation's QA instance. The vendor domain
# itself is fine to mention; the tenant is not.
SITE_TENANTS='geaero''spaceqa'
SITE_PATTERNS="${SITE_HOST}|${SITE_DOMAIN}|\bwjs\b|${SITE_NAME}|${SITE_NETS}|${SITE_TENANTS}"
SITE_HITS=$(grep -rPni "$SITE_PATTERNS" "${EXCLUDES[@]}" \
--include='*.py' --include='*.vue' --include='*.js' \
--include='*.ps1' --include='*.psm1' --include='*.sh' --include='*.iss' \
--include='*.json' --include='*.html' \
plugins/ shopdb/ scripts/ deploy/ frontend/src/ tools/ 2>/dev/null \
| grep -v '/tests\?/' \
| grep -v '\.spec\.js:' \
| grep -v 'site_imports/' \
| grep -v 'installer/bundle/' \
| grep -v '\.plugins-staged/' \

View File

@@ -0,0 +1,136 @@
"""Remove credentials from backup revisions already stored in the database.
WHY THIS EXISTS
The NTLARS/DNC registry carries an FTP password in cleartext. The codec now
redacts credentials at PARSE, so nothing new reaches the database - but every
revision recorded before that still holds the password verbatim, where the
revision panel renders it and the download route hands it back as a
ready-to-import .reg. Anyone with backups.view could read it off a web page.
Rotating the password does not fix these rows: it makes them stale AND still
teaches anyone reading them what the old one was.
WHAT IT CHANGES: backuprevisions.payloadjson only, and only the values whose
NAME looks like a credential (the same test the codec applies). The value name
is kept and recorded in the revision's `redacted` list, so a restore from a
scrubbed revision says plainly which field must be set by hand rather than
quietly producing a machine that cannot log in.
WHAT IT DOES NOT CHANGE: contenthash. The hash identifies the CONFIGURATION a
bay reported, and dedup compares the next post against it. Rewriting hashes here
would make every bay's next post look like a change and insert a duplicate
revision across the fleet.
Usage (on the target instance, in the app dir):
venv\\Scripts\\python scripts\\redact_backup_secrets.py
venv\\Scripts\\python scripts\\redact_backup_secrets.py --commit
Dry run by default: it reports which revisions hold a credential and stops.
Take a database backup before --commit.
"""
import argparse
import json
import os
import sys
# Run from anywhere: put the repo root (parent of scripts/) on the path.
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from dotenv import load_dotenv
# Load .env before anything reads config. Without it DATABASE_URL stays unset
# and config falls back to a hardcoded root@localhost URL, which fails with a
# bare 1045 Access denied naming a user the site does not use - a database
# credential error that is really "nothing told the app which database".
load_dotenv(os.path.join(os.getcwd(), '.env'))
from shopdb import create_app # noqa: E402
from shopdb.extensions import db # noqa: E402
def scrub(projection, issecretname, marker):
"""Redact credential values in one projection. Returns (changed, names)."""
if not isinstance(projection, dict):
return False, []
names = list(projection.get('redacted') or [])
changed = False
for key in projection.get('keys') or []:
if not isinstance(key, dict):
continue
path = key.get('path', '')
values = key.get('values')
if not isinstance(values, dict):
continue
for name, entry in values.items():
if not issecretname(name) or not isinstance(entry, dict):
continue
if entry.get('data') in ('', None, marker):
continue
entry['data'] = marker
fullname = '{}\\{}'.format(path, name) if path else name
if fullname not in names:
names.append(fullname)
changed = True
if changed:
projection['redacted'] = sorted(names)
return changed, names
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--commit', action='store_true',
help='Apply the changes. Dry run by default.')
args = parser.parse_args()
app = create_app()
with app.app_context():
from plugins.backups.models import BackupRevision
from plugins.backups.services.ntlars import issecretname, REDACTEDMARKER
rows = BackupRevision.query.order_by(BackupRevision.backuprevisionid).all()
hits = []
for revision in rows:
projection = revision.payload
if not projection:
continue
# Work on a copy so a dry run cannot mutate a loaded instance into
# something a later flush would write.
working = json.loads(json.dumps(projection))
changed, names = scrub(working, issecretname, REDACTEDMARKER)
if not changed:
continue
hits.append((revision, working, names))
if not hits:
print('No stored revision holds a credential. Nothing to do.')
return 0
print('{} revision(s) hold a credential:'.format(len(hits)))
for revision, _, names in hits:
print(' revision {:<6} asset {:<6} {:<10} {}'.format(
revision.backuprevisionid, revision.assetid,
revision.backupkind, ', '.join(names)))
if not args.commit:
print()
print('Dry run. Nothing was changed. Re-run with --commit to write.')
print('Take a database backup first.')
return 0
for revision, working, _ in hits:
revision.payload = working
db.session.commit()
print()
print('Redacted {} revision(s). contenthash left untouched, so no bay '
'will re-post as changed.'.format(len(hits)))
return 0
if __name__ == '__main__':
raise SystemExit(main())