Filter pagination reset; catalog model retype tool
This commit is contained in:
230
scripts/retype_models.py
Normal file
230
scripts/retype_models.py
Normal file
@@ -0,0 +1,230 @@
|
||||
"""Give catalog models a useful type, so their assets can inherit it.
|
||||
|
||||
The catalog types every printer model as "Printer" - true, and useless. It says
|
||||
the product is a printer, not whether it is a laser, a plotter or a label
|
||||
printer. That finer answer is a property of the MODEL (every VersaLink C405 ever
|
||||
made is a laser MFP), but nothing records it, so nothing can derive it.
|
||||
|
||||
Recording it in the catalog rather than on each printer means:
|
||||
|
||||
- backfill_vendor_from_model.py then fills every printer automatically, since
|
||||
it matches an asset's type to its model's type BY EXACT NAME;
|
||||
- a printer added later inherits the right type as soon as its model is
|
||||
chosen, with no second step, ever;
|
||||
- the catalog stops claiming a plotter and a label printer are the same thing.
|
||||
|
||||
Two passes, because guessing at a model's type unreviewed is how a plotter ends
|
||||
up filed as a laser:
|
||||
|
||||
python scripts/retype_models.py --export printer-types.csv
|
||||
... open it, correct the newtype column, save ...
|
||||
python scripts/retype_models.py --apply printer-types.csv
|
||||
python scripts/retype_models.py --apply printer-types.csv --commit
|
||||
|
||||
A newtype is REFUSED unless it already exists in that asset class's own type
|
||||
vocabulary - "Laser" is accepted for printers because printertypes has it,
|
||||
"Laserjet" is not. That is what keeps the name match working afterwards.
|
||||
|
||||
Defaults to printers. --class machines|computers|network for the others.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
# Each asset class, its own type vocabulary, and the modeltypes.category that a
|
||||
# newly created model type should carry.
|
||||
CLASSES = {
|
||||
'printers': ('printers', 'printertypes', 'printertype', 'Printer'),
|
||||
'machines': ('machines', 'machinetypes', 'machinetype', 'Equipment'),
|
||||
'computers': ('computers', 'computertypes', 'computertype', 'PC'),
|
||||
'network': ('networkdevices', 'networkdevicetypes', 'networkdevicetype', 'Network'),
|
||||
}
|
||||
|
||||
# Model-number keywords that suggest a type. A SUGGESTION only - it lands in the
|
||||
# CSV for a person to accept or correct, and is never applied on its own.
|
||||
# ORDER MATTERS - first match wins, so the specific brands come before the
|
||||
# generic patterns. A bare t\d{3,4} for plotters used to match "Zebra ZT411" and
|
||||
# file a label printer as a plotter, which is exactly the sort of confident
|
||||
# mistake the review step exists to catch. The pattern is now explicit instead.
|
||||
HINTS = [
|
||||
(r'zebra|\bzt\d|\bzd\d|\bgk\d|\bgx\d|label|datamax|intermec', 'Label'),
|
||||
(r'designjet|plotter|latex|\bdj\s?t\d{3,4}', 'Plotter'),
|
||||
(r'datacard|zxp|card|cr80', 'Card'),
|
||||
(r'thermal|tsp\d|tm-t', 'Thermal'),
|
||||
(r'deskjet|officejet|inkjet|pixma', 'Inkjet'),
|
||||
(r'\bml-\d|dot ?matrix|lq-\d|fx-\d', 'Dot Matrix'),
|
||||
# Multifunction before plain laser: an MFP is also a laser, and the more
|
||||
# specific answer is the useful one.
|
||||
(r'mfp|mfc|workcentre|altalink|versalink|\bm\d{3}f', 'MFP'),
|
||||
(r'laserjet|laser|phaser|\bls\b|\bp\d{4}', 'Laser'),
|
||||
]
|
||||
|
||||
# Types the catalog uses that say nothing about the product. LocationOnly is the
|
||||
# classic-ASP placeholder that also left 134 machines untyped.
|
||||
PLACEHOLDERS = {'printer', 'locationonly', 'pc', 'equipment', 'network', 'computer', ''}
|
||||
|
||||
|
||||
def suggest(modelnumber, description=''):
|
||||
haystack = ('%s %s' % (modelnumber or '', description or '')).lower()
|
||||
for pattern, answer in HINTS:
|
||||
if re.search(pattern, haystack):
|
||||
return answer
|
||||
return ''
|
||||
|
||||
|
||||
def vocabulary(connection, typetable, typename):
|
||||
from sqlalchemy import text
|
||||
rows = connection.execute(text('SELECT %s FROM %s ORDER BY 1' % (typename, typetable))).fetchall()
|
||||
return [r[0] for r in rows if r[0]]
|
||||
|
||||
|
||||
def export(connection, classname, path, include_all):
|
||||
from sqlalchemy import text
|
||||
assettable, typetable, typename, _ = CLASSES[classname]
|
||||
rows = connection.execute(text("""
|
||||
SELECT mo.modelnumberid AS modelnumberid,
|
||||
mo.modelnumber AS modelnumber,
|
||||
COALESCE(v.vendor, '') AS vendor,
|
||||
COALESCE(mo.description, '') AS description,
|
||||
COALESCE(mt.modeltype, '') AS currenttype,
|
||||
COUNT(a.modelnumberid) AS assetcount
|
||||
FROM {assettable} a
|
||||
JOIN models mo ON a.modelnumberid = mo.modelnumberid
|
||||
LEFT JOIN vendors v ON mo.vendorid = v.vendorid
|
||||
LEFT JOIN modeltypes mt ON mo.modeltypeid = mt.modeltypeid
|
||||
GROUP BY mo.modelnumberid, mo.modelnumber, v.vendor, mo.description, mt.modeltype
|
||||
ORDER BY assetcount DESC, mo.modelnumber
|
||||
""".format(assettable=assettable))).fetchall()
|
||||
|
||||
known = {v.lower() for v in vocabulary(connection, typetable, typename)}
|
||||
out = []
|
||||
for row in rows:
|
||||
current = (row.currenttype or '').strip()
|
||||
# Already a real answer in this class's vocabulary: nothing to decide.
|
||||
if not include_all and current.lower() in known:
|
||||
continue
|
||||
out.append({
|
||||
'modelnumberid': row.modelnumberid,
|
||||
'modelnumber': row.modelnumber,
|
||||
'vendor': row.vendor,
|
||||
'assets': row.assetcount,
|
||||
'currenttype': current,
|
||||
'suggested': suggest(row.modelnumber, row.description),
|
||||
'newtype': suggest(row.modelnumber, row.description),
|
||||
})
|
||||
|
||||
with open(path, 'w', newline='', encoding='utf-8-sig') as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=[
|
||||
'modelnumberid', 'modelnumber', 'vendor', 'assets',
|
||||
'currenttype', 'suggested', 'newtype'])
|
||||
writer.writeheader()
|
||||
writer.writerows(out)
|
||||
|
||||
print('Wrote %s with %d model(s) to review.' % (path, len(out)))
|
||||
print('Valid values for newtype (%s):' % classname)
|
||||
print(' %s' % ', '.join(vocabulary(connection, typetable, typename)))
|
||||
print('')
|
||||
print('The newtype column is pre-filled with a guess from the model number.')
|
||||
print('CHECK EVERY ROW - a wrong guess types every asset using that model.')
|
||||
print('Clear newtype to leave a model alone.')
|
||||
return 0
|
||||
|
||||
|
||||
def apply(connection, classname, path, commit):
|
||||
from sqlalchemy import text
|
||||
_, typetable, typename, category = CLASSES[classname]
|
||||
known = {v.lower(): v for v in vocabulary(connection, typetable, typename)}
|
||||
|
||||
with open(path, newline='', encoding='utf-8-sig') as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
|
||||
planned, refused = [], []
|
||||
for row in rows:
|
||||
newtype = (row.get('newtype') or '').strip()
|
||||
if not newtype:
|
||||
continue
|
||||
if newtype.lower() not in known:
|
||||
refused.append((row.get('modelnumber', '?'), newtype))
|
||||
continue
|
||||
planned.append((int(row['modelnumberid']), row.get('modelnumber', '?'),
|
||||
known[newtype.lower()], int(row.get('assets') or 0)))
|
||||
|
||||
for modelnumber, newtype in refused:
|
||||
print(' REFUSED %-28s "%s" is not in %s' % (modelnumber, newtype, typetable))
|
||||
if refused:
|
||||
print(' (add it to %s first if it is a real type, or correct the spelling)' % typetable)
|
||||
print('')
|
||||
|
||||
print('%d model(s) would be retyped, covering %d asset(s):'
|
||||
% (len(planned), sum(p[3] for p in planned)))
|
||||
for _, modelnumber, newtype, count in planned[:40]:
|
||||
print(' %-32s -> %-12s (%d asset(s))' % (modelnumber[:32], newtype, count))
|
||||
if len(planned) > 40:
|
||||
print(' ... and %d more' % (len(planned) - 40))
|
||||
|
||||
if not commit:
|
||||
print('')
|
||||
print('DRY RUN - nothing written. Re-run with --commit to apply.')
|
||||
return 0
|
||||
|
||||
for modelnumberid, _, newtype, _ in planned:
|
||||
typeid = connection.execute(
|
||||
text('SELECT modeltypeid FROM modeltypes WHERE modeltype = :n'),
|
||||
{'n': newtype}).scalar()
|
||||
if typeid is None:
|
||||
# The name is valid for this asset class but the catalog has no such
|
||||
# model type yet. Create it, tagged with this class's category.
|
||||
connection.execute(
|
||||
text('INSERT INTO modeltypes (modeltype, category) VALUES (:n, :c)'),
|
||||
{'n': newtype, 'c': category})
|
||||
typeid = connection.execute(
|
||||
text('SELECT modeltypeid FROM modeltypes WHERE modeltype = :n'),
|
||||
{'n': newtype}).scalar()
|
||||
connection.execute(
|
||||
text('UPDATE models SET modeltypeid = :t WHERE modelnumberid = :m'),
|
||||
{'t': typeid, 'm': modelnumberid})
|
||||
|
||||
print('')
|
||||
print('Committed. %d model(s) retyped.' % len(planned))
|
||||
print('Now run: python scripts/backfill_vendor_from_model.py --commit')
|
||||
print('which copies the type onto every asset using those models.')
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('--class', dest='classname', default='printers',
|
||||
choices=sorted(CLASSES), help='asset class (default printers)')
|
||||
parser.add_argument('--export', metavar='CSV', help='write models needing a type')
|
||||
parser.add_argument('--apply', metavar='CSV', help='apply a reviewed file')
|
||||
parser.add_argument('--all', action='store_true',
|
||||
help='with --export, include models that already have a valid type')
|
||||
parser.add_argument('--commit', action='store_true', help='write the changes')
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.export and not args.apply:
|
||||
parser.error('give --export or --apply')
|
||||
|
||||
from shopdb import create_app
|
||||
from shopdb.extensions import db
|
||||
|
||||
app = create_app()
|
||||
with app.app_context():
|
||||
connection = db.session.connection()
|
||||
if args.export:
|
||||
code = export(connection, args.classname, args.export, args.all)
|
||||
else:
|
||||
code = apply(connection, args.classname, args.apply, args.commit)
|
||||
if args.commit:
|
||||
db.session.commit()
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user