shopdb-flask/plugins/printers/services/supply_history.py

"""Read a supply's level history and say what it means.

Two questions come out of the same series of readings:

  * how fast is it draining, and when does it hit empty
  * how many times has it been replaced

Both hang on one observation: a cartridge only ever goes DOWN while it is in
use, so a rise is a replacement. Everything here is built on locating those
rises and treating each stretch between them as one cartridge's life.

The maths is deliberately kept away from Zabbix so it can be tested against a
list of numbers. The service layer fetches; this decides.
"""

from datetime import datetime, timezone

# A reading can wobble by a point without anything happening - SNMP rounding,
# a gauge settling after a power cycle. A rise has to clear this to count as a
# new cartridge rather than noise.
REPLACEMENT_RISE = 10

# A replacement must also LAND high. A fresh cartridge reads near full, so a
# rise that stops mid-range is a gauge bouncing, not a swap. Without this the
# count was any +10 between readings, and the two common noise shapes both
# scored: a supply reading 0 or near-0 while it was out of the machine, then the
# real level again (0 -> 60 counts as +60), and a coarse gauge ticking back up
# after a reseat or a power cycle. That is how one cartridge claimed five
# changes in ninety days.
#
# A site that fits PART-USED cartridges will under-count with this rule. That is
# the right way round: a missed swap widens the run and slows the estimate,
# while a phantom swap resets the run and throws the estimate away entirely.
NEW_CARTRIDGE_LEVEL = 80

# How far ABOVE its previous level a reading may come back and still count as a
# recovery rather than a new cartridge. Small, because it exists for gauge
# noise: a real swap returns near full, well past this.
RECOVERY_TOLERANCE = 2

# How long a dip may last and still be a bad reading rather than real use.
# Levels alone cannot separate the two: a swap to full, ordinary consumption,
# then another swap has the same SHAPE as an outage that recovers. What differs
# is elapsed time. A supply out of the machine, a door open or a bad poll spans
# minutes to an hour or two at the few-minute polling these items use; a level
# that stays down for days is genuinely down, and deleting those readings would
# hide a real empty period.
MAX_DIP_HOURS = 6

# Below this many readings a slope is arithmetic, not evidence. Two points
# through a coarse gauge can "prove" any rate at all.
MIN_POINTS_FOR_ESTIMATE = 4

# A rate needs TIME behind it, not just readings. Supply items are often polled
# every few minutes, so four readings can span a quarter of an hour - and a 2
# point drop across fifteen minutes extrapolates to nearly 200 percent a day,
# which is how a cartridge sitting at 82 percent gets forecast to run out in a
# fortnight. The history window can be short for the same reason: the fetch
# keeps the newest rows up to a per-item cap, so a fast-polled item returns days
# rather than the ninety asked for.
#
# Two days is the smallest span that survives a printer's daily rhythm - a
# single heavy morning does not become the whole picture.
MIN_DAYS_FOR_ESTIMATE = 2

# Many printers report in 10% steps, so a fortnight can pass on one plateau.
# Without a minimum observed drop the slope reads as zero and the forecast
# says "never", which is worse than saying nothing.
MIN_DROP_FOR_ESTIMATE = 2

# How far the fastest and slowest intervals may differ before a single rate
# stops being a fair summary. A cartridge that ran at 10 percent/day for two
# days and 0.2 percent/day since is not described by any one number, and a
# precise-looking figure invites more trust than it has earned.
RATE_SPREAD_FACTOR = 5

# Empty means empty. This was 5, on the reasoning that a printer at 1% is out
# of toner as far as anyone standing at it is concerned - but flooring the
# days-left of everything at or below 5% threw away the only number the row
# exists to give. A cartridge at 2% draining a point a day has two days left,
# and two days is worth knowing; reporting it as already gone puts it beside
# cartridges that genuinely are, and there is no way back to the difference.
#
# Anything above zero gets its real estimate, in hours if that is what it comes
# to. Only a reading of zero is empty.
EMPTY_LEVEL = 0


def _asfloat(value):
    try:
        return float(value)
    except (TypeError, ValueError):
        return None


def normalise(points):
    """[(clock, value)] -> sorted [(datetime, float)], junk dropped.

    Zabbix returns clock as a unix string and value as a string; a history
    table can also carry the odd unparseable row.
    """
    out = []
    for clock, value in points:
        seconds = _asfloat(clock)
        level = _asfloat(value)
        if seconds is None or level is None:
            continue
        out.append((datetime.fromtimestamp(seconds, tz=timezone.utc), level))
    out.sort(key=lambda p: p[0])
    return drop_spikes(out)


def _recovered(previous, following, tolerance=RECOVERY_TOLERANCE):
    """Did the level come BACK to where it was, rather than up to a new one?

    Toner only falls, so a dip that recovers returns to at or below the level it
    left - consumption carried on while the reading was junk. A cartridge that
    was CHANGED comes back HIGHER than the level before the dip.

    That asymmetry is the whole test. Comparing the absolute difference instead
    treated a genuine swap of a nearly-full cartridge (95, then 5, then 100) as
    a recovery and deleted the evidence, because 100 and 95 are close. The small
    upward tolerance is for gauge noise, not for swaps.
    """
    return following <= previous + tolerance


def drop_spikes(points, rise=REPLACEMENT_RISE):
    """Remove dips that RECOVER to where they came from, however long they run.

    A single reading far below both neighbours, then a recovery, is a big
    upward step that scores as a cartridge change. That is how a cartridge
    claimed five changes in ninety days.

    NOT a Zabbix timeout: an item that does not answer records nothing, it does
    not write a zero. The dip is a value the device really reported - a supply
    pulled out to be shaken and reseated, a door open mid-poll, or a site whose
    preprocessing maps the Printer MIB's "unknown" sentinels (-1/-2/-3, which
    cannot land in the unsigned history table) onto 0.

    The filter keys on SHAPE rather than cause, which is why it holds for all of
    them: a level that comes back to where it was did not get a new cartridge.

    ANY LENGTH, not just one reading. At the few-minute polling these items
    often use, a door left open or a supply out of the machine spans several
    polls, and a filter that only removed single readings left the original
    failure in place for every cartridge above NEW_CARTRIDGE_LEVEL.

    A genuine near-empty reading before a swap (30, 5, 100) does not recover -
    it comes back HIGHER than 30 - so it is kept and the swap still counts.
    """
    if len(points) < 3:
        return points
    out = [points[0]]
    index = 1
    while index < len(points) - 1:
        previous = points[index - 1][1]
        if (previous - points[index][1]) < rise:
            out.append(points[index])
            index += 1
            continue
        # A dip starts here. Take every consecutive reading that stays down.
        end = index
        while end < len(points) - 1 and (previous - points[end][1]) >= rise:
            end += 1
        following = points[end][1]
        # ONE bad sample is a candidate whatever the cadence: the reading is an
        # instant, and the gap to its neighbours says nothing about how long the
        # supply was actually out. SEVERAL consecutive low readings only count
        # as one outage if they are close together - spread over days they are a
        # real absence, and they also have the same shape as a swap, ordinary
        # consumption, then another swap.
        spanhours = (points[end][0] - points[index - 1][0]).total_seconds() / 3600
        brief = (end - index) == 1 or spanhours <= MAX_DIP_HOURS
        if (brief
                and (following - points[end - 1][1]) >= rise
                and _recovered(previous, following)):
            index = end          # drop the whole stretch
            continue
        for keep in range(index, end):
            out.append(points[keep])
        index = end
    out.append(points[-1])
    return out


def _is_replacement(previous, current, rise=REPLACEMENT_RISE,
                    newlevel=NEW_CARTRIDGE_LEVEL):
    """One definition of "a cartridge was changed", used by every caller.

    The rise must be big enough to clear gauge noise AND land near full, which
    is what a new cartridge reads. Both conditions, because either alone admits
    a shape that is not a swap.
    """
    return (current - previous) >= rise and current >= newlevel


def find_replacements(points, rise=REPLACEMENT_RISE,
                      newlevel=NEW_CARTRIDGE_LEVEL):
    """Timestamps where the level jumped up - one per cartridge change.

    Returns [] for a series that only falls.
    """
    replacements = []
    for (_, previous), (when, current) in zip(points, points[1:]):
        if _is_replacement(previous, current, rise, newlevel):
            replacements.append(when)
    return replacements


def current_run(points, rise=REPLACEMENT_RISE):
    """The readings since the last replacement.

    Fitting across a replacement averages a spent cartridge with a fresh one
    and produces a slope that describes neither.
    """
    if not points:
        return []
    start = 0
    for index in range(1, len(points)):
        if _is_replacement(points[index - 1][1], points[index][1], rise):
            start = index
    return points[start:]


def interval_rates(points):
    """Percent-per-day for each consecutive pair, falling intervals only.

    A rise inside a run is gauge noise (a swap would have ended the run), and
    a flat interval is real information - a cartridge that did not move - so it
    stays in at zero.
    """
    rates = []
    for (whenprev, prev), (when, current) in zip(points, points[1:]):
        days = (when - whenprev).total_seconds() / 86400
        if days <= 0:
            continue
        drop = prev - current
        if drop < 0:
            continue
        rates.append(drop / days)
    return rates


def _median(values):
    ordered = sorted(values)
    count = len(ordered)
    if not count:
        return None
    middle = count // 2
    if count % 2:
        return ordered[middle]
    return (ordered[middle - 1] + ordered[middle]) / 2


def burn_rate(points):
    """Percent consumed per day over these readings, or None.

    THE MEDIAN OF THE PER-INTERVAL RATES, not the slope between the first and
    last reading. Two endpoints cannot tell "steady" from "burst then stopped":
    a cartridge that lost 20 percent in two days and then barely moved for a
    month reads as 0.83 percent/day forever after, so the report keeps promising
    it will run out long after printing slowed. The burst is one interval among
    many to a median, and one of two points to a secant.

    None means "no honest estimate": too few readings, no elapsed time, or a
    drop too small to distinguish from a gauge that has not moved yet.
    """
    if len(points) < MIN_POINTS_FOR_ESTIMATE:
        return None
    total_days = (points[-1][0] - points[0][0]).total_seconds() / 86400
    if total_days < MIN_DAYS_FOR_ESTIMATE:
        return None
    # The overall drop still gates the estimate: a gauge sitting on one plateau
    # has not proved anything yet, whatever the intervals say.
    if points[0][1] - points[-1][1] < MIN_DROP_FOR_ESTIMATE:
        return None
    rate = _median(interval_rates(points))
    if not rate:
        # Every interval flat or rising, yet the run dropped overall - the
        # movement is all in intervals the median discarded. Fall back to the
        # whole-run slope rather than reporting nothing.
        return (points[0][1] - points[-1][1]) / total_days
    return rate


def rate_is_unstable(points, factor=RATE_SPREAD_FACTOR):
    """True when the intervals disagree enough that one number oversells it."""
    rates = [r for r in interval_rates(points) if r > 0]
    if len(rates) < 2:
        return False
    return max(rates) >= min(rates) * factor


def analyse(points, rise=REPLACEMENT_RISE, currentlevel=None):
    """Everything the report needs about one supply.

    `currentlevel` is the live reading from the printer, when the caller has
    one. It is what the days-left figure is computed against, because it is
    what the report displays: taking the level from the display and the
    countdown from the last stored history point lets the two disagree, and a
    row reading "20% - 4 days left" is read as a broken report, correctly.

    Returns:
      currentlevel     live reading if given, else the latest stored one
      daysleft         at the current rate, or None when there is no estimate
      burnrateperday   percent per day, or None
      reason           why there is no estimate - shown rather than hidden, so
                       a missing forecast is explained instead of looking broken
      replacements     how many times it has been changed in this window
      lastreplaced     when, or None
      basisdays        span of readings the estimate rests on
      points           the run since the last replacement, for the chart
    """
    points = normalise(points)
    result = {
        'currentlevel': currentlevel, 'daysleft': None, 'burnrateperday': None,
        'rateunstable': False,
        'reason': None, 'replacements': 0, 'lastreplaced': None,
        'basisdays': 0, 'points': [],
    }
    if not points:
        # A live level with no history is still worth acting on when it is low.
        if currentlevel is not None and currentlevel <= EMPTY_LEVEL:
            result['daysleft'] = 0
        else:
            result['reason'] = 'no history'
        return result

    replacements = find_replacements(points, rise=rise)
    result['replacements'] = len(replacements)
    result['lastreplaced'] = replacements[-1].isoformat() if replacements else None

    run = current_run(points, rise=rise)
    stored = run[-1][1]
    level = stored if currentlevel is None else currentlevel
    result['currentlevel'] = level
    result['points'] = [(when.isoformat(), level) for when, level in run]
    if len(run) >= 2:
        result['basisdays'] = round(
            (run[-1][0] - run[0][0]).total_seconds() / 86400, 1)

    # A live level well above the stored run means it was swapped since the
    # last stored reading. The run describes the cartridge that came out.
    if currentlevel is not None and currentlevel - stored >= rise:
        result['replacements'] += 1
        result['reason'] = 'replaced recently'
        return result

    rate = burn_rate(run)

    # Empty is empty, and only zero is empty. Ordering by a rate at this level
    # ranks a dead cartridge behind a healthy one that happens to drain faster.
    if level <= EMPTY_LEVEL:
        result['daysleft'] = 0
        result['burnrateperday'] = round(rate, 2) if rate is not None else None
        return result

    if rate is None:
        # Say which of the three it is; "no estimate" alone invites a bug report.
        basis = result.get('basisdays') or 0
        if len(run) < MIN_POINTS_FOR_ESTIMATE or basis < MIN_DAYS_FOR_ESTIMATE:
            result['reason'] = ('replaced recently' if replacements
                                else 'not enough history yet')
        else:
            result['reason'] = 'level has not moved enough to estimate'
        return result

    result['burnrateperday'] = round(rate, 2)
    # Fractional, NOT int(). Truncating meant every cartridge with less than a
    # full day left reported 0, which the bands read as empty and the view
    # printed as "empty" - a cartridge with six hours in it was indistinguishable
    # from one with nothing. Two decimal places is a quarter of an hour, which is
    # finer than the estimate deserves but costs nothing and keeps the ordering
    # of two nearly-spent cartridges meaningful.
    result['daysleft'] = round(max(0.0, level / rate), 2)
    # Say so when the intervals disagree wildly. The number is still the best
    # estimate available; the flag stops it reading as a measurement.
    result['rateunstable'] = rate_is_unstable(run)
    return result


# Urgency bands. The report groups by these and the order list is drawn from
# the first two, so they are defined once here rather than in the view - a
# heading that disagrees with what got added to the list is worse than either.
SOON_DAYS = 14
MONTH_DAYS = 30

# What goes on the order list. Two weeks is the horizon that survives a
# delivery: ordering only what is already empty means running empty.
ORDER_HORIZON_DAYS = SOON_DAYS

BANDS = ('empty', 'soon', 'month', 'later')


def band(daysleft):
    """Which urgency band a cartridge belongs in, or None with no estimate."""
    if daysleft is None:
        return None
    if daysleft <= 0:
        return 'empty'
    if daysleft <= SOON_DAYS:
        return 'soon'
    if daysleft <= MONTH_DAYS:
        return 'month'
    return 'later'


def orderlist(cartridges, horizon=ORDER_HORIZON_DAYS):
    """What to buy, grouped by part number: [{partnumber, quantity, ...}].

    The report's whole purpose reduced to a list someone can hand to
    purchasing. Two cartridges of the same part in different printers is a
    quantity of two, which is the number an order needs and the one a
    per-printer table makes you count by hand.

    A cartridge with no part mapped is still listed, under its printer's model
    and colour. Dropping it would quietly shorten the order.
    """
    groups = {}
    for cartridge in cartridges:
        if cartridge.get('daysleft') is None or cartridge['daysleft'] > horizon:
            continue
        parts = cartridge.get('partnumbers') or []
        # Several capacity tiers can match; the first is the standard one and
        # is what the low-supplies report shows first too.
        partnumber = parts[0]['partnumber'] if parts else None
        key = (partnumber, cartridge.get('color'), cartridge.get('model'))
        group = groups.setdefault(key, {
            'partnumber': partnumber,
            'color': cartridge.get('color'),
            'supplytype': cartridge.get('supplytype'),
            'model': cartridge.get('model'),
            'marketingname': parts[0].get('marketingname') if parts else None,
            'alternates': [p['partnumber'] for p in parts[1:]],
            'quantity': 0,
            'printers': [],
        })
        group['quantity'] += 1
        group['printers'].append({
            'printerid': cartridge.get('printerid'),
            'printername': cartridge.get('printername'),
            'daysleft': cartridge.get('daysleft'),
        })

    # Unmapped parts last: they need a decision before they can be ordered.
    return sorted(groups.values(),
                  key=lambda g: (g['partnumber'] is None,
                                 -g['quantity'],
                                 g['partnumber'] or ''))


def soonest(supplies):
    """Days-left of the supply that runs out first, or None if none estimate.

    The report sorts printers by this: a printer is as urgent as its most
    pressing cartridge, and listing it once per supply would scatter it down
    the page.
    """
    days = [s['daysleft'] for s in supplies if s.get('daysleft') is not None]
    return min(days) if days else None