Shadow-mode client script and cutover doc correction

plugins/printers/services/zabbix_service.py

@@ -220,37 +220,119 @@ class ZabbixService:
             })
         return supplies
 
-    def gethistory(self, itemids, days=90, limit=5000):
+    # Per item, per call. Zabbix applies `limit` to the WHOLE result set, not
+    # per item, so the budget is multiplied by the number of items asked for.
+    HISTORY_PER_ITEM = 2000
+
+    def gethistory(self, itemids, days=90, peritem=None):
         """Raw level history for supply items: {itemid: [(clock, value), ...]}.
 
-        Zabbix keeps this already - we simply never asked for it. history=3 is
-        the unsigned-integer table, which is where a percent-remaining item
-        lands; a site that types the item as float would need history=0, so a
-        miss falls back rather than erroring.
+        Ordered oldest-first, which is what the analysis expects.
+
+        Two things about the Zabbix API make the obvious call return the wrong
+        rows, and both bit us:
+
+          * `limit` caps the whole answer, not each item. A four-cartridge
+            printer polled every five minutes writes over 100k rows in 90 days.
+          * combined with `sortorder: ASC`, the cap keeps the OLDEST rows - so a
+            "90 day" request came back holding the first few days of the window
+            and nothing since. Every rate was computed from history a quarter of
+            a year stale.
+
+        So: sort DESC to keep the NEWEST rows, scale the budget by item count,
+        and hand back ascending.
+
+        history=3 is the unsigned-integer table, where a percent-remaining item
+        lands; a site that types the item as float needs history=0. Items can be
+        split across the two, so both are asked and the results merged - an
+        earlier version stopped as soon as either answered, which silently lost
+        every float-typed supply on a printer that also had an integer one.
 
         Returns {} when Zabbix is off or unreachable, so callers degrade to
         "no estimate" instead of failing.
         """
         if not itemids:
             return {}
+        itemids = [str(i) for i in itemids]
+        peritem = peritem or self.HISTORY_PER_ITEM
         timefrom = int(time.time()) - days * 86400
         collected = {}
+
         for historytype in (3, 0):
+            outstanding = [i for i in itemids if i not in collected]
+            if not outstanding:
+                break
             rows = self._apicall("history.get", {
                 "output": "extend",
                 "history": historytype,
-                "itemids": list(itemids),
+                "itemids": outstanding,
                 "time_from": timefrom,
                 "sortfield": "clock",
-                "sortorder": "ASC",
-                "limit": limit,
+                "sortorder": "DESC",
+                "limit": peritem * len(outstanding),
             }) or []
             for row in rows:
                 collected.setdefault(str(row.get("itemid")), []).append(
                     (row.get("clock"), row.get("value")))
-            # Items live in one table or the other; stop once something answered.
-            if collected:
-                break
+
+        for itemid, points in collected.items():
+            points.reverse()          # DESC came back; analysis wants ASC
+        return collected
+
+    # Trends are hourly aggregates and are kept far longer than raw history.
+    # 90 days of raw 5-minute readings is 26k rows an item; the same window in
+    # trends is 2160. For a burn rate over months that is the right table -
+    # nothing is gained by fitting a line through every individual poll.
+    TRENDS_MIN_DAYS = 7
+    TRENDS_PER_ITEM = 3000
+
+    def gettrends(self, itemids, days=90, peritem=None):
+        """Hourly level averages: {itemid: [(clock, value), ...]}, oldest first.
+
+        Same `limit` caveat as gethistory - it caps the whole answer, so the
+        budget is scaled and the sort is DESC.
+
+        Returns {} if trends are not kept (a site can set the trend period to
+        0), which is a real configuration, not an error. Callers fall back to
+        raw history.
+        """
+        if not itemids:
+            return {}
+        itemids = [str(i) for i in itemids]
+        peritem = peritem or self.TRENDS_PER_ITEM
+        timefrom = int(time.time()) - days * 86400
+        rows = self._apicall("trend.get", {
+            "output": ["itemid", "clock", "value_avg"],
+            "itemids": itemids,
+            "time_from": timefrom,
+            "sortfield": "clock",
+            "sortorder": "DESC",
+            "limit": peritem * len(itemids),
+        }) or []
+
+        collected = {}
+        for row in rows:
+            collected.setdefault(str(row.get("itemid")), []).append(
+                (row.get("clock"), row.get("value_avg")))
+        for itemid, points in collected.items():
+            points.reverse()
+        return collected
+
+    def getlevelhistory(self, itemids, days=90):
+        """Level history for a forecast, from whichever table serves it best.
+
+        Trends for a long window, raw history for a short one, and raw history
+        for any item the trend table has nothing for.
+        """
+        if not itemids:
+            return {}
+        if days < self.TRENDS_MIN_DAYS:
+            return self.gethistory(itemids, days=days)
+
+        collected = self.gettrends(itemids, days=days)
+        missing = [i for i in (str(i) for i in itemids) if not collected.get(i)]
+        if missing:
+            collected.update(self.gethistory(missing, days=days))
         return collected
 
     def getpingstatus(self, ip: str) -> str: