Poll for run completion every 5 minutes instead of checking once

The one-shot VERIFY_DELAY_MINUTES check was too rigid: MassPrice's
processing time varies widely in practice (observed ~1 to over 20
minutes, and its queue can back up further under load — confirmed
live, a run still sitting in "Queued" 15+ minutes in), so a single
check window either misses slow-but-fine completions or has to be set
needlessly long.

Replaced with a poll loop: scheduler.py reschedules another one-off
check every VERIFY_POLL_MINUTES (default 5) until automation's
check_completion() reports "complete", up to VERIFY_MAX_ATTEMPTS
(default 12, ~1 hour) before giving up and marking "not_confirmed" for
good. verify_run() is split into a pure check_completion() (no storage
side effects) so scheduler.py's loop owns the retry/give-up decision;
automation.py just answers "is it done yet".

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01H6TgffxTvYhqxykAbVQg2v

scheduler.py

@@ -5,10 +5,14 @@ APScheduler-triggered runs and manual "Run Now" clicks are pushed onto
 one queue.Queue so exactly one browser session is ever active at a time.
 This must run in exactly one process; see gunicorn.conf.py's workers=1.
 
-The same worker also handles delayed "verify_run" tasks: after a rule
-run is confirmed sent, a one-off APScheduler job (not a recurring cron
-job) fires VERIFY_DELAY_MINUTES later and enqueues a check of whether it
-actually completed on MassPrice's side — see automation.verify_run().
+The same worker also drives a poll loop that confirms a run actually
+completed on MassPrice's side: after a run is confirmed sent, a one-off
+APScheduler job (not a recurring cron job) fires VERIFY_POLL_MINUTES
+later and enqueues a check (automation.check_completion()). If it's not
+complete yet, another one-off job reschedules the next check the same
+interval later — up to VERIFY_MAX_ATTEMPTS times, after which it's
+marked "not_confirmed" for good (which can mean "still processing
+somewhere past our patience", not necessarily "failed").
 """
 import logging
 import os
@@ -43,12 +47,14 @@ _started = False
 
 VALID_DAYS = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"]
 
-# How long to wait after a run is confirmed sent before checking whether
-# it actually completed on MassPrice's side. MassPrice's own processing
-# time varies a lot in practice (observed anywhere from ~1 to ~22
-# minutes) — this is a reasonable middle ground, not a guarantee; a
-# "not_confirmed" result can mean "still processing", not just "failed".
-VERIFY_DELAY_MINUTES = int(os.environ.get("VERIFY_DELAY_MINUTES", "10"))
+# How often to re-check whether a sent run has completed on MassPrice's
+# side, and how many times to try before giving up. MassPrice's own
+# processing time varies a lot in practice (observed anywhere from ~1 to
+# over 20 minutes, and its queue can back up further under load) — 5
+# minutes x 12 attempts gives an hour of patience before a stuck/failed
+# run is reported as "not_confirmed" instead of polling forever.
+VERIFY_POLL_MINUTES = int(os.environ.get("VERIFY_POLL_MINUTES", "5"))
+VERIFY_MAX_ATTEMPTS = int(os.environ.get("VERIFY_MAX_ATTEMPTS", "12"))
 
 
 def _worker_loop():
@@ -64,31 +70,49 @@ def _worker_loop():
                         result["entry_id"], rule_name, result["watermark_activity_id"]
                     )
             elif kind == "verify":
-                _, entry_id, rule_name, watermark_id = item
-                automation.verify_run(entry_id, rule_name, watermark_id)
+                _, entry_id, rule_name, watermark_id, attempt = item
+                _run_verification_check(entry_id, rule_name, watermark_id, attempt)
         except Exception:
             logger.exception("Unhandled error processing %r", item)
         finally:
             _work_queue.task_done()
 
 
+def _run_verification_check(entry_id, rule_name, watermark_id, attempt):
+    result, message = automation.check_completion(rule_name, watermark_id)
+    if result == "complete":
+        automation.update_verification(entry_id, "complete", "")
+    elif attempt >= VERIFY_MAX_ATTEMPTS:
+        elapsed = attempt * VERIFY_POLL_MINUTES
+        automation.update_verification(
+            entry_id,
+            "not_confirmed",
+            f"Still not confirmed after {attempt} checks over ~{elapsed} minutes "
+            f"— last check: {message}",
+        )
+    else:
+        # Not done yet, but not giving up — keep "pending", just record
+        # the latest note, and queue the next check.
+        automation.set_verification_message(entry_id, message)
+        schedule_verification(entry_id, rule_name, watermark_id, attempt=attempt + 1)
+
+
 def enqueue_run(rule_id, rule_name=None):
     _work_queue.put(("run", rule_id, rule_name))
 
 
-def enqueue_verify(entry_id, rule_name, watermark_id):
-    _work_queue.put(("verify", entry_id, rule_name, watermark_id))
+def enqueue_verify(entry_id, rule_name, watermark_id, attempt):
+    _work_queue.put(("verify", entry_id, rule_name, watermark_id, attempt))
 
 
-def schedule_verification(entry_id, rule_name, watermark_id, delay_minutes=None):
-    delay = VERIFY_DELAY_MINUTES if delay_minutes is None else delay_minutes
-    run_date = datetime.now(_TZ) + timedelta(minutes=delay)
+def schedule_verification(entry_id, rule_name, watermark_id, attempt=1):
+    run_date = datetime.now(_TZ) + timedelta(minutes=VERIFY_POLL_MINUTES)
     _scheduler.add_job(
         enqueue_verify,
         trigger="date",
         run_date=run_date,
-        args=[entry_id, rule_name, watermark_id],
-        id=f"verify-{entry_id}",
+        args=[entry_id, rule_name, watermark_id, attempt],
+        id=f"verify-{entry_id}-{attempt}",
         misfire_grace_time=600,
     )
 
@@ -102,8 +126,8 @@ def rebuild_jobs():
 
     Only touches "rule-*" jobs — one-off "verify-*" jobs (see
     schedule_verification()) must survive a schedule edit elsewhere,
-    since they're tracking an already-in-flight run, not a recurring
-    schedule.
+    since they're tracking an already-in-flight run's poll loop, not a
+    recurring schedule.
     """
     for job in _scheduler.get_jobs():
         if job.id.startswith("rule-"):