Add two-stage run verification and 24-hour Activity filter
All checks were successful
Deploy / deploy (push) Successful in 57s
All checks were successful
Deploy / deploy (push) Successful in 57s
Status now reads "Sent" when the confirmation click lands, since that only means MassPrice accepted the run request, not that it finished — their own processing is async and takes anywhere from ~1 to ~22 minutes in observed practice. A new "Complete" column tracks that separately: run_rule() captures a watermark (the target rule's current top data-activitylogid on /Activity/List, not the displayed timestamp string, to sidestep fragile date/timezone parsing) before submitting, and scheduler.py schedules a one-off verify_run() job VERIFY_DELAY_MINUTES later to check whether a new "Complete" entry appeared past that watermark. rebuild_jobs() now only clears recurring "rule-*" jobs, not the one-off "verify-*" jobs, so editing a schedule can't cancel a verification that's already in flight for an earlier run. Also scopes the Activity page to the last 24 hours per request. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01H6TgffxTvYhqxykAbVQg2v
This commit is contained in:
parent
988db5d5c4
commit
8b58dadc5a
8 changed files with 249 additions and 20 deletions
61
scheduler.py
61
scheduler.py
|
|
@ -4,11 +4,17 @@ A single background worker thread owns every Playwright run — both
|
|||
APScheduler-triggered runs and manual "Run Now" clicks are pushed onto
|
||||
one queue.Queue so exactly one browser session is ever active at a time.
|
||||
This must run in exactly one process; see gunicorn.conf.py's workers=1.
|
||||
|
||||
The same worker also handles delayed "verify_run" tasks: after a rule
|
||||
run is confirmed sent, a one-off APScheduler job (not a recurring cron
|
||||
job) fires VERIFY_DELAY_MINUTES later and enqueues a check of whether it
|
||||
actually completed on MassPrice's side — see automation.verify_run().
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import pytz
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
|
@ -29,7 +35,7 @@ _work_queue: "queue.Queue" = queue.Queue()
|
|||
# wiring). Passing it to the scheduler alone is NOT enough — a
|
||||
# CronTrigger built without its own explicit timezone still resolves
|
||||
# get_localzone() itself, regardless of what the scheduler is configured
|
||||
# with, and crashes the same way. Every CronTrigger(...) below must pass
|
||||
# with, and crashes the same way. Every trigger built below must pass
|
||||
# timezone=_TZ explicitly.
|
||||
_TZ = pytz.timezone(os.environ.get("TZ", "UTC"))
|
||||
_scheduler = BackgroundScheduler(timezone=_TZ)
|
||||
|
|
@ -37,20 +43,54 @@ _started = False
|
|||
|
||||
VALID_DAYS = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"]
|
||||
|
||||
# How long to wait after a run is confirmed sent before checking whether
|
||||
# it actually completed on MassPrice's side. MassPrice's own processing
|
||||
# time varies a lot in practice (observed anywhere from ~1 to ~22
|
||||
# minutes) — this is a reasonable middle ground, not a guarantee; a
|
||||
# "not_confirmed" result can mean "still processing", not just "failed".
|
||||
VERIFY_DELAY_MINUTES = int(os.environ.get("VERIFY_DELAY_MINUTES", "10"))
|
||||
|
||||
|
||||
def _worker_loop():
|
||||
while True:
|
||||
rule_id, rule_name = _work_queue.get()
|
||||
item = _work_queue.get()
|
||||
kind = item[0]
|
||||
try:
|
||||
automation.run_rule(rule_id, rule_name)
|
||||
if kind == "run":
|
||||
_, rule_id, rule_name = item
|
||||
result = automation.run_rule(rule_id, rule_name)
|
||||
if result.get("will_verify"):
|
||||
schedule_verification(
|
||||
result["entry_id"], rule_name, result["watermark_activity_id"]
|
||||
)
|
||||
elif kind == "verify":
|
||||
_, entry_id, rule_name, watermark_id = item
|
||||
automation.verify_run(entry_id, rule_name, watermark_id)
|
||||
except Exception:
|
||||
logger.exception("Unhandled error running rule %s", rule_id)
|
||||
logger.exception("Unhandled error processing %r", item)
|
||||
finally:
|
||||
_work_queue.task_done()
|
||||
|
||||
|
||||
def enqueue_run(rule_id, rule_name=None):
|
||||
_work_queue.put((rule_id, rule_name))
|
||||
_work_queue.put(("run", rule_id, rule_name))
|
||||
|
||||
|
||||
def enqueue_verify(entry_id, rule_name, watermark_id):
|
||||
_work_queue.put(("verify", entry_id, rule_name, watermark_id))
|
||||
|
||||
|
||||
def schedule_verification(entry_id, rule_name, watermark_id, delay_minutes=None):
|
||||
delay = VERIFY_DELAY_MINUTES if delay_minutes is None else delay_minutes
|
||||
run_date = datetime.now(_TZ) + timedelta(minutes=delay)
|
||||
_scheduler.add_job(
|
||||
enqueue_verify,
|
||||
trigger="date",
|
||||
run_date=run_date,
|
||||
args=[entry_id, rule_name, watermark_id],
|
||||
id=f"verify-{entry_id}",
|
||||
misfire_grace_time=600,
|
||||
)
|
||||
|
||||
|
||||
def _job_id(rule_id, index):
|
||||
|
|
@ -58,9 +98,16 @@ def _job_id(rule_id, index):
|
|||
|
||||
|
||||
def rebuild_jobs():
|
||||
"""Clear and re-add every APScheduler job from schedules.json."""
|
||||
"""Clear and re-add every recurring cron job from schedules.json.
|
||||
|
||||
Only touches "rule-*" jobs — one-off "verify-*" jobs (see
|
||||
schedule_verification()) must survive a schedule edit elsewhere,
|
||||
since they're tracking an already-in-flight run, not a recurring
|
||||
schedule.
|
||||
"""
|
||||
for job in _scheduler.get_jobs():
|
||||
job.remove()
|
||||
if job.id.startswith("rule-"):
|
||||
job.remove()
|
||||
|
||||
schedules = storage.load("schedules.json", {})
|
||||
for rule_id, rule in schedules.items():
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue