Expire stored source photos after 7 days, keeping grades forever

Storing originals for Regrade grew the database ~16MB per grade at the
upload cap. Photos now expire on a configurable window (default 7 days,
counted from last grade so regrading resets it) and the file is VACUUMed so
the space is actually returned — clearing the column alone only moves pages
to the freelist, which would have made the whole feature a no-op on disk.
Grades, thumbnails and measurements are never pruned; a pruned card's
Regrade falls back to the existing re-pick path, now with a banner saying
why.

README.md

@@ -72,6 +72,33 @@ All your data lives in `grades.db` in this folder — your settings and your
 entire grading history, thumbnails included. Back up that one file and
 you've backed up everything.
 
+### What's kept, and for how long
+
+Grades are kept **forever**: the estimate, every category finding, the
+measurements, and the thumbnail. Nothing expires there.
+
+The **original photos** are a different matter. Each grade stores the
+photo(s) that produced it so **Regrade** can re-run without asking for them
+again — which is worth most right after a grading-logic change, and worth
+progressively less as time passes. They're also what makes the database
+grow: up to ~16MB per grade at the upload cap, so a few hundred grades
+would otherwise run into gigabytes.
+
+So stored photos expire after **7 days**, and the database is compacted to
+actually give the space back. Past that window a card's Regrade button asks
+you to pick the photo again instead of re-running instantly — and once you
+do, that card starts the clock over.
+
+Change the window with `CARD_GRADER_IMAGE_RETENTION_DAYS` (set it to `0` to
+keep photos forever, if disk isn't a concern):
+
+```bash
+CARD_GRADER_IMAGE_RETENTION_DAYS=30 python3 app.py
+```
+
+The countdown runs from when a card was last graded, not when it was first
+graded — so regrading a card keeps its photos for another full window.
+
 ### Optional packages
 
 Two packages unlock real functionality. Each is checked for at runtime, so

app.py

@@ -21,6 +21,7 @@ import io
 import json
 import os
 import sys
+import threading
 import time
 import urllib.parse
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -43,6 +44,11 @@ MAX_UPLOAD_BYTES = 12 * 1024 * 1024
 # ~4/3 that as base64, plus JSON overhead. Kept in step with nginx's
 # client_max_body_size in nginx/card-grader.conf.
 MAX_BODY_BYTES = 20 * 1024 * 1024
+# How often to expire stored source photos (see store.prune_source_images).
+# Six-hourly rather than daily so a long-running container doesn't hold a
+# week's worth of extra images for up to another day past the window, and
+# so the first sweep after a restart isn't the only one that ever runs.
+PRUNE_INTERVAL_SECONDS = 6 * 60 * 60
 # More angles help — front, back, corner close-ups — but past a handful the
 # extra photos cost tokens without adding evidence.
 MAX_GRADE_IMAGES = 6
@@ -490,6 +496,26 @@ class Server(ThreadingHTTPServer):
     daemon_threads = True
 
 
+def _prune_forever():
+    """Expire stored source photos on a timer, for the life of the process.
+
+    A daemon thread rather than a cron entry so the retention window holds
+    on any host this runs on, without a second thing to install and keep in
+    step. Never lets an exception end the loop: a failed sweep should cost
+    one cycle, not disable pruning until the next restart.
+    """
+    while True:
+        try:
+            pruned = store.prune_source_images()
+            if pruned:
+                print("[prune] dropped stored photos from {} grade(s) older "
+                      "than {} days".format(pruned, store.SOURCE_IMAGE_RETENTION_DAYS),
+                      flush=True)
+        except Exception as exc:
+            print("[prune] failed: {}".format(exc), flush=True)
+        time.sleep(PRUNE_INTERVAL_SECONDS)
+
+
 def main():
     store.init()
     try:
@@ -507,8 +533,17 @@ def main():
     if ip:
         print("  On your phone: http://{}:{}  (same Wi-Fi)".format(ip, PORT))
     print("  Database:     {}".format(store.DB_PATH))
+    if store.SOURCE_IMAGE_RETENTION_DAYS > 0:
+        print("  Photos kept:  {} days (grades kept forever)".format(
+            store.SOURCE_IMAGE_RETENTION_DAYS))
+    else:
+        print("  Photos kept:  forever (pruning disabled)")
     print("\n  Ctrl-C to stop.\n")
 
+    # Sweep once at boot so a container that restarts often still expires
+    # things, then hand off to the timer.
+    threading.Thread(target=_prune_forever, daemon=True).start()
+
     try:
         server.serve_forever()
     except KeyboardInterrupt:

docker-compose.yml

@@ -35,3 +35,10 @@ services:
       # This app is reverse-proxied at hippofam.com/cards, not the domain
       # root — see app.py's BASE_PATH handling.
       - CARD_GRADER_BASE_PATH=/cards
+      # How long a grade keeps the original photo(s) that produced it, so
+      # Regrade can re-run without asking for them again. Grades, their
+      # thumbnails and every measurement are kept FOREVER regardless — only
+      # the photos expire, since they're what makes the database grow (up
+      # to ~16MB per grade at the upload cap). Past the window, Regrade
+      # falls back to asking for the photo. 0 disables pruning.
+      - CARD_GRADER_IMAGE_RETENTION_DAYS=7

static/app.js

@@ -483,6 +483,10 @@ function startRegrade(id, hasImages) {
   if (hasImages) {
     runRegrade(id, {});
   } else {
+    // Say why the picker is opening. Otherwise a Regrade that silently
+    // asks for a photo looks broken — especially on a card that regraded
+    // instantly last week, before its stored photo aged out.
+    banner("This card's photo is no longer stored — pick it again to regrade.");
     regradeTargetId = id;
     $('#regrade-file').click();
   }

store.py

@@ -58,6 +58,20 @@ DEFAULT_SETTINGS = {
     "vision_effort": "low",
 }
 
+# How long a grade keeps the original photo(s) that produced it. They exist
+# so Regrade can re-run without asking for the photo again, which is worth
+# most right after a grading-logic change — a value that decays fast. What
+# doesn't decay is the grade record itself, so only the images are dropped
+# here; the history row, its thumbnail and every measurement stay forever.
+# A pruned card's Regrade falls back to asking for the photo, exactly as a
+# card graded before images were stored does.
+# Counted from created_at, which update_grade_result bumps — so a card you
+# regraded yesterday keeps its photos for another week, rather than being
+# pruned on the age of its first grading.
+# 0 or negative disables pruning entirely.
+SOURCE_IMAGE_RETENTION_DAYS = int(
+    os.environ.get("CARD_GRADER_IMAGE_RETENTION_DAYS", "7"))
+
 
 def connect():
     # timeout: how long to wait for a writer's lock before giving up. The
@@ -256,6 +270,46 @@ def update_grade_result(grade_id, grade, thumbnail=None, source_images=None):
     return get_grade(grade_id)
 
 
+def prune_source_images(days=None):
+    """Drop stored photos older than the retention window. Returns the count.
+
+    Only source_images_json is cleared — the grade, its thumbnail and its
+    measurements are untouched, so history stays complete and only the
+    expensive part expires.
+
+    VACUUM afterwards because clearing a column returns its pages to
+    SQLite's freelist without shrinking the file: without it the database
+    would keep every byte this is meant to reclaim, and the whole feature
+    would silently do nothing to disk usage. It rewrites the file, so it's
+    run only when something was actually pruned, and on its own connection
+    since VACUUM cannot execute inside a transaction.
+    """
+    days = SOURCE_IMAGE_RETENTION_DAYS if days is None else days
+    if days <= 0:
+        return 0
+    cutoff = time.strftime("%Y-%m-%dT%H:%M:%S",
+                           time.localtime(time.time() - days * 86400))
+    conn = connect()
+    try:
+        cur = conn.execute(
+            "UPDATE grades SET source_images_json = NULL "
+            "WHERE source_images_json IS NOT NULL AND created_at < ?",
+            (cutoff,),
+        )
+        pruned = cur.rowcount or 0
+        conn.commit()
+    finally:
+        conn.close()
+
+    if pruned:
+        vac = sqlite3.connect(DB_PATH, timeout=60.0, isolation_level=None)
+        try:
+            vac.execute("VACUUM")
+        finally:
+            vac.close()
+    return pruned
+
+
 def get_grade_images(grade_id):
     """The original (bytes, filename) pairs for Regrade, or None if this
     grade never had them stored (a card graded before this feature existed,