Initial commit: N64 Tracker source from server (excludes Coverart images)

scrape_prices.py

@@ -0,0 +1,163 @@
+import requests
+from bs4 import BeautifulSoup
+import json
+import re
+import time
+
+BASE_URL = "https://www.pricecharting.com/console/nintendo-64"
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Referer": "https://www.pricecharting.com/console/nintendo-64",
+}
+
+def parse_price(price_str):
+    if not price_str:
+        return None
+    cleaned = re.sub(r"[^\d.]", "", price_str.strip())
+    try:
+        return float(cleaned) if cleaned else None
+    except ValueError:
+        return None
+
+def parse_table(soup):
+    """Return list of (title, loose_price) tuples from games_table."""
+    table = soup.find("table", id="games_table")
+    if not table:
+        return []
+    tbody = table.find("tbody") or table
+    rows = tbody.find_all("tr")
+    games = []
+    for row in rows:
+        title_cell = row.find("td", class_=lambda c: c and "title" in c.split())
+        if not title_cell:
+            continue
+        a_tag = title_cell.find("a")
+        title = a_tag.get_text(strip=True) if a_tag else None
+        if not title:
+            continue
+        loose_cell = row.find("td", class_=lambda c: c and "used_price" in c.split())
+        if loose_cell:
+            span = loose_cell.find("span", class_="js-price")
+            price_text = span.get_text(strip=True) if span else loose_cell.get_text(strip=True)
+        else:
+            price_text = None
+        games.append((title, parse_price(price_text)))
+    return games
+
+def get_cursor_form(soup):
+    """Find the 'more results' form and extract its hidden fields + cursor value."""
+    for form in soup.find_all("form"):
+        submit = form.find("input", type="submit")
+        if submit and "more" in (submit.get("value") or "").lower():
+            fields = {}
+            for inp in form.find_all("input"):
+                name = inp.get("name")
+                value = inp.get("value", "")
+                if name:
+                    fields[name] = value
+            return fields
+    return None
+
+# ---- Initial GET ----
+print("=" * 60)
+print("N64 PRICE SCRAPER - PriceCharting.com")
+print("=" * 60)
+
+params = {"q": "", "sort": "name", "dir": "asc", "status": "", "minimum-rating": ""}
+print(f"\nFetching initial page...")
+r = requests.get(BASE_URL, params=params, headers=HEADERS, timeout=30)
+print(f"  Status: {r.status_code}  |  {len(r.text)} bytes")
+
+soup = BeautifulSoup(r.text, "html.parser")
+
+# Extract reported total
+total_hint = None
+for m in re.finditer(r"(\d+)\s*items", soup.get_text()):
+    total_hint = int(m.group(1))
+print(f"  Reported total items on page: {total_hint}")
+
+all_results = {}
+batch = parse_table(soup)
+for title, price in batch:
+    all_results[title] = price
+print(f"  Parsed {len(batch)} games from initial page | Running total: {len(all_results)}")
+
+# ---- Cursor-based pagination via POST ----
+fetch_num = 1
+while True:
+    cursor_fields = get_cursor_form(soup)
+    if not cursor_fields:
+        print("\n  No 'more results' form found - done paginating.")
+        break
+
+    cursor = cursor_fields.get("cursor", "?")
+    fetch_num += 1
+    print(f"\nFetching batch {fetch_num} (cursor={cursor})...")
+
+    # POST with form fields (same URL)
+    post_headers = {**HEADERS, "Content-Type": "application/x-www-form-urlencoded"}
+    r = requests.post(BASE_URL, data=cursor_fields, headers=post_headers, timeout=30)
+    print(f"  Status: {r.status_code}  |  {len(r.text)} bytes")
+
+    soup = BeautifulSoup(r.text, "html.parser")
+    batch = parse_table(soup)
+    if not batch:
+        print("  No games in this response - stopping.")
+        break
+
+    new_count = 0
+    for title, price in batch:
+        if title not in all_results:
+            all_results[title] = price
+            new_count += 1
+
+    print(f"  Parsed {len(batch)} games ({new_count} new) | Running total: {len(all_results)}")
+
+    if new_count == 0:
+        print("  No new games - stopping.")
+        break
+
+    time.sleep(0.5)
+
+# ---- Results ----
+print(f"\n{'=' * 60}")
+print(f"SCRAPE COMPLETE - {len(all_results)} unique games collected")
+print(f"{'=' * 60}\n")
+
+print("HTML STRUCTURE OF FIRST 3 ROWS (already shown above in detail)")
+print("Key structure confirmed:")
+print("  <tr data-product='ID'>")
+print("    <td class='title'><a href='...'>GAME NAME</a></td>")
+print("    <td class='price numeric used_price'><span class='js-price'>$XX.XX</span></td>")
+print("    <td class='price numeric cib_price'>...</td>")
+print("    <td class='price numeric new_price'>...</td>")
+print("  </tr>\n")
+
+print(f"{'=' * 60}")
+print("FIRST 20 RESULTS:")
+print(f"{'=' * 60}")
+for i, (title, price) in enumerate(list(all_results.items())[:20]):
+    price_str = f"${price:.2f}" if price is not None else "N/A"
+    print(f"{i+1:3}. {title:<55} {price_str}")
+
+# Save JSON
+output_path = "/tmp/pc_prices.json"
+with open(output_path, "w", encoding="utf-8") as f:
+    json.dump(all_results, f, indent=2, ensure_ascii=False)
+
+prices_only = [p for p in all_results.values() if p is not None]
+no_price = sum(1 for p in all_results.values() if p is None)
+
+print(f"\n{'=' * 60}")
+print("SUMMARY STATS")
+print(f"{'=' * 60}")
+print(f"  Total games saved:       {len(all_results)}")
+print(f"  Games with price data:   {len(prices_only)}")
+print(f"  Games with N/A price:    {no_price}")
+if prices_only:
+    print(f"  Cheapest game:           ${min(prices_only):.2f}")
+    print(f"  Most expensive game:     ${max(prices_only):.2f}")
+    print(f"  Average loose price:     ${sum(prices_only)/len(prices_only):.2f}")
+print(f"\nSaved to: {output_path}")