n64-tracker/scrape_prices.py

import requests
from bs4 import BeautifulSoup
import json
import re
import time

BASE_URL = "https://www.pricecharting.com/console/nintendo-64"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Referer": "https://www.pricecharting.com/console/nintendo-64",
}

def parse_price(price_str):
    if not price_str:
        return None
    cleaned = re.sub(r"[^\d.]", "", price_str.strip())
    try:
        return float(cleaned) if cleaned else None
    except ValueError:
        return None

def parse_table(soup):
    """Return list of (title, loose_price) tuples from games_table."""
    table = soup.find("table", id="games_table")
    if not table:
        return []
    tbody = table.find("tbody") or table
    rows = tbody.find_all("tr")
    games = []
    for row in rows:
        title_cell = row.find("td", class_=lambda c: c and "title" in c.split())
        if not title_cell:
            continue
        a_tag = title_cell.find("a")
        title = a_tag.get_text(strip=True) if a_tag else None
        if not title:
            continue
        loose_cell = row.find("td", class_=lambda c: c and "used_price" in c.split())
        if loose_cell:
            span = loose_cell.find("span", class_="js-price")
            price_text = span.get_text(strip=True) if span else loose_cell.get_text(strip=True)
        else:
            price_text = None
        games.append((title, parse_price(price_text)))
    return games

def get_cursor_form(soup):
    """Find the 'more results' form and extract its hidden fields + cursor value."""
    for form in soup.find_all("form"):
        submit = form.find("input", type="submit")
        if submit and "more" in (submit.get("value") or "").lower():
            fields = {}
            for inp in form.find_all("input"):
                name = inp.get("name")
                value = inp.get("value", "")
                if name:
                    fields[name] = value
            return fields
    return None

# ---- Initial GET ----
print("=" * 60)
print("N64 PRICE SCRAPER - PriceCharting.com")
print("=" * 60)

params = {"q": "", "sort": "name", "dir": "asc", "status": "", "minimum-rating": ""}
print(f"\nFetching initial page...")
r = requests.get(BASE_URL, params=params, headers=HEADERS, timeout=30)
print(f"  Status: {r.status_code}  |  {len(r.text)} bytes")

soup = BeautifulSoup(r.text, "html.parser")

# Extract reported total
total_hint = None
for m in re.finditer(r"(\d+)\s*items", soup.get_text()):
    total_hint = int(m.group(1))
print(f"  Reported total items on page: {total_hint}")

all_results = {}
batch = parse_table(soup)
for title, price in batch:
    all_results[title] = price
print(f"  Parsed {len(batch)} games from initial page | Running total: {len(all_results)}")

# ---- Cursor-based pagination via POST ----
fetch_num = 1
while True:
    cursor_fields = get_cursor_form(soup)
    if not cursor_fields:
        print("\n  No 'more results' form found - done paginating.")
        break

    cursor = cursor_fields.get("cursor", "?")
    fetch_num += 1
    print(f"\nFetching batch {fetch_num} (cursor={cursor})...")

    # POST with form fields (same URL)
    post_headers = {**HEADERS, "Content-Type": "application/x-www-form-urlencoded"}
    r = requests.post(BASE_URL, data=cursor_fields, headers=post_headers, timeout=30)
    print(f"  Status: {r.status_code}  |  {len(r.text)} bytes")

    soup = BeautifulSoup(r.text, "html.parser")
    batch = parse_table(soup)
    if not batch:
        print("  No games in this response - stopping.")
        break

    new_count = 0
    for title, price in batch:
        if title not in all_results:
            all_results[title] = price
            new_count += 1

    print(f"  Parsed {len(batch)} games ({new_count} new) | Running total: {len(all_results)}")

    if new_count == 0:
        print("  No new games - stopping.")
        break

    time.sleep(0.5)

# ---- Results ----
print(f"\n{'=' * 60}")
print(f"SCRAPE COMPLETE - {len(all_results)} unique games collected")
print(f"{'=' * 60}\n")

print("HTML STRUCTURE OF FIRST 3 ROWS (already shown above in detail)")
print("Key structure confirmed:")
print("  <tr data-product='ID'>")
print("    <td class='title'><a href='...'>GAME NAME</a></td>")
print("    <td class='price numeric used_price'><span class='js-price'>$XX.XX</span></td>")
print("    <td class='price numeric cib_price'>...</td>")
print("    <td class='price numeric new_price'>...</td>")
print("  </tr>\n")

print(f"{'=' * 60}")
print("FIRST 20 RESULTS:")
print(f"{'=' * 60}")
for i, (title, price) in enumerate(list(all_results.items())[:20]):
    price_str = f"${price:.2f}" if price is not None else "N/A"
    print(f"{i+1:3}. {title:<55} {price_str}")

# Save JSON
output_path = "/tmp/pc_prices.json"
with open(output_path, "w", encoding="utf-8") as f:
    json.dump(all_results, f, indent=2, ensure_ascii=False)

prices_only = [p for p in all_results.values() if p is not None]
no_price = sum(1 for p in all_results.values() if p is None)

print(f"\n{'=' * 60}")
print("SUMMARY STATS")
print(f"{'=' * 60}")
print(f"  Total games saved:       {len(all_results)}")
print(f"  Games with price data:   {len(prices_only)}")
print(f"  Games with N/A price:    {no_price}")
if prices_only:
    print(f"  Cheapest game:           ${min(prices_only):.2f}")
    print(f"  Most expensive game:     ${max(prices_only):.2f}")
    print(f"  Average loose price:     ${sum(prices_only)/len(prices_only):.2f}")
print(f"\nSaved to: {output_path}")