#!/usr/bin/env python3
"""Download every bulk dataset the toolchain needs. Run once per machine.

    python3 tools/fetch_data.py            # ~1.2GB, skips what already exists
    python3 tools/fetch_data.py --force    # re-download everything

Scryfall bulk URLs are timestamped and change, so they're resolved live from
the API rather than hardcoded. Note the field is `jsonl_download_uri` -- Scryfall
serves JSONL now, not the older JSON array format.
"""
import sys, os, json, gzip, shutil, urllib.request

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA = os.path.join(ROOT, "data")
UA = {"User-Agent": "reaper-king-deckbuilder/1.0", "Accept": "*/*"}

STATIC = [
    ("variants.json",
     "https://json.commanderspellbook.com/variants.json",
     "Commander Spellbook: every combo (~108k)"),
    ("AtomicCards.json",
     "https://mtgjson.com/api/v5/AtomicCards.json.gz",
     "MTGJSON: edhrecRank per card"),
]
SCRYFALL = [("oracle_cards", "oracle-cards.jsonl", "every unique card"),
            ("oracle_tags", "oracle-tags.jsonl", "4,369 community tags"),
            ("rulings", "rulings.jsonl", "all rulings")]


def get(url, dest, force=False):
    if os.path.exists(dest) and not force:
        print(f"  have  {os.path.basename(dest)}")
        return False
    tmp = dest + ".part"
    print(f"  get   {os.path.basename(dest)} ...", end="", flush=True)
    req = urllib.request.Request(url, headers=UA)
    with urllib.request.urlopen(req) as r, open(tmp, "wb") as f:
        shutil.copyfileobj(r, f)
    if url.endswith(".gz"):
        with gzip.open(tmp) as g, open(dest, "wb") as f:
            shutil.copyfileobj(g, f)
        os.remove(tmp)
    else:
        os.replace(tmp, dest)
    print(f" {os.path.getsize(dest)/1048576:.0f}MB")
    return True


def main():
    force = "--force" in sys.argv
    os.makedirs(DATA, exist_ok=True)
    print("Scryfall bulk (URLs resolved live):")
    meta = json.load(urllib.request.urlopen(
        urllib.request.Request("https://api.scryfall.com/bulk-data", headers=UA)))
    by_type = {b["type"]: b for b in meta["data"]}
    for typ, fname, _ in SCRYFALL:
        b = by_type.get(typ)
        if not b:
            print(f"  MISSING bulk type {typ}"); continue
        get(b["jsonl_download_uri"], os.path.join(DATA, fname), force)

    print("other sources:")
    for fname, url, _ in STATIC:
        get(url, os.path.join(DATA, fname), force)

    emb = os.path.join(DATA, "embeddings.npz")
    print("embeddings:")
    if os.path.exists(emb) and not force:
        print("  have  embeddings.npz")
    else:
        print("  run:  python3 tools/embed.py   (needs a GPU or ~20min on CPU)")

    print(f"\ndata/ is {sum(os.path.getsize(os.path.join(DATA,f)) for f in os.listdir(DATA))/1073741824:.1f}GB")


if __name__ == "__main__":
    main()