#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
DailyMed harvester for the Gluten Screen research series.

One call per drug name against spls.json gives us three things at once:
  * existence      — does any current SPL match this name?
  * labelers       — the "[LABELER NAME]" bracket in each title
  * recency        — published_date per SPL

That is enough for the Plogsted decay study and the labeler-drift analysis
without pulling full SPL XML for every product (which would be ~10x the
traffic for data we do not need).

Politeness: 6 workers, 0.25s stagger, on-disk JSON cache so re-runs are free.
"""
import json, os, re, sys, time, urllib.parse, urllib.request
from concurrent.futures import ThreadPoolExecutor

BASE = "https://dailymed.nlm.nih.gov/dailymed/services/v2/spls.json"
UA = "gluten-screen-research/1.0 (+https://glutenscreen.org/; research contact via site)"
CACHE = "dm_cache.json"
LABELER_RE = re.compile(r'\[([^\]]+)\]\s*$')

cache = json.load(open(CACHE)) if os.path.exists(CACHE) else {}
lock_writes = []


def fetch(name: str) -> dict:
    key = name.lower().strip()
    if key in cache:
        return cache[key]
    url = f"{BASE}?drug_name={urllib.parse.quote(key)}&pagesize=25"
    out = {"query": key, "ok": False, "n": 0, "labelers": [], "titles": [], "dates": []}
    for attempt in range(3):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            with urllib.request.urlopen(req, timeout=30) as r:
                d = json.loads(r.read().decode())
            rows = d.get("data", [])
            out["ok"] = True
            out["n"] = len(rows)
            for row in rows:
                t = row.get("title", "")
                out["titles"].append(t)
                out["dates"].append(row.get("published_date", ""))
                m = LABELER_RE.search(t)
                if m:
                    out["labelers"].append(m.group(1).strip())
            break
        except Exception as e:
            out["error"] = str(e)[:120]
            time.sleep(1.5 * (attempt + 1))
    out["labelers"] = sorted(set(out["labelers"]))
    cache[key] = out
    return out


def run(names):
    todo = [n for n in names if n.lower().strip() not in cache]
    print(f"cached: {len(names)-len(todo)} | to fetch: {len(todo)}", file=sys.stderr)
    done = 0
    with ThreadPoolExecutor(max_workers=6) as ex:
        futs = []
        for n in todo:
            futs.append(ex.submit(fetch, n))
            time.sleep(0.25)
        for f in futs:
            f.result()
            done += 1
            if done % 50 == 0:
                print(f"  {done}/{len(todo)}", file=sys.stderr)
                json.dump(cache, open(CACHE, "w"))
    json.dump(cache, open(CACHE, "w"))
    print(f"cache size: {len(cache)}", file=sys.stderr)


if __name__ == "__main__":
    names = json.load(open(sys.argv[1]))
    run(names)