#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Paired US/EU excipient disclosure harvester.

For molecules present in BOTH the EMA centrally-authorised register and
DailyMed, pull the US inactive-ingredient list (classCode="IACT") with its
UNII codes, so US and EU disclosure can be compared WITHIN molecule.

Why within-molecule matters: comparing all EU products against all US products
confounds regime with product mix (the EU central register skews toward
biologics and novel agents; DailyMed is dominated by small-molecule generics).
Pairing on the active ingredient removes that confound.

US SPLs carry FDA SRS UNII codes and controlled substance names; EU leaflets
carry free text. That asymmetry is itself a finding, not just a nuisance.
"""
import json, os, re, sys, time, urllib.parse, urllib.request
from concurrent.futures import ThreadPoolExecutor

UA = "gluten-screen-research/1.0 (+https://glutenscreen.org/)"
SEARCH = "https://dailymed.nlm.nih.gov/dailymed/services/v2/spls.json"
SPL = "https://dailymed.nlm.nih.gov/dailymed/services/v2/spls/{}.xml"
CACHE = "us_excip_cache.json"
IACT = re.compile(
    r'<ingredient classCode="IACT">\s*<ingredientSubstance>\s*'
    r'(?:<code code="([^"]*)"[^>]*/>\s*)?<name>(.*?)</name>', re.S)

cache = json.load(open(CACHE)) if os.path.exists(CACHE) else {}


def get(url, binary=False):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=45) as r:
        return r.read().decode(errors="replace")


def harvest(inn: str) -> dict:
    key = inn.lower().strip()
    if key in cache:
        return cache[key]
    out = {"inn": key, "ok": False, "setids": [], "excipients": [], "uniis": [],
           "n_labels": 0}
    try:
        d = json.loads(get(f"{SEARCH}?drug_name={urllib.parse.quote(key)}&pagesize=4"))
        rows = d.get("data", [])
        out["n_labels"] = len(rows)
        for row in rows[:3]:                       # up to 3 labels per molecule
            sid = row.get("setid")
            if not sid:
                continue
            try:
                xml = get(SPL.format(sid))
            except Exception:
                continue
            out["setids"].append(sid)
            for unii, name in IACT.findall(xml):
                nm = re.sub(r'\s+', ' ', name).strip()
                if nm:
                    out["excipients"].append(nm)
                    out["uniis"].append(unii or "")
            time.sleep(0.2)
        out["ok"] = bool(out["setids"])
    except Exception as e:
        out["error"] = str(e)[:120]
    cache[key] = out
    return out


if __name__ == "__main__":
    inns = json.load(open(sys.argv[1]))
    todo = [i for i in inns if i.lower().strip() not in cache]
    print(f"cached {len(inns)-len(todo)} | fetching {len(todo)}", file=sys.stderr)
    done = 0
    with ThreadPoolExecutor(max_workers=4) as ex:
        futs = []
        for i in todo:
            futs.append(ex.submit(harvest, i))
            time.sleep(0.3)
        for f in futs:
            f.result()
            done += 1
            if done % 20 == 0:
                print(f"  {done}/{len(todo)}", file=sys.stderr)
                json.dump(cache, open(CACHE, "w"))
    json.dump(cache, open(CACHE, "w"))
    print(f"cache {len(cache)}", file=sys.stderr)