#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Plogsted list decay analysis.

Compares glutenfreedrugs.com (structured list, last updated 2019-04-23) against
current DailyMed SPLs harvested 2026-08-02 via dm_fetch.py.

Three measures:
  1. Name resolvability  — does the entry's drug name find any current US label?
  2. List data quality   — how many failures are typos/formatting in the list
                           itself rather than genuine product absence?
  3. Attribution drift   — is the manufacturer the list names still among the
                           current labelers for that product?

METHODOLOGICAL NOTE ON (3): the DailyMed query returns at most 25 SPLs. For a
generic with dozens of labelers the set is truncated, and absence from a
truncated page proves nothing. Drift is therefore computed ONLY over entries
where the result set is complete (n < 25). This is the single most important
guard in this script; removing it inflates the drift figure by ~10 points.

SECOND CAVEAT: DailyMed labelers include repackagers (Aphena, Physicians Total
Care, etc.), which are legitimate labelers a patient will encounter but are not
original manufacturers. This analysis therefore measures ATTRIBUTION STALENESS
— whether the contact the list points you to is still associated with the
current label set — not corporate ownership per se.
"""
import json, re, csv, sys, collections

TYPO = {'buproprion': 'bupropion', 'ipratroprium': 'ipratropium',
        'tabslets': 'tablets', 'forfevo': 'fortevo',
        'quillivant xr powder for': 'quillivant xr'}

CORP = re.compile(r'\b(inc|llc|ltd|corp|corporation|company|co|plc|usa|us|america|'
                  r'american|pharmaceuticals?|pharma|laboratories|labs?|healthcare|'
                  r'health|group|holdings?|gmbh|sa|ag|nv|bv|limited)\b')


def canon(s: str) -> str:
    s = CORP.sub('', s.lower())
    return ' '.join(re.sub(r'[^a-z ]', ' ', s).split())


def variants(n: str) -> set:
    s = n.lower()
    for a, b in TYPO.items():
        s = s.replace(a, b)
    s = re.sub(r'\(.*?\)', '', s)
    s = re.sub(r'\b(all strengths?|all types?|tabs?|tablets?|caplet|liquid|powder for)\b', '', s)
    s = re.sub(r'\s+', ' ', re.sub(r'[-/]', ' ', s)).strip()
    v = {s}
    if s.split():
        v.add(s.split()[0])
    if ' ' in s:
        v.add(' '.join(s.split()[:2]))
    return {x for x in v if len(x) > 2}


def main():
    g = json.load(open('gfd_data.json'))
    dm = json.load(open('dm_cache.json'))

    byname = collections.OrderedDict()
    for d in g['drugs']:
        byname.setdefault(d['name'].strip().lower(), []).append(d)

    direct, recovered, absent = [], {}, []
    for n in byname:
        r = dm.get(n)
        if not (r and r.get('ok')):
            continue
        if r['n'] > 0:
            direct.append(n)
            continue
        hit = next((v for v in variants(n)
                    if dm.get(v, {}).get('ok') and dm[v]['n'] > 0), None)
        if hit:
            recovered[n] = hit
        else:
            absent.append(n)

    tot = len(byname)

    # attribution drift, complete result sets only
    seen, match, drift, drift_rows = set(), 0, 0, []
    for d in g['drugs']:
        n = d['name'].strip().lower()
        mfr = (d.get('manufacturer') or '').strip()
        r = dm.get(n)
        if not (r and r.get('ok') and 0 < r['n'] < 25 and mfr):
            continue
        key = (n, mfr.lower())
        if key in seen:
            continue
        seen.add(key)
        cm, cur = canon(mfr), [canon(x) for x in r['labelers']]
        if any(cm and (cm in c or c in cm) for c in cur if c):
            match += 1
        else:
            drift += 1
            drift_rows.append({'drug': d['name'], 'listed_manufacturer': mfr,
                               'current_labelers': '; '.join(r['labelers']),
                               'n_current_spls': r['n']})

    dtot = match + drift
    out = {
        'source_list': {
            'name': 'glutenfreedrugs.com structured list (Plogsted)',
            'last_updated': g['source_updated_structured'],
            'alpha_list_last_updated': g['source_updated_alpha'],
            'entries': len(g['drugs']),
            'distinct_drug_names': tot,
        },
        'comparator': {'source': 'DailyMed SPL API', 'harvested': '2026-08-02'},
        'name_resolution': {
            'resolves_as_written': len(direct),
            'resolves_as_written_pct': round(100 * len(direct) / tot, 1),
            'resolves_only_after_correcting_list': len(recovered),
            'resolves_only_after_correcting_list_pct': round(100 * len(recovered) / tot, 1),
            'no_current_us_label': len(absent),
            'no_current_us_label_pct': round(100 * len(absent) / tot, 1),
            'fails_naive_lookup_pct': round(100 * (len(recovered) + len(absent)) / tot, 1),
        },
        'attribution_drift': {
            'assessable_pairs': dtot,
            'manufacturer_still_current': match,
            'manufacturer_still_current_pct': round(100 * match / dtot, 1),
            'manufacturer_no_longer_listed': drift,
            'manufacturer_no_longer_listed_pct': round(100 * drift / dtot, 1),
        },
        'absent_products': sorted(byname[n][0]['name'] for n in absent),
        'list_errors': {byname[n][0]['name']: v for n, v in sorted(recovered.items())},
    }

    with open('plogsted-drift-2026.csv', 'w', newline='', encoding='utf-8') as f:
        w = csv.DictWriter(f, fieldnames=['drug', 'listed_manufacturer',
                                          'current_labelers', 'n_current_spls'])
        w.writeheader()
        w.writerows(sorted(drift_rows, key=lambda r: r['drug']))

    json.dump(out, sys.stdout, indent=2, ensure_ascii=False)
    print()


if __name__ == '__main__':
    main()