#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Gluten Screen — EU excipient census.
Reproduces every figure cited in the accompanying paper from ema_data.json.

Usage:  python3 analysis.py > results.json

Design notes
------------
The normalisation function is deliberately conservative. It strips
parentheticals, hydration-state qualifiers and E-numbers, then sorts tokens so
that "iron oxide yellow" and "yellow iron oxide" collapse to one key. It does
NOT attempt synonym resolution across chemically distinct names (e.g.
"hypromellose" vs "hydroxypropyl methylcellulose"), so the fragmentation figure
reported here is a LOWER BOUND on true naming inconsistency.
"""
import json, re, collections, sys

SRC = ('maize', 'corn', 'potato', 'wheat', 'rice', 'tapioca', 'pea')
STARCH_RE = re.compile(r'starch|amylum|dextrin|dextrate', re.I)
# 'malt' is excluded from the gluten regex because 'isomalt' — a sugar alcohol
# with no gluten relationship — otherwise dominates the matches.
GLUTEN_RE = re.compile(r'gluten|wheat|triticum|barley|hordeum|\brye\b|\bmalt\b(?!o)', re.I)


def norm(s: str) -> str:
    s = s.lower().strip()
    s = re.sub(r'\(.*?\)', '', s)
    s = re.sub(r'\b(for ph adjustment|e\s?\d{3,4}|anhydrous|monohydrate|dihydrate)\b', '', s)
    s = re.sub(r'[^a-z ]', ' ', s)
    return ' '.join(sorted(t for t in s.split() if len(t) > 2))


def main():
    data = json.load(open('ema_data.json'))
    rec, stats = data['records'], data['stats']

    out = {
        'corpus': {
            'source': 'EMA centrally-authorised medicines, EPAR product information leaflets',
            'indexed': stats['total_indexed'],
            'parsed': stats['by_status']['done'],
            'parse_errors': stats['by_status']['error'],
            'not_found': stats['by_status']['not_found'],
            'extraction_date': data.get('generated_at'),
        }
    }

    # ---- excipient vocabulary -------------------------------------------
    raw = collections.Counter()
    nrm = collections.Counter()
    groups = collections.defaultdict(set)
    per_product = []
    for k, v in rec.items():
        xs = [x.strip() for x in v.get('excipients', []) if x and len(x.strip()) > 2]
        per_product.append(len(xs))
        for x in xs:
            raw[x.lower()] += 1
            n = norm(x)
            if n:
                nrm[n] += 1
                groups[n].add(x.lower())

    n_products = len(rec)
    out['vocabulary'] = {
        'total_excipient_mentions': sum(raw.values()),
        'distinct_raw_strings': len(raw),
        'distinct_after_normalisation': len(nrm),
        'fragmentation_rate_pct': round(100 * (1 - len(nrm) / len(raw)), 1),
        'substances_with_multiple_spellings': sum(1 for v in groups.values() if len(v) > 1),
        'mean_excipients_per_product': round(sum(per_product) / n_products, 1),
        'products_with_zero_parsed_excipients': sum(1 for p in per_product if p == 0),
    }
    out['top_excipients'] = [
        {'name': x, 'products': n, 'pct_of_corpus': round(100 * n / n_products, 1)}
        for x, n in raw.most_common(25)
    ]

    # ---- starch census ---------------------------------------------------
    starch_str = collections.Counter()
    starch_prod = collections.defaultdict(set)
    for k, v in rec.items():
        for x in v.get('excipients', []):
            xl = x.lower().strip()
            if STARCH_RE.search(xl):
                starch_str[xl] += 1
                starch_prod[xl].add(k)

    all_starch = set().union(*starch_prod.values()) if starch_prod else set()
    sourced = set()
    for x, ps in starch_prod.items():
        if any(s in x for s in SRC):
            sourced |= ps
    unsourced = all_starch - sourced

    out['starch'] = {
        'products_with_any_starch_or_dextrin': len(all_starch),
        'pct_of_corpus': round(100 * len(all_starch) / n_products, 1),
        'distinct_starch_strings': len(starch_str),
        'products_naming_botanical_source': len(sourced),
        'products_with_source_undisclosed': len(unsourced),
        'source_disclosure_rate_pct': round(100 * len(sourced) / len(all_starch), 1),
        'by_source': {
            s: len(set().union(*[starch_prod[x] for x in starch_str if s in x]) or set())
            for s in SRC
            if any(s in x for x in starch_str)
        },
    }

    # ---- wheat / gluten declarations -------------------------------------
    wheat_decl, gf_annot = [], []
    for k, v in rec.items():
        for x in v.get('excipients', []):
            if re.search(r'wheat|triticum', x, re.I):
                wheat_decl.append({'product': v['name'], 'text': x})
            if re.search(r'gluten', x, re.I):
                gf_annot.append({'product': v['name'], 'text': x.strip()})

    out['gluten_declarations'] = {
        'products_declaring_wheat_starch': len(wheat_decl),
        'products_voluntarily_annotating_gluten_free': len(gf_annot),
        'pct_of_starch_products_annotated': round(100 * len(gf_annot) / len(all_starch), 1),
        'annotated_products': gf_annot,
    }

    # ---- the single most source-ambiguous excipient ------------------------
    ssg = {x: n for x, n in starch_str.items() if 'starch glycol' in x}
    ssg_prod = set().union(*[starch_prod[x] for x in ssg]) if ssg else set()
    ssg_sourced = {x for x in ssg if any(s in x for s in SRC)}
    out['sodium_starch_glycolate'] = {
        'products': len(ssg_prod),
        'distinct_spellings': len(ssg),
        'spellings_naming_a_source': len(ssg_sourced),
    }

    json.dump(out, sys.stdout, indent=2, ensure_ascii=False)
    print()


if __name__ == '__main__':
    main()