#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
US food allergen disclosure sampler (Open Food Facts).

Measures how often US food products disclose wheat/gluten, to compare against
the equivalent measurement for drug labels.

METHOD NOTE THAT MATTERS
------------------------
Open Food Facts is crowd-sourced. A missing allergen tag can mean "the label
says nothing" OR "nobody has transcribed it yet." Those are completely
different facts and conflating them would wreck the comparison.

So the denominator is restricted to products that actually have transcribed
ingredient text, and the wheat/gluten determination is made by reading that
text directly rather than trusting the derived allergen tag. A product whose
ingredients list names wheat is counted as disclosing wheat regardless of
whether a contributor populated the allergens field.

This makes the measurement conservative in the right direction: it asks "when
we can see the label, does the label name the grain?"
"""
import json, re, sys, time, urllib.parse, urllib.request

UA = "gluten-research/1.0 (+https://glutenscreen.org)"
BASE = "https://world.openfoodfacts.org/api/v2/search"
FIELDS = "code,product_name,ingredients_text,allergens_tags,traces_tags,labels_tags"

# Grain terms that carry gluten, and the explicit-declaration markers FALCPA
# produces ("CONTAINS: WHEAT", "(wheat)").
GLUTEN_GRAIN = re.compile(
    r'\b(wheat|barley|rye|malt|triticale|semolina|durum|spelt|farro|kamut|einkorn|'
    r'graham|bulgur|couscous|seitan|brewer\'?s yeast)\b', re.I)
CONTAINS_STMT = re.compile(r'\bcontains\b[^.]{0,120}\b(wheat|barley|rye)\b', re.I)
GF_CLAIM = re.compile(r'gluten[\s-]?free', re.I)
# Source-ambiguous terms: the food analogue of "sodium starch glycolate".
AMBIGUOUS = re.compile(
    r'\b(modified food starch|food starch|starch|dextrin|maltodextrin|'
    r'natural flavor|artificial flavor|flavoring|caramel color|'
    r'hydrolyzed (?:vegetable |plant )?protein|yeast extract|vinegar)\b', re.I)
# ...but only ambiguous if the source is NOT named next to it.
SOURCE_NAMED = re.compile(
    r'\b(corn|maize|potato|tapioca|rice|pea|cassava|arrowroot|wheat)\b', re.I)


def fetch(page: int, page_size: int = 100) -> dict:
    qs = urllib.parse.urlencode({
        "countries_tags_en": "united-states",
        "page_size": page_size,
        "page": page,
        "fields": FIELDS,
    })
    req = urllib.request.Request(f"{BASE}?{qs}", headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=60) as r:
        return json.loads(r.read().decode())


def main(pages: int = 30):
    stats = {
        "sampled": 0, "with_ingredients": 0,
        "gluten_grain_named": 0, "contains_statement": 0,
        "gf_claim": 0, "ambiguous_term": 0,
        "ambiguous_unsourced": 0, "allergen_tag_populated": 0,
        "traces_populated": 0,
    }
    seen = set()
    for p in range(1, pages + 1):
        try:
            d = fetch(p)
        except Exception as e:
            print(f"  page {p}: {e}", file=sys.stderr)
            continue
        prods = d.get("products", [])
        if not prods:
            break
        for x in prods:
            code = x.get("code")
            if not code or code in seen:
                continue
            seen.add(code)
            stats["sampled"] += 1

            txt = (x.get("ingredients_text") or "").strip()
            if not txt:
                continue
            stats["with_ingredients"] += 1

            if x.get("allergens_tags"):
                stats["allergen_tag_populated"] += 1
            if x.get("traces_tags"):
                stats["traces_populated"] += 1
            if GLUTEN_GRAIN.search(txt):
                stats["gluten_grain_named"] += 1
            if CONTAINS_STMT.search(txt):
                stats["contains_statement"] += 1

            labels = " ".join(x.get("labels_tags") or [])
            if GF_CLAIM.search(txt) or GF_CLAIM.search(labels):
                stats["gf_claim"] += 1

            amb = AMBIGUOUS.findall(txt)
            if amb:
                stats["ambiguous_term"] += 1
                # unsourced if no botanical source word appears anywhere nearby
                if not SOURCE_NAMED.search(txt):
                    stats["ambiguous_unsourced"] += 1
        print(f"  page {p}: {stats['sampled']} sampled, "
              f"{stats['with_ingredients']} with ingredients", file=sys.stderr)
        time.sleep(0.4)

    n = max(1, stats["with_ingredients"])
    out = {
        "source": "Open Food Facts API v2, countries_tags_en=united-states",
        "harvested": time.strftime("%Y-%m-%d"),
        "us_products_in_database": None,
        "counts": stats,
        "rates_of_products_with_transcribed_ingredients": {
            "gluten_grain_named_pct": round(100 * stats["gluten_grain_named"] / n, 1),
            "explicit_contains_statement_pct": round(100 * stats["contains_statement"] / n, 1),
            "gluten_free_claim_pct": round(100 * stats["gf_claim"] / n, 1),
            "source_ambiguous_term_present_pct": round(100 * stats["ambiguous_term"] / n, 1),
            "ambiguous_and_no_source_named_pct": round(100 * stats["ambiguous_unsourced"] / n, 1),
            "allergen_field_populated_pct": round(100 * stats["allergen_tag_populated"] / n, 1),
        },
    }
    json.dump(out, sys.stdout, indent=2)
    print()


if __name__ == "__main__":
    main(int(sys.argv[1]) if len(sys.argv) > 1 else 30)