#!/usr/bin/env python3 """One-off QA probe of USDA Branded serving sizes and category energy densities (read-only). python3 -I probe_usda_branded.py """ import collections import csv import json import os import re import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import normalize_usda as nu # noqa: E402 csv.field_size_limit(1 << 30) def pct(v, p): return v[min(len(v) - 1, int(p * len(v)))] if v else None def main(): bf, bjl, out = sys.argv[1:4] recs = {} with open(bjl, encoding="utf-8") as f: for line in f: d = json.loads(line) recs[d["id"][2:]] = d res = collections.defaultdict(list) cnt = collections.Counter() cat_k = collections.defaultdict(list) pkg_fmt = collections.Counter() with open(bf, encoding="utf-8", newline="") as fh: for r in csv.DictReader(fh): d = recs.get(r["fdc_id"]) if d is None: continue cat_k[(d["category"], d["unit"])].append((d["kcal"], d["id"], d["name"][:50])) pw = r["package_weight"].strip() pkg_fmt[re.sub(r"[\d.]+", "N", pw)[:30]] += 1 info = nu.serving_problems(r["serving_size"], r["serving_size_unit"], r["household_serving_fulltext"], pw, d["unit"], d["name"]) for k in info: cnt[k] += 1 if len(res[k]) < 60: res[k].append([d["id"], d["kcal"], d["unit"], r["serving_size"], r["household_serving_fulltext"][:50], pw, d["name"][:50], d["category"][:30]]) cats = {} for (c, u), v in sorted(cat_k.items(), key=lambda kv: -len(kv[1])): if len(v) < 300: continue vv = sorted(v) cats[f"{c}|{u}"] = {"n": len(v), "p01": pct(vv, .01)[0], "p02": pct(vv, .02)[0], "p05": pct(vv, .05)[0], "p50": pct(vv, .5)[0], "p99": pct(vv, .99)[0], "low": vv[:15]} with open(out, "w", encoding="utf-8") as f: json.dump({"counts": cnt.most_common(), "examples": res, "cats": cats, "pkg_fmt": pkg_fmt.most_common(80)}, f, ensure_ascii=False, indent=0) if __name__ == "__main__": main()