#!/usr/bin/env python3 """One-off QA probe of the raw Open Food Facts export (read-only), used to design normalize_off.py rules. python3 -I probe_off_rules.py Collects, in one pass: - per category tag (keyword-matched): kcal percentiles of records that are in off.jsonl, and low/high examples - records dropped by data-quality tags: which tags, our own Atwater error, popularity - serving_quantity vs serving text contradictions, big servings, ml records above 845 kcal """ import collections import gzip import json import re import sys sys.path.insert(0, __import__("os").path.dirname(__import__("os").path.abspath(__file__))) import normalize_off as no # noqa: E402 KW = re.compile(r"(chees|fromage|queso|butter|beurre|oil|huile|peanut|nut-butter|nuts|almond|cashew|walnut|hazelnut|" r"pistach|chocolate|chocolat|cocoa|milk|lait|egg|oeuf|flour|farine|sugar|sucre|honey|miel|" r"ice-cream|popcorn|pizza|pancake|baking-mix|cake-mix|mixes|gravy|soup|broth|stock|spread|margarine|" r"mayonnaise|seeds|rice|pasta|cereal|oat|crisps|chips|bread|yogurt|yoghurt)") def pct(v, p): if not v: return None return v[min(len(v) - 1, int(p * len(v)))] def main(): src, offjl, out = sys.argv[1:4] kept = {} with open(offjl, encoding="utf-8") as f: for line in f: d = json.loads(line) kept[d["barcode"]] = (d["kcal"], d["unit"], d["pop"] or 0, d["name"], d["portions"]) print("kept", len(kept), file=sys.stderr) tagk = collections.defaultdict(list) # tag -> [(kcal, unit, pop, code, name)] dq = collections.Counter() dq_rows = [] big_serv = [] contra = [] ml845 = [] rows = 0 with gzip.open(src, "rb") as f: header = f.readline().decode("utf-8").rstrip("\r\n").split("\t") H = {c: i for i, c in enumerate(header)} no.H = H ncol = len(header) for raw in f: rows += 1 try: row = raw.decode("utf-8").rstrip("\n").rstrip("\r").split("\t") except UnicodeDecodeError: continue if len(row) != ncol: continue g = lambda c: no.unq(row[H[c]]) # noqa: E731 code = g("code").strip() if not code.isdigit(): continue bc = no.normalize_barcode(code) if bc is None: continue k = kept.get(bc) cats = g("categories_tags") ok, kcal = no.num(g("energy-kcal_100g")) okp, pop = no.num(g("unique_scans_n")) pop = pop or 0 name = no.clean_text(g("product_name"))[:80] if k is not None: for t in cats.split(","): if t and KW.search(t): tagk[t].append((k[0], k[1], int(k[2]), bc, k[3][:60])) sq_ok, sq = no.num(g("serving_quantity")) st = no.collapse(g("serving_size")) if sq_ok and sq and sq >= 1000: big_serv.append([bc, k[0], k[1], sq, st[:60], k[3][:60], int(k[2]), cats[:120]]) if sq_ok and sq and sq > 0: am = no.amounts(st) fam = [a for a in am if a[1] == k[1]] if fam: r = max(fam, key=lambda a: a[0])[0] lo = min(fam, key=lambda a: a[0])[0] if sq > 2 * r or sq < lo / 2: contra.append([bc, k[0], k[1], sq, st[:60], k[3][:60], int(k[2])]) if k[1] == "ml" and k[0] > 845: ml845.append([bc, k[0], k[3][:60], cats[:100]]) else: dqt = [t for t in g("data_quality_errors_tags").split(",") if t] bad = [t for t in dqt if no.DQ_DROP_RE.match(t) or no.DQ_DROP_RE2.match(t)] if bad and ok and kcal is not None and name: for t in bad: dq[t] += 1 vals = {} for c in ("fat_100g", "carbohydrates_100g", "proteins_100g", "fiber_100g", "alcohol_100g", "polyols_100g", "saturated-fat_100g", "sugars_100g"): vals[c] = no.num(g(c))[1] dq_rows.append([bc, kcal, int(pop), name[:50], vals, bad, no.collapse(g("serving_size"))[:40], g("countries_tags")[:40]]) res = {"rows": rows, "tags": {}, "dq_tag_counts": dq.most_common(), "dq_rows_n": len(dq_rows)} for t, v in sorted(tagk.items(), key=lambda kv: -len(kv[1])): if len(v) < 200: continue for unit in ("g", "ml"): vv = sorted(x[0] for x in v if x[1] == unit) if len(vv) < 50: continue lows = sorted([x for x in v if x[1] == unit], key=lambda x: (x[0], -x[2]))[:12] res["tags"][f"{t}|{unit}"] = {"n": len(vv), "p01": pct(vv, .01), "p05": pct(vv, .05), "p10": pct(vv, .1), "p50": pct(vv, .5), "p90": pct(vv, .9), "p99": pct(vv, .99), "low": [[x[0], x[2], x[3], x[4]] for x in lows]} dq_rows.sort(key=lambda x: -x[2]) res["dq_rows_top"] = dq_rows[:400] res["dq_rows_all_compact"] = [[x[0], x[1], x[2], x[4], [t[3:] for t in x[5]]] for x in dq_rows] big_serv.sort(key=lambda x: -x[6]) res["big_serving_n"] = len(big_serv) res["big_serving"] = big_serv[:3000] res["contra_n"] = len(contra) res["contra"] = sorted(contra, key=lambda x: -x[6])[:3000] res["ml845_n"] = len(ml845) res["ml845"] = ml845[:300] with open(out, "w", encoding="utf-8") as f: json.dump(res, f, ensure_ascii=False) print("done", rows, file=sys.stderr) if __name__ == "__main__": main()