#!/usr/bin/env python3 """One-off probe before writing build_index.py: B/O barcode overlap, kcal agreement, OFF popularity spread. Run: python3 -I FoodIndex/scripts/probe_merge.py Read-only; prints a JSON summary. """ import collections import json import math import os ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) NORM = os.path.join(ROOT, "normalized") def upce_to_upca(code): """Same as the app's BarcodeScan.expandUPCE for the 8-digit form; None when not a valid UPC-E.""" if len(code) != 8 or not code.isdigit() or code[0] not in "01": return None d = [int(c) for c in code] s, b, chk = d[0], d[1:7], d[7] d1, d2, d3, d4, d5, d6 = b if d6 in (0, 1, 2): mid = [d1, d2, d6, 0, 0, 0, 0, d3, d4, d5] elif d6 == 3: mid = [d1, d2, d3, 0, 0, 0, 0, 0, d4, d5] elif d6 == 4: mid = [d1, d2, d3, d4, 0, 0, 0, 0, 0, d5] else: mid = [d1, d2, d3, d4, d5, 0, 0, 0, 0, d6] a = [s] + mid tot = sum(x * 3 if i % 2 == 0 else x for i, x in enumerate(a)) c = (10 - tot % 10) % 10 if c != chk: return None return "".join(map(str, a + [c])) def ean8_ok(code): d = [int(c) for c in code] tot = sum(x * 3 if i % 2 == 0 else x for i, x in enumerate(d[:7])) return (10 - tot % 10) % 10 == d[7] def main(): flags = {} with open(os.path.join(ROOT, "qa", "usda_atwater_flags.jsonl"), encoding="utf-8") as f: for line in f: d = json.loads(line) flags[d["id"]] = d b = {} with open(os.path.join(NORM, "usda_branded.jsonl"), encoding="utf-8") as f: for line in f: d = json.loads(line) if d["barcode"]: b[d["barcode"]] = (d["id"], d["kcal"], d["unit"], d["name"], d["brand"]) out = collections.Counter() pops = collections.Counter() examples = collections.defaultdict(list) upce_only = upce_both = 0 upce_hits_b = 0 with open(os.path.join(NORM, "off.jsonl"), encoding="utf-8") as f: for line in f: d = json.loads(line) p = d["pop"] pops["none" if p is None else int(math.log10(1 + p))] += 1 if p is not None and p > out["max_pop"]: out["max_pop"] = p bc = d["barcode"] if len(bc) == 8: up = upce_to_upca(bc) if up: if ean8_ok(bc): upce_both += 1 else: upce_only += 1 if "0" + up in b: upce_hits_b += 1 t = b.get(bc) if not t: continue out["overlap"] += 1 bid, bk, bu, bn, bb = t ok = d["kcal"] if bu != d["unit"]: out["overlap_unit_differs"] += 1 tol = max(25.0, 0.35 * max(bk, ok)) else: tol = max(20.0, 0.30 * max(bk, ok)) if abs(bk - ok) > tol: key = "conflict_" + ("flagged" if bid in flags else "unflagged") out[key] += 1 if len(examples[key]) < 12: examples[key].append([bid, bn, bb, bk, bu, d["id"], d["name"], ok, d["unit"], d["pop"]]) elif abs(bk - ok) > max(5.0, 0.10 * max(bk, ok)): out["mild_10pct"] += 1 out["upce_only"] = upce_only out["upce_and_ean8"] = upce_both out["upce_expansion_in_b"] = upce_hits_b print(json.dumps({"counts": out, "pop_log10_hist": {str(k): v for k, v in sorted(pops.items(), key=str)}, "examples": examples}, ensure_ascii=False, indent=1)) if __name__ == "__main__": main()