"""Validate normalized/usda_*.jsonl against the field rules in CONTRACT.md (stdlib only).""" import json, os, re, sys, collections root = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.dirname(os.path.abspath(__file__))) KEYS = ["id", "name", "brand", "kcal", "unit", "portions", "barcode", "src", "generic", "pop", "date", "countries", "lang", "category"] ids = set(); bcs = collections.Counter(); per = collections.Counter(); probs = collections.Counter(); ex = collections.defaultdict(list) def bad(k, r): probs[k] += 1 if len(ex[k]) < 3: ex[k].append(r["id"] if isinstance(r, dict) else r) for fn, gen in (("usda_generic.jsonl", True), ("usda_branded.jsonl", False)): path = os.path.join(root, "normalized", fn) with open(path, "rb") as fh: raw = fh.read() if raw.startswith(b"\xef\xbb\xbf"): bad("bom", fn) for line in raw.decode("utf-8").splitlines(): r = json.loads(line) if list(r) != KEYS: bad("keys", r) per[r["src"]] += 1 if r["id"] in ids: bad("dup_id", r) ids.add(r["id"]) if not re.fullmatch(r"[FSNB]:\d+", r["id"]) or r["id"][0] != r["src"]: bad("id_format", r) if r["generic"] is not gen or (gen and r["brand"] != "") or (gen and r["barcode"]): bad("generic_flag", r) if not (2 <= len(r["name"]) <= 120) or r["name"] != " ".join(r["name"].split()): bad("name", r) if not isinstance(r["kcal"], float) or not (0 <= r["kcal"] <= 900) or round(r["kcal"], 1) != r["kcal"]: bad("kcal", r) if r["unit"] not in ("g", "ml") or (gen and r["unit"] != "g"): bad("unit", r) if len(r["portions"]) > 6: bad("portions_n", r) labs = set() for lab, amt in r["portions"]: if not lab or not (0 < amt <= 5000): bad("portion_amount", r) if re.fullmatch(r"100\s*" + r["unit"], lab.strip(), re.I): bad("portion_plain_100", r) if re.search(r"\d\s*(g|grams?|gm|grm)\b", lab, re.I): bad("portion_label_has_grams", r) if lab.lower() in labs: bad("portion_dup_label", r) labs.add(lab.lower()) b = r["barcode"] if b: if not b.isdigit() or not (6 <= len(b) <= 14) or len(b) == 12 or (len(b) == 14 and b[0] == "0"): bad("barcode", r) bcs[b] += 1 elif not gen: bad("branded_no_barcode", r) if r["date"] and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", r["date"]): bad("date", r) if "&" in r["name"] or "&#" in r["name"]: bad("html_entity", r) letters = [c for c in r["name"] if c.isalpha()] if len(letters) > 6 and all(c.isupper() for c in letters): bad("still_all_caps", r) dups = sum(1 for v in bcs.values() if v > 1) print("records per src:", dict(per), "total", sum(per.values())) print("duplicate barcodes:", dups) print("problems:", dict(probs)) for k, v in ex.items(): print(" ", k, v) for fn in ("usda_generic.jsonl", "usda_branded.jsonl"): print(fn, os.path.getsize(os.path.join(root, "normalized", fn)), "bytes")