#!/usr/bin/env python3 """Validate normalized extra-source JSONL files against FoodIndex/CONTRACT.md rules. Usage: python3 -I validate_extra.py [...] """ import json import re import sys from collections import Counter KEYS = ["id", "name", "brand", "kcal", "unit", "portions", "barcode", "src", "generic", "pop", "date", "countries", "lang", "category"] def check(path: str) -> int: errs = Counter() ids = set() n = 0 src_seen = set() with open(path, encoding="utf-8") as f: for ln, line in enumerate(f, 1): n += 1 r = json.loads(line) if list(r.keys()) != KEYS: errs["keys"] += 1 src = r["src"] src_seen.add(src) if not (isinstance(src, str) and len(src) == 1 and src.isupper() and src not in "FSNBO"): errs["src"] += 1 if not r["id"].startswith(src + ":") or len(r["id"]) <= 2: errs["id_prefix"] += 1 if r["id"] in ids: errs["dup_id"] += 1 ids.add(r["id"]) nm = r["name"] if not (2 <= len(nm) <= 120) or nm != re.sub(r"\s+", " ", nm).strip(): errs["name"] += 1 if not isinstance(r["kcal"], (int, float)) or not (0 <= r["kcal"] <= 900) or round(r["kcal"], 1) != r["kcal"]: errs["kcal"] += 1 if r["unit"] not in ("g", "ml"): errs["unit"] += 1 p = r["portions"] if not isinstance(p, list) or len(p) > 6: errs["portions_len"] += 1 for lab, amt in p: if not isinstance(lab, str) or not lab.strip(): errs["portion_label"] += 1 if re.search(r"\d\s*(g|ml)\b", lab, re.I): errs["portion_label_has_amount"] += 1 if not (isinstance(amt, (int, float)) and 0 < amt <= 5000): errs["portion_amount"] += 1 if abs(amt - 100) < 1e-9 and re.fullmatch(r"100\s*(g|ml)", lab.strip(), re.I): errs["portion_plain_100"] += 1 if r["barcode"] != "": errs["barcode"] += 1 if not isinstance(r["generic"], bool): errs["generic"] += 1 if r["generic"] and r["brand"]: errs["generic_with_brand"] += 1 if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", r["date"] or ""): errs["date"] += 1 print(path, "records:", n, "src:", sorted(src_seen), "errors:", dict(errs) or "none") return sum(errs.values()) def main() -> None: bad = sum(check(p) for p in sys.argv[1:]) sys.exit(1 if bad else 0) if __name__ == "__main__": main()