#!/usr/bin/env python3 """Validate normalized/off.jsonl against CONTRACT.md + the stage brief and print summary numbers + samples. Usage: python3 -I validate_off.py [samples_per_country] """ import collections import json import os import random import re import sys path = sys.argv[1] per_country = int(sys.argv[2]) if len(sys.argv) > 2 else 1 KEYS = ['id', 'name', 'brand', 'kcal', 'unit', 'portions', 'barcode', 'src', 'generic', 'pop', 'date', 'countries', 'lang', 'category'] LABEL_AMOUNT_RE = re.compile(r'\d\s*(g|gr|grams?|kg|mg|ml|cl|dl|l)\b', re.I) errors = collections.Counter() examples = collections.defaultdict(list) ids = set() barcodes = set() units = collections.Counter() countries = collections.Counter() en_countries = {'united-states', 'united-kingdom', 'canada', 'australia', 'ireland', 'new-zealand'} n = us = en_any = with_portion = with_pop = with_date = zero = 0 by_country = collections.defaultdict(list) kcal_hist = collections.Counter() def err(k, r): errors[k] += 1 if len(examples[k]) < 3: examples[k].append(r.get('id')) with open(path, encoding='utf-8') as f: for line in f: n += 1 r = json.loads(line) if list(r.keys()) != KEYS: err('keys', r) if r['id'] in ids: err('dup_id', r) ids.add(r['id']) if r['barcode'] in barcodes: err('dup_barcode', r) barcodes.add(r['barcode']) if r['id'] != 'O:' + r['barcode']: err('id_barcode', r) b = r['barcode'] if not re.fullmatch(r'\d{8,14}', b) or len(b) == 12 or (len(b) == 14 and b[0] == '0'): err('barcode_format', r) nm = r['name'] if not (2 <= len(nm) <= 120) or nm != ' '.join(nm.split()): err('name', r) k = r['kcal'] if not isinstance(k, float) or not (0 <= k <= 900) or round(k, 1) != k: err('kcal', r) if k == 0: zero += 1 kcal_hist[min(int(k // 100) * 100, 900)] += 1 if r['unit'] not in ('g', 'ml'): err('unit', r) units[r['unit']] += 1 ps = r['portions'] if not isinstance(ps, list) or len(ps) > 6: err('portions_len', r) for p in ps: if not (isinstance(p, list) and len(p) == 2 and isinstance(p[0], str) and isinstance(p[1], (int, float))): err('portion_shape', r) continue if not (0 < p[1] <= 5000): err('portion_amount', r) if LABEL_AMOUNT_RE.search(p[0]): err('portion_label_has_metric_amount', r) if p[0] == '1 serving' and abs(p[1] - 100) < 0.05: err('portion_plain_100', r) if not p[0] or len(p[0]) > 40: err('portion_label_len', r) if ps: with_portion += 1 if r['src'] != 'O' or r['generic'] is not False: err('src_generic', r) if r['pop'] is not None: with_pop += 1 if not isinstance(r['pop'], int) or r['pop'] < 0: err('pop', r) if r['date']: with_date += 1 if not re.fullmatch(r'\d{4}-\d{2}-\d{2}', r['date']): err('date', r) cs = r['countries'] for c in cs: countries[c] += 1 if 'united-states' in cs: us += 1 if en_countries & set(cs): en_any += 1 for c in cs[:1]: by_country[c].append(line.rstrip('\n')) print(json.dumps({ 'records': n, 'file_bytes': os.path.getsize(path), 'unit_split': dict(units), 'us_records': us, 'english_speaking_country_records': en_any, 'with_portion': with_portion, 'with_pop': with_pop, 'with_date': with_date, 'kcal_zero': zero, 'kcal_hist_per_100': dict(sorted(kcal_hist.items())), 'top_countries': countries.most_common(25), 'errors': dict(errors), 'error_examples': dict(examples)}, ensure_ascii=False, indent=1)) random.seed(20261115) print('SAMPLES') for c in ['united-states', 'france', 'germany', 'united-kingdom', 'spain', 'italy', 'canada', 'switzerland', 'belgium', 'netherlands', 'australia', 'mexico', 'japan', 'brazil', 'india', 'poland']: lst = by_country.get(c, []) for line in random.sample(lst, min(per_country, len(lst))): print(line)