#!/usr/bin/env python3 """Check a built food index site and offline bundle against FoodIndex/CONTRACT.md (contract v2), and run the reference search. Checks: manifest keys ("format": 1, "contract": 2, every source with a 1-120 character "credit", no source "L"); every token shard (bucket of each token, tokens are normalize() fixed points, index lists valid, sorted by score desc / id asc, capped, each listed record really has that token, every record referenced); every barcode shard (keys normalized and in the right bucket, record barcode = key or a UPC-E alias of it, B wins over O); every record valid (fields, ranges, portions, barcode, src letter, score); the same id is the same record in every file; file count and size limits; the bundle (format, contract, sources, generic only, barcode "", unique ids, size). Tokens are rebuilt with v2 stemming (foodtok) for a sample of records (1 in 32 by CRC-32 of the id): each of a sampled record's tokens must be in its bucket's shard and list the record, unless that list is at the cap and the record ranks below its last entry. v2 stems that must exist (cooky, browny, quich, pie, fry) and v1-only forms that must not (cookie, brownie, quiche: not normalize() fixed points) are checked too. Then the contract's reference search (a second Python twin, written separately from search_ref.py) for a few queries: online shards, offline bundle, merged with the online-required filter. Run: python3 -I FoodIndex/scripts/verify_index.py [--site FoodIndex/out/site] [--bundle LosingIt/Resources/GenericFoods.json] """ import argparse import collections import hashlib import json import math import os import re import sys import zlib HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import foodtok # noqa: E402 ROOT = os.path.dirname(HERE) PROJECT = os.path.dirname(ROOT) METRIC = re.compile(r"(? 6: bad("portions") else: for p in portions: if not (isinstance(p, list) and len(p) == 2 and isinstance(p[0], str) and p[0].strip() and isinstance(p[1], (int, float)) and not isinstance(p[1], bool) and 0 < p[1] <= 5000): bad("portion") continue if METRIC.search(p[0]): bad("portion_label_has_metric") if p[1] == 100 and p[0].lower().replace(" ", "") in ("100g", "100ml"): bad("portion_plain_100") if not isinstance(bc, str) or (bc and foodtok.normalize_barcode(bc) != bc): bad("barcode") if src == "L": bad("src_L_reserved") if src not in sources: bad("src") elif sources[src]["generic"] and (bc or brand): bad("generic_with_barcode_or_brand") if not (isinstance(score, int) and not isinstance(score, bool) and 0 <= score <= 1000): bad("score") return ok NEUTRAL = set("raw nfs ns plain regular fresh whole cooked prepared".split()) SOFT = set("slice piece cup serving small medium large oz ounce gram ml glass can bottle scoop tbsp tsp tablespoon " "teaspoon portion lb kcal calory half quarter dozen handful pint quart liter litre packet fl pinch".split()) UNIT_WORDS = "g gr kg mg oz ml l cl dl lb lbs cal kcal ct pc pcs in inch".split() CLASS = set("fish beef pork lamb veal chicken turkey cheese yogurt pepper nut seed beverage crustacean mollusk game " "alcoholic spice".split()) DERIVED = set("sauce seasoning mix dip syrup filling extract flavored meatless imitation substitute vegetarian vegan " "giblet liver gizzard pickled puffed".split()) ASCII_DIGITS = re.compile(r"[0-9]+") QUANTITY = re.compile(r"[0-9]+(?:" + "|".join(sorted(UNIT_WORDS, key=len, reverse=True)) + r")?") def digits(t): return ASCII_DIGITS.fullmatch(t) is not None def quantity(t): return QUANTITY.fullmatch(t) is not None def ref_plan(query): """CONTRACT.md "Reference search" 1: (q, hard, soft).""" q = foodtok.normalize(query) hard = [t for t in q if not (quantity(t) or t in SOFT)] soft = [t for t in q if quantity(t) or t in SOFT] if len(hard) == 0: hard = q[:] soft = [] if len(hard) > 5: order = sorted(range(len(hard)), key=lambda i: (-len(hard[i]), i)) hard = [hard[i] for i in sorted(order[:5])] soft = soft[:5] for p in foodtok.split(query): if len(p) == 1 and p != "s" and p not in soft: soft.append(p) return q, hard, soft def ref_final(rec, hard, soft, required, generic_codes, q=()): """CONTRACT.md "Reference search" 3. `q` is normalize(query) (for DERIVED).""" name, brand = rec[1], rec[2] N = foodtok.normalize(name) Br = foodtok.normalize(brand) P = set() for piece in foodtok.split(name + " " + brand): P.update((piece, foodtok.stem(piece))) soft_hit = len([t for t in soft if t in P]) segs = [] second_part = None for part in name.split(","): toks = foodtok.normalize(part) if toks: segs.append(toks) if len(segs) == 2: second_part = part neutral = set(NEUTRAL) lead = N led_by_stopword = False if second_part is not None: pieces = foodtok.split(second_part) led_by_stopword = bool(pieces) and len(pieces[0]) > 1 and foodtok.stem(pieces[0]) in foodtok.STOPWORDS if len(segs) >= 2 and len(segs[0]) == 1 and segs[0][0] in CLASS and segs[0][0] not in hard \ and not led_by_stopword: neutral.add(segs[0][0]) del segs[0] lead = segs[0] core = set(t for t in N if t not in NEUTRAL and not digits(t) and t not in soft) qcore = set(t for t in hard if t not in neutral) bonus = 0 if all(t in N for t in required): bonus += 250 if core and core == qcore and soft_hit == len(soft): bonus += 200 if lead and lead[0] == hard[0]: bonus += 80 extra = len([t for t in N if t not in hard and t not in neutral and not digits(t) and t not in soft]) bonus -= 12 * min(extra, 10) if segs and set(segs[0]) <= set(hard) | neutral and set(segs[0]) & set(hard): bonus += 40 bonus += 40 * soft_hit bonus -= 80 * len(set(t for t in N if t in DERIVED) - set(q)) brand_only = any(t in Br and t not in N for t in hard) if rec[7] in generic_codes and not brand_only: bonus += 120 return rec[8] + bonus def ref_sort(hits): return sorted(hits, key=lambda x: (-x[0], len(foodtok.normalize(x[1][1])), len(x[1][1]), x[1][0].encode())) def ref_search(query, get_list, generic_codes, limit=50): """CONTRACT.md "Reference search" 1-4. get_list(token) -> list of records (L_t). Returns (top `limit` [(final, rec)], required).""" q, hard, soft = ref_plan(query) if not hard: return [], [] lists = {t: get_list(t) for t in hard} required = [t for t in hard if lists[t]] if not required: return [], required best = {} for rec in (r for t in required for r in lists[t]): toks = set(foodtok.normalize(rec[1])) | set(foodtok.normalize(rec[2])) if any(t not in toks for t in required): continue s = ref_final(rec, hard, soft, required, generic_codes, q) if rec[0] not in best or best[rec[0]][0] < s: best[rec[0]] = (s, rec) return ref_sort(best.values())[:limit], required def ref_merge(online, online_required, offline, limit=50): """CONTRACT.md "Reference search" 5: offline hits need every online-required token; one per id; top 50.""" best = {} for s, rec in online: best[rec[0]] = (s, rec) for s, rec in offline: toks = set(foodtok.normalize(rec[1])) | set(foodtok.normalize(rec[2])) if not set(online_required) <= toks: continue if rec[0] not in best or best[rec[0]][0] < s: best[rec[0]] = (s, rec) return ref_sort(best.values())[:limit] def main(): ap = argparse.ArgumentParser() ap.add_argument("--site", default=os.path.join(ROOT, "out", "site")) ap.add_argument("--bundle", default=os.path.join(PROJECT, "LosingIt", "Resources", "GenericFoods.json")) ap.add_argument("--queries", nargs="*", default=QUERIES) args = ap.parse_args() site = args.site errs = collections.Counter() info = collections.Counter() with open(os.path.join(site, "manifest.json"), encoding="utf-8") as f: man = json.load(f) for k in ("format", "contract", "build", "path", "tokenBuckets", "barcodeBuckets", "maxPerToken", "counts", "sources"): if k not in man: errs["manifest_missing_" + k] += 1 if man.get("format") != 1: errs["manifest_format_not_1"] += 1 if man.get("contract") != CONTRACT_VERSION: errs["manifest_contract_not_2"] += 1 for s in man.get("sources", []): c = s.get("credit") if not isinstance(c, str) or not c.strip() or len(c) > 120 or c != " ".join(c.split()): errs["source_credit_bad"] += 1 else: info["credit_max_len"] = max(info["credit_max_len"], len(c)) if s.get("code") == "L": errs["source_L_reserved"] += 1 TB, BB, CAP = man["tokenBuckets"], man["barcodeBuckets"], man["maxPerToken"] sources = {s["code"]: s for s in man["sources"]} generic = {c for c, s in sources.items() if s.get("generic")} data = os.path.join(site, man["path"]) seen_rec = {} # id -> crc32 of its JSON src_count = collections.Counter() def note(rec): key = zlib.crc32(json.dumps(rec, ensure_ascii=False, separators=(",", ":")).encode()) old = seen_rec.get(rec[0]) if old is None: seen_rec[rec[0]] = key src_count[rec[7]] += 1 check_record(rec, sources, errs) elif old != key: errs["same_id_different_record"] += 1 tdir = os.path.join(data, "t") tfiles = sorted(os.listdir(tdir), key=lambda n: int(n.split(".")[0])) sample_needs = {} # (token, id) -> rank key of a sampled record, from its v2 tokens sample_member = set() # (token, id) pairs of sampled records found in a token list list_end = {} # token -> (list length, rank key of its last record) sampled = set() for n in tfiles: b = int(n.split(".")[0]) if not (0 <= b < TB): errs["token_bucket_out_of_range"] += 1 with open(os.path.join(tdir, n), "rb") as f: raw = f.read() if raw.startswith(b"\xef\xbb\xbf"): errs["bom"] += 1 sh = json.loads(raw) w, r = sh["w"], sh["r"] used = set() keys = [(-rec[8], rec[0]) for rec in r] if keys != sorted(keys): errs["token_shard_r_not_sorted"] += 1 for t, lst in w.items(): info["tokens"] += 1 if foodtok.fnv1a(t) % TB != b: errs["token_wrong_bucket"] += 1 if foodtok.normalize(t) != [t]: errs["token_not_fixed_point"] += 1 if not lst or len(lst) > CAP or len(set(lst)) != len(lst): errs["token_list_bad"] += 1 if any(not (isinstance(i, int) and 0 <= i < len(r)) for i in lst): errs["token_index_out_of_range"] += 1 continue if lst != sorted(lst): errs["token_list_not_sorted"] += 1 if len(lst) == CAP: info["tokens_at_cap"] += 1 for i in lst: used.add(i) rec = r[i] if t not in foodtok.normalize(rec[1] + " " + rec[2]): errs["record_lacks_token"] += 1 if zlib.crc32(rec[0].encode()) % SAMPLE_MOD == 0: sample_member.add((t, rec[0])) if lst and all(isinstance(i, int) and 0 <= i < len(r) for i in lst): last = r[lst[-1]] list_end[t] = (len(lst), (-last[8], last[0])) if len(used) != len(r): errs["unreferenced_record_in_shard"] += 1 ids = [rec[0] for rec in r] if len(set(ids)) != len(ids): errs["duplicate_record_in_shard"] += 1 for rec in r: note(rec) if rec[0] not in sampled and zlib.crc32(rec[0].encode()) % SAMPLE_MOD == 0: sampled.add(rec[0]) for t in foodtok.normalize(rec[1] + " " + rec[2]): sample_needs[(t, rec[0])] = (-rec[8], rec[0]) info["token_files"] = len(tfiles) # sampled records: every v2 token is in its bucket's shard and lists the record (or the list is capped above it) for (t, rid), key in sample_needs.items(): info["sample_postings_checked"] += 1 if (t, rid) in sample_member: continue end = list_end.get(t) if end is None: errs["sample_token_missing_from_bucket"] += 1 elif not (end[0] >= CAP and key > end[1]): errs["sample_record_missing_from_token_list"] += 1 else: info["sample_postings_below_cap"] += 1 info["sample_records"] = len(sampled) for t in ("cooky", "browny", "quich", "pie", "fry", "berry", "tomato", "peach"): if t not in list_end: errs["v2_stem_token_missing_" + t] += 1 for t in ("cookie", "brownie", "quiche", "cookies", "pies", "fries"): if t in list_end: errs["v1_or_unstemmed_token_present_" + t] += 1 bdir = os.path.join(data, "b") bfiles = sorted(os.listdir(bdir), key=lambda n: int(n.split(".")[0])) for n in bfiles: b = int(n.split(".")[0]) with open(os.path.join(bdir, n), "rb") as f: sh = json.loads(f.read()) for k, rec in sh.items(): info["barcode_keys"] += 1 if foodtok.normalize_barcode(k) != k: errs["barcode_key_not_normalized"] += 1 if foodtok.fnv1a(k) % BB != b: errs["barcode_wrong_bucket"] += 1 if rec[6] == k: info["barcode_direct"] += 1 else: up = foodtok.expand_upce(rec[6]) if len(rec[6]) == 8 else None up2 = foodtok.expand_upce(k) if len(k) == 8 else None if up and foodtok.normalize_barcode(up) == k: info["barcode_alias_upce_expanded"] += 1 elif up2 and foodtok.normalize_barcode(up2) == rec[6]: info["barcode_alias_upce_8digit_to_b"] += 1 else: errs["barcode_key_record_mismatch"] += 1 info["barcode_src_" + rec[7]] += 1 note(rec) info["barcode_files"] = len(bfiles) nfiles, total, largest = 0, 0, (0, "") for dp, dn, fn in os.walk(site): for n in fn: p = os.path.join(dp, n) sz = os.path.getsize(p) nfiles += 1 total += sz largest = max(largest, (sz, os.path.relpath(p, site))) if nfiles > 19500: errs["too_many_files"] += 1 if largest[0] >= 5 * 1024 * 1024: errs["file_too_big"] += 1 for need in ("manifest.json", "_headers", "index.html", "attribution.html", "404.html", "scripts/index.html"): if not os.path.exists(os.path.join(site, need)): errs["missing_" + need] += 1 tree = hashlib.sha256() for dp, dn, fn in sorted(os.walk(site)): dn.sort() for n in sorted(fn): p = os.path.join(dp, n) with open(p, "rb") as f: tree.update(os.path.relpath(p, site).encode() + b"\0" + hashlib.sha256(f.read()).digest()) # bundle with open(args.bundle, "rb") as f: braw = f.read() bundle = json.loads(braw) if bundle.get("format") != 1 or bundle.get("build") != man["build"] or bundle.get("contract") != CONTRACT_VERSION: errs["bundle_header"] += 1 if bundle.get("sources") != man.get("sources"): errs["bundle_sources_differ_from_manifest"] += 1 bids = set() bsrc = collections.Counter() for rec in bundle["r"]: check_record(rec, sources, errs) if rec[7] not in generic or rec[6] != "": errs["bundle_not_generic"] += 1 if rec[0] in bids: errs["bundle_duplicate_id"] += 1 bids.add(rec[0]) bsrc[rec[7]] += 1 old = seen_rec.get(rec[0]) if old is not None and old != zlib.crc32(json.dumps(rec, ensure_ascii=False, separators=(",", ":")).encode()): errs["bundle_record_differs_from_site"] += 1 if len(braw) >= 4_000_000: errs["bundle_over_4MB"] += 1 # reference search, online (shards) and offline (bundle) cache = {} def online_list(t): b = foodtok.fnv1a(t) % TB if b not in cache: p = os.path.join(tdir, f"{b}.json") cache[b] = json.load(open(p, encoding="utf-8")) if os.path.exists(p) else {"w": {}, "r": []} sh = cache[b] return [sh["r"][i] for i in sh["w"].get(t, [])[:CAP]] off_post = collections.defaultdict(list) for rec in bundle["r"]: for t in foodtok.normalize(rec[1] + " " + rec[2]): off_post[t].append(rec) searches = {} for q in args.queries: on, on_req = ref_search(q, online_list, generic) off, off_req = ref_search(q, lambda t: off_post.get(t, []), generic) top = ref_merge(on, on_req, off)[:8] _, hard, soft = ref_plan(q) searches[q] = {"hard": hard, "soft": soft, "online_required": on_req, "online_hits": len(on), "offline_hits": len(off), "top": [f"{s} {rec[0]} | {rec[1]} | {rec[2]} | {rec[3]} kcal/100{rec[4]}" for s, rec in top]} out = {"errors": dict(sorted(errs.items())), "info": dict(sorted(info.items())), "records_by_src": dict(sorted(src_count.items())), "records_total": len(seen_rec), "site_files": nfiles, "site_bytes": total, "largest_file": largest, "site_sha256": tree.hexdigest(), "bundle_bytes": len(braw), "bundle_records_by_src": dict(sorted(bsrc.items())), "searches": searches} print(json.dumps(out, ensure_ascii=False, indent=1)) sys.exit(1 if errs else 0) if __name__ == "__main__": main()