#!/usr/bin/env python3 """Independent contract and limits audit of the Losing It food index site and the offline bundle. Checks FoodIndex/out/site and LosingIt/Resources/GenericFoods.json against FoodIndex/CONTRACT.md (contract v2): manifest "format": 1 and "contract": 2, a 1-120 character "credit" on every source, the reserved source letter L, v2 token normalization (every record's tokens recomputed and checked against every token list), and the "Duplicate hiding" rule for every hidden record (qa/merge_hidden_duplicates.jsonl). The token normalization, FNV-1a hash, barcode normalization and UPC-E expansion below are written from the contract text and the GS1 UPC-E rules. They do not import foodtok.py. foodtok.py is loaded only to compare the two. Run (stdlib only): python3 -I qa_contract.py [--site DIR] [--bundle FILE] [--out FILE] [--workers N] Writes a JSON report (default FoodIndex/qa/qa_contract.json) and every violation sample to FoodIndex/qa/qa_contract_violations.jsonl, and prints a summary. Exit status 1 when any error-level violation exists. """ import argparse import collections import hashlib import html import html.parser import importlib.util import json import math import multiprocessing as mp import os import random import re import sys import time import unicodedata import zlib from pathlib import Path HERE = Path(__file__).resolve().parent # FoodIndex/scripts ROOT = HERE.parent # FoodIndex REPO = ROOT.parent # app repository DEFAULT_SITE = ROOT / "out" / "site" DEFAULT_BUNDLE = REPO / "LosingIt" / "Resources" / "GenericFoods.json" DEFAULT_OUT = ROOT / "qa" / "qa_contract.json" CONTRACT_BASE_URL = "https://losingit-food.pages.dev" MAX_FILES = 19500 MAX_FILE_BYTES = 5 * 1024 * 1024 CF_MAX_FILE_BYTES = 25 * 1024 * 1024 TARGET_FILE_BYTES = 400 * 1024 BUNDLE_TARGET_BYTES = 4_000_000 SAMPLES_PER_CODE_WORKER = 25 SAMPLES_PER_CODE_TOTAL = 5000 # Codes that are not contract violations (data quality / informational). Everything else is an error. WARN_CODES = { "brand.untrimmed", "brand.whitespace", "name.allcaps_B", "name.allcaps_O", "name.mojibake", "name.replacement_char", "portion.dup_label", "portion.label_whitespace", "token.not_idempotent", "lower.final_sigma", "barcode.nonstandard_length", "barcode.collision_ok", "bundle.over_target", "bundle.unsearchable", "bundle.extra_key", "html.link_pretty_url", "attr.usda_credit_wording", "attr.license_url_not_license", "headers.no_trailing_newline", "manifest.extra_field", "scripts.post_build_missing", "kcal.negative_zero", "size.over_target", "file.junk", "bundle.generic_not_in_index", "index.generic_not_in_bundle", "name.lone_surrogate_ok", "search.capped_out_of_every_list", "barcode.key_shorter_than_app_accepts", "barcode.only_key_not_lookupable_by_app", "search.hidden_and_kept_twin_capped_out", "barcode.barcode_only_and_unscannable", "attr.source_credit_not_on_page", } CONTRACT_VERSION = 2 CREDIT_MAX = 120 # --------------------------------------------------------------------------------------- contract re-implementation STOPWORDS = frozenset("and with of the in an for or to on de la le et en con mit und from".split()) _AZ4 = re.compile(r"[a-z]{4,}") _ASCII_NON_ALNUM = re.compile(r"[^a-z0-9]+") def stem(tok): """Contract v2 step 5: only all-ASCII a-z tokens of at least 4 scalars (n); the first matching rule, once.""" if not _AZ4.fullmatch(tok): return tok n = len(tok) if n >= 5 and tok[-3:] == "ies": # 1 return tok[:-3] + "y" if tok[-4:] == "sses": # 2 return tok[:-2] if tok[-3:] in ("oes", "xes", "zes") or tok[-4:] in ("ches", "shes"): # 3 return tok[:-2] if tok[-2:] == "ie": # 4 return tok[:-2] + "y" if tok[-3:] == "che" or tok[-2:] in ("oe", "ze"): # 5 return tok[:-1] if tok[-1] == "s" and tok[-2:] not in ("ss", "us", "is"): # 6 return tok[:-1] return tok def _finish(pieces): """Steps 4, 5, 6, 7.""" out = [] seen = set() for p in pieces: if len(p) < 2: continue t = stem(p) if t in STOPWORDS: continue if t in seen: continue seen.add(t) out.append(t) return out def tokens_slow(text): """The contract's normalize(), literally: NFKD, drop Mn, lower(), split on non-L/N scalars, then steps 4-7.""" s = unicodedata.normalize("NFKD", text) s = "".join(c for c in s if unicodedata.category(c) != "Mn") s = s.lower() pieces = [] cur = [] for c in s: if unicodedata.category(c)[0] in ("L", "N"): cur.append(c) else: if cur: pieces.append("".join(cur)) cur = [] if cur: pieces.append("".join(cur)) return _finish(pieces) def tokens(text): """Same result as tokens_slow. For pure ASCII text, NFKD and Mn removal change nothing and the only L*/N* scalars are A-Z, a-z and 0-9, so a regex split on the lowercased text is equivalent (checked in self-test).""" if text.isascii(): return _finish([p for p in _ASCII_NON_ALNUM.split(text.lower()) if p]) return tokens_slow(text) def fnv32(text): h = 0x811C9DC5 for b in text.encode("utf-8"): h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF return h def norm_barcode(raw): """Contract: digits only; 12 -> '0'+code; 14 with leading '0' -> drop it; 8 and 13 as is; other 6..14 as is; else None.""" d = "".join(c for c in raw if c in "0123456789") n = len(d) if n == 12: return "0" + d if n == 14 and d.startswith("0"): return d[1:] if 6 <= n <= 14: return d return None def gs1_check_digit(body): """GS1 mod-10 check digit for the digits in body (weights 3,1,3,... from the rightmost body digit).""" total = 0 for i, ch in enumerate(reversed(body)): total += int(ch) * (3 if i % 2 == 0 else 1) return str((10 - total % 10) % 10) def upce_to_upca(code): """GS1 UPC-E to UPC-A for an 8-digit code (number system 0/1 + 6 digits + check). None if not valid UPC-E.""" if len(code) != 8 or not code.isdigit() or code[0] not in "01": return None ns, x, chk = code[0], code[1:7], code[7] last = x[5] if last in "012": manuf, prod = x[0] + x[1] + last + "00", "00" + x[2] + x[3] + x[4] elif last == "3": manuf, prod = x[0] + x[1] + x[2] + "00", "000" + x[3] + x[4] elif last == "4": manuf, prod = x[0] + x[1] + x[2] + x[3] + "0", "0000" + x[4] else: manuf, prod = x[0:5], "0000" + last body = ns + manuf + prod if gs1_check_digit(body) != chk: return None return body + chk # --------------------------------------------------------------------------------------------- record rules ID_RE = re.compile(r"([A-Z]):(\S+)\Z") BAD_WS = re.compile(r"\s\s|[^\S ]") # two whitespace in a row, or any whitespace other than ' ' # U+200C/U+200D (ZWNJ/ZWJ) are allowed: they are part of emoji sequences and of Persian/Indic spelling. CTRL = re.compile("[\\x00-\\x1f\\x7f-\\x9f\\u200b\\u200e\\u200f\\u2028\\u2029\\ufeff]") MOJIBAKE = re.compile(r"Ã[\x80-\xbf]|â€|Â[\xa0-\xbf]") _METRIC = r"(?:kg|g|gr|grs|gram|grams|gramm|grammes?|mg|ml|cl|dl|l|lt|ltr|litre|liter|litres|liters|millilit(?:er|re)s?)" METRIC_IN_LABEL = re.compile(r"(? 0: lst.extend(v[:room]) def is_num(x): return type(x) in (int, float) def check_record(rec, where, E, declared, generic): """Validate one positional record. Returns True when the record is well-formed enough to index.""" if type(rec) is not list: E.add("rec.not_array", {"where": where, "rec": repr(rec)[:200]}) return False if len(rec) != 9: E.add("rec.length", {"where": where, "len": len(rec), "rec": rec[:3]}) return False rid, name, brand, kcal, unit, portions, barcode, src, score = rec ok = True # 7 src if type(src) is not str or len(src) != 1 or not ("A" <= src <= "Z"): E.add("src.invalid", {"where": where, "id": rid, "src": src}) ok = False elif src == "L": E.add("src.reserved_L", {"where": where, "id": rid}) elif src not in declared: E.add("src.undeclared", {"where": where, "id": rid, "src": src}) # 0 id if type(rid) is not str: E.add("id.type", {"where": where, "id": repr(rid)}) ok = False else: m = ID_RE.match(rid) if not m: E.add("id.format", {"where": where, "id": rid}) elif type(src) is str and m.group(1) != src: E.add("id.src_mismatch", {"where": where, "id": rid, "src": src}) # 1 name if type(name) is not str: E.add("name.type", {"where": where, "id": rid}) ok = False else: if not (2 <= len(name) <= 120): E.add("name.length", {"where": where, "id": rid, "name": name, "len": len(name)}) if name != name.strip(): E.add("name.untrimmed", {"where": where, "id": rid, "name": name}) if BAD_WS.search(name): E.add("name.whitespace", {"where": where, "id": rid, "name": name}) if CTRL.search(name): E.add("name.control_char", {"where": where, "id": rid, "name": name}) if not name.isascii(): try: name.encode("utf-8") except UnicodeEncodeError: E.add("name.lone_surrogate", {"where": where, "id": rid, "name": repr(name)}) if "�" in name: E.add("name.replacement_char", {"where": where, "id": rid, "name": name}) if MOJIBAKE.search(name): E.add("name.mojibake", {"where": where, "id": rid, "name": name}) letters = [c for c in name if c.isalpha()] if len(letters) >= 4 and all(c.isupper() for c in letters): if src == "B": E.add("name.allcaps_B", {"where": where, "id": rid, "name": name}) elif src == "O": E.add("name.allcaps_O", {"where": where, "id": rid, "name": name}) # 2 brand if type(brand) is not str: E.add("brand.type", {"where": where, "id": rid}) ok = False else: if brand != brand.strip(): E.add("brand.untrimmed", {"where": where, "id": rid, "brand": brand}) if BAD_WS.search(brand): E.add("brand.whitespace", {"where": where, "id": rid, "brand": brand}) if CTRL.search(brand): E.add("brand.control_char", {"where": where, "id": rid, "brand": brand}) if not brand.isascii(): try: brand.encode("utf-8") except UnicodeEncodeError: E.add("brand.lone_surrogate", {"where": where, "id": rid, "brand": repr(brand)}) # 3 kcal if not is_num(kcal): E.add("kcal.type", {"where": where, "id": rid, "kcal": repr(kcal)}) elif not math.isfinite(kcal) or kcal < 0 or kcal > 900: E.add("kcal.range", {"where": where, "id": rid, "kcal": kcal}) else: if round(kcal, 1) != kcal: E.add("kcal.decimals", {"where": where, "id": rid, "kcal": kcal}) if kcal == 0 and math.copysign(1.0, kcal) < 0: E.add("kcal.negative_zero", {"where": where, "id": rid}) # 4 unit if unit not in ("g", "ml") or type(unit) is not str: E.add("unit.invalid", {"where": where, "id": rid, "unit": repr(unit)}) # 5 portions if type(portions) is not list: E.add("portions.type", {"where": where, "id": rid}) else: if len(portions) > 6: E.add("portions.too_many", {"where": where, "id": rid, "n": len(portions)}) labels = set() for p in portions: if type(p) is not list or len(p) != 2: E.add("portion.shape", {"where": where, "id": rid, "portion": repr(p)[:120]}) continue lab, amt = p if type(lab) is not str or not lab.strip(): E.add("portion.label_type", {"where": where, "id": rid, "portion": p}) else: if lab != lab.strip() or BAD_WS.search(lab): E.add("portion.label_whitespace", {"where": where, "id": rid, "label": lab}) if CTRL.search(lab): E.add("portion.label_control_char", {"where": where, "id": rid, "label": lab}) pm = PLAIN_METRIC.match(lab) if pm and is_num(amt) and abs(amt - 100) < 1e-9: E.add("portion.plain_100", {"where": where, "id": rid, "portion": p}) elif METRIC_IN_LABEL.search(lab): E.add("portion.label_has_metric_amount", {"where": where, "id": rid, "unit": unit, "portion": p}) if lab in labels: E.add("portion.dup_label", {"where": where, "id": rid, "label": lab}) labels.add(lab) if not is_num(amt): E.add("portion.amount_type", {"where": where, "id": rid, "portion": repr(p)[:120]}) elif not math.isfinite(amt) or not (0 < amt <= 5000): E.add("portion.amount_range", {"where": where, "id": rid, "portion": p}) # 6 barcode if type(barcode) is not str: E.add("barcode.type", {"where": where, "id": rid, "barcode": repr(barcode)}) ok = False elif barcode: nb = norm_barcode(barcode) if nb != barcode: E.add("barcode.not_normalized", {"where": where, "id": rid, "barcode": barcode, "normalized": nb}) elif len(barcode) not in (8, 13, 14): E.add("barcode.nonstandard_length", {"where": where, "id": rid, "barcode": barcode}) # 8 score if type(score) is not int: E.add("score.type", {"where": where, "id": rid, "score": repr(score)}) ok = False elif not (0 <= score <= 1000): E.add("score.range", {"where": where, "id": rid, "score": score}) # generic rules if type(src) is str and src in generic: if brand != "": E.add("generic.has_brand", {"where": where, "id": rid, "brand": brand}) if barcode != "": E.add("generic.has_barcode", {"where": where, "id": rid, "barcode": barcode}) return ok def digest(rec): return hashlib.blake2b(json.dumps(rec, ensure_ascii=False, separators=(",", ":")).encode("utf-8"), digest_size=8).digest() def dup_sig(rec): """Contract "Duplicate hiding" key: normalized name, normalized brand and unit (kcal is compared separately).""" return hashlib.blake2b(json.dumps([tokens(rec[1]), tokens(rec[2]), rec[4]], ensure_ascii=False).encode("utf-8"), digest_size=8).digest() _JSON_STRING = re.compile(r'"(?:[^"\\]|\\.)*"', re.S) def strict_parse(raw, where, E): """Bytes -> object. Rejects BOM, invalid UTF-8, NaN/Infinity, duplicate keys; checks compact separators.""" if raw.startswith(b"\xef\xbb\xbf"): E.add("json.bom", {"where": where}) raw = raw[3:] try: text = raw.decode("utf-8") except UnicodeDecodeError as e: E.add("json.invalid_utf8", {"where": where, "err": str(e)[:200]}) return None dupes = [] def hook(pairs): d = {} for k, v in pairs: if k in d: dupes.append(k) d[k] = v return d def bad_constant(c): raise ValueError("non-finite number " + c) try: obj = json.loads(text, object_pairs_hook=hook, parse_constant=bad_constant) except ValueError as e: E.add("json.parse_error", {"where": where, "err": str(e)[:200]}) return None for k in dupes[:5]: E.add("json.duplicate_key", {"where": where, "key": k}) if len(dupes) > 5: E.counts["json.duplicate_key"] += len(dupes) - 5 compact = json.dumps(obj, ensure_ascii=False, separators=(",", ":")) if compact != text: if json.dumps(obj, ensure_ascii=True, separators=(",", ":")) != text: outside = _JSON_STRING.sub('""', text) if re.search(r"\s", outside): E.add("json.not_compact", {"where": where}) return obj # ------------------------------------------------------------------------------------------------ workers _CFG = {} def _init(cfg): _CFG.update(cfg) def check_token_shard(path_bucket): path, bucket = path_bucket cfg = _CFG TB, CAP = cfg["tokenBuckets"], cfg["maxPerToken"] declared, generic = cfg["declared"], cfg["generic"] where = f"t/{bucket}" E = Errs(SAMPLES_PER_CODE_WORKER) out = {"bucket": bucket, "records": [], "capped": {}, "tokens": 0, "postings": 0, "r": 0, "list_lens": collections.Counter(), "sample": [], "sample_nonascii": [], "sigma": []} raw = Path(path).read_bytes() if not raw: E.add("file.empty", {"where": where}) obj = strict_parse(raw, where, E) if obj is None: out["err"] = (E.counts, dict(E.samples)) return out if type(obj) is not dict or set(obj.keys()) != {"w", "r"}: E.add("tshard.top_keys", {"where": where, "keys": sorted(obj.keys()) if type(obj) is dict else type(obj).__name__}) w = obj.get("w") if type(obj) is dict else None r = obj.get("r") if type(obj) is dict else None if type(w) is not dict: E.add("tshard.w_type", {"where": where}) w = {} if type(r) is not list: E.add("tshard.r_type", {"where": where}) r = [] if not w or not r: E.add("tshard.empty", {"where": where, "w": len(w), "r": len(r)}) out["r"] = len(r) out["tokens"] = len(w) # records toks = [None] * len(r) tokset = [None] * len(r) keys = [None] * len(r) id_at = {} for i, rec in enumerate(r): if not check_record(rec, where, E, declared, generic): continue rid = rec[0] if rid in id_at: E.add("tshard.r_duplicate_id", {"where": where, "id": rid, "at": [id_at[rid], i]}) id_at[rid] = i tk = tokens(rec[1] + " " + rec[2]) toks[i] = tk tokset[i] = set(tk) keys[i] = (-rec[8], rid) out["records"].append((rid, digest(rec), tuple(tk), rec[8], rec[7], rec[6], dup_sig(rec), rec[3])) c = zlib.crc32(rid.encode("utf-8")) if c % 128 == 0: out["sample"].append((rid, rec[1], rec[2])) text = rec[1] + " " + rec[2] if not text.isascii(): if c % 16 == 0: out["sample_nonascii"].append((rid, rec[1], rec[2])) # Python str.lower() applies the Final_Sigma context rule; Swift lowercased() maps per scalar. s = "".join(ch for ch in unicodedata.normalize("NFKD", text) if unicodedata.category(ch) != "Mn") if "Σ" in s and s.lower() != "".join(ch.lower() for ch in s): out["sigma"].append((rid, rec[1], rec[2])) # token lists referenced = bytearray(len(r)) members = {} for t, lst in w.items(): if type(t) is not str or not t: E.add("token.key_invalid", {"where": where, "token": repr(t)}) continue if fnv32(t) % TB != bucket: E.add("token.wrong_bucket", {"where": where, "token": t, "bucket": fnv32(t) % TB}) if (len(t) < 2 or t in STOPWORDS or t != t.lower() or any(unicodedata.category(c)[0] not in ("L", "N") for c in t)): E.add("token.malformed", {"where": where, "token": t}) elif tokens(t) != [t]: E.add("token.not_idempotent", {"where": where, "token": t, "renormalized": tokens(t)}) if type(lst) is not list or not lst: E.add("list.empty_or_type", {"where": where, "token": t}) continue n = len(lst) out["postings"] += n out["list_lens"][n if n < 10 else (10 ** int(math.log10(n)))] += 1 if n > CAP: E.add("list.over_cap", {"where": where, "token": t, "len": n}) prev = None mem = set() for idx in lst: if type(idx) is not int or not (0 <= idx < len(r)): E.add("list.bad_index", {"where": where, "token": t, "index": repr(idx)}) continue if idx in mem: E.add("list.duplicate_index", {"where": where, "token": t, "index": idx}) continue mem.add(idx) referenced[idx] = 1 if tokset[idx] is None: continue if t not in tokset[idx]: rec = r[idx] E.add("posting.token_not_in_record", {"where": where, "token": t, "id": rec[0], "name": rec[1], "brand": rec[2], "tokens": toks[idx]}) k = keys[idx] if prev is not None and k <= prev: E.add("list.unsorted", {"where": where, "token": t, "prev": list(prev), "next": list(k)}) prev = k members[t] = mem if n >= CAP and prev is not None: out["capped"][t] = prev for i in range(len(r)): if not referenced[i] and r[i] is not None: rec = r[i] E.add("tshard.r_unreferenced", {"where": where, "index": i, "id": rec[0] if type(rec) is list and rec else None}) # every token of every record whose bucket is this shard must list that record (unless the list is capped # and the record ranks below the cutoff) for i, tk in enumerate(toks): if tk is None: continue for t in tk: if fnv32(t) % TB != bucket: continue mem = members.get(t) if mem is None: E.add("completeness.token_list_missing", {"where": where, "token": t, "id": r[i][0]}) elif i not in mem: lst = w[t] last_key = keys[lst[-1]] if type(lst[-1]) is int and 0 <= lst[-1] < len(r) else None if not (len(lst) >= CAP and last_key is not None and keys[i] > last_key): E.add("completeness.posting_missing", {"where": where, "token": t, "id": r[i][0], "score": r[i][8], "listLen": len(lst)}) out["err"] = (E.counts, dict(E.samples)) return out def check_barcode_shard(path_bucket): path, bucket = path_bucket cfg = _CFG BB = cfg["barcodeBuckets"] declared, generic = cfg["declared"], cfg["generic"] where = f"b/{bucket}" E = Errs(SAMPLES_PER_CODE_WORKER) out = {"bucket": bucket, "entries": [], "kinds": collections.Counter()} raw = Path(path).read_bytes() if not raw: E.add("file.empty", {"where": where}) obj = strict_parse(raw, where, E) if obj is None: out["err"] = (E.counts, dict(E.samples)) return out if type(obj) is not dict: E.add("bshard.type", {"where": where}) out["err"] = (E.counts, dict(E.samples)) return out if not obj: E.add("bshard.empty", {"where": where}) for key, rec in obj.items(): if norm_barcode(key) != key: E.add("bkey.not_normalized", {"where": where, "key": key, "normalized": norm_barcode(key)}) if fnv32(key) % BB != bucket: E.add("bkey.wrong_bucket", {"where": where, "key": key, "bucket": fnv32(key) % BB}) if not check_record(rec, where, E, declared, generic): continue rb = rec[6] if rb == key: kind = "own" elif rb == "": kind = "no_barcode" E.add("bkey.record_has_no_barcode", {"where": where, "key": key, "id": rec[0]}) elif len(rb) == 8 and upce_to_upca(rb) and norm_barcode(upce_to_upca(rb)) == key: kind = "upce_record_under_upca_key" elif len(key) == 8 and upce_to_upca(key) and norm_barcode(upce_to_upca(key)) == rb: kind = "upce_key_to_upca_record" else: kind = "mismatch" E.add("bkey.record_barcode_mismatch", {"where": where, "key": key, "id": rec[0], "recordBarcode": rb}) out["kinds"][kind] += 1 out["entries"].append((key, rec[0], digest(rec), rec[7], rec[8], rb, kind, tuple(tokens(rec[1] + " " + rec[2])), dup_sig(rec), rec[3])) out["err"] = (E.counts, dict(E.samples)) return out # ------------------------------------------------------------------------------------------------ helpers def pct(sorted_vals, p): if not sorted_vals: return None k = max(0, min(len(sorted_vals) - 1, math.ceil(p / 100 * len(sorted_vals)) - 1)) return sorted_vals[k] def size_stats(vals): v = sorted(vals) return {"n": len(v), "total": sum(v), "min": v[0] if v else None, "p50": pct(v, 50), "p90": pct(v, 90), "p95": pct(v, 95), "p99": pct(v, 99), "max": v[-1] if v else None, "over_400KiB": sum(1 for x in v if x > TARGET_FILE_BYTES), "over_1MiB": sum(1 for x in v if x > 1024 * 1024), "ge_5MiB": sum(1 for x in v if x >= MAX_FILE_BYTES), "over_25MiB": sum(1 for x in v if x > CF_MAX_FILE_BYTES)} class LinkParser(html.parser.HTMLParser): def __init__(self): super().__init__(convert_charrefs=True) self.links = [] self._href = None self._text = [] self.text = [] self.tags = collections.Counter() def handle_starttag(self, tag, attrs): self.tags[tag] += 1 if tag == "a": self._href = dict(attrs).get("href") self._text = [] def handle_endtag(self, tag): if tag == "a" and self._href is not None: self.links.append((self._href, "".join(self._text).strip())) self._href = None def handle_data(self, data): self.text.append(data) if self._href is not None: self._text.append(data) def parse_html(path, where, E): raw = path.read_bytes() try: text = raw.decode("utf-8") except UnicodeDecodeError as e: E.add("html.invalid_utf8", {"where": where, "err": str(e)[:200]}) return None, None p = LinkParser() try: p.feed(text) p.close() except Exception as e: # HTMLParser is lenient; anything raised here is a real problem E.add("html.parse_error", {"where": where, "err": str(e)[:200]}) return text, None if not text.lstrip().lower().startswith(""): E.add("html.no_doctype", {"where": where}) if "" not in text.lower(): E.add("html.no_charset", {"where": where}) return text, p def load_foodtok(): spec = importlib.util.spec_from_file_location("foodtok_under_test", HERE / "foodtok.py") mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod KNOWN_LICENSES = { # canonical license URL -> required words in the license name "https://creativecommons.org/publicdomain/zero/1.0/": ["CC0", "1.0"], "https://opendatacommons.org/licenses/odbl/1-0/": ["Open Database License", "1.0"], "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/": ["Open Government Licence", "3.0"], "https://www.etalab.gouv.fr/licence-ouverte-open-licence/": ["Licence Ouverte", "2.0"], } ODBL_URL = "https://opendatacommons.org/licenses/odbl/1-0/" DBCL_URL = "https://opendatacommons.org/licenses/dbcl/1-0/" CONTRACT_SOURCES = {"F": "USDA Foundation", "S": "USDA SR Legacy", "N": "USDA FNDDS", "B": "USDA Branded", "O": "Open Food Facts"} # ------------------------------------------------------------------------------------------------ main def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--site", default=str(DEFAULT_SITE)) ap.add_argument("--bundle", default=str(DEFAULT_BUNDLE)) ap.add_argument("--out", default=str(DEFAULT_OUT)) ap.add_argument("--workers", type=int, default=max(1, (os.cpu_count() or 2) - 2)) ap.add_argument("--names", type=int, default=20000, help="random index names for the foodtok comparison") args = ap.parse_args() t0 = time.time() site = Path(args.site) E = Errs(SAMPLES_PER_CODE_TOTAL) report = {"site": "", "checks": {}} def log(msg): print(f"[{time.time() - t0:7.1f}s] {msg}", flush=True) # ---------------- self-test of the re-implementation (fast path vs literal path, contract examples) for text, want in [("berries", ["berry"]), ("fries", ["fry"]), ("glasses", ["glass"]), ("tomatoes", ["tomato"]), ("peaches", ["peach"]), ("eggs", ["egg"]), ("oats", ["oat"]), ("chips", ["chip"]), ("Peaches & Cream Oats", ["peach", "cream", "oat"]), ("Bananas, raw", ["banana", "raw"]), ("bus hummus kiwis", ["bus", "hummus", "kiwis"]), ("a b cd", ["cd"]), ("Crème brûlée", ["creme", "brulee"]), ("salt and pepper of the", ["salt", "pepper"]), ("eggs EGGS egg", ["egg"]), ("fish", ["fish"]), ("fish½", ["fish1"]), # contract v2 stemming: rules 1-6 and their boundaries ("cookies cookie", ["cooky"]), ("pies pie", ["pie"]), ("brownie brownies", ["browny"]), ("brie bries", ["bry"]), ("quiche quiches", ["quich"]), ("glaze glazes", ["glaz"]), ("shoe shoes", ["sho"]), ("size sizes", ["siz"]), ("sloe", ["slo"]), ("toes", []), ("ties", ["tie"]), ("spies dies", ["spy", "die"]), ("tie roe ze", ["tie", "roe", "ze"]), ("hummus couscous swiss kiwis", ["hummus", "couscous", "swiss", "kiwis"]), ("crêpes œufs straße", ["crepe", "œufs", "straße"]), ("2pies", ["2pies"])]: got = tokens_slow(text) if got != want or tokens(text) != got: E.add("selftest.normalize", {"text": text, "want": want, "slow": got, "fast": tokens(text)}) if upce_to_upca("04252614") != "042100005264": E.add("selftest.upce", {"code": "04252614", "got": upce_to_upca("04252614")}) # ---------------- inventory of every file all_files = [] for dirpath, dirnames, filenames in os.walk(site): for fn in filenames: p = Path(dirpath) / fn all_files.append((p, p.stat().st_size)) rel = {p.relative_to(site).as_posix(): sz for p, sz in all_files} report["checks"]["files"] = {"total": len(all_files), "limit": MAX_FILES} if len(all_files) > MAX_FILES: E.add("limits.too_many_files", {"total": len(all_files), "limit": MAX_FILES}) for rp, sz in rel.items(): name = rp.rsplit("/", 1)[-1] if name.startswith(".") or name in ("Thumbs.db", "desktop.ini"): E.add("file.junk", {"file": rp}) if sz == 0: E.add("file.empty", {"file": rp}) if sz >= MAX_FILE_BYTES: E.add("limits.file_too_large", {"file": rp, "bytes": sz}) elif sz > TARGET_FILE_BYTES: E.add("size.over_target", {"file": rp, "bytes": sz}) log(f"inventory: {len(all_files)} files") # ---------------- manifest mpath = site / "manifest.json" if not mpath.exists(): E.add("manifest.missing", {}) print("manifest.json missing; cannot continue") return 2 manifest = strict_parse(mpath.read_bytes(), "manifest.json", E) if type(manifest) is not dict: E.add("manifest.type", {}) return 2 required = {"format": int, "contract": int, "build": str, "path": str, "tokenBuckets": int, "barcodeBuckets": int, "maxPerToken": int, "counts": dict, "sources": list} for k, ty in required.items(): if k not in manifest: E.add("manifest.missing_field", {"field": k}) elif type(manifest[k]) is not ty: E.add("manifest.field_type", {"field": k, "type": type(manifest[k]).__name__}) for k in manifest: if k not in required: E.add("manifest.extra_field", {"field": k}) if manifest.get("format") != 1: E.add("manifest.format", {"format": manifest.get("format")}) if manifest.get("contract") != CONTRACT_VERSION: E.add("manifest.contract", {"contract": manifest.get("contract"), "want": CONTRACT_VERSION}) build = manifest.get("build", "") if not re.fullmatch(r"\d{8}", str(build)): E.add("manifest.build_format", {"build": build}) dpath = manifest.get("path", "") if dpath != f"d/{build}": E.add("manifest.path_not_d_build", {"path": dpath, "build": build}) TB, BB, CAP = manifest.get("tokenBuckets"), manifest.get("barcodeBuckets"), manifest.get("maxPerToken") for k, v in (("tokenBuckets", TB), ("barcodeBuckets", BB), ("maxPerToken", CAP)): if type(v) is not int or v <= 0: E.add("manifest.bad_number", {"field": k, "value": v}) if manifest.get("baseUrl") not in (None, CONTRACT_BASE_URL): E.add("manifest.base_url", {"baseUrl": manifest.get("baseUrl"), "contract": CONTRACT_BASE_URL}) sources = manifest.get("sources") or [] declared, generic = set(), set() for s in sources: if type(s) is not dict: E.add("manifest.source_type", {"source": repr(s)[:100]}) continue for k in ("code", "name", "license", "url", "attribution", "credit"): if type(s.get(k)) is not str or not s.get(k).strip(): E.add("manifest.source_field", {"code": s.get("code"), "field": k}) cr = s.get("credit") if type(cr) is str and (len(cr) > CREDIT_MAX or cr != " ".join(cr.split()) or CTRL.search(cr)): E.add("manifest.source_credit", {"code": s.get("code"), "credit": cr, "len": len(cr)}) code = s.get("code") if code == "L": E.add("manifest.source_L_reserved", {}) if type(code) is not str or not re.fullmatch(r"[A-Z]", code): E.add("manifest.source_code", {"code": code}) continue if code in declared: E.add("manifest.source_duplicate", {"code": code}) declared.add(code) if s.get("generic") is True or code in ("F", "S", "N"): generic.add(code) if code in ("F", "S", "N") and s.get("generic") is False: E.add("manifest.contract_generic_marked_false", {"code": code}) if code in ("B", "O", "L") and s.get("generic") is True: E.add("manifest.branded_marked_generic", {"code": code}) if "generic" in s and type(s["generic"]) is not bool: E.add("manifest.generic_type", {"code": code}) data_dir = site / dpath if not data_dir.is_dir(): E.add("manifest.path_missing", {"path": dpath}) ddirs = sorted(p.name for p in (site / "d").iterdir()) if (site / "d").is_dir() else [] if ddirs != [str(build)]: E.add("layout.d_folders", {"found": ddirs, "expected": [build]}) report["checks"]["manifest"] = {"build": build, "path": dpath, "tokenBuckets": TB, "barcodeBuckets": BB, "maxPerToken": CAP, "declared": sorted(declared), "generic": sorted(generic)} # ---------------- layout of the data folder tdir, bdir = data_dir / "t", data_dir / "b" for p in sorted(data_dir.iterdir()) if data_dir.is_dir() else []: if p.name not in ("t", "b"): E.add("layout.unexpected_in_data_dir", {"name": p.name}) name_re = re.compile(r"(0|[1-9][0-9]*)\.json\Z") tjobs, bjobs = [], [] for d, jobs, nb, kind in ((tdir, tjobs, TB, "t"), (bdir, bjobs, BB, "b")): if not d.is_dir(): E.add("layout.missing_dir", {"dir": kind}) continue for p in d.iterdir(): m = name_re.match(p.name) if not p.is_file() or not m: E.add("layout.bad_shard_name", {"dir": kind, "name": p.name}) continue b = int(m.group(1)) if b >= nb: E.add("layout.bucket_out_of_range", {"dir": kind, "bucket": b, "buckets": nb}) continue jobs.append((str(p), b)) tjobs.sort(key=lambda x: x[1]) bjobs.sort(key=lambda x: x[1]) t_missing = sorted(set(range(TB)) - {b for _, b in tjobs}) b_missing = sorted(set(range(BB)) - {b for _, b in bjobs}) report["checks"]["layout"] = {"tokenShards": len(tjobs), "barcodeShards": len(bjobs), "tokenBucketsWithoutFile": len(t_missing), "barcodeBucketsWithoutFile": len(b_missing), "barcodeBucketsWithoutFileList": b_missing[:100], "tokenBucketsWithoutFileList": t_missing[:100]} tsizes = [rel[f"{dpath}/t/{b}.json"] for _, b in tjobs] bsizes = [rel[f"{dpath}/b/{b}.json"] for _, b in bjobs] other = [sz for rp, sz in rel.items() if not rp.startswith(dpath + "/")] report["checks"]["sizes"] = {"all": size_stats([sz for sz in rel.values()]), "token": size_stats(tsizes), "barcode": size_stats(bsizes), "other": size_stats(other), "largest": sorted(((sz, rp) for rp, sz in rel.items()), reverse=True)[:5]} log(f"layout: {len(tjobs)} token shards, {len(bjobs)} barcode shards") cfg = {"tokenBuckets": TB, "barcodeBuckets": BB, "maxPerToken": CAP, "declared": declared, "generic": generic} # ---------------- the build's duplicate-hiding log (hidden id -> kept id), needed while reading the shards hidden = {} hpath_q = ROOT / "qa" / "merge_hidden_duplicates.jsonl" if hpath_q.exists(): with hpath_q.open(encoding="utf-8") as f: for line in f: try: d = json.loads(line) hidden[d["hidden"]] = d.get("kept") except (ValueError, KeyError, TypeError): E.add("qa.hidden_log_unreadable_line", {"line": line[:120]}) dup_need = set(hidden) | {k for k in hidden.values() if k} dupinfo = {} # id -> (dup signature, kcal) for hidden records and their kept twins # ---------------- token shards (parallel) ids = {} # id -> [digest, tokens or None, score, src, barcode, set(buckets) or None] capped = {} tot_tokens = tot_post = tot_r = 0 list_lens = collections.Counter() sample, sample_na, sigma = {}, {}, {} all_tokens = set() ctx = mp.get_context("spawn") with ctx.Pool(args.workers, initializer=_init, initargs=(cfg,)) as pool: for k, res in enumerate(pool.imap_unordered(check_token_shard, tjobs, chunksize=16)): E.merge(*res["err"]) b = res["bucket"] tot_tokens += res["tokens"] tot_post += res["postings"] tot_r += res["r"] list_lens.update(res["list_lens"]) for t, key in res["capped"].items(): if t in capped: E.add("token.in_two_shards", {"token": t}) capped[t] = key for rid, dg, tk, score, src, bc, sig, kcal in res["records"]: if rid in dup_need: dupinfo[rid] = (sig, kcal) e = ids.get(rid) if e is None: ids[rid] = [dg, tk, score, src, bc, {b}] all_tokens.update(tk) else: if e[0] != dg: E.add("global.id_content_conflict", {"id": rid, "where": f"t/{b}"}) e[5].add(b) for rid, n, br in res["sample"]: sample[rid] = (n, br) for rid, n, br in res["sample_nonascii"]: sample_na[rid] = (n, br) for rid, n, br in res["sigma"]: sigma[rid] = (n, br) if (k + 1) % 2000 == 0: log(f"token shards {k + 1}/{len(tjobs)}") log("token shards done") bmap = {} kinds = collections.Counter() for k, res in enumerate(pool.imap_unordered(check_barcode_shard, bjobs, chunksize=8)): E.merge(*res["err"]) kinds.update(res["kinds"]) for key, rid, dg, src, score, rb, kind, tk, sig, kcal in res["entries"]: if rid in dup_need: dupinfo[rid] = (sig, kcal) if key in bmap: E.add("bkey.duplicate_across_shards", {"key": key}) bmap[key] = (rid, src, score, kind) e = ids.get(rid) if e is None: ids[rid] = [dg, None, score, src, rb, None, tk] elif e[0] != dg: E.add("global.id_content_conflict", {"id": rid, "where": f"b/{res['bucket']}", "key": key}) log("barcode shards done") token_ids = sum(1 for e in ids.values() if e[1] is not None) report["checks"]["tokenIndex"] = {"tokens": tot_tokens, "postings": tot_post, "rEntries": tot_r, "distinctRecords": token_ids, "cappedTokens": len(capped), "listLengthBuckets": {str(k): v for k, v in sorted(list_lens.items())}} # ---------------- cross-shard completeness: a token-searchable record must be in every one of its token lists bucket_of = {} missing_cross = 0 for rid, e in ids.items(): tk = e[1] if tk is None: continue seen = e[5] need = collections.defaultdict(list) for t in tk: c = bucket_of.get(t) if c is None: c = bucket_of[t] = fnv32(t) % TB need[c].append(t) for c in seen: if c not in need: E.add("completeness.record_in_foreign_shard", {"id": rid, "bucket": c}) for c, ts in need.items(): if c in seen: continue for t in ts: cut = capped.get(t) if cut is None or not ((-e[2], rid) > tuple(cut)): missing_cross += 1 E.add("completeness.missing_from_shard", {"id": rid, "token": t, "bucket": c, "score": e[2], "capped": cut is not None}) log("cross-shard completeness done") # ---------------- barcode index vs records barcode_only = collections.Counter() unsearchable_barcode_only = 0 collisions = collections.Counter() for rid, e in ids.items(): if e[1] is None: barcode_only[e[3]] += 1 rb = e[4] if not rb: continue m = bmap.get(rb) if m is None: E.add("barcode.record_not_in_barcode_shards", {"id": rid, "barcode": rb, "src": e[3]}) continue if m[0] == rid: continue win_src, win_score = m[1], m[2] if win_src == "O" and e[3] == "B": E.add("barcode.O_preferred_over_B", {"barcode": rb, "winner": m[0], "loser": rid}) elif win_src == e[3] and win_score < e[2]: E.add("barcode.lower_score_preferred", {"barcode": rb, "winner": m[0], "winnerScore": win_score, "loser": rid, "loserScore": e[2]}) else: collisions[f"{win_src}>{e[3]}"] += 1 E.add("barcode.collision_ok", {"barcode": rb, "winner": m[0], "loser": rid}) # Records that are only in barcode shards: explain why each one is missing from every token list. keys_of = collections.defaultdict(list) for key, (rid, src, score, kind) in bmap.items(): keys_of[rid].append(key) why = collections.Counter() why_src = collections.Counter() reason_of = {} def scannable(k): if len(k) in (13, 14) or (len(k) == 8 and upce_to_upca(k) is None): return gs1_check_digit(k[:-1]) == k[-1] return False unscannable = collections.Counter() for rid, e in ids.items(): if e[1] is not None: if rid in hidden: E.add("hidden.listed_in_token_index", {"id": rid}) if e[4] and not any(scannable(k) for k in keys_of.get(rid, [])): unscannable["searchable:" + e[3]] += 1 continue tk = e[6] if rid in hidden: reason = "hidden_duplicate" elif not tk: reason = "no_tokens" elif all(t in capped and (-e[2], rid) > tuple(capped[t]) for t in tk): reason = "capped_out_of_every_list" E.add("search.capped_out_of_every_list", {"id": rid, "score": e[2], "tokens": list(tk)}) else: reason = "unexplained" E.add("completeness.barcode_only_unexplained", {"id": rid, "score": e[2], "tokens": list(tk), "uncappedTokens": [t for t in tk if t not in capped], "aboveCutoff": [t for t in tk if t in capped and not ((-e[2], rid) > tuple(capped[t]))]}) why[reason] += 1 why_src[f"{reason}:{e[3]}"] += 1 reason_of[rid] = reason ks = keys_of.get(rid, []) if not any(scannable(k) for k in ks): unscannable[f"barcodeOnly:{reason}:{e[3]}"] += 1 E.add("barcode.barcode_only_and_unscannable", {"id": rid, "keys": ks, "reason": reason}) if ks and all(not (8 <= len(k) <= 14) for k in ks): E.add("barcode.only_key_not_lookupable_by_app", {"id": rid, "keys": ks, "reason": reason}) kept_why = collections.Counter() dup_rule = collections.Counter() for rid, kept in hidden.items(): if rid not in ids: E.add("hidden.hidden_record_not_in_index", {"hidden": rid}) continue # contract "Duplicate hiding": a hidden record has a barcode, and its kept twin has the same normalized name, # brand and unit, kcal within 2 %, and the higher (score, then lower id) he, ke = ids[rid], ids.get(kept) if not he[4]: E.add("hidden.no_barcode", {"hidden": rid}) if ke is not None: hs, hk = dupinfo.get(rid, (None, None)) ks, kk = dupinfo.get(kept, (None, None)) if hs is None or ks is None or hs != ks: E.add("hidden.rule_name_brand_unit_differ", {"hidden": rid, "kept": kept}) elif abs(hk - kk) > 0.02 * max(hk, kk) + 1e-9: E.add("hidden.rule_kcal_over_2pct", {"hidden": rid, "kept": kept, "kcal": hk, "keptKcal": kk}) elif not (ke[2] > he[2] or (ke[2] == he[2] and kept.encode() < rid.encode())): E.add("hidden.rule_kept_not_higher", {"hidden": rid, "kept": kept, "score": he[2], "keptScore": ke[2]}) else: dup_rule["ok"] += 1 if kept in hidden: E.add("hidden.rule_kept_is_hidden", {"hidden": rid, "kept": kept}) ke = ids.get(kept) if ke is None: kept_why["kept_missing"] += 1 E.add("hidden.kept_record_missing", {"hidden": rid, "kept": kept}) elif ke[1] is None: r_ = reason_of.get(kept, "?") kept_why["kept_" + r_] += 1 code = "hidden.kept_also_hidden" if r_ == "hidden_duplicate" else "search.hidden_and_kept_twin_capped_out" E.add(code, {"hidden": rid, "kept": kept, "keptReason": r_}) he = ids[rid] # Would the hidden twin itself have made the cut in at least one of its token lists? if he[6] and any(t not in capped or (-he[2], rid) < tuple(capped[t]) for t in he[6]): kept_why["hidden_twin_would_have_been_listed"] += 1 E.add("hidden.dedupe_hid_the_listable_twin", {"hidden": rid, "hiddenScore": he[2], "kept": kept, "keptScore": ke[2]}) else: kept_why["kept_searchable"] += 1 # Barcode key lengths and GS1 check digits (scanners only return codes whose check digit is right). klen = collections.Counter() kbad = collections.Counter() for key, (rid, src, score, kind) in bmap.items(): klen[f"{src}:{len(key)}"] += 1 if len(key) in (8, 13, 14) and gs1_check_digit(key[:-1]) != key[-1]: kbad[f"{src}:{len(key)}" + (":validUPCE" if len(key) == 8 and upce_to_upca(key) else "")] += 1 if len(key) < 8: E.add("barcode.key_shorter_than_app_accepts", {"key": key, "id": rid}) report["checks"]["barcodeIndex"] = {"keys": len(bmap), "kinds": dict(kinds), "collisions": dict(collisions), "barcodeOnlyRecordsBySource": dict(barcode_only), "barcodeOnlyWhy": dict(why), "barcodeOnlyWhyBySource": dict(sorted(why_src.items())), "hiddenDuplicatesInQaLog": len(hidden), "hiddenKeptTwin": dict(kept_why), "hiddenPairsMeetingContractRule": dup_rule["ok"], "recordsWithoutScannableKey": dict(sorted(unscannable.items())), "keyLengthsBySource": dict(sorted(klen.items())), "badCheckDigitBySourceLength": dict(sorted(kbad.items()))} # ---------------- counts vs manifest by_src = collections.Counter(e[3] for e in ids.values()) counts = manifest.get("counts") or {} actual = {"records": len(ids), "bySource": dict(sorted(by_src.items())), "tokenSearchable": token_ids, "barcodes": len(bmap), "genericInIndex": sum(v for k, v in by_src.items() if k in generic)} report["checks"]["counts"] = {"manifest": counts, "actual": actual} if counts.get("records") != len(ids): E.add("manifest.count_records", {"manifest": counts.get("records"), "actual": len(ids)}) mbs = counts.get("bySource") or {} for code in sorted(set(mbs) | set(by_src)): if mbs.get(code) != by_src.get(code, 0): E.add("manifest.count_by_source", {"src": code, "manifest": mbs.get(code), "actual": by_src.get(code, 0)}) if sum(v for v in mbs.values() if type(v) is int) != counts.get("records"): E.add("manifest.count_sum", {"sum": sum(mbs.values()), "records": counts.get("records")}) if "tokenSearchable" in counts and counts["tokenSearchable"] != token_ids: E.add("manifest.count_token_searchable", {"manifest": counts["tokenSearchable"], "actual": token_ids, "barcodeOnlyWhy": dict(why)}) if "barcodes" in counts and counts["barcodes"] != len(bmap): E.add("manifest.count_barcodes", {"manifest": counts["barcodes"], "actual": len(bmap)}) for code in by_src: if code not in declared: E.add("manifest.source_not_declared", {"src": code}) for code in declared: if code not in by_src: E.add("manifest.declared_source_without_records", {"src": code}) # ---------------- bundle bpath = Path(args.bundle) bundle_ids = {} if not bpath.exists(): E.add("bundle.missing", {}) else: braw = bpath.read_bytes() bundle = strict_parse(braw, "bundle", E) binfo = {"bytes": len(braw), "target": BUNDLE_TARGET_BYTES} if len(braw) >= BUNDLE_TARGET_BYTES: E.add("bundle.over_target", {"bytes": len(braw)}) if type(bundle) is dict: for k in bundle: if k not in ("format", "contract", "build", "sources", "r"): E.add("bundle.extra_key", {"key": k}) if bundle.get("format") != 1: E.add("bundle.format", {"format": bundle.get("format")}) if bundle.get("contract") != CONTRACT_VERSION: E.add("bundle.contract", {"contract": bundle.get("contract"), "want": CONTRACT_VERSION}) if bundle.get("build") != build: E.add("bundle.build_mismatch", {"bundle": bundle.get("build"), "manifest": build}) if bundle.get("sources") != manifest.get("sources"): E.add("bundle.sources_differ_from_manifest", {}) br = bundle.get("r") if type(br) is not list: E.add("bundle.r_type", {}) br = [] bsrc = collections.Counter() for i, rec in enumerate(br): if not check_record(rec, "bundle", E, declared, generic): continue rid = rec[0] if rid in bundle_ids: E.add("bundle.duplicate_id", {"id": rid}) bundle_ids[rid] = i bsrc[rec[7]] += 1 if rec[7] not in generic: E.add("bundle.non_generic_record", {"id": rid, "src": rec[7]}) if rec[6] != "": E.add("bundle.record_has_barcode", {"id": rid, "barcode": rec[6]}) if not tokens(rec[1] + " " + rec[2]): E.add("bundle.unsearchable", {"id": rid, "name": rec[1]}) e = ids.get(rid) if e is None: E.add("bundle.generic_not_in_index", {"id": rid}) elif e[0] != digest(rec): E.add("bundle.differs_from_index", {"id": rid}) for rid, e in ids.items(): if e[3] in generic and rid not in bundle_ids: E.add("index.generic_not_in_bundle", {"id": rid}) binfo.update({"records": len(br), "bySource": dict(bsrc)}) if "generic" in counts and counts["generic"] != len(bundle_ids): E.add("manifest.count_generic", {"manifest": counts["generic"], "bundle": len(bundle_ids)}) else: E.add("bundle.type", {}) report["checks"]["bundle"] = binfo log("bundle done") # ---------------- _headers hpath = site / "_headers" rules = [] if not hpath.exists(): E.add("headers.missing", {}) else: htext = hpath.read_bytes().decode("utf-8", errors="replace") if not htext.endswith("\n"): E.add("headers.no_trailing_newline", {}) cur = None for ln_no, ln in enumerate(htext.splitlines(), 1): if len(ln) > 2000: E.add("headers.line_too_long", {"line": ln_no}) if not ln.strip() or ln.lstrip().startswith("#"): continue if ln[0] not in " \t": u = ln.strip() if not (u.startswith("/") or re.match(r"https://", u)): E.add("headers.bad_url_line", {"line": ln_no, "text": ln}) if u.count("*") > 1: E.add("headers.multiple_splats", {"line": ln_no, "text": ln}) cur = [u, []] rules.append(cur) else: if cur is None: E.add("headers.header_before_url", {"line": ln_no, "text": ln}) continue s = ln.strip() if s.startswith("!"): cur[1].append(("!", s[1:].strip())) continue hname, sep, value = s.partition(":") if not sep or not re.fullmatch(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+", hname) or not value.strip(): E.add("headers.bad_header_line", {"line": ln_no, "text": ln}) continue cur[1].append((hname, value.strip())) if len(rules) > 100: E.add("headers.too_many_rules", {"rules": len(rules)}) want = {"/d/*": ("Cache-Control", "public, max-age=31536000, immutable"), "/manifest.json": ("Cache-Control", "public, max-age=3600")} got = {u: hs for u, hs in rules} for u, (hn, hv) in want.items(): hs = got.get(u) if hs is None: E.add("headers.rule_missing", {"url": u}) elif not any(n.lower() == hn.lower() and v == hv for n, v in hs): E.add("headers.rule_value", {"url": u, "got": hs, "want": [hn, hv]}) for u in got: if u not in want: E.add("headers.extra_rule", {"url": u, "headers": got[u]}) if not dpath.startswith("d/"): E.add("headers.data_path_not_under_d", {"path": dpath}) report["checks"]["headers"] = {"rules": [[u, hs] for u, hs in rules]} # ---------------- HTML pages, links, attribution pages = {} for rp in sorted(rel): if rp.endswith(".html"): text, parser = parse_html(site / rp, rp, E) pages[rp] = (text, parser) if parser is None: continue for href, _ in parser.links: if not href or re.match(r"(https?:|mailto:)", href) or href.startswith("#"): continue base = "" if href.startswith("/") else (rp.rsplit("/", 1)[0] + "/" if "/" in rp else "") target = (base + href.lstrip("/")).split("#")[0] parts = [] for seg in target.split("/"): if seg == "..": if parts: parts.pop() elif seg not in ("", "."): parts.append(seg) target = "/".join(parts) + ("/" if target.endswith("/") and parts else "") if target.endswith("/") or target == "": target += "index.html" if target not in rel: E.add("html.broken_link", {"page": rp, "href": href, "resolved": target}) elif target.endswith(".html") and target != "404.html": E.add("html.link_pretty_url", {"page": rp, "href": href}) for need in ("index.html", "attribution.html", "404.html"): if need not in rel: E.add("site.missing_page", {"page": need}) atext, ap_ = pages.get("attribution.html", (None, None)) attr_info = {} if atext and ap_: plain = html.unescape(" ".join("".join(ap_.text).split())) links = ap_.links hrefs = {h for h, _ in links} for s in sources: code = s.get("code") info = {} info["name"] = html.unescape(" ".join(s.get("name", "").split())) in plain info["code"] = f"{code}" in atext info["license_link"] = any(h == s.get("licenseUrl") and t == " ".join(s.get("license", "").split()) for h, t in links) if s.get("licenseUrl") else (s.get("license", "") in plain) info["attribution"] = " ".join(s.get("attribution", "").split()) in plain info["url_link"] = s.get("url") in hrefs if s.get("credit") and " ".join(s["credit"].split()) not in plain: E.add("attr.source_credit_not_on_page", {"code": code}) n = (counts.get("bySource") or {}).get(code) info["count"] = n is not None and f"{n:,} records" in plain for k, v in info.items(): if not v: E.add("attr.source_" + k, {"code": code}) lurl = s.get("licenseUrl") words = KNOWN_LICENSES.get(lurl) if words is None: E.add("attr.license_url_not_license", {"code": code, "licenseUrl": lurl, "license": s.get("license")}) elif not all(wd in s.get("license", "") for wd in words): E.add("attr.license_name_url_mismatch", {"code": code, "license": s.get("license"), "licenseUrl": lurl}) if code in ("F", "S", "N", "B"): a = s.get("attribution", "") if lurl != "https://creativecommons.org/publicdomain/zero/1.0/": E.add("attr.usda_not_cc0", {"code": code}) if "USDA, Agricultural Research Service, FoodData Central" not in a: if ("Agricultural Research Service" in a and "FoodData Central" in a and ("USDA" in a or "U.S. Department of Agriculture" in a)): E.add("attr.usda_credit_wording", {"code": code, "attribution": a}) else: E.add("attr.usda_credit_missing", {"code": code, "attribution": a}) if code == "O": if lurl != ODBL_URL: E.add("attr.off_not_odbl", {"licenseUrl": lurl}) if "DbCL" not in s.get("license", "") and "Database Contents License" not in s.get("license", ""): E.add("attr.off_no_dbcl", {}) attr_info[code] = info for code, label in CONTRACT_SOURCES.items(): if code not in declared: E.add("manifest.contract_source_missing", {"code": code, "source": label}) # ODbL share-alike notice odbl = {"odbl_link": ODBL_URL in hrefs, "dbcl_link": DBCL_URL in hrefs, "off_link": any(h.rstrip("/") == "https://world.openfoodfacts.org" for h in hrefs), "derived_db_under_odbl": bool(re.search(r"(derivative|derived) database[^.]*\.?[^.]*made\s+available\s+under\s+the\s+Open Database License", plain)), "scripts_link": any(h in ("scripts/", "/scripts/", "scripts/index.html") for h in hrefs), "manifest_link": any(h.endswith("manifest.json") for h in hrefs)} for k, v in odbl.items(): if not v: E.add("attr.odbl_" + k, {}) attr_info["odbl"] = odbl else: E.add("attr.unreadable", {}) itext, ip_ = pages.get("index.html", (None, None)) if ip_: ih = {h for h, _ in ip_.links} if "attribution.html" not in ih: E.add("index.no_attribution_link", {}) if ODBL_URL not in ih: E.add("index.no_odbl_link", {}) report["checks"]["attribution"] = attr_info # ---------------- privacy scan of the non-shard text files emails = collections.Counter() home = str(Path.home()) for rp in rel: if rp.startswith(dpath + "/"): continue text = (site / rp).read_bytes().decode("utf-8", errors="replace") for m in re.finditer(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", text): emails[m.group(0)] += 1 if "/" + "Users/" in text or home in text or "/" + "home/" in text: # split so this file's own copy passes E.add("privacy.local_path", {"file": rp}) if re.search(r"/(?:private/)?tmp/claude-", text): E.add("privacy.tmp_path", {"file": rp}) report["checks"]["emailsInSiteText"] = dict(emails) for em in emails: if em.lower() != "support@powerupgaming.dev": E.add("privacy.other_email", {"email": em}) # ---------------- script copies sdir = site / "scripts" repo_s, home_s = str(REPO), str(Path.home()) def sanitized(text): text = text.replace(repo_s, "") text = re.sub(r"/(?:private/)?tmp/claude-[^\s'\"`)]*", "", text) return text.replace(home_s, "") build_time = mpath.stat().st_mtime copies = {} for f in sorted(HERE.glob("*.py")): cp = sdir / (f.name + ".txt") if not cp.exists(): code = "scripts.post_build_missing" if f.stat().st_mtime > build_time else "scripts.copy_missing" E.add(code, {"script": f.name}) copies[f.name] = "missing" continue same = sanitized(f.read_text(encoding="utf-8")) == cp.read_text(encoding="utf-8") copies[f.name] = "same" if same else "differs" if not same: E.add("scripts.copy_differs", {"script": f.name}) cc = sdir / "CONTRACT.md.txt" if not cc.exists(): E.add("scripts.contract_missing", {}) elif sanitized((ROOT / "CONTRACT.md").read_text(encoding="utf-8")) != cc.read_text(encoding="utf-8"): E.add("scripts.contract_differs", {}) if cc.exists() and not cc.read_text(encoding="utf-8").startswith("# Losing It food index: contract v2"): E.add("scripts.contract_not_v2", {}) for p in sdir.glob("*.txt"): stem_ = p.name[:-4] if stem_ != "CONTRACT.md" and not (HERE / stem_).exists(): E.add("scripts.orphan_copy", {"file": p.name}) stext, sp_ = pages.get("scripts/index.html", (None, None)) if sp_: listed = {h for h, _ in sp_.links} for p in sdir.glob("*.txt"): if p.name not in listed: E.add("scripts.not_listed", {"file": p.name}) else: E.add("scripts.index_missing", {}) report["checks"]["scriptCopies"] = copies # ---------------- tokenizer agreement: foodtok.py vs this re-implementation ft = load_foodtok() tv = json.loads((HERE / "tok_vectors.json").read_text(encoding="utf-8")) tv_bad = {"mine_vs_vector": 0, "foodtok_vs_vector": 0, "mine_vs_foodtok": 0, "fast_vs_slow": 0, "buckets": 0} for v in tv: text = v["text"] mine, fast, theirs = tokens_slow(text), tokens(text), ft.normalize(text) if mine != v["tokens"]: tv_bad["mine_vs_vector"] += 1 E.add("tok.vector_mine", {"text": text, "want": v["tokens"], "got": mine}) if theirs != v["tokens"]: tv_bad["foodtok_vs_vector"] += 1 E.add("tok.vector_foodtok", {"text": text, "want": v["tokens"], "got": theirs}) if mine != theirs: tv_bad["mine_vs_foodtok"] += 1 E.add("tok.mine_vs_foodtok", {"text": text, "mine": mine, "foodtok": theirs}) if fast != mine: tv_bad["fast_vs_slow"] += 1 E.add("tok.fast_vs_slow", {"text": text}) mb = [fnv32(t) % TB for t in mine] if "buckets" in v and (mb != v["buckets"] or [ft.token_bucket(t, TB) for t in mine] != mb): tv_bad["buckets"] += 1 E.add("tok.vector_buckets", {"text": text, "want": v.get("buckets"), "got": mb}) bv = json.loads((HERE / "barcode_vectors.json").read_text(encoding="utf-8")) bv_bad = 0 for v in bv: mine = norm_barcode(v["text"]) theirs = ft.normalize_barcode(v["text"]) mb = fnv32(mine) % BB if mine else None if mine != v["barcode"] or theirs != mine or mb != v["barcodeBucket"]: bv_bad += 1 E.add("tok.barcode_vector", {"text": v["text"], "want": [v["barcode"], v["barcodeBucket"]], "mine": [mine, mb], "foodtok": theirs}) rng = random.Random(20261115) pool_ids = sorted(sample) picked = rng.sample(pool_ids, min(args.names, len(pool_ids))) na_ids = sorted(sample_na) picked_na = rng.sample(na_ids, min(args.names, len(na_ids))) name_stats = collections.Counter() for group, idl, src_map in (("random", picked, sample), ("nonascii", picked_na, sample_na)): for rid in idl: n, br = src_map[rid] for text in (n, n + " " + br): name_stats[group + "_texts"] += 1 mine, fast, theirs = tokens_slow(text), tokens(text), ft.normalize(text) if mine != theirs: name_stats[group + "_mismatch"] += 1 E.add("tok.names_mine_vs_foodtok", {"id": rid, "text": text, "mine": mine, "foodtok": theirs}) if fast != mine: name_stats[group + "_fast_vs_slow"] += 1 E.add("tok.fast_vs_slow", {"id": rid, "text": text}) for t in mine: if fnv32(t) != ft.fnv1a(t): name_stats[group + "_hash_mismatch"] += 1 E.add("tok.fnv_mismatch", {"token": t}) hash_bad = 0 for t in all_tokens: if fnv32(t) != ft.fnv1a(t): hash_bad += 1 E.add("tok.fnv_mismatch", {"token": t}) upce_codes = {k for k in bmap if len(k) == 8} | {e[4] for e in ids.values() if len(e[4]) == 8} upce_bad = 0 for c in upce_codes: if upce_to_upca(c) != ft.expand_upce(c): upce_bad += 1 E.add("tok.upce_mismatch", {"code": c, "mine": upce_to_upca(c), "foodtok": ft.expand_upce(c)}) for s_ in sigma.items(): E.add("lower.final_sigma", {"id": s_[0], "name": s_[1][0], "brand": s_[1][1]}) report["checks"]["tokenizerAgreement"] = { "tokVectors": len(tv), "tokVectorProblems": tv_bad, "barcodeVectors": len(bv), "barcodeVectorProblems": bv_bad, "randomNames": len(picked), "nonAsciiNames": len(picked_na), "nameStats": dict(name_stats), "distinctTokensHashed": len(all_tokens), "hashMismatches": hash_bad, "upceCodesCompared": len(upce_codes), "upceMismatches": upce_bad, "recordsWithFinalSigmaCaseDifference": len(sigma)} log("tokenizer agreement done") # ---------------- write report errors = {k: v for k, v in E.counts.items() if k not in WARN_CODES} warns = {k: v for k, v in E.counts.items() if k in WARN_CODES} report["errors"] = dict(sorted(errors.items())) report["warnings"] = dict(sorted(warns.items())) report["samples"] = {k: v[:10] for k, v in sorted(E.samples.items())} report["seconds"] = round(time.time() - t0, 1) out = Path(args.out) out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(report, ensure_ascii=False, indent=1, default=list), encoding="utf-8") vpath = out.with_name(out.stem + "_violations.jsonl") with vpath.open("w", encoding="utf-8") as f: for code in sorted(E.samples): level = "warn" if code in WARN_CODES else "error" for d in E.samples[code]: f.write(json.dumps({"level": level, "code": code, **d}, ensure_ascii=False, default=list) + "\n") print(json.dumps({"errors": report["errors"], "warnings": report["warnings"], "files": report["checks"]["files"], "sizes": report["checks"]["sizes"], "counts": report["checks"]["counts"], "barcodeIndex": report["checks"]["barcodeIndex"], "tokenIndex": report["checks"]["tokenIndex"], "layout": report["checks"]["layout"], "tokenizerAgreement": report["checks"]["tokenizerAgreement"], "bundle": report["checks"].get("bundle"), "seconds": report["seconds"]}, ensure_ascii=False, indent=1, default=list)) return 1 if errors else 0 if __name__ == "__main__": sys.exit(main())