#!/usr/bin/env python3
"""Independent contract and limits audit of the Losing It food index site and the offline bundle.
Checks FoodIndex/out/site and LosingIt/Resources/GenericFoods.json against FoodIndex/CONTRACT.md (contract v2):
manifest "format": 1 and "contract": 2, a 1-120 character "credit" on every source, the reserved source letter L,
v2 token normalization (every record's tokens recomputed and checked against every token list), and the
"Duplicate hiding" rule for every hidden record (qa/merge_hidden_duplicates.jsonl).
The token normalization, FNV-1a hash, barcode normalization and UPC-E expansion below are written from the
contract text and the GS1 UPC-E rules. They do not import foodtok.py. foodtok.py is loaded only to compare the two.
Run (stdlib only):
python3 -I qa_contract.py [--site DIR] [--bundle FILE] [--out FILE] [--workers N]
Writes a JSON report (default FoodIndex/qa/qa_contract.json) and every violation sample to
FoodIndex/qa/qa_contract_violations.jsonl, and prints a summary. Exit status 1 when any error-level violation exists.
"""
import argparse
import collections
import hashlib
import html
import html.parser
import importlib.util
import json
import math
import multiprocessing as mp
import os
import random
import re
import sys
import time
import unicodedata
import zlib
from pathlib import Path
HERE = Path(__file__).resolve().parent # FoodIndex/scripts
ROOT = HERE.parent # FoodIndex
REPO = ROOT.parent # app repository
DEFAULT_SITE = ROOT / "out" / "site"
DEFAULT_BUNDLE = REPO / "LosingIt" / "Resources" / "GenericFoods.json"
DEFAULT_OUT = ROOT / "qa" / "qa_contract.json"
CONTRACT_BASE_URL = "https://losingit-food.pages.dev"
MAX_FILES = 19500
MAX_FILE_BYTES = 5 * 1024 * 1024
CF_MAX_FILE_BYTES = 25 * 1024 * 1024
TARGET_FILE_BYTES = 400 * 1024
BUNDLE_TARGET_BYTES = 4_000_000
SAMPLES_PER_CODE_WORKER = 25
SAMPLES_PER_CODE_TOTAL = 5000
# Codes that are not contract violations (data quality / informational). Everything else is an error.
WARN_CODES = {
"brand.untrimmed", "brand.whitespace", "name.allcaps_B", "name.allcaps_O", "name.mojibake", "name.replacement_char",
"portion.dup_label", "portion.label_whitespace", "token.not_idempotent", "lower.final_sigma",
"barcode.nonstandard_length", "barcode.collision_ok", "bundle.over_target", "bundle.unsearchable",
"bundle.extra_key", "html.link_pretty_url", "attr.usda_credit_wording", "attr.license_url_not_license",
"headers.no_trailing_newline", "manifest.extra_field", "scripts.post_build_missing", "kcal.negative_zero",
"size.over_target", "file.junk", "bundle.generic_not_in_index", "index.generic_not_in_bundle",
"name.lone_surrogate_ok", "search.capped_out_of_every_list", "barcode.key_shorter_than_app_accepts",
"barcode.only_key_not_lookupable_by_app", "search.hidden_and_kept_twin_capped_out",
"barcode.barcode_only_and_unscannable", "attr.source_credit_not_on_page",
}
CONTRACT_VERSION = 2
CREDIT_MAX = 120
# --------------------------------------------------------------------------------------- contract re-implementation
STOPWORDS = frozenset("and with of the in an for or to on de la le et en con mit und from".split())
_AZ4 = re.compile(r"[a-z]{4,}")
_ASCII_NON_ALNUM = re.compile(r"[^a-z0-9]+")
def stem(tok):
"""Contract v2 step 5: only all-ASCII a-z tokens of at least 4 scalars (n); the first matching rule, once."""
if not _AZ4.fullmatch(tok):
return tok
n = len(tok)
if n >= 5 and tok[-3:] == "ies": # 1
return tok[:-3] + "y"
if tok[-4:] == "sses": # 2
return tok[:-2]
if tok[-3:] in ("oes", "xes", "zes") or tok[-4:] in ("ches", "shes"): # 3
return tok[:-2]
if tok[-2:] == "ie": # 4
return tok[:-2] + "y"
if tok[-3:] == "che" or tok[-2:] in ("oe", "ze"): # 5
return tok[:-1]
if tok[-1] == "s" and tok[-2:] not in ("ss", "us", "is"): # 6
return tok[:-1]
return tok
def _finish(pieces):
"""Steps 4, 5, 6, 7."""
out = []
seen = set()
for p in pieces:
if len(p) < 2:
continue
t = stem(p)
if t in STOPWORDS:
continue
if t in seen:
continue
seen.add(t)
out.append(t)
return out
def tokens_slow(text):
"""The contract's normalize(), literally: NFKD, drop Mn, lower(), split on non-L/N scalars, then steps 4-7."""
s = unicodedata.normalize("NFKD", text)
s = "".join(c for c in s if unicodedata.category(c) != "Mn")
s = s.lower()
pieces = []
cur = []
for c in s:
if unicodedata.category(c)[0] in ("L", "N"):
cur.append(c)
else:
if cur:
pieces.append("".join(cur))
cur = []
if cur:
pieces.append("".join(cur))
return _finish(pieces)
def tokens(text):
"""Same result as tokens_slow. For pure ASCII text, NFKD and Mn removal change nothing and the only L*/N*
scalars are A-Z, a-z and 0-9, so a regex split on the lowercased text is equivalent (checked in self-test)."""
if text.isascii():
return _finish([p for p in _ASCII_NON_ALNUM.split(text.lower()) if p])
return tokens_slow(text)
def fnv32(text):
h = 0x811C9DC5
for b in text.encode("utf-8"):
h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF
return h
def norm_barcode(raw):
"""Contract: digits only; 12 -> '0'+code; 14 with leading '0' -> drop it; 8 and 13 as is; other 6..14 as is; else None."""
d = "".join(c for c in raw if c in "0123456789")
n = len(d)
if n == 12:
return "0" + d
if n == 14 and d.startswith("0"):
return d[1:]
if 6 <= n <= 14:
return d
return None
def gs1_check_digit(body):
"""GS1 mod-10 check digit for the digits in body (weights 3,1,3,... from the rightmost body digit)."""
total = 0
for i, ch in enumerate(reversed(body)):
total += int(ch) * (3 if i % 2 == 0 else 1)
return str((10 - total % 10) % 10)
def upce_to_upca(code):
"""GS1 UPC-E to UPC-A for an 8-digit code (number system 0/1 + 6 digits + check). None if not valid UPC-E."""
if len(code) != 8 or not code.isdigit() or code[0] not in "01":
return None
ns, x, chk = code[0], code[1:7], code[7]
last = x[5]
if last in "012":
manuf, prod = x[0] + x[1] + last + "00", "00" + x[2] + x[3] + x[4]
elif last == "3":
manuf, prod = x[0] + x[1] + x[2] + "00", "000" + x[3] + x[4]
elif last == "4":
manuf, prod = x[0] + x[1] + x[2] + x[3] + "0", "0000" + x[4]
else:
manuf, prod = x[0:5], "0000" + last
body = ns + manuf + prod
if gs1_check_digit(body) != chk:
return None
return body + chk
# --------------------------------------------------------------------------------------------- record rules
ID_RE = re.compile(r"([A-Z]):(\S+)\Z")
BAD_WS = re.compile(r"\s\s|[^\S ]") # two whitespace in a row, or any whitespace other than ' '
# U+200C/U+200D (ZWNJ/ZWJ) are allowed: they are part of emoji sequences and of Persian/Indic spelling.
CTRL = re.compile("[\\x00-\\x1f\\x7f-\\x9f\\u200b\\u200e\\u200f\\u2028\\u2029\\ufeff]")
MOJIBAKE = re.compile(r"Ã[\x80-\xbf]|â€|Â[\xa0-\xbf]")
_METRIC = r"(?:kg|g|gr|grs|gram|grams|gramm|grammes?|mg|ml|cl|dl|l|lt|ltr|litre|liter|litres|liters|millilit(?:er|re)s?)"
METRIC_IN_LABEL = re.compile(r"(? 0:
lst.extend(v[:room])
def is_num(x):
return type(x) in (int, float)
def check_record(rec, where, E, declared, generic):
"""Validate one positional record. Returns True when the record is well-formed enough to index."""
if type(rec) is not list:
E.add("rec.not_array", {"where": where, "rec": repr(rec)[:200]})
return False
if len(rec) != 9:
E.add("rec.length", {"where": where, "len": len(rec), "rec": rec[:3]})
return False
rid, name, brand, kcal, unit, portions, barcode, src, score = rec
ok = True
# 7 src
if type(src) is not str or len(src) != 1 or not ("A" <= src <= "Z"):
E.add("src.invalid", {"where": where, "id": rid, "src": src})
ok = False
elif src == "L":
E.add("src.reserved_L", {"where": where, "id": rid})
elif src not in declared:
E.add("src.undeclared", {"where": where, "id": rid, "src": src})
# 0 id
if type(rid) is not str:
E.add("id.type", {"where": where, "id": repr(rid)})
ok = False
else:
m = ID_RE.match(rid)
if not m:
E.add("id.format", {"where": where, "id": rid})
elif type(src) is str and m.group(1) != src:
E.add("id.src_mismatch", {"where": where, "id": rid, "src": src})
# 1 name
if type(name) is not str:
E.add("name.type", {"where": where, "id": rid})
ok = False
else:
if not (2 <= len(name) <= 120):
E.add("name.length", {"where": where, "id": rid, "name": name, "len": len(name)})
if name != name.strip():
E.add("name.untrimmed", {"where": where, "id": rid, "name": name})
if BAD_WS.search(name):
E.add("name.whitespace", {"where": where, "id": rid, "name": name})
if CTRL.search(name):
E.add("name.control_char", {"where": where, "id": rid, "name": name})
if not name.isascii():
try:
name.encode("utf-8")
except UnicodeEncodeError:
E.add("name.lone_surrogate", {"where": where, "id": rid, "name": repr(name)})
if "�" in name:
E.add("name.replacement_char", {"where": where, "id": rid, "name": name})
if MOJIBAKE.search(name):
E.add("name.mojibake", {"where": where, "id": rid, "name": name})
letters = [c for c in name if c.isalpha()]
if len(letters) >= 4 and all(c.isupper() for c in letters):
if src == "B":
E.add("name.allcaps_B", {"where": where, "id": rid, "name": name})
elif src == "O":
E.add("name.allcaps_O", {"where": where, "id": rid, "name": name})
# 2 brand
if type(brand) is not str:
E.add("brand.type", {"where": where, "id": rid})
ok = False
else:
if brand != brand.strip():
E.add("brand.untrimmed", {"where": where, "id": rid, "brand": brand})
if BAD_WS.search(brand):
E.add("brand.whitespace", {"where": where, "id": rid, "brand": brand})
if CTRL.search(brand):
E.add("brand.control_char", {"where": where, "id": rid, "brand": brand})
if not brand.isascii():
try:
brand.encode("utf-8")
except UnicodeEncodeError:
E.add("brand.lone_surrogate", {"where": where, "id": rid, "brand": repr(brand)})
# 3 kcal
if not is_num(kcal):
E.add("kcal.type", {"where": where, "id": rid, "kcal": repr(kcal)})
elif not math.isfinite(kcal) or kcal < 0 or kcal > 900:
E.add("kcal.range", {"where": where, "id": rid, "kcal": kcal})
else:
if round(kcal, 1) != kcal:
E.add("kcal.decimals", {"where": where, "id": rid, "kcal": kcal})
if kcal == 0 and math.copysign(1.0, kcal) < 0:
E.add("kcal.negative_zero", {"where": where, "id": rid})
# 4 unit
if unit not in ("g", "ml") or type(unit) is not str:
E.add("unit.invalid", {"where": where, "id": rid, "unit": repr(unit)})
# 5 portions
if type(portions) is not list:
E.add("portions.type", {"where": where, "id": rid})
else:
if len(portions) > 6:
E.add("portions.too_many", {"where": where, "id": rid, "n": len(portions)})
labels = set()
for p in portions:
if type(p) is not list or len(p) != 2:
E.add("portion.shape", {"where": where, "id": rid, "portion": repr(p)[:120]})
continue
lab, amt = p
if type(lab) is not str or not lab.strip():
E.add("portion.label_type", {"where": where, "id": rid, "portion": p})
else:
if lab != lab.strip() or BAD_WS.search(lab):
E.add("portion.label_whitespace", {"where": where, "id": rid, "label": lab})
if CTRL.search(lab):
E.add("portion.label_control_char", {"where": where, "id": rid, "label": lab})
pm = PLAIN_METRIC.match(lab)
if pm and is_num(amt) and abs(amt - 100) < 1e-9:
E.add("portion.plain_100", {"where": where, "id": rid, "portion": p})
elif METRIC_IN_LABEL.search(lab):
E.add("portion.label_has_metric_amount", {"where": where, "id": rid, "unit": unit, "portion": p})
if lab in labels:
E.add("portion.dup_label", {"where": where, "id": rid, "label": lab})
labels.add(lab)
if not is_num(amt):
E.add("portion.amount_type", {"where": where, "id": rid, "portion": repr(p)[:120]})
elif not math.isfinite(amt) or not (0 < amt <= 5000):
E.add("portion.amount_range", {"where": where, "id": rid, "portion": p})
# 6 barcode
if type(barcode) is not str:
E.add("barcode.type", {"where": where, "id": rid, "barcode": repr(barcode)})
ok = False
elif barcode:
nb = norm_barcode(barcode)
if nb != barcode:
E.add("barcode.not_normalized", {"where": where, "id": rid, "barcode": barcode, "normalized": nb})
elif len(barcode) not in (8, 13, 14):
E.add("barcode.nonstandard_length", {"where": where, "id": rid, "barcode": barcode})
# 8 score
if type(score) is not int:
E.add("score.type", {"where": where, "id": rid, "score": repr(score)})
ok = False
elif not (0 <= score <= 1000):
E.add("score.range", {"where": where, "id": rid, "score": score})
# generic rules
if type(src) is str and src in generic:
if brand != "":
E.add("generic.has_brand", {"where": where, "id": rid, "brand": brand})
if barcode != "":
E.add("generic.has_barcode", {"where": where, "id": rid, "barcode": barcode})
return ok
def digest(rec):
return hashlib.blake2b(json.dumps(rec, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
digest_size=8).digest()
def dup_sig(rec):
"""Contract "Duplicate hiding" key: normalized name, normalized brand and unit (kcal is compared separately)."""
return hashlib.blake2b(json.dumps([tokens(rec[1]), tokens(rec[2]), rec[4]], ensure_ascii=False).encode("utf-8"),
digest_size=8).digest()
_JSON_STRING = re.compile(r'"(?:[^"\\]|\\.)*"', re.S)
def strict_parse(raw, where, E):
"""Bytes -> object. Rejects BOM, invalid UTF-8, NaN/Infinity, duplicate keys; checks compact separators."""
if raw.startswith(b"\xef\xbb\xbf"):
E.add("json.bom", {"where": where})
raw = raw[3:]
try:
text = raw.decode("utf-8")
except UnicodeDecodeError as e:
E.add("json.invalid_utf8", {"where": where, "err": str(e)[:200]})
return None
dupes = []
def hook(pairs):
d = {}
for k, v in pairs:
if k in d:
dupes.append(k)
d[k] = v
return d
def bad_constant(c):
raise ValueError("non-finite number " + c)
try:
obj = json.loads(text, object_pairs_hook=hook, parse_constant=bad_constant)
except ValueError as e:
E.add("json.parse_error", {"where": where, "err": str(e)[:200]})
return None
for k in dupes[:5]:
E.add("json.duplicate_key", {"where": where, "key": k})
if len(dupes) > 5:
E.counts["json.duplicate_key"] += len(dupes) - 5
compact = json.dumps(obj, ensure_ascii=False, separators=(",", ":"))
if compact != text:
if json.dumps(obj, ensure_ascii=True, separators=(",", ":")) != text:
outside = _JSON_STRING.sub('""', text)
if re.search(r"\s", outside):
E.add("json.not_compact", {"where": where})
return obj
# ------------------------------------------------------------------------------------------------ workers
_CFG = {}
def _init(cfg):
_CFG.update(cfg)
def check_token_shard(path_bucket):
path, bucket = path_bucket
cfg = _CFG
TB, CAP = cfg["tokenBuckets"], cfg["maxPerToken"]
declared, generic = cfg["declared"], cfg["generic"]
where = f"t/{bucket}"
E = Errs(SAMPLES_PER_CODE_WORKER)
out = {"bucket": bucket, "records": [], "capped": {}, "tokens": 0, "postings": 0, "r": 0,
"list_lens": collections.Counter(), "sample": [], "sample_nonascii": [], "sigma": []}
raw = Path(path).read_bytes()
if not raw:
E.add("file.empty", {"where": where})
obj = strict_parse(raw, where, E)
if obj is None:
out["err"] = (E.counts, dict(E.samples))
return out
if type(obj) is not dict or set(obj.keys()) != {"w", "r"}:
E.add("tshard.top_keys", {"where": where, "keys": sorted(obj.keys()) if type(obj) is dict else type(obj).__name__})
w = obj.get("w") if type(obj) is dict else None
r = obj.get("r") if type(obj) is dict else None
if type(w) is not dict:
E.add("tshard.w_type", {"where": where})
w = {}
if type(r) is not list:
E.add("tshard.r_type", {"where": where})
r = []
if not w or not r:
E.add("tshard.empty", {"where": where, "w": len(w), "r": len(r)})
out["r"] = len(r)
out["tokens"] = len(w)
# records
toks = [None] * len(r)
tokset = [None] * len(r)
keys = [None] * len(r)
id_at = {}
for i, rec in enumerate(r):
if not check_record(rec, where, E, declared, generic):
continue
rid = rec[0]
if rid in id_at:
E.add("tshard.r_duplicate_id", {"where": where, "id": rid, "at": [id_at[rid], i]})
id_at[rid] = i
tk = tokens(rec[1] + " " + rec[2])
toks[i] = tk
tokset[i] = set(tk)
keys[i] = (-rec[8], rid)
out["records"].append((rid, digest(rec), tuple(tk), rec[8], rec[7], rec[6], dup_sig(rec), rec[3]))
c = zlib.crc32(rid.encode("utf-8"))
if c % 128 == 0:
out["sample"].append((rid, rec[1], rec[2]))
text = rec[1] + " " + rec[2]
if not text.isascii():
if c % 16 == 0:
out["sample_nonascii"].append((rid, rec[1], rec[2]))
# Python str.lower() applies the Final_Sigma context rule; Swift lowercased() maps per scalar.
s = "".join(ch for ch in unicodedata.normalize("NFKD", text) if unicodedata.category(ch) != "Mn")
if "Σ" in s and s.lower() != "".join(ch.lower() for ch in s):
out["sigma"].append((rid, rec[1], rec[2]))
# token lists
referenced = bytearray(len(r))
members = {}
for t, lst in w.items():
if type(t) is not str or not t:
E.add("token.key_invalid", {"where": where, "token": repr(t)})
continue
if fnv32(t) % TB != bucket:
E.add("token.wrong_bucket", {"where": where, "token": t, "bucket": fnv32(t) % TB})
if (len(t) < 2 or t in STOPWORDS or t != t.lower()
or any(unicodedata.category(c)[0] not in ("L", "N") for c in t)):
E.add("token.malformed", {"where": where, "token": t})
elif tokens(t) != [t]:
E.add("token.not_idempotent", {"where": where, "token": t, "renormalized": tokens(t)})
if type(lst) is not list or not lst:
E.add("list.empty_or_type", {"where": where, "token": t})
continue
n = len(lst)
out["postings"] += n
out["list_lens"][n if n < 10 else (10 ** int(math.log10(n)))] += 1
if n > CAP:
E.add("list.over_cap", {"where": where, "token": t, "len": n})
prev = None
mem = set()
for idx in lst:
if type(idx) is not int or not (0 <= idx < len(r)):
E.add("list.bad_index", {"where": where, "token": t, "index": repr(idx)})
continue
if idx in mem:
E.add("list.duplicate_index", {"where": where, "token": t, "index": idx})
continue
mem.add(idx)
referenced[idx] = 1
if tokset[idx] is None:
continue
if t not in tokset[idx]:
rec = r[idx]
E.add("posting.token_not_in_record",
{"where": where, "token": t, "id": rec[0], "name": rec[1], "brand": rec[2], "tokens": toks[idx]})
k = keys[idx]
if prev is not None and k <= prev:
E.add("list.unsorted", {"where": where, "token": t, "prev": list(prev), "next": list(k)})
prev = k
members[t] = mem
if n >= CAP and prev is not None:
out["capped"][t] = prev
for i in range(len(r)):
if not referenced[i] and r[i] is not None:
rec = r[i]
E.add("tshard.r_unreferenced", {"where": where, "index": i, "id": rec[0] if type(rec) is list and rec else None})
# every token of every record whose bucket is this shard must list that record (unless the list is capped
# and the record ranks below the cutoff)
for i, tk in enumerate(toks):
if tk is None:
continue
for t in tk:
if fnv32(t) % TB != bucket:
continue
mem = members.get(t)
if mem is None:
E.add("completeness.token_list_missing", {"where": where, "token": t, "id": r[i][0]})
elif i not in mem:
lst = w[t]
last_key = keys[lst[-1]] if type(lst[-1]) is int and 0 <= lst[-1] < len(r) else None
if not (len(lst) >= CAP and last_key is not None and keys[i] > last_key):
E.add("completeness.posting_missing",
{"where": where, "token": t, "id": r[i][0], "score": r[i][8], "listLen": len(lst)})
out["err"] = (E.counts, dict(E.samples))
return out
def check_barcode_shard(path_bucket):
path, bucket = path_bucket
cfg = _CFG
BB = cfg["barcodeBuckets"]
declared, generic = cfg["declared"], cfg["generic"]
where = f"b/{bucket}"
E = Errs(SAMPLES_PER_CODE_WORKER)
out = {"bucket": bucket, "entries": [], "kinds": collections.Counter()}
raw = Path(path).read_bytes()
if not raw:
E.add("file.empty", {"where": where})
obj = strict_parse(raw, where, E)
if obj is None:
out["err"] = (E.counts, dict(E.samples))
return out
if type(obj) is not dict:
E.add("bshard.type", {"where": where})
out["err"] = (E.counts, dict(E.samples))
return out
if not obj:
E.add("bshard.empty", {"where": where})
for key, rec in obj.items():
if norm_barcode(key) != key:
E.add("bkey.not_normalized", {"where": where, "key": key, "normalized": norm_barcode(key)})
if fnv32(key) % BB != bucket:
E.add("bkey.wrong_bucket", {"where": where, "key": key, "bucket": fnv32(key) % BB})
if not check_record(rec, where, E, declared, generic):
continue
rb = rec[6]
if rb == key:
kind = "own"
elif rb == "":
kind = "no_barcode"
E.add("bkey.record_has_no_barcode", {"where": where, "key": key, "id": rec[0]})
elif len(rb) == 8 and upce_to_upca(rb) and norm_barcode(upce_to_upca(rb)) == key:
kind = "upce_record_under_upca_key"
elif len(key) == 8 and upce_to_upca(key) and norm_barcode(upce_to_upca(key)) == rb:
kind = "upce_key_to_upca_record"
else:
kind = "mismatch"
E.add("bkey.record_barcode_mismatch", {"where": where, "key": key, "id": rec[0], "recordBarcode": rb})
out["kinds"][kind] += 1
out["entries"].append((key, rec[0], digest(rec), rec[7], rec[8], rb, kind, tuple(tokens(rec[1] + " " + rec[2])),
dup_sig(rec), rec[3]))
out["err"] = (E.counts, dict(E.samples))
return out
# ------------------------------------------------------------------------------------------------ helpers
def pct(sorted_vals, p):
if not sorted_vals:
return None
k = max(0, min(len(sorted_vals) - 1, math.ceil(p / 100 * len(sorted_vals)) - 1))
return sorted_vals[k]
def size_stats(vals):
v = sorted(vals)
return {"n": len(v), "total": sum(v), "min": v[0] if v else None, "p50": pct(v, 50), "p90": pct(v, 90),
"p95": pct(v, 95), "p99": pct(v, 99), "max": v[-1] if v else None,
"over_400KiB": sum(1 for x in v if x > TARGET_FILE_BYTES),
"over_1MiB": sum(1 for x in v if x > 1024 * 1024),
"ge_5MiB": sum(1 for x in v if x >= MAX_FILE_BYTES),
"over_25MiB": sum(1 for x in v if x > CF_MAX_FILE_BYTES)}
class LinkParser(html.parser.HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.links = []
self._href = None
self._text = []
self.text = []
self.tags = collections.Counter()
def handle_starttag(self, tag, attrs):
self.tags[tag] += 1
if tag == "a":
self._href = dict(attrs).get("href")
self._text = []
def handle_endtag(self, tag):
if tag == "a" and self._href is not None:
self.links.append((self._href, "".join(self._text).strip()))
self._href = None
def handle_data(self, data):
self.text.append(data)
if self._href is not None:
self._text.append(data)
def parse_html(path, where, E):
raw = path.read_bytes()
try:
text = raw.decode("utf-8")
except UnicodeDecodeError as e:
E.add("html.invalid_utf8", {"where": where, "err": str(e)[:200]})
return None, None
p = LinkParser()
try:
p.feed(text)
p.close()
except Exception as e: # HTMLParser is lenient; anything raised here is a real problem
E.add("html.parse_error", {"where": where, "err": str(e)[:200]})
return text, None
if not text.lstrip().lower().startswith(""):
E.add("html.no_doctype", {"where": where})
if "" not in text.lower():
E.add("html.no_charset", {"where": where})
return text, p
def load_foodtok():
spec = importlib.util.spec_from_file_location("foodtok_under_test", HERE / "foodtok.py")
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
KNOWN_LICENSES = {
# canonical license URL -> required words in the license name
"https://creativecommons.org/publicdomain/zero/1.0/": ["CC0", "1.0"],
"https://opendatacommons.org/licenses/odbl/1-0/": ["Open Database License", "1.0"],
"https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/": ["Open Government Licence", "3.0"],
"https://www.etalab.gouv.fr/licence-ouverte-open-licence/": ["Licence Ouverte", "2.0"],
}
ODBL_URL = "https://opendatacommons.org/licenses/odbl/1-0/"
DBCL_URL = "https://opendatacommons.org/licenses/dbcl/1-0/"
CONTRACT_SOURCES = {"F": "USDA Foundation", "S": "USDA SR Legacy", "N": "USDA FNDDS", "B": "USDA Branded",
"O": "Open Food Facts"}
# ------------------------------------------------------------------------------------------------ main
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
ap.add_argument("--site", default=str(DEFAULT_SITE))
ap.add_argument("--bundle", default=str(DEFAULT_BUNDLE))
ap.add_argument("--out", default=str(DEFAULT_OUT))
ap.add_argument("--workers", type=int, default=max(1, (os.cpu_count() or 2) - 2))
ap.add_argument("--names", type=int, default=20000, help="random index names for the foodtok comparison")
args = ap.parse_args()
t0 = time.time()
site = Path(args.site)
E = Errs(SAMPLES_PER_CODE_TOTAL)
report = {"site": "", "checks": {}}
def log(msg):
print(f"[{time.time() - t0:7.1f}s] {msg}", flush=True)
# ---------------- self-test of the re-implementation (fast path vs literal path, contract examples)
for text, want in [("berries", ["berry"]), ("fries", ["fry"]), ("glasses", ["glass"]), ("tomatoes", ["tomato"]),
("peaches", ["peach"]), ("eggs", ["egg"]), ("oats", ["oat"]), ("chips", ["chip"]),
("Peaches & Cream Oats", ["peach", "cream", "oat"]), ("Bananas, raw", ["banana", "raw"]),
("bus hummus kiwis", ["bus", "hummus", "kiwis"]), ("a b cd", ["cd"]),
("Crème brûlée", ["creme", "brulee"]), ("salt and pepper of the", ["salt", "pepper"]),
("eggs EGGS egg", ["egg"]), ("fish", ["fish"]), ("fish½", ["fish1"]),
# contract v2 stemming: rules 1-6 and their boundaries
("cookies cookie", ["cooky"]), ("pies pie", ["pie"]), ("brownie brownies", ["browny"]),
("brie bries", ["bry"]), ("quiche quiches", ["quich"]), ("glaze glazes", ["glaz"]),
("shoe shoes", ["sho"]), ("size sizes", ["siz"]), ("sloe", ["slo"]), ("toes", []),
("ties", ["tie"]), ("spies dies", ["spy", "die"]), ("tie roe ze", ["tie", "roe", "ze"]),
("hummus couscous swiss kiwis", ["hummus", "couscous", "swiss", "kiwis"]),
("crêpes œufs straße", ["crepe", "œufs", "straße"]), ("2pies", ["2pies"])]:
got = tokens_slow(text)
if got != want or tokens(text) != got:
E.add("selftest.normalize", {"text": text, "want": want, "slow": got, "fast": tokens(text)})
if upce_to_upca("04252614") != "042100005264":
E.add("selftest.upce", {"code": "04252614", "got": upce_to_upca("04252614")})
# ---------------- inventory of every file
all_files = []
for dirpath, dirnames, filenames in os.walk(site):
for fn in filenames:
p = Path(dirpath) / fn
all_files.append((p, p.stat().st_size))
rel = {p.relative_to(site).as_posix(): sz for p, sz in all_files}
report["checks"]["files"] = {"total": len(all_files), "limit": MAX_FILES}
if len(all_files) > MAX_FILES:
E.add("limits.too_many_files", {"total": len(all_files), "limit": MAX_FILES})
for rp, sz in rel.items():
name = rp.rsplit("/", 1)[-1]
if name.startswith(".") or name in ("Thumbs.db", "desktop.ini"):
E.add("file.junk", {"file": rp})
if sz == 0:
E.add("file.empty", {"file": rp})
if sz >= MAX_FILE_BYTES:
E.add("limits.file_too_large", {"file": rp, "bytes": sz})
elif sz > TARGET_FILE_BYTES:
E.add("size.over_target", {"file": rp, "bytes": sz})
log(f"inventory: {len(all_files)} files")
# ---------------- manifest
mpath = site / "manifest.json"
if not mpath.exists():
E.add("manifest.missing", {})
print("manifest.json missing; cannot continue")
return 2
manifest = strict_parse(mpath.read_bytes(), "manifest.json", E)
if type(manifest) is not dict:
E.add("manifest.type", {})
return 2
required = {"format": int, "contract": int, "build": str, "path": str, "tokenBuckets": int, "barcodeBuckets": int,
"maxPerToken": int, "counts": dict, "sources": list}
for k, ty in required.items():
if k not in manifest:
E.add("manifest.missing_field", {"field": k})
elif type(manifest[k]) is not ty:
E.add("manifest.field_type", {"field": k, "type": type(manifest[k]).__name__})
for k in manifest:
if k not in required:
E.add("manifest.extra_field", {"field": k})
if manifest.get("format") != 1:
E.add("manifest.format", {"format": manifest.get("format")})
if manifest.get("contract") != CONTRACT_VERSION:
E.add("manifest.contract", {"contract": manifest.get("contract"), "want": CONTRACT_VERSION})
build = manifest.get("build", "")
if not re.fullmatch(r"\d{8}", str(build)):
E.add("manifest.build_format", {"build": build})
dpath = manifest.get("path", "")
if dpath != f"d/{build}":
E.add("manifest.path_not_d_build", {"path": dpath, "build": build})
TB, BB, CAP = manifest.get("tokenBuckets"), manifest.get("barcodeBuckets"), manifest.get("maxPerToken")
for k, v in (("tokenBuckets", TB), ("barcodeBuckets", BB), ("maxPerToken", CAP)):
if type(v) is not int or v <= 0:
E.add("manifest.bad_number", {"field": k, "value": v})
if manifest.get("baseUrl") not in (None, CONTRACT_BASE_URL):
E.add("manifest.base_url", {"baseUrl": manifest.get("baseUrl"), "contract": CONTRACT_BASE_URL})
sources = manifest.get("sources") or []
declared, generic = set(), set()
for s in sources:
if type(s) is not dict:
E.add("manifest.source_type", {"source": repr(s)[:100]})
continue
for k in ("code", "name", "license", "url", "attribution", "credit"):
if type(s.get(k)) is not str or not s.get(k).strip():
E.add("manifest.source_field", {"code": s.get("code"), "field": k})
cr = s.get("credit")
if type(cr) is str and (len(cr) > CREDIT_MAX or cr != " ".join(cr.split()) or CTRL.search(cr)):
E.add("manifest.source_credit", {"code": s.get("code"), "credit": cr, "len": len(cr)})
code = s.get("code")
if code == "L":
E.add("manifest.source_L_reserved", {})
if type(code) is not str or not re.fullmatch(r"[A-Z]", code):
E.add("manifest.source_code", {"code": code})
continue
if code in declared:
E.add("manifest.source_duplicate", {"code": code})
declared.add(code)
if s.get("generic") is True or code in ("F", "S", "N"):
generic.add(code)
if code in ("F", "S", "N") and s.get("generic") is False:
E.add("manifest.contract_generic_marked_false", {"code": code})
if code in ("B", "O", "L") and s.get("generic") is True:
E.add("manifest.branded_marked_generic", {"code": code})
if "generic" in s and type(s["generic"]) is not bool:
E.add("manifest.generic_type", {"code": code})
data_dir = site / dpath
if not data_dir.is_dir():
E.add("manifest.path_missing", {"path": dpath})
ddirs = sorted(p.name for p in (site / "d").iterdir()) if (site / "d").is_dir() else []
if ddirs != [str(build)]:
E.add("layout.d_folders", {"found": ddirs, "expected": [build]})
report["checks"]["manifest"] = {"build": build, "path": dpath, "tokenBuckets": TB, "barcodeBuckets": BB,
"maxPerToken": CAP, "declared": sorted(declared), "generic": sorted(generic)}
# ---------------- layout of the data folder
tdir, bdir = data_dir / "t", data_dir / "b"
for p in sorted(data_dir.iterdir()) if data_dir.is_dir() else []:
if p.name not in ("t", "b"):
E.add("layout.unexpected_in_data_dir", {"name": p.name})
name_re = re.compile(r"(0|[1-9][0-9]*)\.json\Z")
tjobs, bjobs = [], []
for d, jobs, nb, kind in ((tdir, tjobs, TB, "t"), (bdir, bjobs, BB, "b")):
if not d.is_dir():
E.add("layout.missing_dir", {"dir": kind})
continue
for p in d.iterdir():
m = name_re.match(p.name)
if not p.is_file() or not m:
E.add("layout.bad_shard_name", {"dir": kind, "name": p.name})
continue
b = int(m.group(1))
if b >= nb:
E.add("layout.bucket_out_of_range", {"dir": kind, "bucket": b, "buckets": nb})
continue
jobs.append((str(p), b))
tjobs.sort(key=lambda x: x[1])
bjobs.sort(key=lambda x: x[1])
t_missing = sorted(set(range(TB)) - {b for _, b in tjobs})
b_missing = sorted(set(range(BB)) - {b for _, b in bjobs})
report["checks"]["layout"] = {"tokenShards": len(tjobs), "barcodeShards": len(bjobs),
"tokenBucketsWithoutFile": len(t_missing), "barcodeBucketsWithoutFile": len(b_missing),
"barcodeBucketsWithoutFileList": b_missing[:100], "tokenBucketsWithoutFileList": t_missing[:100]}
tsizes = [rel[f"{dpath}/t/{b}.json"] for _, b in tjobs]
bsizes = [rel[f"{dpath}/b/{b}.json"] for _, b in bjobs]
other = [sz for rp, sz in rel.items() if not rp.startswith(dpath + "/")]
report["checks"]["sizes"] = {"all": size_stats([sz for sz in rel.values()]), "token": size_stats(tsizes),
"barcode": size_stats(bsizes), "other": size_stats(other),
"largest": sorted(((sz, rp) for rp, sz in rel.items()), reverse=True)[:5]}
log(f"layout: {len(tjobs)} token shards, {len(bjobs)} barcode shards")
cfg = {"tokenBuckets": TB, "barcodeBuckets": BB, "maxPerToken": CAP, "declared": declared, "generic": generic}
# ---------------- the build's duplicate-hiding log (hidden id -> kept id), needed while reading the shards
hidden = {}
hpath_q = ROOT / "qa" / "merge_hidden_duplicates.jsonl"
if hpath_q.exists():
with hpath_q.open(encoding="utf-8") as f:
for line in f:
try:
d = json.loads(line)
hidden[d["hidden"]] = d.get("kept")
except (ValueError, KeyError, TypeError):
E.add("qa.hidden_log_unreadable_line", {"line": line[:120]})
dup_need = set(hidden) | {k for k in hidden.values() if k}
dupinfo = {} # id -> (dup signature, kcal) for hidden records and their kept twins
# ---------------- token shards (parallel)
ids = {} # id -> [digest, tokens or None, score, src, barcode, set(buckets) or None]
capped = {}
tot_tokens = tot_post = tot_r = 0
list_lens = collections.Counter()
sample, sample_na, sigma = {}, {}, {}
all_tokens = set()
ctx = mp.get_context("spawn")
with ctx.Pool(args.workers, initializer=_init, initargs=(cfg,)) as pool:
for k, res in enumerate(pool.imap_unordered(check_token_shard, tjobs, chunksize=16)):
E.merge(*res["err"])
b = res["bucket"]
tot_tokens += res["tokens"]
tot_post += res["postings"]
tot_r += res["r"]
list_lens.update(res["list_lens"])
for t, key in res["capped"].items():
if t in capped:
E.add("token.in_two_shards", {"token": t})
capped[t] = key
for rid, dg, tk, score, src, bc, sig, kcal in res["records"]:
if rid in dup_need:
dupinfo[rid] = (sig, kcal)
e = ids.get(rid)
if e is None:
ids[rid] = [dg, tk, score, src, bc, {b}]
all_tokens.update(tk)
else:
if e[0] != dg:
E.add("global.id_content_conflict", {"id": rid, "where": f"t/{b}"})
e[5].add(b)
for rid, n, br in res["sample"]:
sample[rid] = (n, br)
for rid, n, br in res["sample_nonascii"]:
sample_na[rid] = (n, br)
for rid, n, br in res["sigma"]:
sigma[rid] = (n, br)
if (k + 1) % 2000 == 0:
log(f"token shards {k + 1}/{len(tjobs)}")
log("token shards done")
bmap = {}
kinds = collections.Counter()
for k, res in enumerate(pool.imap_unordered(check_barcode_shard, bjobs, chunksize=8)):
E.merge(*res["err"])
kinds.update(res["kinds"])
for key, rid, dg, src, score, rb, kind, tk, sig, kcal in res["entries"]:
if rid in dup_need:
dupinfo[rid] = (sig, kcal)
if key in bmap:
E.add("bkey.duplicate_across_shards", {"key": key})
bmap[key] = (rid, src, score, kind)
e = ids.get(rid)
if e is None:
ids[rid] = [dg, None, score, src, rb, None, tk]
elif e[0] != dg:
E.add("global.id_content_conflict", {"id": rid, "where": f"b/{res['bucket']}", "key": key})
log("barcode shards done")
token_ids = sum(1 for e in ids.values() if e[1] is not None)
report["checks"]["tokenIndex"] = {"tokens": tot_tokens, "postings": tot_post, "rEntries": tot_r,
"distinctRecords": token_ids, "cappedTokens": len(capped),
"listLengthBuckets": {str(k): v for k, v in sorted(list_lens.items())}}
# ---------------- cross-shard completeness: a token-searchable record must be in every one of its token lists
bucket_of = {}
missing_cross = 0
for rid, e in ids.items():
tk = e[1]
if tk is None:
continue
seen = e[5]
need = collections.defaultdict(list)
for t in tk:
c = bucket_of.get(t)
if c is None:
c = bucket_of[t] = fnv32(t) % TB
need[c].append(t)
for c in seen:
if c not in need:
E.add("completeness.record_in_foreign_shard", {"id": rid, "bucket": c})
for c, ts in need.items():
if c in seen:
continue
for t in ts:
cut = capped.get(t)
if cut is None or not ((-e[2], rid) > tuple(cut)):
missing_cross += 1
E.add("completeness.missing_from_shard",
{"id": rid, "token": t, "bucket": c, "score": e[2], "capped": cut is not None})
log("cross-shard completeness done")
# ---------------- barcode index vs records
barcode_only = collections.Counter()
unsearchable_barcode_only = 0
collisions = collections.Counter()
for rid, e in ids.items():
if e[1] is None:
barcode_only[e[3]] += 1
rb = e[4]
if not rb:
continue
m = bmap.get(rb)
if m is None:
E.add("barcode.record_not_in_barcode_shards", {"id": rid, "barcode": rb, "src": e[3]})
continue
if m[0] == rid:
continue
win_src, win_score = m[1], m[2]
if win_src == "O" and e[3] == "B":
E.add("barcode.O_preferred_over_B", {"barcode": rb, "winner": m[0], "loser": rid})
elif win_src == e[3] and win_score < e[2]:
E.add("barcode.lower_score_preferred", {"barcode": rb, "winner": m[0], "winnerScore": win_score,
"loser": rid, "loserScore": e[2]})
else:
collisions[f"{win_src}>{e[3]}"] += 1
E.add("barcode.collision_ok", {"barcode": rb, "winner": m[0], "loser": rid})
# Records that are only in barcode shards: explain why each one is missing from every token list.
keys_of = collections.defaultdict(list)
for key, (rid, src, score, kind) in bmap.items():
keys_of[rid].append(key)
why = collections.Counter()
why_src = collections.Counter()
reason_of = {}
def scannable(k):
if len(k) in (13, 14) or (len(k) == 8 and upce_to_upca(k) is None):
return gs1_check_digit(k[:-1]) == k[-1]
return False
unscannable = collections.Counter()
for rid, e in ids.items():
if e[1] is not None:
if rid in hidden:
E.add("hidden.listed_in_token_index", {"id": rid})
if e[4] and not any(scannable(k) for k in keys_of.get(rid, [])):
unscannable["searchable:" + e[3]] += 1
continue
tk = e[6]
if rid in hidden:
reason = "hidden_duplicate"
elif not tk:
reason = "no_tokens"
elif all(t in capped and (-e[2], rid) > tuple(capped[t]) for t in tk):
reason = "capped_out_of_every_list"
E.add("search.capped_out_of_every_list", {"id": rid, "score": e[2], "tokens": list(tk)})
else:
reason = "unexplained"
E.add("completeness.barcode_only_unexplained",
{"id": rid, "score": e[2], "tokens": list(tk),
"uncappedTokens": [t for t in tk if t not in capped],
"aboveCutoff": [t for t in tk if t in capped and not ((-e[2], rid) > tuple(capped[t]))]})
why[reason] += 1
why_src[f"{reason}:{e[3]}"] += 1
reason_of[rid] = reason
ks = keys_of.get(rid, [])
if not any(scannable(k) for k in ks):
unscannable[f"barcodeOnly:{reason}:{e[3]}"] += 1
E.add("barcode.barcode_only_and_unscannable", {"id": rid, "keys": ks, "reason": reason})
if ks and all(not (8 <= len(k) <= 14) for k in ks):
E.add("barcode.only_key_not_lookupable_by_app", {"id": rid, "keys": ks, "reason": reason})
kept_why = collections.Counter()
dup_rule = collections.Counter()
for rid, kept in hidden.items():
if rid not in ids:
E.add("hidden.hidden_record_not_in_index", {"hidden": rid})
continue
# contract "Duplicate hiding": a hidden record has a barcode, and its kept twin has the same normalized name,
# brand and unit, kcal within 2 %, and the higher (score, then lower id)
he, ke = ids[rid], ids.get(kept)
if not he[4]:
E.add("hidden.no_barcode", {"hidden": rid})
if ke is not None:
hs, hk = dupinfo.get(rid, (None, None))
ks, kk = dupinfo.get(kept, (None, None))
if hs is None or ks is None or hs != ks:
E.add("hidden.rule_name_brand_unit_differ", {"hidden": rid, "kept": kept})
elif abs(hk - kk) > 0.02 * max(hk, kk) + 1e-9:
E.add("hidden.rule_kcal_over_2pct", {"hidden": rid, "kept": kept, "kcal": hk, "keptKcal": kk})
elif not (ke[2] > he[2] or (ke[2] == he[2] and kept.encode() < rid.encode())):
E.add("hidden.rule_kept_not_higher", {"hidden": rid, "kept": kept, "score": he[2], "keptScore": ke[2]})
else:
dup_rule["ok"] += 1
if kept in hidden:
E.add("hidden.rule_kept_is_hidden", {"hidden": rid, "kept": kept})
ke = ids.get(kept)
if ke is None:
kept_why["kept_missing"] += 1
E.add("hidden.kept_record_missing", {"hidden": rid, "kept": kept})
elif ke[1] is None:
r_ = reason_of.get(kept, "?")
kept_why["kept_" + r_] += 1
code = "hidden.kept_also_hidden" if r_ == "hidden_duplicate" else "search.hidden_and_kept_twin_capped_out"
E.add(code, {"hidden": rid, "kept": kept, "keptReason": r_})
he = ids[rid]
# Would the hidden twin itself have made the cut in at least one of its token lists?
if he[6] and any(t not in capped or (-he[2], rid) < tuple(capped[t]) for t in he[6]):
kept_why["hidden_twin_would_have_been_listed"] += 1
E.add("hidden.dedupe_hid_the_listable_twin", {"hidden": rid, "hiddenScore": he[2], "kept": kept,
"keptScore": ke[2]})
else:
kept_why["kept_searchable"] += 1
# Barcode key lengths and GS1 check digits (scanners only return codes whose check digit is right).
klen = collections.Counter()
kbad = collections.Counter()
for key, (rid, src, score, kind) in bmap.items():
klen[f"{src}:{len(key)}"] += 1
if len(key) in (8, 13, 14) and gs1_check_digit(key[:-1]) != key[-1]:
kbad[f"{src}:{len(key)}" + (":validUPCE" if len(key) == 8 and upce_to_upca(key) else "")] += 1
if len(key) < 8:
E.add("barcode.key_shorter_than_app_accepts", {"key": key, "id": rid})
report["checks"]["barcodeIndex"] = {"keys": len(bmap), "kinds": dict(kinds), "collisions": dict(collisions),
"barcodeOnlyRecordsBySource": dict(barcode_only),
"barcodeOnlyWhy": dict(why), "barcodeOnlyWhyBySource": dict(sorted(why_src.items())),
"hiddenDuplicatesInQaLog": len(hidden), "hiddenKeptTwin": dict(kept_why),
"hiddenPairsMeetingContractRule": dup_rule["ok"],
"recordsWithoutScannableKey": dict(sorted(unscannable.items())),
"keyLengthsBySource": dict(sorted(klen.items())),
"badCheckDigitBySourceLength": dict(sorted(kbad.items()))}
# ---------------- counts vs manifest
by_src = collections.Counter(e[3] for e in ids.values())
counts = manifest.get("counts") or {}
actual = {"records": len(ids), "bySource": dict(sorted(by_src.items())), "tokenSearchable": token_ids,
"barcodes": len(bmap), "genericInIndex": sum(v for k, v in by_src.items() if k in generic)}
report["checks"]["counts"] = {"manifest": counts, "actual": actual}
if counts.get("records") != len(ids):
E.add("manifest.count_records", {"manifest": counts.get("records"), "actual": len(ids)})
mbs = counts.get("bySource") or {}
for code in sorted(set(mbs) | set(by_src)):
if mbs.get(code) != by_src.get(code, 0):
E.add("manifest.count_by_source", {"src": code, "manifest": mbs.get(code), "actual": by_src.get(code, 0)})
if sum(v for v in mbs.values() if type(v) is int) != counts.get("records"):
E.add("manifest.count_sum", {"sum": sum(mbs.values()), "records": counts.get("records")})
if "tokenSearchable" in counts and counts["tokenSearchable"] != token_ids:
E.add("manifest.count_token_searchable", {"manifest": counts["tokenSearchable"], "actual": token_ids,
"barcodeOnlyWhy": dict(why)})
if "barcodes" in counts and counts["barcodes"] != len(bmap):
E.add("manifest.count_barcodes", {"manifest": counts["barcodes"], "actual": len(bmap)})
for code in by_src:
if code not in declared:
E.add("manifest.source_not_declared", {"src": code})
for code in declared:
if code not in by_src:
E.add("manifest.declared_source_without_records", {"src": code})
# ---------------- bundle
bpath = Path(args.bundle)
bundle_ids = {}
if not bpath.exists():
E.add("bundle.missing", {})
else:
braw = bpath.read_bytes()
bundle = strict_parse(braw, "bundle", E)
binfo = {"bytes": len(braw), "target": BUNDLE_TARGET_BYTES}
if len(braw) >= BUNDLE_TARGET_BYTES:
E.add("bundle.over_target", {"bytes": len(braw)})
if type(bundle) is dict:
for k in bundle:
if k not in ("format", "contract", "build", "sources", "r"):
E.add("bundle.extra_key", {"key": k})
if bundle.get("format") != 1:
E.add("bundle.format", {"format": bundle.get("format")})
if bundle.get("contract") != CONTRACT_VERSION:
E.add("bundle.contract", {"contract": bundle.get("contract"), "want": CONTRACT_VERSION})
if bundle.get("build") != build:
E.add("bundle.build_mismatch", {"bundle": bundle.get("build"), "manifest": build})
if bundle.get("sources") != manifest.get("sources"):
E.add("bundle.sources_differ_from_manifest", {})
br = bundle.get("r")
if type(br) is not list:
E.add("bundle.r_type", {})
br = []
bsrc = collections.Counter()
for i, rec in enumerate(br):
if not check_record(rec, "bundle", E, declared, generic):
continue
rid = rec[0]
if rid in bundle_ids:
E.add("bundle.duplicate_id", {"id": rid})
bundle_ids[rid] = i
bsrc[rec[7]] += 1
if rec[7] not in generic:
E.add("bundle.non_generic_record", {"id": rid, "src": rec[7]})
if rec[6] != "":
E.add("bundle.record_has_barcode", {"id": rid, "barcode": rec[6]})
if not tokens(rec[1] + " " + rec[2]):
E.add("bundle.unsearchable", {"id": rid, "name": rec[1]})
e = ids.get(rid)
if e is None:
E.add("bundle.generic_not_in_index", {"id": rid})
elif e[0] != digest(rec):
E.add("bundle.differs_from_index", {"id": rid})
for rid, e in ids.items():
if e[3] in generic and rid not in bundle_ids:
E.add("index.generic_not_in_bundle", {"id": rid})
binfo.update({"records": len(br), "bySource": dict(bsrc)})
if "generic" in counts and counts["generic"] != len(bundle_ids):
E.add("manifest.count_generic", {"manifest": counts["generic"], "bundle": len(bundle_ids)})
else:
E.add("bundle.type", {})
report["checks"]["bundle"] = binfo
log("bundle done")
# ---------------- _headers
hpath = site / "_headers"
rules = []
if not hpath.exists():
E.add("headers.missing", {})
else:
htext = hpath.read_bytes().decode("utf-8", errors="replace")
if not htext.endswith("\n"):
E.add("headers.no_trailing_newline", {})
cur = None
for ln_no, ln in enumerate(htext.splitlines(), 1):
if len(ln) > 2000:
E.add("headers.line_too_long", {"line": ln_no})
if not ln.strip() or ln.lstrip().startswith("#"):
continue
if ln[0] not in " \t":
u = ln.strip()
if not (u.startswith("/") or re.match(r"https://", u)):
E.add("headers.bad_url_line", {"line": ln_no, "text": ln})
if u.count("*") > 1:
E.add("headers.multiple_splats", {"line": ln_no, "text": ln})
cur = [u, []]
rules.append(cur)
else:
if cur is None:
E.add("headers.header_before_url", {"line": ln_no, "text": ln})
continue
s = ln.strip()
if s.startswith("!"):
cur[1].append(("!", s[1:].strip()))
continue
hname, sep, value = s.partition(":")
if not sep or not re.fullmatch(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+", hname) or not value.strip():
E.add("headers.bad_header_line", {"line": ln_no, "text": ln})
continue
cur[1].append((hname, value.strip()))
if len(rules) > 100:
E.add("headers.too_many_rules", {"rules": len(rules)})
want = {"/d/*": ("Cache-Control", "public, max-age=31536000, immutable"),
"/manifest.json": ("Cache-Control", "public, max-age=3600")}
got = {u: hs for u, hs in rules}
for u, (hn, hv) in want.items():
hs = got.get(u)
if hs is None:
E.add("headers.rule_missing", {"url": u})
elif not any(n.lower() == hn.lower() and v == hv for n, v in hs):
E.add("headers.rule_value", {"url": u, "got": hs, "want": [hn, hv]})
for u in got:
if u not in want:
E.add("headers.extra_rule", {"url": u, "headers": got[u]})
if not dpath.startswith("d/"):
E.add("headers.data_path_not_under_d", {"path": dpath})
report["checks"]["headers"] = {"rules": [[u, hs] for u, hs in rules]}
# ---------------- HTML pages, links, attribution
pages = {}
for rp in sorted(rel):
if rp.endswith(".html"):
text, parser = parse_html(site / rp, rp, E)
pages[rp] = (text, parser)
if parser is None:
continue
for href, _ in parser.links:
if not href or re.match(r"(https?:|mailto:)", href) or href.startswith("#"):
continue
base = "" if href.startswith("/") else (rp.rsplit("/", 1)[0] + "/" if "/" in rp else "")
target = (base + href.lstrip("/")).split("#")[0]
parts = []
for seg in target.split("/"):
if seg == "..":
if parts:
parts.pop()
elif seg not in ("", "."):
parts.append(seg)
target = "/".join(parts) + ("/" if target.endswith("/") and parts else "")
if target.endswith("/") or target == "":
target += "index.html"
if target not in rel:
E.add("html.broken_link", {"page": rp, "href": href, "resolved": target})
elif target.endswith(".html") and target != "404.html":
E.add("html.link_pretty_url", {"page": rp, "href": href})
for need in ("index.html", "attribution.html", "404.html"):
if need not in rel:
E.add("site.missing_page", {"page": need})
atext, ap_ = pages.get("attribution.html", (None, None))
attr_info = {}
if atext and ap_:
plain = html.unescape(" ".join("".join(ap_.text).split()))
links = ap_.links
hrefs = {h for h, _ in links}
for s in sources:
code = s.get("code")
info = {}
info["name"] = html.unescape(" ".join(s.get("name", "").split())) in plain
info["code"] = f"{code}" in atext
info["license_link"] = any(h == s.get("licenseUrl") and t == " ".join(s.get("license", "").split())
for h, t in links) if s.get("licenseUrl") else (s.get("license", "") in plain)
info["attribution"] = " ".join(s.get("attribution", "").split()) in plain
info["url_link"] = s.get("url") in hrefs
if s.get("credit") and " ".join(s["credit"].split()) not in plain:
E.add("attr.source_credit_not_on_page", {"code": code})
n = (counts.get("bySource") or {}).get(code)
info["count"] = n is not None and f"{n:,} records" in plain
for k, v in info.items():
if not v:
E.add("attr.source_" + k, {"code": code})
lurl = s.get("licenseUrl")
words = KNOWN_LICENSES.get(lurl)
if words is None:
E.add("attr.license_url_not_license", {"code": code, "licenseUrl": lurl, "license": s.get("license")})
elif not all(wd in s.get("license", "") for wd in words):
E.add("attr.license_name_url_mismatch", {"code": code, "license": s.get("license"), "licenseUrl": lurl})
if code in ("F", "S", "N", "B"):
a = s.get("attribution", "")
if lurl != "https://creativecommons.org/publicdomain/zero/1.0/":
E.add("attr.usda_not_cc0", {"code": code})
if "USDA, Agricultural Research Service, FoodData Central" not in a:
if ("Agricultural Research Service" in a and "FoodData Central" in a
and ("USDA" in a or "U.S. Department of Agriculture" in a)):
E.add("attr.usda_credit_wording", {"code": code, "attribution": a})
else:
E.add("attr.usda_credit_missing", {"code": code, "attribution": a})
if code == "O":
if lurl != ODBL_URL:
E.add("attr.off_not_odbl", {"licenseUrl": lurl})
if "DbCL" not in s.get("license", "") and "Database Contents License" not in s.get("license", ""):
E.add("attr.off_no_dbcl", {})
attr_info[code] = info
for code, label in CONTRACT_SOURCES.items():
if code not in declared:
E.add("manifest.contract_source_missing", {"code": code, "source": label})
# ODbL share-alike notice
odbl = {"odbl_link": ODBL_URL in hrefs, "dbcl_link": DBCL_URL in hrefs,
"off_link": any(h.rstrip("/") == "https://world.openfoodfacts.org" for h in hrefs),
"derived_db_under_odbl": bool(re.search(r"(derivative|derived) database[^.]*\.?[^.]*made\s+available\s+under\s+the\s+Open Database License", plain)),
"scripts_link": any(h in ("scripts/", "/scripts/", "scripts/index.html") for h in hrefs),
"manifest_link": any(h.endswith("manifest.json") for h in hrefs)}
for k, v in odbl.items():
if not v:
E.add("attr.odbl_" + k, {})
attr_info["odbl"] = odbl
else:
E.add("attr.unreadable", {})
itext, ip_ = pages.get("index.html", (None, None))
if ip_:
ih = {h for h, _ in ip_.links}
if "attribution.html" not in ih:
E.add("index.no_attribution_link", {})
if ODBL_URL not in ih:
E.add("index.no_odbl_link", {})
report["checks"]["attribution"] = attr_info
# ---------------- privacy scan of the non-shard text files
emails = collections.Counter()
home = str(Path.home())
for rp in rel:
if rp.startswith(dpath + "/"):
continue
text = (site / rp).read_bytes().decode("utf-8", errors="replace")
for m in re.finditer(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", text):
emails[m.group(0)] += 1
if "/" + "Users/" in text or home in text or "/" + "home/" in text: # split so this file's own copy passes
E.add("privacy.local_path", {"file": rp})
if re.search(r"/(?:private/)?tmp/claude-", text):
E.add("privacy.tmp_path", {"file": rp})
report["checks"]["emailsInSiteText"] = dict(emails)
for em in emails:
if em.lower() != "support@powerupgaming.dev":
E.add("privacy.other_email", {"email": em})
# ---------------- script copies
sdir = site / "scripts"
repo_s, home_s = str(REPO), str(Path.home())
def sanitized(text):
text = text.replace(repo_s, "")
text = re.sub(r"/(?:private/)?tmp/claude-[^\s'\"`)]*", "", text)
return text.replace(home_s, "")
build_time = mpath.stat().st_mtime
copies = {}
for f in sorted(HERE.glob("*.py")):
cp = sdir / (f.name + ".txt")
if not cp.exists():
code = "scripts.post_build_missing" if f.stat().st_mtime > build_time else "scripts.copy_missing"
E.add(code, {"script": f.name})
copies[f.name] = "missing"
continue
same = sanitized(f.read_text(encoding="utf-8")) == cp.read_text(encoding="utf-8")
copies[f.name] = "same" if same else "differs"
if not same:
E.add("scripts.copy_differs", {"script": f.name})
cc = sdir / "CONTRACT.md.txt"
if not cc.exists():
E.add("scripts.contract_missing", {})
elif sanitized((ROOT / "CONTRACT.md").read_text(encoding="utf-8")) != cc.read_text(encoding="utf-8"):
E.add("scripts.contract_differs", {})
if cc.exists() and not cc.read_text(encoding="utf-8").startswith("# Losing It food index: contract v2"):
E.add("scripts.contract_not_v2", {})
for p in sdir.glob("*.txt"):
stem_ = p.name[:-4]
if stem_ != "CONTRACT.md" and not (HERE / stem_).exists():
E.add("scripts.orphan_copy", {"file": p.name})
stext, sp_ = pages.get("scripts/index.html", (None, None))
if sp_:
listed = {h for h, _ in sp_.links}
for p in sdir.glob("*.txt"):
if p.name not in listed:
E.add("scripts.not_listed", {"file": p.name})
else:
E.add("scripts.index_missing", {})
report["checks"]["scriptCopies"] = copies
# ---------------- tokenizer agreement: foodtok.py vs this re-implementation
ft = load_foodtok()
tv = json.loads((HERE / "tok_vectors.json").read_text(encoding="utf-8"))
tv_bad = {"mine_vs_vector": 0, "foodtok_vs_vector": 0, "mine_vs_foodtok": 0, "fast_vs_slow": 0, "buckets": 0}
for v in tv:
text = v["text"]
mine, fast, theirs = tokens_slow(text), tokens(text), ft.normalize(text)
if mine != v["tokens"]:
tv_bad["mine_vs_vector"] += 1
E.add("tok.vector_mine", {"text": text, "want": v["tokens"], "got": mine})
if theirs != v["tokens"]:
tv_bad["foodtok_vs_vector"] += 1
E.add("tok.vector_foodtok", {"text": text, "want": v["tokens"], "got": theirs})
if mine != theirs:
tv_bad["mine_vs_foodtok"] += 1
E.add("tok.mine_vs_foodtok", {"text": text, "mine": mine, "foodtok": theirs})
if fast != mine:
tv_bad["fast_vs_slow"] += 1
E.add("tok.fast_vs_slow", {"text": text})
mb = [fnv32(t) % TB for t in mine]
if "buckets" in v and (mb != v["buckets"] or [ft.token_bucket(t, TB) for t in mine] != mb):
tv_bad["buckets"] += 1
E.add("tok.vector_buckets", {"text": text, "want": v.get("buckets"), "got": mb})
bv = json.loads((HERE / "barcode_vectors.json").read_text(encoding="utf-8"))
bv_bad = 0
for v in bv:
mine = norm_barcode(v["text"])
theirs = ft.normalize_barcode(v["text"])
mb = fnv32(mine) % BB if mine else None
if mine != v["barcode"] or theirs != mine or mb != v["barcodeBucket"]:
bv_bad += 1
E.add("tok.barcode_vector", {"text": v["text"], "want": [v["barcode"], v["barcodeBucket"]],
"mine": [mine, mb], "foodtok": theirs})
rng = random.Random(20261115)
pool_ids = sorted(sample)
picked = rng.sample(pool_ids, min(args.names, len(pool_ids)))
na_ids = sorted(sample_na)
picked_na = rng.sample(na_ids, min(args.names, len(na_ids)))
name_stats = collections.Counter()
for group, idl, src_map in (("random", picked, sample), ("nonascii", picked_na, sample_na)):
for rid in idl:
n, br = src_map[rid]
for text in (n, n + " " + br):
name_stats[group + "_texts"] += 1
mine, fast, theirs = tokens_slow(text), tokens(text), ft.normalize(text)
if mine != theirs:
name_stats[group + "_mismatch"] += 1
E.add("tok.names_mine_vs_foodtok", {"id": rid, "text": text, "mine": mine, "foodtok": theirs})
if fast != mine:
name_stats[group + "_fast_vs_slow"] += 1
E.add("tok.fast_vs_slow", {"id": rid, "text": text})
for t in mine:
if fnv32(t) != ft.fnv1a(t):
name_stats[group + "_hash_mismatch"] += 1
E.add("tok.fnv_mismatch", {"token": t})
hash_bad = 0
for t in all_tokens:
if fnv32(t) != ft.fnv1a(t):
hash_bad += 1
E.add("tok.fnv_mismatch", {"token": t})
upce_codes = {k for k in bmap if len(k) == 8} | {e[4] for e in ids.values() if len(e[4]) == 8}
upce_bad = 0
for c in upce_codes:
if upce_to_upca(c) != ft.expand_upce(c):
upce_bad += 1
E.add("tok.upce_mismatch", {"code": c, "mine": upce_to_upca(c), "foodtok": ft.expand_upce(c)})
for s_ in sigma.items():
E.add("lower.final_sigma", {"id": s_[0], "name": s_[1][0], "brand": s_[1][1]})
report["checks"]["tokenizerAgreement"] = {
"tokVectors": len(tv), "tokVectorProblems": tv_bad, "barcodeVectors": len(bv), "barcodeVectorProblems": bv_bad,
"randomNames": len(picked), "nonAsciiNames": len(picked_na), "nameStats": dict(name_stats),
"distinctTokensHashed": len(all_tokens), "hashMismatches": hash_bad,
"upceCodesCompared": len(upce_codes), "upceMismatches": upce_bad,
"recordsWithFinalSigmaCaseDifference": len(sigma)}
log("tokenizer agreement done")
# ---------------- write report
errors = {k: v for k, v in E.counts.items() if k not in WARN_CODES}
warns = {k: v for k, v in E.counts.items() if k in WARN_CODES}
report["errors"] = dict(sorted(errors.items()))
report["warnings"] = dict(sorted(warns.items()))
report["samples"] = {k: v[:10] for k, v in sorted(E.samples.items())}
report["seconds"] = round(time.time() - t0, 1)
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(report, ensure_ascii=False, indent=1, default=list), encoding="utf-8")
vpath = out.with_name(out.stem + "_violations.jsonl")
with vpath.open("w", encoding="utf-8") as f:
for code in sorted(E.samples):
level = "warn" if code in WARN_CODES else "error"
for d in E.samples[code]:
f.write(json.dumps({"level": level, "code": code, **d}, ensure_ascii=False, default=list) + "\n")
print(json.dumps({"errors": report["errors"], "warnings": report["warnings"],
"files": report["checks"]["files"], "sizes": report["checks"]["sizes"],
"counts": report["checks"]["counts"], "barcodeIndex": report["checks"]["barcodeIndex"],
"tokenIndex": report["checks"]["tokenIndex"], "layout": report["checks"]["layout"],
"tokenizerAgreement": report["checks"]["tokenizerAgreement"],
"bundle": report["checks"].get("bundle"), "seconds": report["seconds"]},
ensure_ascii=False, indent=1, default=list))
return 1 if errors else 0
if __name__ == "__main__":
sys.exit(main())