#!/usr/bin/env python3 """Independent Python twin of the contract's reference search, for search-quality QA and Swift parity. FoodIndex/CONTRACT.md (contract v2), "Reference search", sections 1-5, over the built site (local files, as the app would fetch them) and the offline bundle (LosingIt/Resources/GenericFoods.json): 1 plan q = normalize(query); soft = quantities (ASCII digits, optionally followed by a UNITS suffix: "2", "100g", "12oz") and SOFT words, hard = the rest (all of q when that leaves nothing); at most the 5 longest hard tokens and the first 5 soft tokens; then every distinct single-scalar piece of the query other than "s" is added to soft. 2 fetch only hard tokens are fetched; required = hard tokens with a non-empty L_t; candidates are the records of every required L_t (once per id) that contain every required token. 3 score score + 250 allTokensInName + 200 exactName + 80 startsWithFirst - 12 min(extraNameTokens, 10) + 40 headSegment + 40 softHit - 80 derived + 120 (generic and not brandOnly), with NEUTRAL words and a USDA CLASS prefix ("Fish, salmon, raw", not "Cheese, with wine") read as neutral when the query doesn't have it; derived = DERIVED words in the name ("sauce", "mix", "meatless", ...) that the query lacks. 4 order one hit per id (the higher final); final desc, then fewer tokens in normalize(name), then fewer Unicode scalars in the raw name, then id by UTF-8 bytes; top 50. 5 offline the bundle with the same plan and rules (uncapped postings). Merged: an offline hit is kept only when its name and brand tokens contain every token the ONLINE search required; one hit per id, ordered as in 4, top 50. The tokenizer here is written from the contract text, not imported from foodtok.py, so the audit is independent; `--selftest` checks it against scripts/tok_vectors.json (the vectors the Swift tokenizer is diffed against) and the contract's own examples. Build-time judgements (the brand-only -80 and the like) are already in each record's score and are NOT applied again here. Modes: contract CONTRACT v2 "Reference search", every word whole, online + offline merged. [default] ("proposed" is an alias kept for old command lines: the v1 proposal became contract v2.) app the contract's informative "App as-you-type behaviour": while the text ends inside a word, the offline search treats the last hard token as a prefix (the token, or a word as written that starts with the typed piece); the prefix counts towards required, allTokensInName and brandOnly only. When nothing the online search found fits the word being typed (and it isn't the only hard word), the online search runs again on the finished words. Online results are narrowed to records that fit the word being typed (all of them when none do), and an offline hit that fits it is exempt from that word in the online-required filter. A lone stopword being typed becomes the prefix. Soft tokens are never prefixes. (The app's online results for an EARLIER text, shown while the next search runs, have no model here.) Run: python3 -I FoodIndex/scripts/search_ref.py "chicken breast" "diet coke" python3 -I FoodIndex/scripts/search_ref.py --audit [--top 5] [--json qa/search_audit.json] [--mode app] python3 -I FoodIndex/scripts/search_ref.py --selftest python3 -I FoodIndex/scripts/search_ref.py --uncapped --json qa/search_uncapped.json (slow: reads every shard) Stdlib only. Never runs anything from downloads/. """ import argparse import gzip import json import os import sys import time import unicodedata HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) PROJECT = os.path.dirname(ROOT) DEFAULT_SITE = os.path.join(ROOT, "out", "site") DEFAULT_BUNDLE = os.path.join(PROJECT, "LosingIt", "Resources", "GenericFoods.json") LIMIT = 50 MAX_HARD = 5 MAX_SOFT = 5 # ---------------------------------------------------------------- contract: token normalization STOPWORDS = frozenset("and with of the in an for or to on de la le et en con mit und from".split()) _CAT = {} def _ln(ch): v = _CAT.get(ch) if v is None: v = _CAT[ch] = unicodedata.category(ch)[0] in ("L", "N") return v def split_pieces(text): """Steps 1-3: NFKD, drop Mn, str.lower(), split on every scalar that is not L*/N*. Returns (pieces, ends_in_token): ends_in_token is True when the text ends inside a piece.""" s = unicodedata.normalize("NFKD", text or "") s = "".join(c for c in s if unicodedata.category(c) != "Mn").lower() pieces, cur = [], [] for c in s: if _ln(c): cur.append(c) elif cur: pieces.append("".join(cur)) cur = [] ends = bool(cur) if cur: pieces.append("".join(cur)) return pieces, ends def _ascii_az(t): return all("a" <= c <= "z" for c in t) def stem(t): """Step 5: only all-ASCII a-z pieces of n >= 4 characters; the first matching rule, once.""" n = len(t) if n < 4 or not _ascii_az(t): return t if t.endswith("ies") and n >= 5: # 1 return t[:-3] + "y" if t.endswith("sses"): # 2 return t[:-2] if t.endswith(("oes", "ches", "shes", "xes", "zes")): # 3 return t[:-2] if t.endswith("ie"): # 4 return t[:-2] + "y" if t.endswith(("che", "oe", "ze")): # 5 return t[:-1] if t.endswith("s") and not t.endswith(("ss", "us", "is")): # 6 return t[:-1] return t def token_from_piece(p): """Steps 4-6 for one piece: None when it is shorter than 2 scalars or a stopword.""" if len(p) < 2: return None t = stem(p) return None if t in STOPWORDS else t _NORM_CACHE = {} def normalize(text): """Steps 1-7: the contract's normalize().""" r = _NORM_CACHE.get(text) if r is not None: return r out, seen = [], set() for p in split_pieces(text)[0]: t = token_from_piece(p) if t is None or t in seen: continue seen.add(t) out.append(t) if len(_NORM_CACHE) < 2_000_000: _NORM_CACHE[text] = out return out def fnv1a(s): h = 0x811C9DC5 for b in s.encode("utf-8"): h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF return h def written_words(s): """The pieces of s that would be tokens, as written (not stemmed), first occurrence (as-you-type matching).""" seen, out = set(), [] for p in split_pieces(s)[0]: if token_from_piece(p) is not None and p not in seen: seen.add(p) out.append(p) return out def begins(word, piece): return word.encode("utf-8").startswith(piece.encode("utf-8")) # ---------------------------------------------------------------- contract: word lists NEUTRAL = frozenset("raw nfs ns plain regular fresh whole cooked prepared".split()) SOFT = frozenset("slice piece cup serving small medium large oz ounce gram ml glass can bottle scoop tbsp tsp " "tablespoon teaspoon portion lb kcal calory half quarter dozen handful pint quart liter litre packet " "fl pinch".split()) UNITS = frozenset("g gr kg mg oz ml l cl dl lb lbs cal kcal ct pc pcs in inch".split()) CLASS = frozenset("fish beef pork lamb veal chicken turkey cheese yogurt pepper nut seed beverage crustacean mollusk " "game alcoholic spice".split()) DERIVED = frozenset("sauce seasoning mix dip syrup filling extract flavored meatless imitation substitute vegetarian " "vegan giblet liver gizzard pickled puffed".split()) DERIVED_PENALTY = 80 def is_digits(t): """'Digits' in the contract: a token made only of ASCII 0-9 (not str.isdigit(), which accepts other scripts).""" return bool(t) and all("0" <= c <= "9" for c in t) def is_quantity(t): """A 'quantity' in the contract: ASCII digits, optionally followed by one of UNITS ("2", "100g", "12oz", "2l").""" i = 0 while i < len(t) and "0" <= t[i] <= "9": i += 1 return i > 0 and (i == len(t) or t[i:] in UNITS) # ---------------------------------------------------------------- 1. plan class Plan: __slots__ = ("query", "q", "hard", "soft", "prefix", "piece") def __init__(self, query, q, hard, soft, prefix=None, piece=None): self.query, self.q, self.hard, self.soft, self.prefix, self.piece = query, q, hard, soft, prefix, piece def as_dict(self): d = {"q": self.q, "hard": self.hard, "soft": self.soft} if self.prefix is not None: d.update(prefix=self.prefix, prefixPiece=self.piece) return d def _single_scalar_pieces(pieces): """Plan step 4: every distinct single-scalar piece other than "s", in order of first appearance.""" seen, out = set(), [] for p in pieces: if len(p) == 1 and p != "s" and p not in seen: seen.add(p) out.append(p) return out def plan_query(query): """Reference search section 1, steps 1-4. Step 5 (empty hard -> nothing) is the caller's check.""" q = normalize(query) soft = [t for t in q if is_quantity(t) or t in SOFT] # step 2 hard = [t for t in q if not (is_quantity(t) or t in SOFT)] if not hard: hard, soft = list(q), [] if len(hard) > MAX_HARD: # step 3 keep = sorted(range(len(hard)), key=lambda i: (-len(hard[i]), i))[:MAX_HARD] hard = [hard[i] for i in sorted(keep)] soft = soft[:MAX_SOFT] soft = soft + _single_scalar_pieces(split_pieces(query)[0]) # step 4 return Plan(query, q, hard, soft) def plan_typing(typed): """The informative as-you-type plan: plan_query plus the prefix (the last hard token while the text ends inside that word; a lone stopword being typed becomes the prefix). Soft tokens are never prefixes.""" p = plan_query(typed) pieces, ends = split_pieces(typed) if ends and pieces: last_piece = pieces[-1] last = token_from_piece(last_piece) if last is not None: # the word being typed is the last hard token (not a soft word, a repeat, or dropped by the 5 limit) if p.hard and last == p.hard[-1] and last not in p.soft: p.prefix, p.piece = last, last_piece elif not p.q and len(last_piece) >= 2: # a lone stopword still being typed: "to" is on its way to tomato, "on" to onion, "la" to lasagna. # It becomes the one hard token and the prefix; q stays normalize(query) (empty), as section 1 defines it. partial = stem(last_piece) p.hard = [partial] p.soft = _single_scalar_pieces(pieces) p.prefix, p.piece = partial, last_piece return p # ---------------------------------------------------------------- 3. score def final_score(rec, plan, required, generic, prefix_in_name=False, prefix_in_brand=False): """Reference search section 3 for one candidate. prefix_in_* only matter in the as-you-type (app) mode.""" name, brand = rec[1], rec[2] N = normalize(name) Br = normalize(brand) hard, soft = plan.hard, plan.soft hset, sset = set(hard), set(soft) nset, bset = set(N), set(Br) prefix = plan.prefix def in_name(t): return t in nset or (t == prefix and prefix_in_name) def in_brand(t): return t in bset or (t == prefix and prefix_in_brand) if soft: P = set() for piece in split_pieces(name + " " + brand)[0]: P.add(piece) P.add(stem(piece)) soft_hit = sum(1 for t in soft if t in P) else: soft_hit = 0 segs, stop_led = [], [] for part in name.split(","): toks = normalize(part) if toks: segs.append(toks) first = split_pieces(part)[0][:1] # the part starts with a stopword ("Cheese, with wine"): its first piece makes no token by being one stop_led.append(bool(first) and len(first[0]) >= 2 and stem(first[0]) in STOPWORDS) neutral = NEUTRAL lead = N if (len(segs) >= 2 and len(segs[0]) == 1 and segs[0][0] in CLASS and segs[0][0] not in hset and not stop_led[1]): neutral = NEUTRAL | {segs[0][0]} segs = segs[1:] lead = segs[0] # the class word stays in core; the query's own soft words don't count against an exact name core = {t for t in N if t not in NEUTRAL and not is_digits(t) and t not in sset} qcore = {t for t in hard if t not in neutral} all_in_name = all(in_name(t) for t in required) exact = bool(core) and core == qcore and soft_hit == len(soft) starts = bool(lead) and lead[0] == hard[0] extra = sum(1 for t in N if t not in hset and t not in neutral and not is_digits(t) and t not in sset) brand_only = any(in_brand(t) and not in_name(t) for t in hard) head = bool(segs) and all(t in hset or t in neutral for t in segs[0]) and any(t in hset for t in segs[0]) qset = set(plan.q) derived = sum(1 for t in nset if t in DERIVED and t not in qset) s = (rec[8] + 250 * all_in_name + 200 * exact + 80 * starts - 12 * min(extra, 10) + 40 * head + 40 * soft_hit - DERIVED_PENALTY * derived) if rec[7] in generic and not brand_only: s += 120 return s # ---------------------------------------------------------------- 4. order def order_key(hit): s, rec = hit return (-s, len(normalize(rec[1])), len(rec[1]), rec[0].encode("utf-8")) def ordered(hits, limit=LIMIT): """One hit per id (the higher final), ordered as in section 4, top `limit`.""" best = {} for s, rec in hits: old = best.get(rec[0]) if old is None or old[0] < s: best[rec[0]] = (s, rec) return sorted(best.values(), key=order_key)[:limit] def record_tokens(rec): return set(normalize(rec[1])) | set(normalize(rec[2])) # ---------------------------------------------------------------- 2. candidates (whole words) def rank_lists(plan, lists, generic): """Sections 2.2-4 given L_t (a list of records) for each hard token. Returns (top 50 [(final, rec)], required, number of candidates, candidate token).""" required = [t for t in plan.hard if lists.get(t)] if not required: return [], required, 0, None # Every required list, each record once: lists are capped, so a record can be missing from a long list it # belongs in, and checking its own tokens finds it from any of them. pool, seen = [], set() for t in required: for rec in lists[t]: if rec[0] not in seen: seen.add(rec[0]) pool.append(rec) hits = [] for rec in pool: toks = record_tokens(rec) if not all(t in toks for t in required): continue hits.append((final_score(rec, plan, required, generic), rec)) return ordered(hits), required, len(pool), "+".join(required) def merge(online, offline, online_required, typing=None): """Section 5: offline hits must contain every online-required token; then one hit per id, ordered, top 50. As typed (`typing`, a plan with a prefix), an offline hit that fits the word being typed is exempt from the prefix token.""" exempt = typing.prefix if typing is not None else None def keep(rec): toks = record_tokens(rec) return all(t in toks or (t == exempt and prefix_fits(rec, typing)) for t in online_required) kept = [h for h in offline if keep(h[1])] if online_required else list(offline) return ordered(list(online) + kept), len(offline) - len(kept) # ---------------------------------------------------------------- data access def _generic_codes(sources): return {"F", "S", "N"} | {s["code"] for s in sources if s.get("generic") is True and s.get("code") not in ("B", "O", "L")} class Site: def __init__(self, site_dir): self.dir = site_dir with open(os.path.join(site_dir, "manifest.json"), encoding="utf-8") as f: self.manifest = json.load(f) m = self.manifest self.tb = int(m["tokenBuckets"]) self.bb = int(m["barcodeBuckets"]) self.cap = int(m.get("maxPerToken") or 10**9) self.path = m["path"].strip("/") self.generic = _generic_codes(m.get("sources", [])) self._shards = {} def shard(self, bucket): """A token shard as the app gets it: (root or None for 404, raw bytes, gzip bytes, parse ms).""" hit = self._shards.get(bucket) if hit is not None: return hit p = os.path.join(self.dir, self.path, "t", f"{bucket}.json") if not os.path.exists(p): hit = (None, 0, 0, 0.0) else: with open(p, "rb") as f: raw = f.read() t0 = time.perf_counter() root = json.loads(raw.decode("utf-8")) ms = (time.perf_counter() - t0) * 1000 hit = (root, len(raw), len(gzip.compress(raw, 6)), ms) if len(self._shards) > 400: self._shards.clear() self._shards[bucket] = hit return hit def token_lists(self, hard): """L_t = shard.w[t] as records, for every hard token (a 404 or a missing token is an empty list). The build caps every list at maxPerToken; `cap` only matters when lowered to simulate a smaller cap.""" lists, fetched = {}, {} for t in hard: b = fnv1a(t) % self.tb root, nraw, ngz, ms = self.shard(b) fetched[b] = {"bytes": nraw, "gzip": ngz, "parse_ms": round(ms, 1), "missing": root is None} if root is None: continue idx = root.get("w", {}).get(t) if not idx: continue r = root.get("r", []) lst = [r[i] for i in idx[: self.cap] if isinstance(i, int) and 0 <= i < len(r)] if lst: lists[t] = lst return lists, fetched def search(self, plan): """Online half: (top 50, required, lists, fetched, candidates, candidate token).""" if not plan.hard: return [], [], {}, {}, 0, None lists, fetched = self.token_lists(plan.hard) top, required, ncand, shortest = rank_lists(plan, lists, self.generic) return top, required, lists, fetched, ncand, shortest class Bundle: def __init__(self, path): with open(path, encoding="utf-8") as f: root = json.load(f) self.header = {k: v for k, v in root.items() if k != "r"} self.generic = _generic_codes(root.get("sources", [])) self.records, ids = [], set() for rec in root["r"]: if rec[0] in ids: continue ids.add(rec[0]) self.records.append(rec) self.post = {} self.wpost = {} self.w_name = [] self.w_brand = [] for i, rec in enumerate(self.records): for t in normalize(rec[1] + " " + rec[2]): self.post.setdefault(t, []).append(i) wn, wb = written_words(rec[1]), written_words(rec[2]) self.w_name.append(wn) self.w_brand.append(wb) for w in dict.fromkeys(wn + wb): self.wpost.setdefault(w, []).append(i) self.vocab = sorted(self.wpost, key=lambda w: w.encode("utf-8")) def search(self, plan, generic=None): """Section 5, offline standalone (whole words): (top 50, required).""" if not plan.hard: return [], [] g = self.generic if generic is None else generic lists = {t: [self.records[i] for i in self.post[t]] for t in plan.hard if t in self.post} top, required, _, _ = rank_lists(plan, lists, g) return top, required def search_typing(self, plan, generic=None): """Offline search as typed (plan from plan_typing): the prefix token matches the token itself or a word as written that starts with the typed piece. Returns (top 50, required).""" if plan.prefix is None: return self.search(plan, generic) if not plan.hard: return [], [] g = self.generic if generic is None else generic prefix, piece = plan.prefix, plan.piece lists = {} pm = set() for t in plan.hard: if t == prefix: pm = set(self.post.get(t, [])) for w in self.vocab: if begins(w, piece): pm.update(self.wpost[w]) if pm: lists[t] = sorted(pm) elif t in self.post: lists[t] = self.post[t] required = [t for t in plan.hard if t in lists] if not required: return [], required shortest = min(required, key=lambda t: len(lists[t])) hits = [] for i in lists[shortest]: rec = self.records[i] toks = record_tokens(rec) if not all((i in pm) if t == prefix else (t in toks) for t in required): continue pin = pib = False if i in pm: pin = any(begins(w, piece) for w in self.w_name[i]) pib = any(begins(w, piece) for w in self.w_brand[i]) hits.append((final_score(rec, plan, required, g, pin, pib), rec)) return ordered(hits), required def prefix_fits(rec, plan): """The record fits the word being typed: a word of its name or brand (as written) starts with the typed piece, or it has the prefix token itself.""" if plan.prefix is None: return True text = rec[1] + " " + rec[2] return any(begins(w, plan.piece) for w in written_words(text)) or plan.prefix in normalize(text) def finished_words(typed): """The text's finished words: its pieces (normalization steps 1-3) without the last one, joined with spaces.""" return " ".join(split_pieces(typed)[0][:-1]) def needs_finished_words(online, plan): """As typed: search again on the finished words when the last hard word is being typed, it isn't the only hard word, and nothing the online search found fits it.""" return plan.prefix is not None and len(plan.hard) > 1 and not any(prefix_fits(h[1], plan) for h in online) # ---------------------------------------------------------------- one query def run_query(site, bundle, query, mode="contract"): if mode == "proposed": mode = "contract" t0 = time.perf_counter() plan = plan_typing(query) if mode == "app" else plan_query(query) res = {"query": query, "mode": mode, "tokens": plan.q, **plan.as_dict()} if not plan.hard: res.update(required=[], offline_required=[], buckets=[], shards={}, shard_bytes=0, shard_gzip=0, max_shard=0, parse_ms=0.0, candidates=0, candidate_token=None, online=0, offline=0, offline_filtered=0, merged=0, top=[], online_top=[], offline_top=[]) return res online, required, lists, fetched, ncand, shortest = site.search(plan) searched = plan if mode == "app": offline, off_required = bundle.search_typing(plan, site.generic) if needs_finished_words(online, plan): # the word being typed is a token of its own ("fairlife mi") or a typo: the finished words' results # and required list are the ones used searched = plan_query(finished_words(query)) online, required, lists, fetched2, ncand, shortest = site.search(searched) fetched = {**fetched, **fetched2} if plan.prefix is not None: # narrowed to the word being typed; when nothing fits, everything the finished words found stays online = [h for h in online if prefix_fits(h[1], plan)] or online merged, nfilt = merge(online, offline, required, typing=plan) else: offline, off_required = bundle.search(plan, site.generic) merged, nfilt = merge(online, offline, required) ms = (time.perf_counter() - t0) * 1000 res.update( required=required, offline_required=off_required, online_query=searched.query, buckets=[{"token": t, "bucket": fnv1a(t) % site.tb, "list": len(lists.get(t, [])), "capped": len(lists.get(t, [])) >= site.cap} for t in searched.hard], shards=fetched, shard_bytes=sum(v["bytes"] for v in fetched.values()), shard_gzip=sum(v["gzip"] for v in fetched.values()), max_shard=max((v["bytes"] for v in fetched.values()), default=0), parse_ms=round(sum(v["parse_ms"] for v in fetched.values()), 1), candidates=ncand, candidate_token=shortest, online=len(online), offline=len(offline), offline_filtered=nfilt, merged=len(merged), elapsed_ms=round(ms, 1), top=[_hit(s, r) for s, r in merged], online_top=[_hit(s, r) for s, r in online], offline_top=[_hit(s, r) for s, r in offline], ) return res def run_query_proposed(site, bundle, query): """Old name: the v1 "proposed" ranking became contract v2.""" return run_query(site, bundle, query, "contract") def _hit(s, r): return {"final": s, "id": r[0], "name": r[1], "brand": r[2], "kcal": r[3], "unit": r[4], "src": r[7], "base": r[8], "portions": r[5][:2]} def fmt(res, top=5): head = f"Q {res['query']!r} hard={res['hard']} soft={res['soft']}" if res.get("prefix"): head += f" prefix={res['prefix']!r}/{res['prefixPiece']!r}" lines = [head] if not res["hard"]: lines.append(" (no hard tokens: no results)") return "\n".join(lines) b = ", ".join(f"{x['token']}->{x['bucket']}({x['list']}{'*' if x['capped'] else ''})" for x in res["buckets"]) lines.append(f" buckets: {b}") lines.append(f" shards: {res['shard_bytes']/1024:.0f} KB raw / {res['shard_gzip']/1024:.0f} KB gzip, " f"max {res['max_shard']/1024:.0f} KB; candidates={res['candidates']} (via {res['candidate_token']}); " f"required={res['required']} online={res['online']} offline={res['offline']} " f"(filtered {res['offline_filtered']})") for i, h in enumerate(res["top"][:top], 1): p = (" ; " + h["portions"][0][0] + f"={h['portions'][0][1]:g}") if h["portions"] else "" lines.append(f" {i}. {h['name']} | {h['brand']} | {h['kcal']:g} kcal/100{h['unit']} | {h['src']} " f"| {h['final']} (base {h['base']}){p}") return "\n".join(lines) # ---------------------------------------------------------------- audit queries AUDIT = { "generic": ["banana", "apple", "egg", "eggs", "scrambled eggs", "chicken breast", "grilled chicken", "white rice", "brown rice cooked", "oatmeal", "avocado", "broccoli", "salmon", "ground beef 90 lean", "bacon", "bread", "whole wheat bread", "pasta", "spaghetti", "pizza slice", "french fries", "hamburger", "burrito", "sushi", "tacos", "ramen", "hummus", "almonds", "peanut butter", "cheddar cheese", "greek yogurt", "milk 2%", "orange juice", "coffee with milk", "latte", "beer", "red wine", "protein shake", # extra everyday foods "strawberries", "grapes", "watermelon", "orange", "potato", "sweet potato", "baked potato", "mashed potatoes", "cottage cheese", "tuna", "shrimp", "steak", "pork chop", "turkey sandwich", "caesar salad", "ice cream", "granola", "popcorn", "cheeseburger", "hot dog", "bagel", "cream cheese", "butter", "olive oil", "honey", "whole milk", "skim milk", "almond milk", "oat milk", "egg whites", "hard boiled egg", "tortilla", "pepperoni pizza", "lasagna", "chili", "chicken noodle soup", "coffee", "chocolate chip cookie", "cookies", "brownie", "salad"], "brand": ["chobani greek yogurt", "fairlife milk", "coca cola", "diet coke", "pepsi", "red bull", "cheerios", "frosted flakes", "oreo", "doritos", "lays classic", "kind bar", "clif bar", "quest bar", "ben & jerry's", "halo top", "nutella", "skippy peanut butter", "kraft mac and cheese", "campbell's tomato soup", "starbucks", "big mac", "chick-fil-a sandwich", "chipotle bowl", "subway", # extra brands "mcdonalds fries", "gatorade", "monster energy", "sprite", "starbucks latte", "taco bell", "premier protein shake", "dannon light and fit", "special k", "pop tarts"], "tricky": ["blueberries", "tomatoes", "jalapeño", "jalapeno", "crème brûlée", "creme brulee", "mac & cheese", "mac n cheese", "2% milk", "M&M's", "m&ms", "coke zero", "gluten free bread", "cookie", "ben and jerrys", "lay's", "chick fil a", "mcdonald's big mac", "pb&j", "a1 sauce", "7up", "sour cream", "half and half", "café au lait"], } # Contract examples and v2 stemming boundaries (selftest). _STEM_CASES = { "berries": "berry", "fries": "fry", "cookies": "cooky", "glasses": "glass", "tomatoes": "tomato", "peaches": "peach", "quiches": "quich", "cookie": "cooky", "brownie": "browny", "brie": "bry", "quiche": "quich", "glaze": "glaz", "shoe": "sho", "eggs": "egg", "oats": "oat", "pies": "pie", "pie": "pie", "ties": "tie", "bries": "bry", "brownies": "browny", "glazes": "glaz", "size": "siz", "sizes": "siz", "shoes": "sho", "toes": "to", "sloe": "slo", "hummus": "hummus", "couscous": "couscous", "swiss": "swiss", "kiwis": "kiwis", "bus": "bus", "ies": "ies", "spies": "spy", "dies": "die", "oes": "oes", "hoes": "ho", "tie": "tie", "zie": "zie", "boxes": "box", "dishes": "dish", "fizzes": "fizz", "cafés": "cafés", } _PLAN_CASES = [ ("milk 2%", ["milk"], ["2"]), ("special k", ["special"], ["k"]), ("ground beef 90 lean", ["ground", "beef", "lean"], ["90"]), ("pizza slice", ["pizza"], ["slice"]), ("1 cup rice", ["rice"], ["cup", "1"]), ("2 cups", ["cup"], ["2"]), ("12 cups", ["12", "cup"], []), ("2 slices pizza 3 cups 1 oz 4 tbsp 9 lb 7 ml 8 g", ["pizza"], ["slice", "cup", "oz", "tbsp", "lb", "2", "3", "1", "4", "9", "7", "8", "g"]), ("M&M's", [], ["m"]), ("m&ms", ["ms"], ["m"]), ("vitamin d milk", ["vitamin", "milk"], ["d"]), ("the and of", [], []), ("one two three four five sixsix", ["one", "three", "four", "five", "sixsix"], []), ("\u0661\u0662 milk", ["\u0661\u0662", "milk"], []), ("\uff12\uff10 milk", ["milk"], ["20"]), ("100g chicken breast", ["chicken", "breast"], ["100g"]), ("12oz coke 2l", ["coke"], ["12oz", "2l"]), ("half cup oatmeal", ["oatmeal"], ["half", "cup"]), ("half and half", ["half"], []), ("2gs milk 1x 100gr", ["2gs", "milk", "1x"], ["100gr"]), ("\u0661\u0662g milk", ["\u0661\u0662g", "milk"], []), ("quarter pounder", ["pounder"], ["quarter"]), ("6in sub", ["sub"], ["6in"]), ] def selftest(): path = os.path.join(HERE, "tok_vectors.json") with open(path, encoding="utf-8") as f: vec = json.load(f) bad = 0 for v in vec: toks = normalize(v["text"]) bks = [fnv1a(t) % 12000 for t in toks] if toks != v["tokens"] or ("buckets" in v and bks != v["buckets"]): bad += 1 print("MISMATCH", v["text"], toks, v["tokens"]) for word, want in _STEM_CASES.items(): if stem(word) != want: bad += 1 print("STEM", word, stem(word), want) for query, hard, soft in _PLAN_CASES: p = plan_query(query) if p.hard != hard or p.soft != soft: bad += 1 print("PLAN", query, p.hard, p.soft, "want", hard, soft) print(f"selftest: {len(vec)} vectors, {len(_STEM_CASES)} stems, {len(_PLAN_CASES)} plans, {bad} mismatches") return bad == 0 # ---------------------------------------------------------------- uncapped ground truth def uncapped(site, queries, top=10, hidden_path=None): """Every token-searchable record, no maxPerToken cap: what each query would return online if lists were uncapped. Records come from the barcode shards (every barcoded record) plus token-shard records with barcode "". Records hidden from token search as duplicates (qa/merge_hidden_duplicates.jsonl) are skipped.""" hidden = set() if hidden_path and os.path.exists(hidden_path): with open(hidden_path, encoding="utf-8") as f: for line in f: try: o = json.loads(line) except ValueError: continue if isinstance(o.get("hidden"), str): hidden.add(o["hidden"]) plans = [] for qq in queries: p = plan_query(qq) if p.hard: plans.append((qq, p, set(p.hard))) tok_any = {t for _, p, _ in plans for t in p.hard} seen = set() present = {t: 0 for t in tok_any} buf = {qq: [] for qq, _, _ in plans} def consider(rec): if rec[0] in seen or rec[0] in hidden: return seen.add(rec[0]) hit = record_tokens(rec) & tok_any if not hit: return for t in hit: present[t] += 1 for qq, p, hs in plans: if hs & hit: buf[qq].append(rec) bdir = os.path.join(site.dir, site.path, "b") for fn in sorted(os.listdir(bdir)): with open(os.path.join(bdir, fn), encoding="utf-8") as f: root = json.load(f) for rec in root.values(): consider(rec) tdir = os.path.join(site.dir, site.path, "t") for fn in sorted(os.listdir(tdir)): with open(os.path.join(tdir, fn), encoding="utf-8") as f: root = json.load(f) for rec in root.get("r", []): if rec[6] == "": consider(rec) out = {} for qq, p, _ in plans: required = [t for t in p.hard if present.get(t)] hits = [(final_score(rec, p, required, site.generic), rec) for rec in buf[qq] if all(t in record_tokens(rec) for t in required)] if required else [] ranked = ordered(hits, limit=top) out[qq] = {"hard": p.hard, "soft": p.soft, "required": required, "matches_uncapped": len(hits), "top": [_hit(s, r) for s, r in ranked], "posting_counts": {t: present.get(t, 0) for t in p.hard}} return out def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("queries", nargs="*") ap.add_argument("--site", default=DEFAULT_SITE) ap.add_argument("--bundle", default=DEFAULT_BUNDLE) ap.add_argument("--mode", choices=("contract", "app", "proposed"), default="contract") ap.add_argument("--audit", action="store_true", help="run the built-in audit query list") ap.add_argument("--top", type=int, default=5) ap.add_argument("--cap", type=int, help="simulate a smaller maxPerToken (lists are score-sorted, so a prefix)") ap.add_argument("--json", help="write full results (top 50 per query) to this JSON file") ap.add_argument("--selftest", action="store_true") ap.add_argument("--uncapped", action="store_true", help="ground truth without the maxPerToken cap (slow)") a = ap.parse_args() if a.selftest: sys.exit(0 if selftest() else 1) site = Site(a.site) if a.cap: site.cap = min(site.cap, a.cap) queries = list(a.queries) groups = {} if a.audit or not queries: for g, qs in AUDIT.items(): for qq in qs: groups[qq] = g queries.append(qq) if a.uncapped: out = uncapped(site, queries, top=max(10, a.top), hidden_path=os.path.join(ROOT, "qa", "merge_hidden_duplicates.jsonl")) if a.json: with open(a.json, "w", encoding="utf-8") as f: json.dump(out, f, ensure_ascii=False, indent=1) for qq, r in out.items(): print(f"Q {qq!r} hard={r['hard']} uncapped matches={r['matches_uncapped']} postings={r['posting_counts']}") for i, h in enumerate(r["top"][: a.top], 1): print(f" {i}. {h['name']} | {h['brand']} | {h['kcal']:g} | {h['src']} | {h['final']}") return bundle = Bundle(a.bundle) results = [] for qq in queries: res = run_query(site, bundle, qq, a.mode) print(fmt(res, a.top)) res["group"] = groups.get(qq, "") results.append(res) if a.json: with open(a.json, "w", encoding="utf-8") as f: json.dump({"mode": "contract" if a.mode == "proposed" else a.mode, "contract": 2, "build": site.manifest.get("build"), "results": results}, f, ensure_ascii=False, indent=1) if __name__ == "__main__": main()