#!/usr/bin/env python3 """Compare the search_ref.py audit outputs (contract, app, proposed, uncapped) for QA. Contract v2: "contract" is the v2 reference search (online + offline merged with the online-required filter) and "proposed" is an alias of it (the v1 proposal became v2), so those two columns are identical. Reads FoodIndex/qa/search_audit_{contract,app,proposed,uncapped}.json and prints, per query: - offline pollution: merged top 5 entries that lack a token the online search required - cap loss: uncapped top 5 ids missing from the capped online top 50 - app vs contract top-5 differences - shard bytes, and a side-by-side of contract and proposed top 5 Run: python3 -I FoodIndex/scripts/search_audit_compare.py [--detail] """ import argparse import json import os import sys HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import search_ref as sr # noqa: E402 QA = os.path.join(os.path.dirname(HERE), "qa") # What a US user most likely wants at the top, as a regex over "name | brand" (case-insensitive). Judgment calls, # written before looking at the proposed ranking's output; hit@1 / hit@3 are counted against them. EXPECT = { "banana": r"^bananas?, raw", "apple": r"^apples?,.*raw", "egg": r"^eggs?, (whole|raw)", "eggs": r"^eggs?, (whole|raw)", "scrambled eggs": r"scrambled", "chicken breast": r"^chicken,? breast", "grilled chicken": r"grilled", "white rice": r"^rice, white, cooked|^white rice, cooked", "brown rice cooked": r"^rice, brown, cooked", "oatmeal": r"^oatmeal|^cereals?, oats", "avocado": r"^avocados?, (raw|hass)", "broccoli": r"^broccoli, (raw|cooked)", "salmon": r"^(fish, )?salmon, (raw|nfs|baked|grilled|cooked)", "ground beef 90 lean": r"90% lean", "bacon": r"^(pork )?bacon,.*cooked|^bacon, (ns|cooked)", "bread": r"^bread, (white|nfs|wheat|whole wheat)", "whole wheat bread": r"^bread, whole wheat", "pasta": r"^pasta,.*cooked", "spaghetti": r"^(spaghetti|pasta)(?! sauce| squash)", "pizza slice": r"^pizza", "french fries": r"french fries", "hamburger": r"^hamburger", "burrito": r"^burrito", "sushi": r"^sushi", "tacos": r"^taco(?! sauce| shell)", "ramen": r"ramen", "hummus": r"^hummus", "almonds": r"^almonds?\b(?! chicken)", "peanut butter": r"^peanut butter( \||, (smooth|creamy|regular))", "cheddar cheese": r"^cheese, cheddar|^cheddar cheese", "greek yogurt": r"^yogurt, greek,.*plain", "milk 2%": r"2%", "orange juice": r"^orange juice", "coffee with milk": r"coffee.*milk", "latte": r"latte", "beer": r"^beer\b", "red wine": r"^wine, red|red wine \|", "protein shake": r"protein.*shake|shake.*protein", "strawberries": r"^strawberr(y|ies), raw", "grapes": r"^grapes?, (raw|red|green)", "watermelon": r"^watermelon, raw", "orange": r"^oranges?, raw", "potato": r"^potato(es)?, (nfs|baked|boiled|raw)", "sweet potato": r"^sweet potato", "baked potato": r"^potato(es)?, baked", "mashed potatoes": r"^potato(es)?, mashed", "cottage cheese": r"cottage", "tuna": r"tuna.*canned|^tuna, (raw|nfs)|^fish, tuna", "shrimp": r"^shrimp, (nfs|cooked|steamed|boiled|grilled)", "steak": r"^beef, steak|^steak, ns|^steak \||steak, (nfs|grilled|broiled)", "pork chop": r"^pork, chop", "turkey sandwich": r"^turkey sandwich", "caesar salad": r"^caesar salad", "ice cream": r"^ice cream, (nfs|vanilla|regular)", "granola": r"^(cereal, )?granola", "popcorn": r"^popcorn, (nfs|air|popped|microwave)", "cheeseburger": r"^cheeseburger", "hot dog": r"^hot dog", "bagel": r"^bagel", "cream cheese": r"^cream cheese|^cheese, cream", "butter": r"^butter, (nfs|salted|stick|regular)", "olive oil": r"olive oil|oil, olive", "honey": r"^honey \|", "whole milk": r"^milk, whole", "skim milk": r"^milk,.*(skim|fat free)", "almond milk": r"^almond milk", "oat milk": r"^oat milk", "egg whites": r"^egg,? white", "hard boiled egg": r"hard.?boiled", "tortilla": r"^tortilla, (nfs|flour|corn)", "pepperoni pizza": r"pepperoni", "lasagna": r"^lasagna", "chili": r"^chili\b", "chicken noodle soup": r"chicken noodle", "coffee": r"^coffee, (brewed|nfs|regular)", "salad": r"^(salad, (green|garden|tossed)|lettuce|green salad|garden salad|tossed salad)", "chocolate chip cookie": r"^cookies?, chocolate chip", "cookies": r"^cookies?, (nfs|chocolate chip)", "brownie": r"brownie", "blueberries": r"^blueberr(y|ies), raw", "tomatoes": r"^tomato(es)?, (raw|red)", "jalapeño": r"jalapeno", "jalapeno": r"jalapeno", "crème brûlée": r"cr[eè]me br[uû]l[eé]e", "creme brulee": r"cr[eè]me br[uû]l[eé]e", "mac & cheese": r"mac(aroni)?\b.*cheese", "mac n cheese": r"mac(aroni)?\b.*cheese", "2% milk": r"2%", "gluten free bread": r"gluten", "cookie": r"^cookie, (nfs|chocolate chip)", "sour cream": r"^sour cream(, (regular|nfs)| \|)|^cream, sour", "half and half": r"half (and|&) half", "chobani greek yogurt": r"chobani", "fairlife milk": r"fairlife", "coca cola": r"coca.?cola(?!.*(diet|zero|light))", "diet coke": r"diet coke", "pepsi": r"^pepsi\b(?!.*(diet|max|zero))", "red bull": r"red bull", "cheerios": r"cheerios", "frosted flakes": r"frosted flakes", "oreo": r"^oreos?\b", "doritos": r"doritos", "lays classic": r"lay.*classic|classic.*lay", "kind bar": r"kind", "clif bar": r"clif bar", "quest bar": r"quest", "ben & jerry's": r"jerry", "ben and jerrys": r"jerry", "halo top": r"halo top", "nutella": r"^nutella \|", "skippy peanut butter": r"skippy", "kraft mac and cheese": r"kraft", "campbell's tomato soup": r"campbell.*tomato|tomato.*campbell", "starbucks": r"starbucks", "big mac": r"big mac", "chick-fil-a sandwich": r"chick-fil-a.*sandwich", "chipotle bowl": r"chipotle", "subway": r"subway", "mcdonalds fries": r"mcdonald.*fries", "gatorade": r"gatorade", "monster energy": r"monster", "sprite": r"sprite", "starbucks latte": r"starbucks.*latte|latte.*starbucks", "taco bell": r"taco bell", "premier protein shake": r"premier protein", "dannon light and fit": r"light.*fit", "special k": r"special k", "pop tarts": r"pop.?tart", "M&M's": r"m&m", "m&ms": r"m&m", "coke zero": r"coke zero", "lay's": r"lay", "chick fil a": r"chick-fil-a", "mcdonald's big mac": r"big mac", "pb&j": r"\bpb\b|peanut butter.*jelly", "a1 sauce": r"\ba1\b", "7up": r"7.?up", "café au lait": r"au lait", } def hits_at(results, q): import re pat = EXPECT.get(q) if pat is None: return None rx = re.compile(pat, re.I) ok = [bool(rx.search(f"{h['name']} | {h['brand']}")) for h in results[:5]] return (ok[:1] == [True], any(ok[:3]), any(ok[:5])) def load(name): with open(os.path.join(QA, f"search_audit_{name}.json"), encoding="utf-8") as f: return json.load(f) def short(h): return f"{h['name']} | {h['brand']} | {h['kcal']:g}/{h['unit']} | {h['src']} | {h['final']}" def main(): ap = argparse.ArgumentParser() ap.add_argument("--detail", action="store_true") a = ap.parse_args() con = {r["query"]: r for r in load("contract")["results"]} app = {r["query"]: r for r in load("app")["results"]} pro = {r["query"]: r for r in load("proposed")["results"]} unc = load("uncapped") stats = {"queries": len(con), "polluted_top5": 0, "polluted_top1": 0, "cap_loss_top5": 0, "app_differs_top5": 0, "no_results": 0} sizes = [] for q, r in con.items(): if not r["tokens"]: stats["no_results"] += 1 print(f"### {q!r}: NO TOKENS -> no results at all") continue req = [b["token"] for b in r["buckets"] if b["list"] > 0] sizes.append((r["shard_bytes"], r["shard_gzip"], r["max_shard"], q)) pol = [] for i, h in enumerate(r["top"][:5]): toks = set(sr.normalize(h["name"])) | set(sr.normalize(h["brand"])) missing = [t for t in req if t not in toks] if missing: pol.append((i + 1, h["name"], missing)) if pol: stats["polluted_top5"] += 1 if pol[0][0] == 1: stats["polluted_top1"] += 1 u = unc.get(q) loss = [] if u: online_ids = {h["id"] for h in r.get("online_top", [])} for i, h in enumerate(u["top"][:5]): if h["id"] not in online_ids: loss.append((i + 1, short(h))) if loss: stats["cap_loss_top5"] += 1 ap_ids = [h["id"] for h in app[q]["top"][:5]] co_ids = [h["id"] for h in r["top"][:5]] if ap_ids != co_ids: stats["app_differs_top5"] += 1 print(f"### {q!r} tokens={r['tokens']} shards={r['shard_bytes']/1024:.0f}KB raw/{r['shard_gzip']/1024:.0f}KB gz " f"cand={r['candidates']} online={r['online']} offline={r['offline']}" + (f" uncapped_matches={u['matches_uncapped']}" if u else "")) if pol: print(" OFFLINE POLLUTION (merged top5 lacking an online-required token):", "; ".join(f"#{i} {n} (no {','.join(m)})" for i, n, m in pol)) if loss: print(" CAP LOSS (uncapped top5 not in capped online top50):", "; ".join(f"#{i} {s}" for i, s in loss)) if ap_ids != co_ids: print(" APP top5:", " || ".join(short(h) for h in app[q]["top"][:5])) if a.detail: print(" ONLINE top5:", " || ".join(short(h) for h in r.get("online_top", [])[:5])) print(" PROPOSED top5:", " || ".join(short(h) for h in pro[q]["top"][:5])) tally = {} misses = {} for mode, data in (("contract", con), ("app", app), ("online_only", {q: {"top": r.get("online_top", [])} for q, r in con.items()}), ("proposed", pro)): t = tally.setdefault(mode, {"queries": 0, "hit@1": 0, "hit@3": 0, "hit@5": 0}) for q in data: h = hits_at(data[q]["top"], q) if h is None: continue t["queries"] += 1 t["hit@1"] += h[0] t["hit@3"] += h[1] t["hit@5"] += h[2] if not h[0]: misses.setdefault(mode, []).append(q) print("\nHIT RATES", json.dumps(tally)) for mode, qs in misses.items(): print(f"MISS@1 {mode} ({len(qs)}):", ", ".join(qs)) sizes.sort(reverse=True) print("\nSTATS", json.dumps(stats)) tot = [s[0] for s in sizes] gz = [s[1] for s in sizes] print(f"shard bytes per query: median {sorted(tot)[len(tot)//2]/1024:.0f} KB raw, max {max(tot)/1024:.0f} KB raw; " f"gzip median {sorted(gz)[len(gz)//2]/1024:.0f} KB, max {max(gz)/1024:.0f} KB") print("largest per-query downloads:", "; ".join(f"{q!r} {b/1024:.0f}KB raw/{g/1024:.0f}KB gz (max file {m/1024:.0f}KB)" for b, g, m, q in sizes[:8])) if __name__ == "__main__": main()