#!/usr/bin/env python3 """Write qa/search_vectors.json: the parity oracle for the app's search engine (contract v2 "Reference search"). For every query it records the plan and the expected results, computed by search_ref.py (the Python twin written from FoodIndex/CONTRACT.md) over the built site and the offline bundle: {"query", "group", "q", "hard", "soft", "required" the ONLINE required list (hard tokens with a non-empty shard list), in hard order "offlineRequired" the same over the bundle's postings "online" top 50 [id, final] of the online search alone (section 2-4) "offline" top 50 [id, final] of the offline bundle search standing alone, whole words (section 5) "merged" top 50 [id, final] of online + offline, offline hits filtered by the online required list} The header holds the build, the bucket and cap parameters, the generic source codes and SHA-256 digests of the manifest and the bundle the results were computed from (a vector file only applies to that build). Queries: every search_ref.AUDIT query, then more than 400 chosen deterministically (FNV-1a ranks over sorted inputs, no randomness): single words and 2-4 word phrases from real names and brands of every source, digits and "%", portion words, single letters, accents, apostrophes and ampersands, USDA class-prefix names, brand-only words, words only the online index has (offline-filter cases), 6+ word queries (5-longest rule), queries with no tokens, stopword-only queries, non-Latin text and the v2 stemming forms. Records are read from the bundle and from every SHARD_STEP-th token shard. Run: python3 -I FoodIndex/scripts/make_search_vectors.py [--site ...] [--bundle ...] [--out ...] Stdlib only. """ import argparse import hashlib import json import os import re import sys HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import search_ref as sr # noqa: E402 ROOT = os.path.dirname(HERE) DEFAULT_OUT = os.path.join(ROOT, "qa", "search_vectors.json") SHARD_STEP = 40 CURATED = { "digits": ["milk 2%", "ground beef 90 lean", "2% milk", "1% milk", "greek yogurt 0%", "85% lean ground beef", "93/7 ground turkey", "70% dark chocolate", "100% whole wheat bread", "7up", "a1 sauce", "coke 2 liter", "3 musketeers", "5 guys burger", "24 oz steak", "v8 juice", "h2o", "b12", "omega 3", "2 eggs", "100 grand bar", "milk 3.25%", "skim milk 0% fat", "1 2 3 milk"], "portion": ["pizza slice", "1 cup rice", "2 slices bread", "1 tbsp peanut butter", "large egg", "medium banana", "small apple", "can of coke", "bottle of water", "glass of milk", "scoop protein powder", "1 oz cheese", "8 oz steak", "1 lb ground beef", "tsp sugar", "tablespoon olive oil", "teaspoon honey", "serving of almonds", "piece of chicken", "portion of fries", "100 kcal", "calories in a banana", "250 ml milk", "cup", "2 cups", "slice", "large", "medium pizza slice", "12 oz can soda", "one cup of coffee", "pieces"], "letters": ["special k", "vitamin d milk", "k cups", "vitamin c", "x large egg", "t bone steak", "plan b", "c4 energy", "j&j", "s", "y", "u.s. cheese", "b vitamins", "baby food stage 2", "ice cream a la mode", "e.l.f.", "k-cup coffee", "o's cereal"], "accents": ["crème brûlée", "jalapeño", "café au lait", "pâté", "crêpe", "açaí bowl", "piña colada", "crème fraîche", "müsli", "smørrebrød", "nøttemix", "gruyère", "brûlée", "éclair", "jalapeños", "pâtes", "fraîche", "tomates séchées", "lait demi écrémé", "Bœuf bourguignon", "straße", "Käsekuchen", "œufs"], "apostrophes": ["ben & jerry's", "m&ms", "M&M's", "reese's", "mcdonald's", "wendy's", "lay's", "hershey's", "pb&j", "a&w root beer", "trader joe's", "kellogg's", "campbell's", "annie's mac & cheese", "chick-fil-a", "dunkin' donuts", "mac 'n cheese", "s&w beans", "ben and jerrys", "hershey's kisses", "reese's pieces", "s'mores", "m & m", "mike's hard lemonade", "nature's own bread", "newman's own", "o'doul's", "land o'lakes butter"], "class_prefix": ["salmon", "tuna", "steak", "cod", "trout", "shrimp", "crab", "lobster", "ground beef", "pork loin", "turkey breast", "cheddar", "mozzarella", "almonds", "sunflower seeds", "black pepper", "cinnamon", "venison", "bison", "beer", "salmon fish", "fish", "chicken", "beef", "cheese", "nut", "yogurt", "pepper", "chicken thigh", "beef jerky", "fish sticks", "cheese pizza", "turkey", "lamb chop", "veal", "squid", "clams", "game hen"], "no_tokens": ["", " ", "!!!", "a b c", "1 2 3", "&", "-- %%", "...", "'", "s s s", "©®™", "é", "x"], "stopwords": ["the", "and", "of the", "with and", "de la", "to", "on", "mit und", "from", "for or in an", "le et en con", "the and of with"], "stemming": ["cookies", "cookie", "brownies", "brownie", "pies", "pie", "quiche", "quiches", "fries", "veggie burger", "veggies", "smoothie", "smoothies", "shoestring fries", "glazed donuts", "glaze", "brie cheese", "pierogies", "tomatoes", "peaches", "berries", "strawberries", "potatoes", "sandwiches", "glasses of milk", "hummus", "couscous", "swiss cheese", "oatmeal cookies", "apple pies", "pot pie", "sizes", "toes", "calories", "brioche", "cherries", "mango smoothie"], "long": ["grilled chicken breast with garlic and herbs", "large black coffee with whole milk and sugar", "chocolate chip cookie dough ice cream half gallon", "whole wheat bread toasted with butter and jam", "spaghetti with meat sauce and parmesan cheese", "one large scrambled egg with cheese and bacon", "turkey sandwich on white bread with lettuce tomato mayo", "organic unsweetened vanilla almond milk original flavor shelf stable", "a b c d e f g h", "1 2 3 4 5 6 chicken", "fat free sugar free vanilla pudding snack cups 4 pack"], "mixed_case_space": [" Chicken BREAST ", "banana ", "BANANA", "Greek Yogurt", "PeAnUt BuTtEr", "\tapple\n", "coca-cola", "coca cola zero", "COKE ZERO", "diet-coke"], "non_latin_curated": ["молоко", "хлеб", "яблоко", "сыр", "γάλα", "γιαούρτι", "牛奶", "米饭", "豆腐", "緑茶", "ラーメン", "おにぎり", "김치", "라면", "حليب", "خبز", "חלב", "ข้าว", "दूध", "chai चाय", "pho phở", "sushi 寿司", "kimchi 김치"], } WORD = re.compile(r"[^\W_]+(?:['’][^\W_]+)?", re.U) def h(s): return sr.fnv1a(s) def pick(items, n, salt): """The n items with the smallest FNV-1a hash of salt|item (ties by the item), from a deduplicated list.""" uniq = sorted(set(items)) return sorted(uniq, key=lambda x: (h(salt + "|" + x), x))[:n] def words_of(name): return WORD.findall(name) def is_latin_text(s): return all(ord(c) < 0x250 or not c.isalpha() for c in s) def load_records(site, bundle): recs = {} for rec in bundle.records: recs[rec[0]] = rec tdir = os.path.join(site.dir, site.path, "t") names = sorted((n for n in os.listdir(tdir) if n.endswith(".json")), key=lambda n: int(n[:-5])) for n in names: if int(n[:-5]) % SHARD_STEP: continue with open(os.path.join(tdir, n), encoding="utf-8") as f: root = json.load(f) for rec in root.get("r", []): recs.setdefault(rec[0], rec) return [recs[k] for k in sorted(recs)] def generated_queries(records, bundle, site): by_src = {} for rec in records: by_src.setdefault(rec[7], []).append(rec) out = {} # single words and 2-4 word phrases from names, per source singles, phrases, brands, brandword_combo, long_names, nonlatin, digit_names, accent_names, class2 = \ [], [], [], [], [], [], [], [], [] for src in sorted(by_src): recs = by_src[src] ws, ps = [], [] for rec in recs: name = rec[1] w = words_of(name) latin = is_latin_text(name) if not latin: if w: nonlatin.append(" ".join(w[:2])) continue for x in w: if len(x) >= 3 and x.isalpha(): ws.append(x.lower()) if len(w) >= 2: k = 2 + h("len|" + rec[0]) % 3 start = h("start|" + rec[0]) % max(1, len(w) - k + 1) ps.append(" ".join(w[start:start + k])) if len(sr.normalize(name)) >= 6: long_names.append(name) if re.search(r"\d", name) and len(w) <= 6: digit_names.append(name) if not name.isascii(): accent_names.append(" ".join(w[:3])) segs = [x.strip() for x in name.split(",")] if len(segs) >= 2 and sr.normalize(segs[0]) and len(sr.normalize(segs[0])) == 1 \ and sr.normalize(segs[0])[0] in sr.CLASS and sr.normalize(segs[1]): class2.append(segs[1]) if rec[2]: brands.append(rec[2]) bt = sr.normalize(rec[2]) nt = [t for t in sr.normalize(name) if t in bundle.post] if bt and nt and bt[0] not in bundle.post and bt[0] not in sr.SOFT and not sr.is_digits(bt[0]): first = rec[2].split()[0] brandword_combo.append((first if sr.normalize(first) == bt[:1] else bt[0]) + " " + nt[0]) singles += pick(ws, 9, "single:" + src) phrases += pick(ps, 14, "phrase:" + src) out["single_word"] = singles out["phrase"] = phrases out["brand"] = pick(brands, 30, "brand") out["online_only_brand_plus_generic"] = pick(brandword_combo, 30, "combo") out["long_name"] = pick(long_names, 20, "long") out["non_latin"] = pick(nonlatin, 25, "nonlatin") out["digits_from_names"] = pick(digit_names, 15, "digits") out["accents_from_names"] = pick(accent_names, 15, "accents") out["class_second_segment"] = pick(class2, 15, "class2") # brand-only words: a brand word that is a whole query (single token, online only or not) bw = [] for b in brands: t = sr.normalize(b) if len(t) == 1 and len(t[0]) >= 3 and t[0].isascii(): bw.append(b) out["brand_only_word"] = pick(bw, 20, "brandword") return out def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--site", default=sr.DEFAULT_SITE) ap.add_argument("--bundle", default=sr.DEFAULT_BUNDLE) ap.add_argument("--out", default=DEFAULT_OUT) a = ap.parse_args() site = sr.Site(a.site) bundle = sr.Bundle(a.bundle) if bundle.generic != site.generic: sys.exit(f"bundle generic sources {sorted(bundle.generic)} != manifest {sorted(site.generic)}") if bundle.header.get("build") != site.manifest.get("build"): sys.exit("bundle and manifest are from different builds") queries = [] # (group, query) seen = set() def add(group, q): if q not in seen: seen.add(q) queries.append((group, q)) for g, qs in sr.AUDIT.items(): for q in qs: add("audit_" + g, q) n_audit = len(queries) for g, qs in CURATED.items(): for q in qs: add(g, q) records = load_records(site, bundle) for g, qs in generated_queries(records, bundle, site).items(): for q in qs: add(g, q) cases = [] stats = {"queries": len(queries), "audit": n_audit, "extra": len(queries) - n_audit, "no_hard": 0, "online_empty": 0, "offline_empty": 0, "merged_empty": 0, "offline_filtered_all": 0, "with_soft": 0, "hard_capped_to_5": 0, "online_required_missing_offline": 0, "groups": {}} for group, query in queries: p = sr.plan_query(query) if p.hard: online, required, _, _, _, _ = site.search(p) offline, off_required = bundle.search(p, site.generic) merged, nfilt = sr.merge(online, offline, required) else: online = offline = merged = [] required = off_required = [] nfilt = 0 cases.append({"query": query, "group": group, "q": p.q, "hard": p.hard, "soft": p.soft, "required": required, "offlineRequired": off_required, "online": [[r[0], s] for s, r in online], "offline": [[r[0], s] for s, r in offline], "merged": [[r[0], s] for s, r in merged]}) stats["groups"][group] = stats["groups"].get(group, 0) + 1 stats["no_hard"] += not p.hard stats["online_empty"] += bool(p.hard) and not online stats["offline_empty"] += bool(p.hard) and not offline stats["merged_empty"] += bool(p.hard) and not merged stats["offline_filtered_all"] += bool(offline) and nfilt == len(offline) stats["with_soft"] += bool(p.soft) stats["hard_capped_to_5"] += len([t for t in p.q if t not in p.soft]) > 5 and len(p.hard) == 5 stats["online_required_missing_offline"] += any(t not in off_required for t in required) def sha(path): with open(path, "rb") as f: return hashlib.sha256(f.read()).hexdigest() m = site.manifest doc = { "about": "Contract v2 reference search results (FoodIndex/CONTRACT.md) for the Swift parity test. Each case: " "query, plan (q, hard, soft), the online required list, the offline required list, and the top 50 " "[id, final] for online alone, offline alone (whole words) and merged (offline hits must contain " "every online-required token). Ordered by final desc, fewer name tokens, shorter name (Unicode " "scalars), id by UTF-8 bytes. Generated by scripts/make_search_vectors.py from search_ref.py.", "format": m.get("format"), "contract": m.get("contract"), "build": m.get("build"), "tokenBuckets": site.tb, "barcodeBuckets": site.bb, "maxPerToken": int(m["maxPerToken"]), "limit": sr.LIMIT, "generic": sorted(site.generic), "manifestSha256": sha(os.path.join(a.site, "manifest.json")), "bundleSha256": sha(a.bundle), "stats": stats, "cases": cases, } os.makedirs(os.path.dirname(a.out), exist_ok=True) tmp = a.out + ".tmp" with open(tmp, "w", encoding="utf-8") as f: f.write("{" + ",\n".join(json.dumps(k) + ":" + json.dumps(v, ensure_ascii=False, separators=(",", ":")) for k, v in doc.items() if k != "cases")) f.write(',\n"cases":[\n' + ",\n".join(json.dumps(c, ensure_ascii=False, separators=(",", ":")) for c in cases) + "\n]}\n") os.replace(tmp, a.out) stats["bytes"] = os.path.getsize(a.out) print(json.dumps(stats, ensure_ascii=False, indent=1)) if __name__ == "__main__": main()