#!/usr/bin/env python3 """Write tokenizer test vectors for the Swift side (LosingIt/Engine/FoodTokenizer.swift), contract v2. scripts/tok_vectors.json {"text", "tokens", "hashes", "buckets"} cases (tokenBuckets = 12000): 400 sampled real names, LEGACY_CASES (earlier samples) and the hand-written v2 stemming cases (V2_CASES) scripts/barcode_vectors.json barcode cases {"text", "barcode", "barcodeBucket"} (barcodeBuckets = 4096) Texts are real names (and "name brand" index texts) sampled from every normalized source, chosen to cover each source and script plus the tricky cases: accents, letters without a decomposition (ø ł đ æ œ ß), Cyrillic, Greek (including a capital sigma at the end of a word), Chinese, Japanese, Korean, Arabic, Hebrew, Thai, Indic scripts, Vietnamese, Turkish dotted/dotless i, numbers and units, punctuation, every plural rule, compatibility characters (fullwidth, ligatures, superscripts, fractions, (TM)), emoji and stopwords. Sampling is deterministic: within each category the records with the smallest FNV-1a hash of "|" are taken. (The plural categories still use the v1 rule order to pick texts, so the 400 sampled texts are the same as in v1; their tokens are v2.) V2_CASES then exercise every v2 stemming rule (contract "Token normalization" step 5) and its boundaries: the 4-letter minimum, "ies" only from 5 letters, the ie/che/oe/ze singular rules, the ss/us/is exceptions, stems that become stopwords ("toes" -> "to", "ands" -> "and"), upper case, accents that NFKD removes before stemming (so "crêpes" stems) and letters that it can't remove (so "œufs", "smørrebrøds" and "straße" don't), compatibility characters, digits glued to letters, and mixed scripts. Check against Swift: tokcheck --vectors FoodIndex/scripts/tok_vectors.json Run: python3 -I FoodIndex/scripts/make_tok_vectors.py """ import heapq import json import os import re import sys import unicodedata HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import foodtok # noqa: E402 ROOT = os.path.dirname(HERE) NORM = os.path.join(ROOT, "normalized") TOKEN_BUCKETS, BARCODE_BUCKETS = 12000, 4096 TOTAL = 400 QUOTA = { "src_F": 12, "src_S": 22, "src_N": 22, "src_B": 35, "src_M": 22, "src_U": 12, "src_C": 12, "src_O": 20, "latin_accents": 30, "eszett_umlaut": 8, "undecomposed_letters": 10, "cyrillic": 20, "greek": 10, "greek_final_capital_sigma": 6, "han": 14, "kana": 12, "hangul": 10, "arabic": 10, "hebrew": 5, "thai": 8, "indic": 6, "vietnamese": 6, "turkish_i": 6, "numbers_units": 14, "punctuation": 14, "plural_ies": 5, "plural_sses": 3, "plural_oes": 3, "plural_ches": 3, "plural_shes": 2, "plural_xes": 2, "plural_zes": 2, "plural_s": 5, "keeps_ss_us_is": 4, "compat_chars": 10, "symbols_emoji": 5, "stopwords": 5, "mixed_scripts": 5, } assert sum(QUOTA.values()) == TOTAL, sum(QUOTA.values()) RANGES = { "cyrillic": [(0x0400, 0x052F)], "greek": [(0x0370, 0x03FF), (0x1F00, 0x1FFF)], "han": [(0x3400, 0x4DBF), (0x4E00, 0x9FFF)], "kana": [(0x3040, 0x30FF), (0x31F0, 0x31FF)], "hangul": [(0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF)], "arabic": [(0x0600, 0x06FF), (0x0750, 0x077F), (0xFB50, 0xFDFF), (0xFE70, 0xFEFF)], "hebrew": [(0x0590, 0x05FF)], "thai": [(0x0E00, 0x0E7F)], "indic": [(0x0900, 0x0DFF)], } COMPAT = re.compile("[!-○ff-st²³¹⁰-₟¼-¾⅐-⅟" "™ºªµℓ㌀-㏿①-⓿]") UNDECOMPOSED = re.compile("[øØłŁđĐæÆœŒħĦŧŦþÞðÐ]") VIETNAMESE = re.compile("[Ạ-ỹơƠưƯ]") TURKISH = re.compile("[İıŞşĞğ]") ESZETT = re.compile("[ßẞäöüÄÖÜ]") FINAL_SIGMA = re.compile(r"[Α-ΩΆ-Ώ]Σ(?![Α-Ωα-ωΆ-ώ])") NUMBERS_UNITS = re.compile(r"\d+(?:[.,]\d+)?\s*(?:%|g|kg|mg|ml|l|cl|oz|fl\.? ?oz|lb|kcal|cal|ct|pk|x)\b", re.I) PUNCT = re.compile(r"[’'&/+\-\"()\[\]!?:;*#.,]") WORDS = re.compile(r"[A-Za-z]+") STOP = foodtok.STOPWORDS def scripts_of(text): found = set() for ch in text: o = ord(ch) if o < 0x80: if ch.isalpha(): found.add("latin") continue for name, rs in RANGES.items(): if any(a <= o <= b for a, b in rs): found.add(name) break else: if unicodedata.category(ch)[0] == "L" and (0x00C0 <= o <= 0x024F or 0x1E00 <= o <= 0x1EFF): found.add("latin") return found def plural_cats(text): cats = set() for w in WORDS.findall(text): w = w.lower() if len(w) < 4: continue if w.endswith("ies"): cats.add("plural_ies") elif w.endswith("sses"): cats.add("plural_sses") elif w.endswith("oes"): cats.add("plural_oes") elif w.endswith("ches"): cats.add("plural_ches") elif w.endswith("shes"): cats.add("plural_shes") elif w.endswith("xes"): cats.add("plural_xes") elif w.endswith("zes"): cats.add("plural_zes") elif w.endswith(("ss", "us", "is")): cats.add("keeps_ss_us_is") elif w.endswith("s"): cats.add("plural_s") return cats def categories(rec, text): cats = {"src_" + rec["src"]} if text.isascii(): cats |= plural_cats(text) if NUMBERS_UNITS.search(text): cats.add("numbers_units") if len(PUNCT.findall(text)) >= 3: cats.add("punctuation") if sum(1 for w in WORDS.findall(text.lower()) if w in STOP) >= 2: cats.add("stopwords") return cats sc = scripts_of(text) for name in RANGES: if name in sc: cats.add(name) if "latin" in sc and len(sc) > 1: cats.add("mixed_scripts") if "latin" in sc: if any(unicodedata.category(c) == "Mn" for c in unicodedata.normalize("NFKD", text)): cats.add("latin_accents") cats |= plural_cats(text) if ESZETT.search(text): cats.add("eszett_umlaut") if UNDECOMPOSED.search(text): cats.add("undecomposed_letters") if VIETNAMESE.search(text): cats.add("vietnamese") if TURKISH.search(text): cats.add("turkish_i") if FINAL_SIGMA.search(text): cats.add("greek_final_capital_sigma") if COMPAT.search(text): cats.add("compat_chars") if any(unicodedata.category(c) == "So" for c in text): cats.add("symbols_emoji") if NUMBERS_UNITS.search(text): cats.add("numbers_units") if len(PUNCT.findall(text)) >= 3 or "’" in text: cats.add("punctuation") return cats LEGACY_CASES = [ # sampled by earlier runs (before the normalized inputs changed); kept so no case is lost "SUNKIST Orange 100% Premium Frozen Concentrate 32floz 4+1 Sunkist", "M\u00fcsli Do Plavek", "Naturell N\u00f8ttemix", "\u6c96\u7e04\u9ed2\u7cd6 Almond Sugar Cane \u5473", ] V2_CASES = [ # the task's word list "pies", "pie", "ties", "fries", "brie", "bries", "cookie", "cookies", "brownie", "brownies", "quiche", "quiches", "glaze", "glazes", "size", "sizes", "shoe", "shoes", "toes", "sloe", "peaches", "tomatoes", "glasses", "hummus", "couscous", "swiss", "berries", # singulars and plurals together (must give one token) "cookie cookies Cookies COOKIES", "pie pies", "brownie brownies", "quiche quiches", "glaze glazes", "shoe shoes", "size sizes", "brie bries", "fry fries", "berry berries", "peach peaches", "tomato tomatoes", "veggie veggies", "smoothie smoothies", "calorie calories", "hoagie hoagies", "brioche brioches", "pierogi pierogies", "potpie potpies", "sloe sloes", "aloe vera", "maize", "breeze", "freeze dried", # rule 1 (ies, n >= 5) and its 4-letter boundary (rule 6 instead) "spies", "dies", "lies", "ties", "pies", "flies", "fries", "cries", "candies", "cherries", "strawberries", "ies", "xies", "abies", # rule 2 (sses) and rule 6 exceptions (ss, us, is) "glasses", "molasses", "bass", "basses", "grass", "swiss", "hummus", "asparagus", "couscous", "citrus", "octopus", "kiwis", "iris", "anis", "chips", "plus", "bus", "gas", "peas", "oats", "eggs", "nuts", # rule 3 (oes, ches, shes, xes, zes) "potatoes", "tomatoes", "mangoes", "shoes", "toes", "hoes", "does", "oes", "peaches", "sandwiches", "quiches", "radishes", "dishes", "boxes", "foxes", "fizzes", "glazes", "sizes", "zes", "ches", # rule 4 (ie) and rule 5 (che, oe, ze) with 3/4/5-letter boundaries "tie", "die", "pie", "zie", "brie", "bie", "nie", "cookie", "brownie", "pixie", "che", "ache", "niche", "cache", "quiche", "creche", "oe", "roe", "toe", "doe", "aloe", "sloe", "shoe", "canoe", "oboe", "ze", "doze", "size", "glaze", "maize", "bronze", "seize", # 3/4/5-letter boundaries for the s rules "ess", "eses", "ass", "uses", "aces", "cakes", "roses", "dates", "kiwi", "kiwis", "fig", "figs", "ham", "hams", # stems that become stopwords (dropped) or repeat earlier tokens "toes", "ands", "fors", "withs", "froms", "ons", "ins", "toes and shoes", "and the of", # case "PIES", "Brownies", "QUICHE", "GlAzEs", "Pierogies", # accents removed by NFKD before stemming: these DO stem "crêpes", "pâtés", "brûlées", "jalapeños", "cafés", "çookies", "piés", "PİES", "quichés", "tôes", # letters NFKD can't remove, other scripts: these must NOT stem "œufs", "smørrebrøds", "straße", "Straßes", "brieł", "cookieß", "pirogiesæ", "ñoquies", "pieś", "пироги", "печенье cookies", "ΣΟΥΒΛΑΚΙΑΣ", "κουλουράκια", "クッキーズ", "饼干 cookies", "쿠키 cookies", "كعكات", "עוגיות", "ขนมปัง", "कुकीज़", "bánh quies", "ıes", "İes", # compatibility characters (NFKD) and digits glued to letters "COOKIES", "fies", "ffies", "flies", "2pies", "pies2", "cookies123", "3 pies", "12 cookies", "½ pies", "pies²", "fries®", "cookies™", # mixed scripts and punctuation "cookies & cream", "M&M's cookies", "pies/quiches", "brownie-ブラウニー", "quiche Лорен", "glaze—donuts", "fries'n'shakes", "shoe-string fries", "Ben & Jerry's Cookies", "Reese's Pieces", "s'mores", "Lay's", "Chick-fil-A", "Pop-Tarts", "Cheez-Its", "Oreos", "Pringles", "Hershey's Kisses", ] def main(): heaps = {c: [] for c in QUOTA} # max-heaps (negated hash) of the smallest hashes spare = [] # fill-ins when a category runs short barcodes = [] files = sorted(f for f in os.listdir(NORM) if f.endswith(".jsonl")) for fname in files: with open(os.path.join(NORM, fname), encoding="utf-8") as f: for line in f: try: rec = json.loads(line) except ValueError: continue name = rec.get("name") or "" brand = rec.get("brand") or "" rid = rec.get("id") or "" text = name + " " + brand if brand and foodtok.fnv1a("brand|" + rid) % 2 == 0 else name for c in categories(rec, text): if c not in heaps: continue h = foodtok.fnv1a(c + "|" + rid) item = (-h, rid, text) hp = heaps[c] if len(hp) < QUOTA[c] * 3: heapq.heappush(hp, item) elif item > hp[0]: heapq.heapreplace(hp, item) if not text.isascii(): h = foodtok.fnv1a("spare|" + rid) item = (-h, rid, text) if len(spare) < 400: heapq.heappush(spare, item) elif item > spare[0]: heapq.heapreplace(spare, item) bc = rec.get("barcode") or "" if bc: h = foodtok.fnv1a("bc|" + rid) item = (-h, rid, bc) if len(barcodes) < 40: heapq.heappush(barcodes, item) elif item > barcodes[0]: heapq.heapreplace(barcodes, item) chosen, texts = [], set() short = {} for c in QUOTA: # fixed category order, smallest hashes first got = 0 for negh, rid, text in sorted(heaps[c], key=lambda t: (-t[0], t[1])): if got >= QUOTA[c]: break if text in texts: continue texts.add(text) chosen.append((c, rid, text)) got += 1 if got < QUOTA[c]: short[c] = QUOTA[c] - got for negh, rid, text in sorted(spare, key=lambda t: (-t[0], t[1])): if len(chosen) >= TOTAL: break if text not in texts: texts.add(text) chosen.append(("fill_non_ascii", rid, text)) vectors = [] for c, rid, text in chosen[:TOTAL]: vectors.append(vector(text)) nsampled = len(vectors) for text in dict.fromkeys(LEGACY_CASES + V2_CASES): if text not in texts: texts.add(text) vectors.append(vector(text)) with open(os.path.join(HERE, "tok_vectors.json"), "w", encoding="utf-8") as f: json.dump(vectors, f, ensure_ascii=False, indent=0) f.write("\n") # Barcode vectors: real normalized barcodes in several raw spellings, plus invalid lengths. bv = [] for negh, rid, bc in sorted(barcodes, key=lambda t: (-t[0], t[1])): variants = [bc] if len(bc) == 13 and bc.startswith("0"): variants += [bc[1:], "0" + bc, bc[1:6] + " " + bc[6:]] elif len(bc) == 13: variants += [bc[:1] + "-" + bc[1:7] + "-" + bc[7:]] elif len(bc) == 8: variants += [bc[:4] + " " + bc[4:]] for v in variants: nb = foodtok.normalize_barcode(v) bv.append({"text": v, "barcode": nb, "barcodeBucket": None if nb is None else foodtok.fnv1a(nb) % BARCODE_BUCKETS}) for v in ["12345", "123456", "1234567", "123456789012345", "00000123456789", "abc", "", "0 12345 67890 5", # ASCII 0-9 only: other digits (Arabic-Indic, fullwidth, superscript, Devanagari) are dropped "\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669\u0660\u0661", "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19\uff10\uff15", "0123456789\u0665\u0661", "301762042200\u0663", "3017620422003\u00b2", "\u0966\u0967 3017620422003", "3017620422003", "301762042200\u00b3", "\uff13017620422003"]: nb = foodtok.normalize_barcode(v) bv.append({"text": v, "barcode": nb, "barcodeBucket": None if nb is None else foodtok.fnv1a(nb) % BARCODE_BUCKETS}) with open(os.path.join(HERE, "barcode_vectors.json"), "w", encoding="utf-8") as f: json.dump(bv, f, ensure_ascii=False, indent=0) f.write("\n") by_cat = {} for c, _, _ in chosen[:TOTAL]: by_cat[c] = by_cat.get(c, 0) + 1 print(json.dumps({"vectors": len(vectors), "sampled": nsampled, "legacy_and_v2_cases": len(vectors) - nsampled, "barcode_vectors": len(bv), "by_category": by_cat, "short_categories": short}, ensure_ascii=False, indent=1)) def vector(text): toks = foodtok.normalize(text) hashes = [foodtok.fnv1a(t) for t in toks] return {"text": text, "tokens": toks, "hashes": hashes, "buckets": [h % TOKEN_BUCKETS for h in hashes]} if __name__ == "__main__": main()