#!/usr/bin/env python3 """Search tokens, FNV-1a hashing and barcode normalization for the Losing It food index. This is FoodIndex/CONTRACT.md (contract v2), sections "Token normalization", "Hash and buckets" and "Barcode normalization", in Python. The app's Swift twin is LosingIt/Engine/FoodTokenizer.swift; the two must agree byte for byte (scripts/tok_vectors.json holds test vectors; build/tokcheck diffs them against Swift). Importable (stdlib only): import foodtok foodtok.normalize("Peaches & Cream Oats") -> ["peach", "cream", "oat"] foodtok.fnv1a("peach") -> 32-bit int foodtok.token_bucket("peach", 12000) -> int foodtok.normalize_barcode("012345678905") -> "0012345678905" foodtok.expand_upce("04252614") -> "042100005264" (same as the app's BarcodeScan.expandUPCE) Run as a script to print the tokens of each argument: python3 -I foodtok.py "Bananas, raw" "Crème brûlée" """ import re import sys import unicodedata STOPWORDS = frozenset(("and", "with", "of", "the", "in", "an", "for", "or", "to", "on", "de", "la", "le", "et", "en", "con", "mit", "und", "from")) _ES_ENDINGS = ("oes", "ches", "shes", "xes", "zes") # rule 3: drop the final "es" _E_ENDINGS = ("che", "oe", "ze") # rule 5: drop the final "e" _S_KEEP = ("ss", "us", "is") # rule 6 exceptions _ASCII_SPLIT = re.compile(r"[^a-z0-9]+") _ASCII_AZ = re.compile(r"[a-z]+") # Per-character cache of "General_Category is L* or N*" (step 3); the alphabet of food names is small. _LN = {} def _is_letter_or_number(ch): v = _LN.get(ch) if v is None: v = unicodedata.category(ch)[0] in "LN" _LN[ch] = v return v def stem(token): """Step 5: light English stemming, only for all-ASCII a-z tokens of 4 or more characters (n = its length). The first matching rule is applied, once: 1. ends with "ies" and n >= 5 -> "ies" becomes "y" (berries -> berry, fries -> fry, cookies -> cooky) 2. ends with "sses" -> drop the final "es" (glasses -> glass) 3. ends with oes, ches, shes, xes, zes -> drop the final "es" (tomatoes -> tomato, quiches -> quich) 4. ends with "ie" -> "ie" becomes "y" (cookie -> cooky, brownie -> browny, brie -> bry) 5. ends with che, oe, ze -> drop the final "e" (quiche -> quich, glaze -> glaz, shoe -> sho) 6. ends with "s", not ss, us or is -> drop the final "s" (eggs -> egg, pies -> pie)""" n = len(token) if n < 4 or not _ASCII_AZ.fullmatch(token): return token if token.endswith("ies") and n >= 5: return token[:-3] + "y" if token.endswith("sses"): return token[:-2] if token.endswith(_ES_ENDINGS): return token[:-2] if token.endswith("ie"): return token[:-2] + "y" if token.endswith(_E_ENDINGS): return token[:-1] if token.endswith("s") and not token.endswith(_S_KEEP): return token[:-1] return token def split(text): """Steps 1-3: NFKD, drop Mn, lowercase (str.lower), split on every scalar that is not L* or N*.""" if text.isascii(): # NFKD and Mn removal are no-ops on ASCII; ASCII letters and digits are the only L*/N* scalars. return [p for p in _ASCII_SPLIT.split(text.lower()) if p] s = unicodedata.normalize("NFKD", text) s = "".join(ch for ch in s if unicodedata.category(ch) != "Mn").lower() pieces, cur = [], [] for ch in s: if _is_letter_or_number(ch): cur.append(ch) elif cur: pieces.append("".join(cur)) cur = [] if cur: pieces.append("".join(cur)) return pieces def normalize(text): """The contract's normalize(): search tokens in order of first appearance.""" out, seen = [], set() for piece in split(text or ""): if len(piece) < 2: # step 4 (len() counts Unicode scalars) continue tok = stem(piece) # step 5 if tok in STOPWORDS or tok in seen: # steps 6 and 7 continue seen.add(tok) out.append(tok) return out def fnv1a(text): """32-bit FNV-1a over the UTF-8 bytes of text.""" h = 0x811C9DC5 for b in text.encode("utf-8"): h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF return h def token_bucket(token, buckets): return fnv1a(token) % buckets def normalize_barcode(raw): """Digits only; 12 -> "0"+12; 14 starting with "0" -> drop it; other lengths 6..14 as they are; else None.""" d = "".join(ch for ch in (raw or "") if "0" <= ch <= "9") n = len(d) if n == 12: return "0" + d if n == 14 and d[0] == "0": return d[1:] if 6 <= n <= 14: return d return None def _upc_check(digits): total = sum(x * 3 if i % 2 == 0 else x for i, x in enumerate(digits)) return (10 - total % 10) % 10 def expand_upce(code): """UPC-E to UPC-A exactly like the app's BarcodeScan.expandUPCE: the 8-digit form (number system 0 or 1, six digits, check digit; None when the check digit is wrong) or the bare six digits. None otherwise.""" if not code or not code.isascii() or not code.isdigit(): return None d = [int(c) for c in code] if len(d) == 8: if d[0] not in (0, 1): return None system, body, check = d[0], d[1:7], d[7] elif len(d) == 6: system, body, check = 0, d, None else: return None d1, d2, d3, d4, d5, d6 = body if d6 in (0, 1, 2): middle = [d1, d2, d6, 0, 0, 0, 0, d3, d4, d5] elif d6 == 3: middle = [d1, d2, d3, 0, 0, 0, 0, 0, d4, d5] elif d6 == 4: middle = [d1, d2, d3, d4, 0, 0, 0, 0, 0, d5] else: middle = [d1, d2, d3, d4, d5, 0, 0, 0, 0, d6] upca = [system] + middle computed = _upc_check(upca) if check is not None and check != computed: return None return "".join(map(str, upca + [computed])) def gs1_check_ok(code): """True when the last digit is the GS1 mod-10 check digit of the others (EAN-8, UPC-A, EAN-13, GTIN-14).""" if not code or not code.isascii() or not code.isdigit() or len(code) < 2: return False body = [int(c) for c in code[:-1]] total = sum(x * (3 if i % 2 == 0 else 1) for i, x in enumerate(reversed(body))) return (10 - total % 10) % 10 == int(code[-1]) if __name__ == "__main__": for arg in sys.argv[1:]: toks = normalize(arg) print(arg, "->", toks, [token_bucket(t, 12000) for t in toks])