#!/usr/bin/env python3 """Merge every normalized source and build the Losing It static food index (FoodIndex/CONTRACT.md, contract v2). Inputs FoodIndex/normalized/*.jsonl one normalized record per line (see the stage reports) FoodIndex/qa/usda_atwater_flags.jsonl macros of USDA records whose energy disagrees with 4P+4C+9F+7A Outputs FoodIndex/out/site/ the Cloudflare Pages site (manifest, shards, pages, scripts) LosingIt/Resources/GenericFoods.json the in-app offline bundle (generic foods only) FoodIndex/out/build_stats.json counts, drops, shard sizes (deterministic, no timings) FoodIndex/qa/merge_*.jsonl every record this stage dropped or hid, with the reason Rerunnable and deterministic: the same inputs give byte-identical outputs (sorted inputs, sorted keys, no timestamps, no set iteration). Python 3.14 stdlib only. Run: python3 -I FoodIndex/scripts/build_index.py [--build 20261115] [--max-per-token 1000] ... Steps 1. Load and validate every record against the contract; clean names and brands; drop and count anything invalid. 2. Merge-time accuracy checks on USDA Branded (B), using the USDA stage's Atwater QA file: - kcal 0 although the label's own macros give >= 30 kcal/100 and protein or fat >= 5 g -> drop - kcal 0 although the macros give >= 30 kcal/100 and nothing in the name or category makes zero believable (the Open Food Facts stage's zero-kcal rule, reused) -> drop - physically impossible label macros (a macro > 100 g, or P+F+C > 105 g, per 100 g) -> drop 3. Barcodes shared by B and Open Food Facts (O): only the B record is kept, everywhere (B inherits the O record's unique_scans_n for its score). When the two disagree on kcal by more than max(30 %, 20 kcal) (same unit; max(35 %, 25 kcal) when the units differ), at most one is right: B is still kept (manufacturer data), unless B fails its own Atwater check (it is in the USDA QA flag file), then O is kept and B dropped. Every conflict is logged in qa/merge_bo_conflicts.jsonl. (--bo-conflict keep-b | drop-both switch policy; dropping both would only hand the app's live Open Food Facts fallback the O value anyway.) An O 8-digit UPC-E whose UPC-A expansion is a B barcode is the same GTIN and is merged the same way when the kcal agree (and kept as a separate product when they do not). 4. Exact duplicates inside O, and inside B (normalized name + brand + unit, kcal within 2 %, barcode present): the one the token lists rank first (score desc, id asc) stays in token search; the others are hidden from token search but keep their own barcode in the barcode shards, so scanning them still works. This is the contract's "Duplicate hiding" (a hidden record has a barcode and a kept twin with the same normalized name, brand and unit, kcal within 2 % of the larger value, and a higher (score, then lower id)); it is applied within one source only and the name key (dup_key) is stricter than normalize(), so it hides a subset of what the contract allows. 5. Score every record 0..1000 (manifest.json "scoring" has the exact rules; summary below), then a consensus check: an O record whose kcal is more than 2x away from the median of same-brand, same-unit, same-name-word B/O records is ranked 150 lower (logged in qa/merge_consensus_outliers.jsonl). 6. Write token shards, barcode shards (plus UPC-E aliases), manifest, _headers, pages, scripts, bundle. Portions (every source): a household measure that contradicts its amount ('1 tsp' = 100 g, '8 fl oz' = 8 ml, '14 cup' for 1/4 cup) becomes '1 serving' (generic lab data: the portion is dropped); '1 serving' of exactly 100 is removed; 'cooked from' portions on raw generic foods are removed; then the most useful portion goes first (a label naming the food, 1 medium/regular, a single item; spoons for spreads and oils; tiny, cubic-inch, dense '1 cup' and > 800 kcal portions last). Records in ml above 845 kcal/100 ml (more than pure fat) are stored per 100 g, ml portions converted at 0.92 g/ml. Scoring (integer, clamped 0..1000) F 735, N 735, S 730 (+10 with portions); +45 canonical generic form ('Banana, raw', 'Milk, NFS'), else +20 USDA head-first name ('Rice, white, ...'); +12 per neutral descriptor after the first word (max 5); -40 ingredient forms (flour, powder, concentrate, ...); S: -120 for SR brand records (ALL-CAPS brand in the name) U, C 560 (-40 ingredient forms) M 550 (+10 with a portion), -150 for drinks B 420 + recency + twinPop + 10 with a portion recency = round(80 * clamp(1 - age_days / (8 * 365.25), 0, 1)), age = build date - record date twinPop = round(150 * min(1, log10(1 + unique_scans_n) / 3.25)) from the O record with the same barcode O 250 + pop + region + completeness pop = round(350 * min(1, log10(1 + unique_scans_n) / 3.25)) (OFF max is about 1,700 scans) region = 60 when countries has united-states (or lang is "en"), 30 for united-kingdom, canada, australia, ireland, new-zealand completeness = 10 brand + 10 portion + 5 category B/O -80 when every name word is in the brand and the product has fewer than 20 scans every source: - min(60, len(name) - 60) when the name is longer than 60 characters """ import argparse import collections import datetime import html import json import math import os import re import shutil import sys import unicodedata HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) # python3 -I does not add the script's own directory import foodtok # noqa: E402 from normalize_usda import fix_mojibake, mostly_caps, smart_title # noqa: E402 import normalize_off as offrules # noqa: E402 (zero-kcal plausibility rules) ROOT = os.path.dirname(HERE) # FoodIndex/ PROJECT = os.path.dirname(ROOT) # the app repository NORM_DIR = os.path.join(ROOT, "normalized") QA_DIR = os.path.join(ROOT, "qa") OUT_DIR = os.path.join(ROOT, "out") SITE_DIR = os.path.join(OUT_DIR, "site") BUNDLE = os.path.join(PROJECT, "LosingIt", "Resources", "GenericFoods.json") FORMAT = 1 CONTRACT = 2 # FoodIndex/CONTRACT.md version: manifest and bundle carry "contract": 2 DEFAULT_BUILD = "20261115" BASE_URL = "https://losingit-food.pages.dev" SUPPORT = "support@powerupgaming.dev" ODBL = "https://opendatacommons.org/licenses/odbl/1-0/" DBCL = "https://opendatacommons.org/licenses/dbcl/1-0/" CC0 = "https://creativecommons.org/publicdomain/zero/1.0/" OGL3 = "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" ETALAB2 = "https://www.etalab.gouv.fr/licence-ouverte-open-licence/" # Source metadata, from the normalization stage reports. Order = manifest order. "credit" (at most 120 characters) # is the short text the app shows next to a source's foods (contract "Licensing and attribution"). CREDIT_USDA = "USDA, Agricultural Research Service, FoodData Central (public domain)" CREDIT_MAX = 120 SOURCES = [ {"code": "F", "name": "USDA FoodData Central (Foundation Foods)", "license": "CC0 1.0 Universal (public domain)", "licenseUrl": CC0, "url": "https://fdc.nal.usda.gov/", "attribution": "USDA, Agricultural Research Service, FoodData Central: Foundation Foods (October 2026 release). " "fdc.nal.usda.gov", "credit": CREDIT_USDA, "generic": True}, {"code": "S", "name": "USDA FoodData Central (SR Legacy)", "license": "CC0 1.0 Universal (public domain)", "licenseUrl": CC0, "url": "https://fdc.nal.usda.gov/", "attribution": "USDA, Agricultural Research Service, FoodData Central: SR Legacy (April 2018). " "fdc.nal.usda.gov", "credit": CREDIT_USDA, "generic": True}, {"code": "N", "name": "USDA FoodData Central (FNDDS 2021-2023, Survey Foods)", "license": "CC0 1.0 Universal (public domain)", "licenseUrl": CC0, "url": "https://fdc.nal.usda.gov/", "attribution": "USDA, Agricultural Research Service, FoodData Central: Food and Nutrient Database for Dietary " "Studies 2021-2023 (October 2024 release). fdc.nal.usda.gov", "credit": CREDIT_USDA, "generic": True}, {"code": "B", "name": "USDA FoodData Central (Global Branded Food Products Database)", "license": "CC0 1.0 Universal (public domain)", "licenseUrl": CC0, "url": "https://fdc.nal.usda.gov/", "attribution": "USDA, Agricultural Research Service, FoodData Central: Branded Foods (October 2026 release). " "fdc.nal.usda.gov", "credit": CREDIT_USDA, "generic": False}, {"code": "O", "name": "Open Food Facts", "license": "Open Database License (ODbL) 1.0 for the database; Database Contents License (DbCL) 1.0 for " "individual contents. Product images are CC BY-SA and are not used.", "licenseUrl": ODBL, "url": "https://world.openfoodfacts.org/", "attribution": "Contains data from Open Food Facts (https://world.openfoodfacts.org), available under the " "Open Database License (https://opendatacommons.org/licenses/odbl/1-0/). Individual contents " "under the Database Contents License (https://opendatacommons.org/licenses/dbcl/1-0/).", "credit": "Contains data from Open Food Facts, available under the Open Database License", "generic": False}, {"code": "M", "name": "NYC DOHMH MenuStat (Historical), restaurant chain menu items", "license": "NYC public data set under NYC Admin. Code 23-502(d): no registration, licence or use " "restrictions; the City may require redistributors to identify the source and version and " "describe modifications", "licenseUrl": "https://data.cityofnewyork.us/Health/DOHMH-MenuStat-Historical-/qgc5-ecnb", "url": "https://data.cityofnewyork.us/Health/DOHMH-MenuStat-Historical-/qgc5-ecnb", "attribution": "Restaurant data: NYC Department of Health and Mental Hygiene (DOHMH), MenuStat (Historical), " "NYC Open Data dataset qgc5-ecnb (data collected January 2017 and January 2018; published " "2019-02-04). Modified by Losing It: converted to kcal per 100 g / 100 ml from the stated " "serving size; items without a serving size, limited-time offers, items no longer listed in " "2018 and internally inconsistent items were removed. Restaurant names are trademarks of " "their owners, who do not endorse this app.", "credit": "Restaurant data: NYC Health Department, MenuStat (NYC Open Data)", "generic": False}, {"code": "U", "name": "UK CoFID 2021 (McCance and Widdowson's The Composition of Foods Integrated Dataset " "2021, Public Health England)", "license": "Open Government Licence v3.0", "licenseUrl": OGL3, "url": "https://www.gov.uk/government/publications/composition-of-foods-integrated-dataset-cofid", "attribution": "Contains public sector information licensed under the Open Government Licence v3.0 " "(https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/). Source: Public " "Health England (2021) McCance and Widdowson's The Composition of Foods Integrated Dataset " "2021. (c) Crown copyright 2021.", "credit": "Contains public sector information licensed under the Open Government Licence v3.0 (UK CoFID 2021)", "generic": True}, {"code": "C", "name": "ANSES CIQUAL 2025 French food composition table (English names)", "license": "Licence Ouverte / Etalab Open Licence 2.0", "licenseUrl": ETALAB2, "url": "https://doi.org/10.57745/RDMHWY", "attribution": "Anses. 2025. Table de composition nutritionnelle des aliments Ciqual (Ciqual 2025, " "https://doi.org/10.57745/RDMHWY, last updated 2025-11-19). Licence Ouverte / Open Licence 2.0 " "(Etalab). Modified: English food names and EU Regulation 1169/2011 energy (kcal/100 g) only.", "credit": "Anses, Ciqual 2025 food composition table (updated 2025-11-19), Licence Ouverte 2.0", "generic": True}, ] SRC = {s["code"]: s for s in SOURCES} assert "L" not in SRC, "source letter L is reserved for the app's own log" assert all(0 < len(s["credit"]) <= CREDIT_MAX for s in SOURCES), "credit must be 1..120 characters" GENERIC = {s["code"] for s in SOURCES if s["generic"]} # Load order: B before O (B/O barcode merge), generic first. Unknown files are loaded after these, sorted. FILE_ORDER = ["usda_generic.jsonl", "usda_branded.jsonl", "extra_M.jsonl", "extra_U.jsonl", "extra_C.jsonl", "off.jsonl"] # ------------------------------------------------------------------------------------------------ scoring GENERIC_BASE = {"F": 735, "N": 735, "S": 730} EXTRA_GENERIC_BASE = 560 RESTAURANT_BASE = 550 RESTAURANT_DRINK_PENALTY = 150 # M fountain drinks (cup volume incl. ice) must not outrank the packaged drink BRAND_ONLY_NAME_PENALTY = 80 # B/O records named only after their brand ("Starbucks | Starbucks"), unless BRAND_ONLY_MIN_POP = 20 # scanned at least this often (then it is the product itself: "Pepsi | Pepsi") CONSENSUS_PENALTY = 150 # O kcal more than 2x away from same-brand, same-name B/O twins CANONICAL_BONUS = 45 # generic "Banana, raw" / "Milk, NFS" / "Rice, white, cooked" forms HEAD_FIRST_BONUS = 20 # USDA "Head, descriptor" names ("Rice, white, ...") over dish names ("Rice cake") INGREDIENT_PENALTY = 40 # generic ingredient forms nobody logs as a meal (flour, dry powder, concentrate) B_BASE, B_RECENCY_MAX, B_RECENCY_YEARS, B_TWINPOP_MAX = 420, 80, 8.0, 150 O_BASE, O_POP_MAX, POP_LOG_FULL = 250, 350, 3.25 O_REGION_US, O_REGION_EN = 60, 30 EN_COUNTRIES = ("united-kingdom", "canada", "australia", "ireland", "new-zealand") LONG_NAME, LONG_NAME_MAX_PENALTY = 60, 60 SCORING_DOC = { "range": "integer, clamped to 0..1000; higher is better; the default rank before the query bonuses", "F": "735 (+10 with portions): USDA Foundation, accurate lab analyses", "N": "735 (+10 with portions): USDA FNDDS survey foods, consumer-friendly names and portions", "S": "730 (+10 with portions): USDA SR Legacy", "generic forms": "F/S/N: +45 when every word after the first is a neutral descriptor (raw, NFS, plain, cooked, " "fresh, whole, regular, brewed, a number ...: 'Banana, raw', 'Milk, NFS'), else +20 for a USDA " "'Head, descriptor' name whose first part is one word ('Rice, white, cooked'); +12 per neutral " "word after the first (max 5); -40 for ingredient forms (flour, dry powder, concentrate, " "dehydrated, baby food, infant formula, pre-gelatinized); S: -120 for SR Legacy brand records " "(an ALL-CAPS brand in the name: 'Beverages, water, bottled, PEPSI, AQUAFINA')", "U,C": "560 (-40 for ingredient forms): generic extras (UK CoFID 2021, French CIQUAL 2025)", "M": "550 (+10 with portions), -150 for drinks (unit ml or category Beverages): restaurant chain items " "(MenuStat, 2018 menus)", "B": "420 + recency + twinPop + 10 with a portion; recency = round(80*clamp(1 - age_days/(8*365.25), 0, 1)) " "with age = build date - record date; twinPop = round(150*min(1, log10(1+unique_scans_n)/3.25)) " "taken from the Open Food Facts record with the same barcode", "O": "250 + pop + region + completeness; pop = round(350*min(1, log10(1+unique_scans_n)/3.25)); region = 60 " "when countries has united-states (or lang is en), 30 for united-kingdom, canada, australia, ireland, " "new-zealand; completeness = 10 brand + 10 portion + 5 category", "all": "minus min(60, len(name) - 60) when the name is longer than 60 characters; B/O: -80 when every name word " "is part of the brand ('Starbucks | Starbucks') and the product has fewer than 20 scans; O: -150 when kcal " "is more than 2x away from the median of the other B/O records with the same brand, unit and name words " "(at least 3 others, or 4 when the name has one word besides the brand)", } NEUTRAL_WORDS = frozenset(foodtok.normalize( "raw nfs ns plain regular fresh whole cooked prepared boiled steamed unprepared ready to eat unsalted salted " "all varieties types type commercial home prepared without added with no fat added drained solids liquids " "skin flesh peeled unpeeled ripe mature immature edible portion as purchased ns as to cooking method form " "whether made brewed")) NEUTRAL_COMP = 12 # per neutral descriptor word after the first (max 5): offsets the contract's # -12 per extra name word, so "Rice, white, cooked, NS as to fat" is not beaten by # "Rice, white, puffed" just for being precise SR_BRAND_PENALTY = 120 # SR Legacy records of a brand ("Beverages, ..., PEPSI, AQUAFINA", "McDONALD'S, ...") SR_BRAND_RE = re.compile(r"(? 'Banana, raw' before 'Banana split'); ingredient forms rank lower ('White rice, flour, pre-gelatinized').""" toks = foodtok.normalize(name) if not toks: return 0 bonus = 0 head = foodtok.normalize(name.split(",")[0]) neutral = [t for t in toks[1:] if t in NEUTRAL_WORDS or t.isdigit()] if len(toks) == 1 or len(neutral) == len(toks) - 1: bonus = CANONICAL_BONUS # "Banana, raw", "Milk, NFS", "Egg, whole, raw, fresh", "Blueberries" elif "," in name and len(head) == 1: bonus = HEAD_FIRST_BONUS # "Rice, white, cooked" (USDA head-noun-first) over "Rice cake" bonus += NEUTRAL_COMP * min(5, len(neutral)) if INGREDIENT_RE.search(name): bonus -= INGREDIENT_PENALTY return bonus def brand_only_name(r): if not r[R_BRAND]: return False nt = foodtok.normalize(r[R_NAME]) return bool(nt) and set(nt) <= set(foodtok.normalize(r[R_BRAND])) def compute_score(r, build_date, twin_pop): src = r[R_SRC] has_portion = bool(r[R_PORT]) if src in GENERIC_BASE: s = GENERIC_BASE[src] + (10 if has_portion else 0) + generic_form_bonus(r[R_NAME]) if src == "S" and SR_BRAND_RE.search(r[R_NAME]): s -= SR_BRAND_PENALTY elif src == "M": drink = r[R_UNIT] == "ml" or r[R_CAT] == "Beverages" s = RESTAURANT_BASE + (10 if has_portion else 0) - (RESTAURANT_DRINK_PENALTY if drink else 0) elif src == "B": s = B_BASE + recency_bonus(r[R_DATE], build_date) + pop_bonus(twin_pop, B_TWINPOP_MAX) \ + (10 if has_portion else 0) elif src == "O": s = O_BASE + pop_bonus(r[R_POP], O_POP_MAX) countries = r[R_COUNTRIES] if "united-states" in countries or r[R_LANG] == "en": s += O_REGION_US elif any(c in countries for c in EN_COUNTRIES): s += O_REGION_EN s += (10 if r[R_BRAND] else 0) + (10 if has_portion else 0) + (5 if r[R_CAT] else 0) elif src in GENERIC: # secondary generic sources: no form bonus, so a CoFID "Blueberries" (exact name +200 at query time) does not # beat USDA "Blueberries, raw" with portions s = EXTRA_GENERIC_BASE - (INGREDIENT_PENALTY if INGREDIENT_RE.search(r[R_NAME]) else 0) else: s = RESTAURANT_BASE if src in ("B", "O") and brand_only_name(r) and (twin_pop if src == "B" else (r[R_POP] or 0)) < BRAND_ONLY_MIN_POP: s -= BRAND_ONLY_NAME_PENALTY n = len(r[R_NAME]) if n > LONG_NAME: s -= min(LONG_NAME_MAX_PENALTY, n - LONG_NAME) return int(max(0, min(1000, s))) # ------------------------------------------------------------------------------------------------ text cleaning _WS = re.compile(r"\s+") _CC = re.compile(r"[\x00-\x1f\x7f-\x9f]") _INVISIBLE = re.compile("[--®™©℠]") _LEAD = re.compile(r"^(?:[\s,;:_~*|\\•·>!?]+|[.\-–—/+=#](?!\d))+") _TRAIL = re.compile(r"(?:[\s,;:_~*#|\\•·<\-–—/=]+|(?<=[^\W\d_]{3})\.)+$") _SPACE_BEFORE = re.compile(r"\s+([,;:)\]])") _SPACE_AFTER_OPEN = re.compile(r"([(\[])\s+") _EMPTY_BRACKETS = re.compile(r"\(\s*\)|\[\s*\]") _REPEAT_COMMA = re.compile(r",(?:\s*,)+") _COMMA_NO_SPACE = re.compile(r",(?=[^\W\d_])") _PAIRS = {'"': '"', "“": "”", "„": "“", "«": "»", "»": "«", "”": "”", "‘": "’", "「": "」"} _LONE_QUOTES = "\"“”„«»" def clean_text(s): """Whitespace collapsed, invisible characters and (R)/(TM)/(C) signs removed, junk punctuation off the ends.""" s = fix_mojibake(s) if "&" in s and ";" in s: s = html.unescape(s) # "Holie's", "Ben & Jerry's" s = s.replace('""', '"') # CSV-style doubled quotes: 1 "" cube, 6"" Ciabatta s = re.sub(r"(\d)\s+%", r"\1%", s) # "70 % Cacao" s = _CC.sub(" ", s) s = _INVISIBLE.sub("", s) s = _WS.sub(" ", s).strip() if not s: return s s = _EMPTY_BRACKETS.sub("", s) s = _SPACE_BEFORE.sub(r"\1", s) s = _SPACE_AFTER_OPEN.sub(r"\1", s) s = _REPEAT_COMMA.sub(",", s) s = _COMMA_NO_SPACE.sub(", ", s) s = _WS.sub(" ", s).strip() for _ in range(4): before = s s = _LEAD.sub("", s) s = _TRAIL.sub("", s) if not s: break if len(s) >= 3 and s[0] in _PAIRS and s[-1] == _PAIRS[s[0]] and s[0] not in s[1:-1] \ and s[-1] not in s[1:-1]: s = s[1:-1].strip() for q in _LONE_QUOTES: # a lone quote at one end, none elsewhere (not an inch mark: 12") if s and s.count(q) == 1 and (s[0] == q or (s[-1] == q and not (len(s) > 1 and s[-2].isdigit()))): s = s.replace(q, "").strip() if not s: break while s and s[-1] in "([{": s = s[:-1].rstrip() if s.endswith(")") and s.count(")") > s.count("("): s = s[:-1].rstrip() if s.startswith("(") and s.count("(") > s.count(")"): s = s[1:].lstrip() if s == before: break return s def cut(s, limit=120): """At most `limit` characters, cut at the last ", " or space (never inside a word).""" if len(s) <= limit: return s head = s[:limit + 1] i = head.rfind(", ") j = head.rfind(" ") k = i if i >= limit * 0.6 else j if k <= 0: k = limit return clean_text(s[:k]) def has_letter_or_number(s): return any(unicodedata.category(ch)[0] in "LN" for ch in s) def dup_key(s): """Duplicate key: NFKD, marks removed, lowercase, alphanumeric runs joined by one space.""" if s.isascii(): return " ".join(re.findall(r"[a-z0-9]+", s.lower())) s = unicodedata.normalize("NFKD", s) s = "".join(ch for ch in s if unicodedata.category(ch) != "Mn").lower() return " ".join(re.findall(r"[^\W_]+", s)) # ------------------------------------------------------------------------------------------------ portions _NUM = r"\d+(?:[.,]\d+)?" _METRIC_UNIT = r"(?:g|gr|grs|gram|grams|gramm|grammes?|gm|grm|kg|ml|mls|millilit(?:er|re)s?|cl|dl|l|ltr|lt|" \ r"lit(?:er|re)s?)" METRIC_IN_LABEL = re.compile(r"(? 2: kind = "tsp" elif u in ("c", "cup", "cups"): kind = "cup" elif u.startswith(("tbs", "tbl", "tablespoon")): kind = "tbsp" elif u.startswith(("tsp", "teaspoon")): kind = "tsp" elif u.startswith(("lb", "pound")): kind = "lb" else: kind = "oz" if kind in SPOON_CUP_ML: ratio = amount / (num * SPOON_CUP_ML[kind]) if num >= 10 and ratio < 0.3: return "lost_slash" if unit == "ml": return "ok" if 0.85 <= ratio <= 1.2 else "mismatch" return "ok" if 0.012 <= ratio <= 1.6 else "mismatch" if kind == "floz" or (kind == "oz" and unit == "ml"): ratio = amount / (num * 29.5735) if num >= 10 and ratio < 0.3: return "lost_slash" if unit == "ml": return "ok" if 0.85 <= ratio <= 1.2 else "mismatch" return "ok" if 0.5 <= ratio <= 1.6 else "mismatch" grams = num * (453.592 if kind == "lb" else 28.3495) ratio = amount / grams if num >= 10 and ratio < 0.2: return "lost_slash" return "ok" if 0.85 <= ratio <= 1.18 else "mismatch" SMALL_PORTION = re.compile(r"\b(miniature|mini|tiny|baby|bite.?size|snack.?size|fun.?size|cubic inch|surface inch|" r"kid'?s|child'?s|sample|taste|cocktail size)\b", re.I) SIZE_PORTION = re.compile(r"^1 (medium|regular|small/regular|regular/medium|average)\b", re.I) ITEM_PORTION = re.compile(r"^1 (large|can or bottle|can|bottle|slice|piece|serving|sandwich|burger|burrito|taco|bar|" r"cookie|muffin|bagel|roll|bun|link|patty|fillet|egg|item|order|bowl|container|package|packet|" r"pouch|stick|scoop|glass|mug|each|whole|fruit|breast|thigh|drumstick|wing|leg|steak|chop|" r"pie|cake|doughnut|donut|biscuit|pancake|waffle|tortilla|pita|pizza|wedge|cup)\b", re.I) SPOON_PORTION = re.compile(r"^1 (tbsp|tsp|pat|teaspoon|tablespoon)\b", re.I) SPREAD_NAME = re.compile(r"\b(honey|syrup|sugar|jam|jelly|butter|oil|margarine|mayonnaise|mayo|dressing|cream cheese|" r"peanut butter|nutella|lard|shortening|ghee|spread|molasses|agave|sauce|ketchup|mustard|" r"vinegar|salt|spice|pepper|seasoning|sweetener)\b", re.I) _MEASURE_WORDS = frozenset(foodtok.normalize( "small medium large extra jumbo regular whole piece pieces slice slices serving cup cups tbsp tsp oz fl inch " "inches long diameter each item about approx raw cooked boneless bone with without skin yield after")) def order_portions(portions, name, kcal): """Most useful portion first: a portion naming the food itself ('1 McDonald's Big Mac'), then 1 medium/regular, then a single item (1 slice, 1 can or bottle, 1 serving); spoons first for spreads, oils and sweeteners; tiny, miniature, cubic-inch portions, a dense food's '1 cup', and any portion over 800 kcal last.""" if len(portions) < 2: return portions ntoks = set(foodtok.normalize(name)) - _MEASURE_WORDS dense = kcal >= 500 or (kcal >= 250 and bool(SPREAD_NAME.search(name))) keyed = [] for i, (label, amt) in enumerate(portions): ltoks = [t for t in foodtok.normalize(label) if not t.isdigit()] food = [t for t in ltoks if t in ntoks] extra = [t for t in ltoks if t not in ntoks and t not in _MEASURE_WORDS] big = amt * kcal / 100.0 > 800 if big: bucket = 3 elif SMALL_PORTION.search(label) or label.lower().startswith("1 fl oz") or \ (dense and re.match(r"^1 cups?\b", label, re.I)): bucket = 2 else: bucket = 0 if dense and SPOON_PORTION.match(label): rank = -1 elif food and len(food) >= len(extra) and not LABEL_MEASURE.match(label): rank = 0 - len(food) * 0.01 + len(extra) * 0.001 elif SIZE_PORTION.match(label): rank = 1 elif ITEM_PORTION.match(label) and not (dense and re.match(r"^1 cups?\b", label, re.I)): rank = 2 else: rank = 3 keyed.append((bucket, rank, i)) keyed.sort() return [portions[i] for _, _, i in keyed] def pex(kind, before, after, amount): if len(PORTION_EXAMPLES[kind]) < 25: PORTION_EXAMPLES[kind].append([before, after, amount]) def is_num(v): return isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v) def clean_portions(raw, st, unit="g", generic=False, raw_food=False): """Contract portions: [[label, amount]], label without a metric amount, 0 < amount <= 5000, no plain 100, no duplicates, at most 6. Bad portions are dropped (the record stays). A household measure that contradicts the amount ('1 tsp' = 100 g, '8 fl oz' = 8 ml, '14 cup' for 1/4 cup) becomes '1 serving' (or, for generic lab data, the portion is dropped).""" if not isinstance(raw, list): st["portion_not_list"] += 1 return [] out, seen = [], set() for p in raw: if not (isinstance(p, (list, tuple)) and len(p) == 2 and isinstance(p[0], str) and is_num(p[1])): st["portion_bad_shape"] += 1 continue amount = round(float(p[1]), 2) if not (0 < amount <= 5000): st["portion_bad_amount"] += 1 continue label = clean_text(p[0]) if not label or not has_letter_or_number(label): st["portion_bad_label"] += 1 continue label = re.sub(r"^(\d+)\.0+(?=\s|$)", r"\1", label) # "1.0 portion", "2.0 tbsp" if "\ufffd" in label or label.lower().strip(" .") in BARE_LABEL or METRIC_GARBLED.search(label): pex("label_unusable_to_1_serving", label, "1 serving", amount) label = "1 serving" st["portion_label_unusable_to_1_serving"] += 1 if raw_food and COOKED_FROM.search(label): st["portion_cooked_weight_on_raw_food_dropped"] += 1 # the grams are the cooked weight continue mt = TRAILING_NUMBER.search(label) if mt and abs(float(mt.group(1)) - amount) <= max(0.6, 0.01 * amount) and len(label[:mt.start()]) >= 3: pex("trailing_amount_removed", label, label[:mt.start()], amount) label = clean_text(label[:mt.start()]) st["portion_label_trailing_amount_removed"] += 1 chk = label_measure_check(label, amount, unit) if chk in ("mismatch", "lost_slash"): if generic: st[f"portion_measure_{chk}_dropped"] += 1 pex(f"measure_{chk}_dropped", label, None, amount) continue pex(f"measure_{chk}_to_1_serving", label, "1 serving", amount) label = "1 serving" st[f"portion_measure_{chk}_to_1_serving"] += 1 compact = label.lower().replace(" ", "") if compact in PLAIN_100 and abs(amount - 100) < 1e-9: st["portion_plain_100_removed"] += 1 continue if METRIC_IN_LABEL.search(label): m = METRIC_ONLY.match(label) if m: stated = float(m.group(1).replace(",", ".")) * UNIT_FACTOR.get(m.group(2).lower(), 1.0) if abs(stated - amount) > 0.03 * max(stated, amount): st["portion_metric_label_mismatch_dropped"] += 1 continue if abs(amount - 100) < 1e-9: st["portion_plain_100_removed"] += 1 continue pex("metric_only_to_1_serving", label, "1 serving", amount) label = "1 serving" st["portion_metric_only_label_to_1_serving"] += 1 else: stripped = clean_text(METRIC_PAREN.sub("", label)) if stripped and not METRIC_IN_LABEL.search(stripped) and has_letter_or_number(stripped): pex("metric_paren_removed", label, stripped, amount) label = stripped st["portion_metric_paren_removed"] += 1 else: stripped = clean_text(METRIC_IN_LABEL.sub(" ", stripped or label)) stripped = clean_text(re.sub(r"\s*[/=]\s*$|^\s*[/=]\s*", "", stripped)) if stripped and any(unicodedata.category(c)[0] == "L" for c in stripped): pex("metric_text_removed", label, stripped, amount) label = stripped st["portion_metric_text_removed"] += 1 else: pex("metric_label_to_1_serving", label, "1 serving", amount) label = "1 serving" st["portion_metric_label_to_1_serving"] += 1 if len(label) > MAX_LABEL: pex("label_cut_80", label, cut(label, MAX_LABEL), amount) label = cut(label, MAX_LABEL) st["portion_label_cut_80"] += 1 if SERVING_100.match(label) and abs(amount - 100) < 1e-9: st["portion_serving_of_exactly_100_removed"] += 1 # duplicates the app's own 100 g / 100 ml continue key = (label, amount) if key in seen: st["portion_duplicate_removed"] += 1 continue seen.add(key) out.append([label, amount]) return out # ------------------------------------------------------------------------------------------------ records # A loaded record is a list with these slots. (R_ID, R_NAME, R_BRAND, R_KCAL, R_UNIT, R_PORT, R_BC, R_SRC, R_SCORE, R_POP, R_DATE, R_COUNTRIES, R_LANG, R_CAT, R_HIDDEN) = range(15) ID_RE = re.compile(r"^([A-Z]):(\S{1,64})$") ML_MAX_KCAL = 845.0 FAT_DENSITY = 0.92 PLACEHOLDER_BRANDS = {"sans marque", "n/a", "na", "nan", "null", "none", "no brand", "unknown", "generic", "brand", "0", "-", "--", "no name", "unbranded", "not applicable", "sin marca", "ohne marke", "marque", "inconnu", "inconnue", "aucune", "aucun", "x", "xx", "xxx", "?", "test"} def validate(d, st, drops): """Contract checks plus cleaning. Returns a record list, or None (reason counted under st["drop_*"]).""" def drop(reason): st["drop_" + reason] += 1 if drops is not None and len(drops[reason]) < 50: drops[reason].append({k: d.get(k) for k in ("id", "name", "brand", "kcal", "unit", "src")} if isinstance(d, dict) else {"raw": str(d)[:200]}) return None if not isinstance(d, dict): return drop("not_object") rid, src = d.get("id"), d.get("src") if not isinstance(src, str) or src not in SRC: return drop("unknown_src") if not isinstance(rid, str) or not ID_RE.match(rid) or rid[0] != src: return drop("bad_id") name = d.get("name") if not isinstance(name, str): return drop("name_not_string") raw_name = name name = clean_text(name) if src == "O" and mostly_caps(name): name = smart_title(name) st["name_title_cased"] += 1 if name and name[0].islower() and len(name[0].upper()) == 1 and not (len(name) > 1 and name[1].isupper()): name = name[0].upper() + name[1:] st["name_first_letter_capitalized"] += 1 if len(name) > 120: name = cut(name) st["name_cut_120"] += 1 if name != raw_name: st["name_changed"] += 1 if not (2 <= len(name) <= 120) or not has_letter_or_number(name): return drop("name_invalid") brand = d.get("brand") if brand is None: brand = "" if not isinstance(brand, str): return drop("brand_not_string") brand = clean_text(brand) if brand.lower().strip(" .") in PLACEHOLDER_BRANDS: st["brand_placeholder_cleared"] += 1 brand = "" if len(brand) > 120: brand = cut(brand) st["brand_cut_120"] += 1 generic = SRC[src]["generic"] if d.get("generic") is not None and bool(d.get("generic")) != generic: return drop("generic_flag_mismatch") if generic and brand: st["generic_brand_cleared"] += 1 brand = "" kcal = d.get("kcal") if not is_num(kcal): return drop("kcal_not_number") kcal = round(float(kcal), 1) if not (0.0 <= kcal <= 900.0): return drop("kcal_out_of_range") unit = d.get("unit") if unit not in ("g", "ml"): return drop("unit_invalid") raw_portions = d.get("portions", []) if unit == "ml" and kcal > ML_MAX_KCAL: # more than pure fat per 100 ml is impossible: the value is per 100 g (oils, 900) or a label-rounding value # (130 kcal per 15 ml = 867); as per 100 g it is within 2 % of the truth. ml portions -> g at 0.92 g/ml. unit = "g" st["unit_ml_over_845_to_g"] += 1 if isinstance(raw_portions, list): raw_portions = [[p[0], round(float(p[1]) * FAT_DENSITY, 1)] if isinstance(p, (list, tuple)) and len(p) == 2 and is_num(p[1]) else p for p in raw_portions] raw_food = generic and bool(re.search(r"\braw\b", name, re.I)) portions = clean_portions(raw_portions, st, unit, generic, raw_food) portions = order_portions(portions, name, kcal) if len(portions) > 6: st["portion_capped_6"] += 1 portions = portions[:6] bc = d.get("barcode") or "" if not isinstance(bc, str): return drop("barcode_not_string") if generic: if bc: st["generic_barcode_cleared"] += 1 bc = "" elif bc: nb = foodtok.normalize_barcode(bc) if nb is None: st["barcode_invalid_cleared"] += 1 bc = "" elif len(nb) < 8: st["barcode_shorter_than_8_cleared"] += 1 # the app looks up 8-14 digits only; the record stays searchable bc = "" elif nb != bc: st["barcode_renormalized"] += 1 bc = nb pop = d.get("pop") if not (is_num(pop) and pop >= 0): pop = None date = d.get("date") if isinstance(d.get("date"), str) else "" countries = d.get("countries") if isinstance(d.get("countries"), list) else [] countries = tuple(c for c in countries if isinstance(c, str)) lang = d.get("lang") if isinstance(d.get("lang"), str) else None cat = d.get("category") if isinstance(d.get("category"), str) else "" return [rid, name, brand, kcal, unit, portions, bc, src, 0, pop, date, countries, lang, cat, False] def zero_plausible(name, cat): """The Open Food Facts stage's zero-kcal rule, on a name and a free-text category.""" lname = offrules.strip_accents_lower(name) lcat = offrules.strip_accents_lower(cat or "") zname = lname for a, b in offrules.ZERO_NEG_SAFE: zname = zname.replace(a, b) if offrules.ZERO_NEG_RE.search(zname): return False if offrules.ZERO_NAME_RE.search(lname) or offrules.ZERO_NAME_RE.search(lcat): return True return bool(offrules.ZERO_DIET_RE.search(lname) and (offrules.DRINK_RE.search(lname) or offrules.DRINK_RE.search(lcat))) def b_accuracy_drop(r, flag): """Reason to drop a B record given its Atwater QA flag (macros per 100 g/ml), or None.""" p = flag.get("protein") or 0.0 f = flag.get("fat") or 0.0 c = flag.get("carb") or 0.0 est = flag.get("atwater_4_4_9_7") or 0.0 if r[R_UNIT] == "g" and (max(p, f, c) > 100.0 or p + f + c > 105.0): return "b_impossible_macros" if r[R_KCAL] == 0.0 and est >= 30.0: if p >= 5.0 or f >= 5.0: return "b_zero_kcal_with_protein_or_fat" if not zero_plausible(r[R_NAME], r[R_CAT]): return "b_zero_kcal_not_plausible" return None def kcal_conflict(a, b, same_unit): if same_unit: return abs(a - b) > max(20.0, 0.30 * max(a, b)) return abs(a - b) > max(25.0, 0.35 * max(a, b)) def iter_jsonl(path): with open(path, encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue try: yield json.loads(line) except ValueError: yield None def jdump(obj): return json.dumps(obj, ensure_ascii=False, separators=(",", ":")) def record_json(r): return jdump([r[R_ID], r[R_NAME], r[R_BRAND], r[R_KCAL], r[R_UNIT], r[R_PORT], r[R_BC], r[R_SRC], r[R_SCORE]]) def write_jsonl(path, rows): tmp = path + ".tmp" with open(tmp, "w", encoding="utf-8") as f: for row in rows: f.write(json.dumps(row, ensure_ascii=False) + "\n") os.replace(tmp, path) # ------------------------------------------------------------------------------------------------ pages CSS = """:root{--bg:#F7F3EC;--card:#FFFDF9;--ink:#2F2A25;--muted:#6E655C;--line:#E4DCCF;--accent:#7A5C3E} @media (prefers-color-scheme:dark){:root{--bg:#1E1B18;--card:#26221E;--ink:#EDE6DC;--muted:#B3A898;--line:#3A342D;--accent:#D9B88F}} *{box-sizing:border-box}html{-webkit-text-size-adjust:100%} body{margin:0;background:var(--bg);color:var(--ink);font:17px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",system-ui,sans-serif} main{max-width:720px;margin:0 auto;padding:48px 16px 64px} h1,h2{font-family:"New York","Iowan Old Style",Georgia,serif;font-weight:600;line-height:1.25} h1{font-size:2rem;margin:0 0 .5rem}h2{font-size:1.3rem;margin:2.2rem 0 .6rem} p,li{color:var(--ink)}.muted{color:var(--muted)}a{color:var(--accent)} .card{background:var(--card);border:1px solid var(--line);border-radius:14px;padding:16px 18px;margin:12px 0} .card h3{margin:0 0 .3rem;font-size:1.05rem}.card p{margin:.35rem 0} table{border-collapse:collapse;width:100%;font-size:.95rem}td,th{text-align:left;padding:6px 8px;border-bottom:1px solid var(--line)} td.n{text-align:right;font-variant-numeric:tabular-nums}code{font:.9em ui-monospace,SFMono-Regular,Menlo,monospace;overflow-wrap:anywhere} ul{padding-left:1.2rem}""" def page(title, body): return ("\n
" "" "" f"This is the open food index used by the Losing It app to search foods and look up barcodes when logging calories. It holds calories per 100 g (or 100 ml) and common portions for generic foods, packaged products and restaurant items, merged from open databases.
It is a set of static JSON files. The app downloads only the few files a search needs and does the matching on the phone, so this server only ever sees a bucket number, never what you type.
| Source | Records |
|---|
The data comes from USDA FoodData Central, Open Food Facts and other open sources. See attribution and licenses for every source and the required notices. The index is made available under the Open Database License (ODbL) 1.0.
Values come from labels and food composition tables and can contain errors. Questions or corrections: {SUPPORT}.
""" return page("Losing It food index", body) def attribution_page(build, counts, path): cards = [] for s in SOURCES: c = s["code"] cards.append(f"""Source code {c} · {fmt_int(counts.get(c, 0))} records in build {html.escape(build)}
· {'generic foods' if s['generic'] else 'branded or restaurant items'}
License: {html.escape(s['license'])}
Attribution: {html.escape(s['attribution'])}
Shown in the app: {html.escape(s['credit'])}
Source: {html.escape(s['url'])}
The Losing It food index (build {html.escape(build)}) is built from the sources below. Each one is credited here with its license and the attribution it asks for. Brand and restaurant names are trademarks of their owners, who do not endorse this app.
This index contains data from Open Food Facts, which is made available under the Open Database License (ODbL) 1.0. Individual contents are under the Database Contents License (DbCL) 1.0.
The Losing It food index is a derivative database of Open Food Facts and the other sources listed here. It is made
available under the Open Database License (ODbL) 1.0, and any rights in its individual contents
under the Database Contents License 1.0. The whole derived database is published openly on this
site: manifest.json names the data folder (/{html.escape(path)}/) of token shards
(t/) and barcode shards (b/).
How it was made: records were converted to kcal per 100 g or 100 ml with portions, checked for impossible or contradictory values (records that failed were removed), merged across sources by barcode, de-duplicated and ranked. The scripts that do this are published at /scripts/ as plain text files, and are also available on request at {SUPPORT}.
Values come from manufacturer labels, restaurant disclosures and food composition tables. They are provided as they are, without warranty, and can contain errors. Questions or corrections: {SUPPORT}.
""" return page("Food index attribution", body) def not_found_page(): return page("Not found", "There is nothing at this address. " "See the Losing It food index.
") _SANITIZE = [ (re.compile(re.escape(PROJECT)), "{html.escape(n)}These Python scripts (standard library only) download, normalize and merge the sources and build this food index.
They are published under the share-alike terms of the Open Database License so the derived database
can be rebuilt. Local file paths were replaced with <project>. build_index.py builds the
site; foodtok.py is the shared tokenizer; CONTRACT.md.txt documents the file formats.
Questions: {SUPPORT} · Attribution and licenses
""" with open(os.path.join(d, "index.html"), "w", encoding="utf-8") as f: f.write(page("Food index scripts", body)) return files # ------------------------------------------------------------------------------------------------ main def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--build", default=DEFAULT_BUILD) ap.add_argument("--token-buckets", type=int, default=12000) ap.add_argument("--barcode-buckets", type=int, default=4096) ap.add_argument("--max-per-token", type=int, default=1000) ap.add_argument("--bo-conflict", choices=["prefer-consistent", "keep-b", "drop-both"], default="prefer-consistent") ap.add_argument("--site", default=SITE_DIR) ap.add_argument("--bundle", default=BUNDLE) ap.add_argument("--bundle-limit", type=int, default=4_000_000) ap.add_argument("--max-files", type=int, default=19_500) ap.add_argument("--max-file-bytes", type=int, default=5 * 1024 * 1024) args = ap.parse_args() build = args.build build_date = datetime.date(int(build[:4]), int(build[4:6]), int(build[6:8])) TB, BB, CAP = args.token_buckets, args.barcode_buckets, args.max_per_token data_path = f"d/{build}" os.makedirs(OUT_DIR, exist_ok=True) stats = collections.OrderedDict() st_by_src = collections.defaultdict(collections.Counter) drops = collections.defaultdict(list) merge_log = collections.defaultdict(list) # qa/merge_