#!/usr/bin/env python3 """Energy-density plausibility bands for packaged foods (Open Food Facts and USDA Branded). Label data is often entered per serving into a per-100 g field ("Cheddar 110", "Peanut butter 180"), or scaled with a wrong serving size ("Butter Lovers popcorn 4 kcal/100 g"). The Atwater check cannot see either error, because the macros are wrong by the same factor. For a few kinds of food the energy per 100 g is pinned by physics or by composition (oil is ~900 kcal/100 g, a hard cheese is never 110), so a value outside these wide bands is a data error and the record is dropped. A band applies only when BOTH the category matches (an OFF categories_tags tag, or the USDA branded_food_category) AND the product name names that kind of food, and none of the band's exclusion words is in the name. Names are matched accent-stripped and lowercased. The bands are deliberately wide: they are meant to catch values that are off by a factor of two or more, not to judge recipes. Used by normalize_off.py and normalize_usda.py. Python stdlib only. """ import re import unicodedata def fold(s): s = unicodedata.normalize("NFKD", s or "") return "".join(ch for ch in s if unicodedata.category(ch) != "Mn").lower() def _rx(words): return re.compile(r"(?:" + words + r")") # Common exclusions: composite foods and products that only carry the food's name as a flavour or an ingredient. _COMPOSITE = (r"\bsauce|\bdip\b|\bdips\b|dressing|vinaigrette|marinade|\bsoup|seasoning|\bmix\b|\bmixes\b|" r"preparation|preparado|preparato|\bkit\b|flavou?r|aroma|\bsnacks?\b|\bbites?\b|\bbars?\b|\bcake|" r"cookie|biscuit|cracker|\bbread|\bpie\b|pizza|pasta|sandwich|\bwrap|salad|burger|omelet|quiche|" r"\bdrink|boisson|bebida|beverage|shake|smoothie|ice cream|yogurt|yoghurt|yaourt|pudding|dessert|" r"candy|bonbon|cereal|granola|muesli|\bdog\b|\bcat\b|\bpet\b") # Chocolate bars and chocolate candies are the food itself. _COMPOSITE_CHOC = _COMPOSITE.replace(r"\bbars?\b|", "").replace(r"candy|bonbon|", "") BANDS = [ # id, OFF tags, USDA categories, name regex, exclusion regex, units, min, max {"id": "hard_cheese", "tags": {"en:cheeses", "en:hard-cheeses", "en:cheddar-cheese", "en:cow-cheeses"}, "cats": {"Cheese"}, "name": _rx(r"\b(cheddar|gouda|emmental|emmentaler|emmenthal|gruyere|parmesan|parmigiano|grana padano|comte|" r"edam|manchego|pecorino|romano|beaufort|cantal|mimolette|maasdam|leerdam|appenzeller|asiago|" r"jarlsberg|provolone|colby|monterey jack|pepper ?jack|swiss cheese|old amsterdam|sbrinz|raclette|" r"tilsit|tilsiter|havarti|fontina)\b"), "not": _rx(_COMPOSITE + r"|light|\blite\b|leger|legere|allege|reduced|less fat|lower fat|low ?fat|fat ?free|" r"non ?fat|\bskim|\d+ ?%|leicht|ligero|ligera|magro|spread|tartiner|streich|fondue|stick|crisp|chip|" r"puff|popcorn|macaroni|\bmac\b|powder|style|alternative|vegan|plant|dairy free|imitation|substitute|" r"bagel|pretzel|tortilla|souffle|cream cheese|cottage|ricotta|whey|grated topping|queso|velveeta|processed|" r"american|carrot|apple|almond|snack pack|lunch|blend of|kit"), "units": ("g",), "min": 180.0}, {"id": "cheese", "tags": {"en:cheeses"}, "cats": {"Cheese"}, "name": _rx(r"\b(cheese|cheeses|fromage|queso|quesillo|kase|formaggio|kaas|cheddar|gouda|emmental|gruyere|" r"parmesan|mozzarella|feta|brie|camembert|halloumi|ricotta|cottage|paneer|burrata|quark|" r"mascarpone|neufchatel|chevre|roquefort|gorgonzola|stilton|edam|manchego|pecorino|provolone)\b"), "not": _rx(_COMPOSITE + r"|whey|serum|lactoserum|petit.lait|fondue"), "units": ("g",), "min": 30.0}, {"id": "butter", "tags": {"en:butters", "en:dairy-spreads", "en:salted-butters", "en:unsalted-butters", "en:spreadable-fats"}, "cats": {"Butter & Spread"}, "name": _rx(r"\b(butter|beurre|mantequilla|burro|boter|smor|maslo|manteiga|vaj)\b"), "not": _rx(_COMPOSITE + r"|peanut|arachide|cacahu|almond|amande|cashew|hazelnut|noisette|\bnut|seed|sesame|" r"apple|pomme|pumpkin|fruit|bleuet|blueberr|cocoa|cacao|shea|karite|body|speculoos|light|\blite\b|" r"leger|legere|allege|reduced|less fat|leicht|ligera|half|spread|tartiner|streich|squeeze|spray|" r"blend|melange|margarin|vegetable|plant|vegan|\boil|huile|cream|creme|honey|miel|cinnamon|maple|" r"chocolate|bean|lentil|tofu|bruschetta|olive|crab|lobster|toffee|caramel|fudge|chicken|buttermilk|" r"butternut|butterscotch|herb|garlic|truffle|sauce|clarified|ghee|dairy.free|seasoned|lilikoi|liliko|" r"lemon|curd|tastes like|like butter|style|potato|balls|\d+ ?%|demi.?ecreme|magra|omega|not butter|believe|" r"berry|movie|theater|popcorn"), "units": ("g",), "min": 300.0}, {"id": "oil", "tags": {"en:vegetable-oils", "en:olive-oils", "en:extra-virgin-olive-oils", "en:virgin-olive-oils", "en:sunflower-oils", "en:rapeseed-oils", "en:coconut-oils", "en:vegetable-fats", "en:fats"}, "cats": {"Vegetable & Cooking Oils"}, "name": _rx(r"\b(oil|oils|huile|aceite|olio|ol|oel|olie|olej|azeite|ghee|lard|saindoux|manteca|shortening|" r"tallow|suet)\b"), "not": _rx(_COMPOSITE + r"|spray|vinaigre|vinegar|vinagre|aceto|extract|essence|pesto|glycerin|cbd|hemp extract|" r"essential|infused water|tuna|sardine|anchov|in oil|en aceite|a l.huile|all.olio|in olio|sun.?dried|" r"tomato|artichoke|pepper|\bolives\b|potato|chips|crisps|popcorn|mayo|balsam|lemon juice|omega|" r"capsule|softgel|supplement|fish oil|cod liver|krill|massage|\bhair|\bskin|tofu|herring|mackerel|" r"chil+i oil|chile oil|crisp|black bean|release|lecithin|pan coating|chili|chilli|spread|buttery|margarin|" r"butter|tartiner|streich|\d+ ?%"), "units": ("g", "ml"), "min": 700.0, "min_ml": 640.0}, {"id": "nut_butter", "tags": {"en:nut-butters", "en:peanut-butters", "en:almond-butters", "en:spreads", "en:nuts-and-their-products", "en:plant-based-spreads"}, "cats": {"Nut & Seed Butters"}, "name": _rx(r"peanut butter|beurre d.arachide|beurre de cacahu|crema de cacahuete|crema de mani|" r"mantequilla de mani|mantequilla de cacahuate|pasta de amendoim|erdnussbutter|erdnussmus|" r"pindakaas|almond butter|beurre d.amande|crema de almendra|mandelmus|cashew butter|nut butter|" r"nutbutter|hazelnut butter|sunflower (?:seed )?butter|sunbutter|tahini|tahin|sesame paste|" r"pistachio butter|walnut butter|pecan butter|macadamia butter|seed butter"), "not": _rx(_COMPOSITE + r"|powder|pb2|pbfit|poudre|polvo|pulver|light|reduced fat|\blite\b|\bcups?\b|filled|" r"crunch bar|protein|milk|puffs|balls|fudge|brownie|wafer|jelly|jam|pretzel|chocolate|based|better.n|" r"schmier|pumpkin|apple|raisin|fruit|blend|cocoa|cacao|maple|honey|cinnamon|hummus|humus|hommus|houmous|clif|" r"\bbar|protein"), "units": ("g",), "min": 300.0}, {"id": "nut_butter_powder", "tags": {"en:nut-butters", "en:peanut-butters", "en:spreads", "en:nuts-and-their-products"}, "cats": {"Nut & Seed Butters"}, "name": _rx(r"(peanut|almond|nut).*(powder|powdered|poudre|polvo|pulver)|pb2|pbfit"), "not": _rx(_COMPOSITE + r"|milk|protein shake"), "units": ("g",), "min": 300.0}, {"id": "hazelnut_spread", "tags": {"en:hazelnut-spreads", "en:cocoa-and-hazelnuts-spreads", "en:chocolate-spreads"}, "cats": set(), "name": _rx(r"nutella|hazelnut|noisette|nocciol|avellana|haselnuss|hazelnoot|pate a tartiner|crema de cacao|" r"chocolate spread|nuss.?nougat"), "not": _rx(_COMPOSITE + r"|light|reduced sugar|fruit|cous|lasagn|fromage|cheese"), "units": ("g",), "min": 250.0}, {"id": "chocolate", "tags": {"en:chocolates", "en:dark-chocolates", "en:milk-chocolates", "en:white-chocolates", "en:chocolates-with-hazelnuts", "en:chocolates-with-almonds", "en:chocolate-bars", "en:extra-dark-chocolates", "en:plain-chocolates"}, "cats": {"Chocolate"}, "name": _rx(r"\b(chocolate|chocolat|schokolade|cioccolato|chocolade|czekolada|cokolada|suklaa)\b"), "not": _rx(_COMPOSITE_CHOC + r"|milk(?! chocolate)|\blait(?! chocolat)|leche(?! chocolate)|latte|powder|poudre|" r"polvo|pulver|hot chocolate|hot cocoa|cocoa mix|syrup|sirop|natilla|\bflan|creme|cream|mousse|" r"spread|tartiner|glace|gelato|protein|whey|brownie|wafer|chili|mole|\btea\b|coffee|cafe|liqueur|" r"liquor|wine|beer|\bdates?\b|fruit|raisin|almonds\b|\bnuts\b|pretzel|popcorn|rice cake|crispy rice|" r"marshmallow|covered|coated|dipped|enrobed|matzo|truffle sauce|baking chips|morsels|chips|apple|\bfigs?\b|" r"orange peel|ginger|cherr|strawberr|blueberr|cranberr|sugar free|no sugar|keto|stevia|galleta|avena|" r"swiss miss|lucky charms|" r"sweetened with|monk|allulose"), "units": ("g",), "min": 300.0}, {"id": "dairy_milk", "tags": {"en:milks", "en:whole-milks", "en:semi-skimmed-milks", "en:skimmed-milks", "en:cow-milks", "en:milks-liquid-and-powder", "en:pasteurized-milks", "en:uht-milks", "en:lactose-free-milks"}, "cats": {"Milk", "Milk/Milk Substitutes"}, "name": _rx(r"\b(milk|lait|leche|latte|milch|melk|maelk|mjolk|maito|leite|mleko|mlijeko|mleka|tej)\b"), "not": _rx(_COMPOSITE + r"|powder|poudre|polvo|pulver|\bdry\b|dried|concentr|condens|evapor|infant|formula|" r"baby|toddler|growing|croissance|almond|amande|almendra|mandel|\bsoy|soja|\boat|avoine|avena|hafer|" r"haver|\brice|\briz\b|arroz|\breis|coconut|\bcoco\b|cashew|hazelnut|noisette|\bpea\b|hemp|chanvre|" r"plant|vegetal|vegan|\bchat\b|chien|goat|chevre|sheep|brebis|camel|choco|cacao|cocoa|strawberr|" r"fraise|banana|vanill|caramel|coffee|\bcafe|\btea\b|kefir|buttermilk|ribot|cultured|protein|" r"cream|creme|\bnata\b|\bhalf\b|egg ?nog|ponche|masala|chai|horchata|sweet|sucre|\bpain|cheese|" r"mousse|cookies|toffee|fudge|\bice\b|helado|dulce|cajeta|manjar|tigre|tres leches|milky|malted|" r"ovomaltine|nesquik|flavou?r|honey|miel|maple|turmeric|golden|ultra.?filtered|a2|jersey|" r"channel island|guernsey|high in|fortified with|nog|sunflower|instant|ricotta|holiday|snickerdoodle|" r"seasonal|pumpkin|peppermint|mint|eggnog|formule|1er age|2e age|crescimento|infantil|\bem po\b|en polvo|" r"\bpo\b"), "units": ("g", "ml"), "min": 25.0, "max": 100.0, "skip_above": 250.0}, {"id": "egg", "tags": {"en:eggs", "en:chicken-eggs", "en:eggs-and-their-products", "en:fresh-eggs", "en:free-range-eggs", "en:free-range-chicken-eggs", "en:organic-eggs", "en:barn-eggs", "en:caged-chicken-eggs"}, "cats": {"Eggs & Egg Substitutes"}, "name": _rx(r"\b(large|medium|small|extra.large|jumbo|grade a|grade aa|dozen|cage.free|free.range|pasture|" r"brown|white|fresh|farm|organic|bio|raised|range|frais|plein air|label rouge|de poules?|calibre|frescos?|" r"camperos?|de gallina|ecologicos?|freiland|bodenhaltung|\d+) (eggs|oeufs|huevos|eier|uova|ovos)\b|" r"\b(eggs|oeufs|huevos|eier|uova) (frais|bio|de poules?|plein air|label rouge|frescos|camperos|" r"de gallina|ecologicos|freiland|bodenhaltung|fresche|free.range|cage.free|large|medium)\b|" r"^(eggs?|oeufs?|huevos?|eier|uova)$"), "not": _rx(_COMPOSITE + r"|white|blanc|clara|eiweiss|eiklar|albumen|albumin|yolk|jaune|yema|dotter|tuorlo|" r"gema|liquid|liquide|liquido|substitute|beaters|replacer|product|scrambl|frittata|muffin|patty|" r"noodle|nouille|pates|fideo|nudel|mayo|powder|poudre|dried|dehydrated|\bdry\b|scotch|pickled|" r"devil|\bnog\b|chocolate|easter|kinder|creme egg|cream egg|surprise|roll|fried rice|curry|caviar|" r"\broe\b|fish|salmon|lump|quail|caille|codorniz|wachtel|duck|canard|pato|ente|goose|\boie\b|turtle|" r"century|balut|tea egg|salted|cooked|boiled|hard|durs?\b|poached|fried|frit"), "units": ("g",), "min": 110.0, "max": 200.0}, {"id": "egg_white", "tags": {"en:eggs", "en:eggs-and-their-products", "en:egg-whites", "en:liquid-egg-whites"}, "cats": {"Eggs & Egg Substitutes"}, "name": _rx(r"egg whites?|blancs? d.oeufs?|claras? de huevo|eiklar"), "skip_above": 250.0, "not": _rx(_COMPOSITE + r"|powder|poudre|dried|dehydrated|deshidrat|\bdry\b|yolk|whole|protein|albumin|albume|" r"\bbrot\b|riegel|mayo|scrambl|burrito|muffin|frittata|delights|sausage|bacon|neige|caramel|bites"), "units": ("g", "ml"), "min": 30.0, "max": 80.0}, {"id": "nuts", "tags": {"en:nuts", "en:almonds", "en:cashew-nuts", "en:walnuts", "en:peanuts", "en:hazelnuts", "en:pistachios", "en:pecan-nuts", "en:macadamia-nuts", "en:brazil-nuts", "en:pine-nuts", "en:mixed-nuts", "en:shelled-nuts", "en:roasted-nuts", "en:salted-nuts", "en:shelled-almonds"}, "cats": {"Popcorn, Peanuts, Seeds & Related Snacks"}, "name": _rx(r"\b(almonds?|amandes?|almendras?|mandeln?|mandorle|cashews?|cajou|anacardos?|cashewkerne|" r"walnuts?|nueces|walnusskerne|walnusse|noci|peanuts?|cacahuetes?|cacahuates?|arachides?|" r"erdnusse?|erdnusskerne|hazelnuts?|noisettes?|avellanas?|haselnusse?|haselnusskerne|nocciole|" r"pistachios?|pistaches?|pistachos?|pistazien|pecans?|pacanes|macadamias?|brazil nuts?|" r"paranusse|pine nuts?|pignons?|pinoli|pinones|mixed nuts|nut mix)\b"), "not": _rx(_COMPOSITE + r"|milk|\blait|leche|latte|extract|essence|\boil\b|huile|butter|beurre|spread|cream|" r"creme|paste|pasta|puree|flour|farine|harina|\bmeal\b|powder|poudre|chestnut|chataigne|castana|" r"marron|kastanie|water chestnut|coconut|\bcoco\b|kola|nutmeg|muscade|tiger|chufa|in shell|" r"en coque|unshelled|with shell|in the shell|in.shell|cascara|caramel|praline|brittle|nougat|chocolate|" r"chocolat|cluster|noodle|\brice\b|frozen|fruit|raisin|cranberr|apricot|\bdates?\b|\bfigs?\b|mango|" r"pineapple|papaya|banana|apple|boiled|green peanuts|candied|glazed|frosted|sugar|honey|coated|" r"covered|yogurt|trail|marzipan|massepain|amaretti|dragee|sliced almonds and|slivered almonds and|jordan|" r"preserve|jam|berry|mozza|cubes|blend|with"), "units": ("g",), "min": 250.0}, {"id": "flour", "tags": {"en:flours", "en:wheat-flours", "en:cereal-flours", "en:legume-flours", "en:corn-flours", "en:rice-flours", "en:rye-flours", "en:spelt-flours", "en:buckwheat-flours"}, "cats": {"Flours & Corn Meal"}, "name": _rx(r"\b(flour|farine|harina|mehl|farina|farinha|bloem|maka|jauho|cornmeal|corn meal|semolina|" r"semoule|semola|griess|masa harina|atta|maida|besan)\b"), "not": _rx(_COMPOSITE + r"|melange|backmischung|pancake|waffle|brownie|batter|breading|coating|tempura|" r"egg replacer|\bflan|curry|pretzel|noodle|tortilla|\bpain\b|\bpan\b|gnocchi|dough|gravy|roux|" r"thickener|yeast|xanthan|\bgum\b|baking powder|levure|porridge|cream of|fecule|maizena|" r"cornstarch|corn starch|arrowroot|tapioca pearl|cricket|konjac|glucomannan|psyllium|fiber|fibre|" r"\bbran\b|son de|salvado|kleie|resistant|cassava|manioc|yuca|banana|plantain|potato|pomme de terre|chia|" r"gluten.free|sans gluten|blend"), "units": ("g",), "min": 250.0}, {"id": "sugar", "tags": {"en:sugars", "en:white-sugars", "en:cane-sugars", "en:brown-sugars", "en:granulated-sugars", "en:powdered-sugars", "en:caster-sugars", "en:beet-sugars", "en:cane-sugar"}, "cats": {"Granulated, Brown & Powdered Sugar"}, "name": _rx(r"\b(sugar|sucre|azucar|zucker|zucchero|suiker|cukier|sokeri|acucar|socker|cassonade|vergeoise|" r"panela|jaggery|piloncillo|muscovado|demerara|turbinado|rapadura)\b"), "not": _rx(_COMPOSITE + r"|free|\bsans\b|\bsin\b|ohne|senza|zero|no sugar|low sugar|reduced|less sugar|light|" r"\blite\b|leger|substitut|sweetener|edulcorant|stevia|truvia|splenda|sucralose|aspartame|monk|" r"erythrit|xylit|allulose|tagatose|sucrazit|blend|syrup|sirop|sirope|jarabe|sciroppo|\bgum\b|" r"chewing|wafer|snap peas|sugar snap|\bcorn\b|soda|cola|lemonade|\btea\b|coffee|coated|glazed|" r"frosted|maple|honey|molasses|melasse|agave|sugar plum|sugar baby|icing|frosting|fondant|decor|" r"sprinkle|confetti|nonpareil|cane juice|\bcane\b(?! sugar)|diabetic|keto|alternative|\balt\b|liquid|" r"simple|browning|colou?ring|infused|date|palm|rock|bouleau|birch|birken|added|\d+ ?%|without|squash|cordial"), "units": ("g",), "min": 280.0}, {"id": "honey", "tags": {"en:honeys", "en:flower-honeys", "en:raw-honeys", "en:creamed-honeys", "en:liquid-honeys", "en:sweet-spreads", "en:bee-products"}, "cats": {"Honey"}, "name": _rx(r"\b(honey|miel|honig|miele|mel|med|honing|miod|hunaja|bal)\b"), "not": _rx(_COMPOSITE + r"|mustard|moutarde|senf|bbq|barbecue|roast|\bnut|peanut|almond|\boat|candy|drops?|" r"lozenge|pastille|cough|throat|lemon|\btea\b|\bthe\b|infusion|tisane|greek|\bham\b|chicken|" r"turkey|graham|wafer|butter|beurre|spread|cream|creme|substitute|imitation|sugar free|" r"sin azucar|vegan|agave|maple|syrup|sirop|sirope|jarabe|melon|honeydew|crisp|wheat|propolis|" r"royal jelly|jelly|pollen|vinegar|vinaigre|liqueur|\bmead\b|hydromel|wine|beer|\bale\b|\boil\b|" r"huile|soap|essential|infused|garlic|chili|chile|hot honey|cinnamon|ginger|bourbon|whiskey|" r"rum\b|comb|panal|rayon"), "units": ("g",), "min": 200.0}, {"id": "popcorn", "tags": {"en:popcorn", "en:salted-popcorn", "en:sweet-popcorn", "en:microwave-popcorn", "en:caramel-popcorn"}, "cats": {"Popcorn, Peanuts, Seeds & Related Snacks", "Popcorn (Shelf Stable)"}, "name": _rx(r"\b(popcorn|pop corn|pop-corn|palomitas|pipoca|popcorns)\b"), "not": _rx(r"\btea\b|\btee\b|\bthe\b|seasoning|\bsalt\b|\boil\b|topping|jelly bean|candy|ice cream|cereal|" r"chicken|shrimp|cauliflower|flavou?r(ed)? (?:oil|salt|spray)|chocolate.*popcorn|popcorn.*chocolate|" r"\bdog\b|\bpet\b"), "units": ("g",), "min": 250.0}, {"id": "ice_cream", "tags": {"en:ice-creams", "en:ice-cream-tubs", "en:ice-creams-and-sorbets", "en:frozen-desserts", "en:ice-cream-bars", "en:ice-cream-cones", "en:frozen-yogurts"}, "cats": {"Ice Cream & Frozen Yogurt", "Other Frozen Desserts"}, "name": _rx(r"ice ?cream|icecream|gelato|helado|eiscreme|speiseeis|frozen custard|\bglace\b|creme glacee"), "not": _rx(r"ice cube|glacon|ice ball|crushed ice|ice pop|icepop|ice lolly|lolly|popsicle|paleta|ghiacciol|" r"\bpolo\b|sorbet|sorbetto|sorbete|granita|granizado|italian ice|water ice|shaved ice|snow cone|" r"slush|fruit bar|juice bar|fruit pop|pickle|tomato|coffee|cafe|latte|glace royale|glacage|glaze|" r"icing|\bcones?\b(?!.*cream)|wafer cup|\bmix\b|powder|poudre|preparation|syrup|sauce|topping|" r"sprinkle|\bdog\b|\bpet\b|zero|0 ?kcal|keto|sugar free|no sugar|light|\blite\b|protein|" r"low ?cal|calorie|halo top|enlightened|arctic zero|nick.s|yasso|skinny|frozen yog|wink|\bthe\b|\btea\b|" r"iced|a l.eau|\beau\b"), "units": ("g", "ml"), "min": 50.0}, {"id": "pizza", "tags": {"en:pizzas", "en:frozen-pizzas", "en:pizzas-pies-and-quiches", "en:pizzas-and-pies"}, "cats": {"Pizza"}, "name": _rx(r"\b(pizzas?|pizze|calzones?|flammkuchen|tarte flambee|stromboli)\b"), "not": _rx(r"\bsauce|salsa|dough|pate a pizza|\bmasa\b|\bbase\b|crust(?! pizza)|\bkit\b|seasoning|spice|" r"topping|cheese blend|shredded|herbs|oregano|\brolls?\b|bites|snacks?|crackers?|chips|crisps|" r"pretzel|goldfish|pringles|\bsoup|pasta|lasagn|cauliflower|veggie crust|zucchini|keto|" r"low carb|protein|plant based|pocket"), "units": ("g",), "min": 100.0}, {"id": "breakfast_cereal", "tags": {"en:breakfast-cereals", "en:extruded-cereals", "en:puffed-cereals", "en:mueslis", "en:granolas", "en:flakes", "en:rolled-oats", "en:oat-flakes", "en:chocolate-cereals"}, "cats": {"Cereal"}, "name": _rx(r"cereal|cereales|flakes|muesli|musli|granola|\boats?\b|avena|avoine|hafer|porridge|crunch|" r"\bpuffs\b|\bpops\b|loops|cheerios|chex|krispies|special k|weetabix|shreddies|shredded wheat|" r"grape.nuts|fitness|chocapic|crispies|clusters|smacks|frosties|copos|fiocchi|flocons|knusper|" r"bran|\bcrisp\b|wheaties|kix|trix|froot|corn flakes|cornflakes"), "not": _rx(r"prepared|cooked|ready to eat porridge|porridge pot|\bcup\b|\bdrink|latte|bebida|\bmilk\b|" r"\blait\b|leche|yogurt|yoghurt|yaourt|overnight|pudding|bircher|soaked|grits|cream of|farina|" r"hot cereal|congee|baby|infant|\bbebe\b|\btea\b|\bbars?\b|liquid|smoothie|shake|\bmix\b|cookie|" r"\bbread|\bdog\b|\bpet\b|fibre one|fiber one|\bpsyllium|oat bran|wheat bran|\bbran\b|pouch|squeeze|" r"creamy oats|fruit &|fruit and|oats &|puree|toddler|kids|refrigerated|ready|\bcups\b"), "units": ("g",), "min": 150.0}, {"id": "crisps", "tags": {"en:crisps", "en:potato-crisps", "en:tortilla-chips", "en:corn-chips", "en:potato-crisps-in-sunflower-oil", "en:salted-potato-crisps", "en:flavoured-potato-crisps", "en:chips-and-fries"}, "cats": {"Chips, Pretzels & Snacks"}, "name": _rx(r"\b(chips|crisps|tortilla chips|nachos|totopos|patatas fritas|papas fritas|chipsy|kartoffelchips|" r"pringles|doritos|ruffles|lay.?s|kettle)\b"), "not": _rx(r"\bfries\b|frites|french fried|frozen|oven|wedges|\bhash|chocolate|\bchoc\b|baking|morsels|cookie|" r"pickle|vegetable|veggie|apple|banana|fruit|coconut|kale|seaweed|nori|cauliflower|broccoli|carrot|" r"\bbeet|\bpeas?\b|salsa|\bdip\b|sauce|guacamole|seasoning|\bwood|smoke|\btea\b|light|\blite\b|" r"reduced|baked|popped|\bpuffs?\b|crispbread|rice cake|crisp rice|crispy rice|rice crisp|cereal|" r"\bbars?\b|chicken|fish|onion|jalapeno slices|sliced jalapeno|nachos sliced|cheese sauce|bean dip|" r"protein|keto|air fried|\bfat free|lentil|chickpea|cassava|plantain|mushroom|jackfruit|tempeh|" r"pork|chicharron|shrimp|prawn|quinoa|millet|flax|hummus|cheese chips"), "units": ("g",), "min": 250.0}, {"id": "cookies", "tags": {"en:biscuits", "en:cookies", "en:sweet-biscuits", "en:chocolate-biscuits", "en:shortbread-cookies", "en:biscuits-and-cakes", "en:dry-biscuits"}, "cats": {"Cookies & Biscuits", "Biscuits/Cookies"}, "name": _rx(r"\b(cookies?|biscuits?|galletas?|keks|kekse|biscotti|sables?|shortbread|oreos?|digestives?|" r"wafers?|gaufrettes?|speculoos|petit beurre|bourbon|hobnobs?|macarons?|macaroons?)\b"), "not": _rx(r"dough|\bpate\b|\bmasa\b|\bmix\b|preparation|\bdog\b|\bpet\b|\bcat\b|chien|\bchat\b|treat|" r"ice cream|cheesecake|pudding|yogurt|cereal|milk|\bdrink|gravy|refrigerated|frozen|buttermilk|" r"baking powder|southern|flaky|grands|scone|sauce|spread|cream|creme|butter$|light|\blite\b|" r"reduced|sugar free|keto|protein|crumbs|crust|teething|baby|infant|rusk|biscotte|crackers?|cakes"), "units": ("g",), "min": 280.0}, {"id": "crackers", "tags": {"en:crackers", "en:salted-crackers", "en:crispbreads", "en:rice-cakes", "en:breadsticks", "en:extruded-flatbreads", "en:puffed-rice-cakes"}, "cats": {"Crackers & Biscotti"}, "name": _rx(r"\b(crackers?|crispbreads?|knackebrod|rice cakes?|galettes de riz|matzo|matzah|grissini|" r"breadsticks?|biscottes?|zwieback|crostini|melba|tostadas?|saltines?|water biscuits?)\b"), "not": _rx(r"\bkit\b|salad|salsa|\bdip\b|cheese sauce|nacho|soup|matzo ball|topper|nori|sushi|lunch|combo|" r"pack with|& cheese|and cheese|with cheese|tuna|chicken|\bdog\b|\bpet\b|crab|lobster|" r"cracker ?jack|firecracker|nutcracker|crackling|crack seed|crust|crumbs|meal|\bmix\b|light|" r"\blite\b|thin|protein|baby|infant|teething|fiber|fibre|bran|\bgg\b|crostini"), "units": ("g",), "min": 200.0}, {"id": "bread", "tags": {"en:breads", "en:sliced-breads", "en:white-breads", "en:wholemeal-breads", "en:special-breads", "en:baguettes", "en:sandwich-breads", "en:rye-breads", "en:multigrain-breads", "en:bagel-breads"}, "cats": {"Breads & Buns"}, "name": _rx(r"\b(bread|pain|brot|brood|pane|chleb|leipa|limpa|baguettes?|demi.baguettes?|loaf|loaves|toast|" r"bagels?|ciabatta|focaccia|brioche|sourdough|levain|wholemeal|batard|ficelle|flute|boule|miche|" r"campagne|fougasse|pumpernickel|vollkornbrot|mischbrot|graubrot|roggenbrot|broodje|panini|" r"buns?|rolls?|naan|pita|pitta)\b"), "not": _rx(r"crumbs?|chapelure|panko|dough|\bmix\b|preparation|\bkit\b|pudding|sauce|spread|\bdip\b|stuffing|" r"\braw\b|uncooked|pizza|soup|salad|french toast|shortbread|gingerbread|sweetbread|ris de veau|" r"cornbread|monkey bread|bread and butter|pickle|keto|low carb|carb|light|\blite\b|diet|\bthin|" r"delightful|protein|fiber|fibre|zero|cloud|cauliflower|\begg ?rolls?|spring rolls?|cinnamon rolls?|" r"sushi|cabbage|lobster|sausage rolls?|hot dog|burger|sandwich|wrap|\bham\b|fruit roll|toaster|" r"pastr|strudel|danish|croissant|donut|doughnut|cake|muffin|scone|bun.*(?:cream|filled)|bao|" r"steamed|frozen|gluten.free|sans gluten|senza glutine|glutenfrei|sin gluten|injera|ethiopian|teff|" r"calorie|\d+ ?cal\b"), "units": ("g",), "min": 140.0}, {"id": "maple_syrup", "tags": {"en:maple-syrups", "en:syrups", "en:sweeteners", "en:simple-syrups"}, "cats": {"Syrups & Molasses"}, "name": _rx(r"\bmaple syrup|sirop d.erable|jarabe de arce|ahornsirup|sciroppo d.acero|pure maple"), "not": _rx(r"sugar free|sans sucre|light|\blite\b|\blow\b|reduced|zero|diet|calorie|keto|flavou?r|imitation|" r"artificial|pancake syrup|table syrup|butter|cream|creme|candy|bonbon|cookie|bacon|sausage|\bham\b|" r"chicken|turkey|salmon|glaze|dressing|vinaigrette|\bbbq\b|sauce|mustard|oatmeal|granola|cereal|" r"\bnuts?\b|pecan|walnut|almond|\bbars?\b|water|\bsap\b|\btea\b|coffee|vinegar|chips?|popcorn|" r"cotton|blend|infused|bourbon|whisk"), "units": ("g", "ml"), "min": 200.0, "min_ml": 250.0}, ] # Name-only mode, for Open Food Facts records that have no category at all (mostly the USDA import; country tags # are unreliable there, "Organic Peanut Butter" by HEB is tagged Ireland). The patterns are English only. The name must END with the food word, so the food is the head noun ("Extra Sharp White Cheddar", # "Vitamin D Whole Milk", "Organic Peanut Butter"), not a flavour or an ingredient ("Broccoli Cheddar Rice"). # The band's own name pattern and exclusions apply as well. _TAIL_CHEESE = r"(?: cheese)?(?: (?:block|slices?|chunks?|wedge|shreds|shredded|loaf|bar|sticks?))?$" NAME_ONLY_TAIL = { "hard_cheese": re.compile(r"\b(cheddar|gouda|emmental|emmentaler|gruyere|parmesan|parmigiano reggiano|edam|" r"manchego|pecorino romano|romano|asiago|jarlsberg|provolone|colby|colby jack|" r"monterey jack|pepper jack|swiss|havarti|fontina)" + _TAIL_CHEESE), "cheese": re.compile(r"\b(cheese|mozzarella|feta|brie|camembert|halloumi|ricotta|paneer|burrata)" r"(?: (?:block|slices?|chunks?|wedge|crumbles|shreds|shredded|log|ball|sticks?))?$"), "butter": re.compile(r"\bbutter(?: (?:sticks?|quarters|block|tub))?$"), "oil": re.compile(r"\boil$"), "nut_butter": re.compile(r"\b(peanut|almond|cashew|nut|sunflower seed|sunflower|hazelnut|pecan|walnut|" r"pistachio|macadamia) butter(?: (?:creamy|crunchy|smooth|chunky|natural))?$|\btahini$"), "chocolate": re.compile(r"\bchocolate(?: (?:bar|bars|block|squares))?$"), "dairy_milk": re.compile(r"\bmilk$"), "egg": re.compile(r"\beggs$"), "egg_white": re.compile(r"\begg whites$"), "nuts": re.compile(r"\b(almonds|cashews|walnuts|peanuts|pecans|pecan halves|pistachios|macadamias|" r"macadamia nuts|hazelnuts|brazil nuts|pine nuts|mixed nuts|walnut halves)$"), "flour": re.compile(r"\bflour$"), "sugar": re.compile(r"\bsugar$"), "honey": re.compile(r"\bhoney$"), "popcorn": re.compile(r"\bpopcorn$"), "ice_cream": re.compile(r"(?:^|, )ice cream(?:,|$)|\bice cream$|\bgelato$"), "maple_syrup": re.compile(r"maple syrup$"), } NAME_ONLY_OK = set(NAME_ONLY_TAIL) # In name-only mode a connector means the food word is a side or a flavour: "Creamed Spinach With Asiago Cheese", # "Brownies arachides et peanuts", "Twix Creamy Peanut Butter", "Almond Milk 0% sugar". NAME_ONLY_CONNECTOR = re.compile(r"\b(with|and|w/|in|on|mit|und|et|avec|au|aux|a la|a l|de|des|du|con|y|filled|" r"stuffed|topped|coated|covered|flavou?r(ed)?|style|sauce|no|not|zero|free|" r"added|without|sans|sin|ohne)\b|&|\+|\d+ ?%|" r"\b(twix|m&m|snickers|reese|butterfinger|kit ?kat|swiss miss|lucky charms|" r"nesquik|ovaltine|cappuccino|believe)") # OFF tags that mean "not this band" even when another tag of the band is present. NOT_TAGS = {"oil": {"en:margarines", "en:spreadable-fats", "en:butters", "en:dairy-spreads", "en:light-margarines"}, "sugar": {"en:xylitol", "en:sugar-substitutes", "en:sweeteners", "en:polyols", "en:erythritol", "en:tabletop-sweeteners", "en:table-top-sweeteners"}, "butter": {"en:margarines", "en:light-margarines", "en:light-butters", "en:unsalted-light-butter", "en:salted-light-butter"}} def check(name, kcal, unit, off_tags=None, usda_category=None, brand="", name_only=False): """(band id, min, max) when kcal per 100 g/ml is outside a band that applies to this product, else None. The band's name pattern is matched on the name; its exclusions on the name and the brand. name_only=True (an OFF record without any category, from North America) uses the NAME_ONLY_TAIL head-noun patterns instead of tags.""" if kcal is None: return None lname = None for b in BANDS: if unit not in b["units"]: continue if name_only: if b["id"] not in NAME_ONLY_OK: continue if lname is None: lname = fold(name).strip(" .") lfull = lname + " | " + fold(brand) head = lname.split(",")[0].strip() # the food must be the head noun of the name's first part ("Fudge Bars, Chocolate" is not chocolate); # USDA-style "Brand, Ice Cream, Flavor" names are matched whole for ice cream if not NAME_ONLY_TAIL[b["id"]].search(lname if b["id"] == "ice_cream" else head): continue if NAME_ONLY_CONNECTOR.search(head if b["id"] != "ice_cream" else lname.replace(",", " ")): continue elif off_tags is not None: if not (b["tags"] & off_tags) or (off_tags & NOT_TAGS.get(b["id"], set())): continue elif usda_category is not None: if usda_category not in b["cats"]: continue else: continue if lname is None: lname = fold(name) lfull = lname + " | " + fold(brand) if not b["name"].search(lname) or b["not"].search(lfull): continue lo = b.get("min_ml", b.get("min")) if unit == "ml" else b.get("min") hi = b.get("max") if b.get("skip_above") is not None and kcal >= b["skip_above"]: continue # e.g. > 250 kcal for "milk" is a powder or condensed milk, not judged here if (lo is not None and kcal < lo) or (hi is not None and kcal > hi): return b["id"], lo, hi return None if __name__ == "__main__": # The audit's examples (category, name, kcal, unit) and some values that must pass. bad = [({"en:cheeses", "en:cheddar-cheese"}, "Simple Truth Sharp White Cheddar", 110, "g"), ({"en:cheeses", "en:hard-cheeses"}, "Kerrygold Extra Sharp Cheddar", 120, "g"), ({"en:peanut-butters"}, "Crunchy Peanut Butter", 126, "g"), ({"en:milks", "en:whole-milks"}, "Vitamin D Whole Milk", 150, "ml"), ({"en:butters"}, "Butter", 90, "g"), ({"en:dark-chocolates", "en:chocolates"}, "Dark chocolate bar", 170, "g"), ({"en:eggs", "en:chicken-eggs"}, "Large brown eggs", 60, "g"), ({"en:olive-oils"}, "Extra Virgin Olive Oil", 80, "ml"), ({"en:maple-syrups"}, "Pure Maple Syrup", 44, "ml"), ({"en:hazelnut-spreads"}, "Nutella", 100, "g"), ({"en:popcorn"}, "Light Butter Flavored Popcorn", 4, "g")] good = [({"en:cheeses", "en:cheddar-cheese"}, "Mature Cheddar", 410, "g"), ({"en:cheeses"}, "Cottage cheese 0%", 60, "g"), ({"en:milks", "en:skimmed-milks"}, "Skimmed milk", 34, "ml"), ({"en:vegetable-oils"}, "Olive oil cooking spray", 0, "g"), ({"en:milks"}, "Eau de source", 0, "ml"), ({"en:nuts"}, "Water chestnuts", 20, "g"), ({"en:ice-creams"}, "Ice cubes", 0, "g"), ({"en:pizzas"}, "Pizza sauce", 35, "g"), ({"en:sugars"}, "Truvia sweetener", 0, "g")] for tags, n, k, u in bad: assert check(n, k, u, off_tags=tags), (n, k) for tags, n, k, u in good: assert not check(n, k, u, off_tags=tags), (n, k, check(n, k, u, off_tags=tags)) assert check("Sharp White Cheddar Cheese", 110, "g", name_only=True) # no category: name alone assert check("Vitamin D Whole Milk", 150, "ml", name_only=True) assert check("Sweet Action, Ice Cream, Vegan Colorado Peach", 10, "g", name_only=True) assert check("Pasture Raised Eggs", 60, "g", name_only=True) assert not check("Cheddar Cheese Crackers", 110, "g", name_only=True) assert not check("Broccoli Cheddar Rice", 173, "g", name_only=True) assert not check("Fudge Bars, Chocolate", 130, "g", name_only=True) assert not check("Creamed Spinach With Asiago Cheese", 91, "g", name_only=True) assert not check("Almond Milk 0% sugar", 14, "ml", name_only=True) assert not check("Twix Creamy Peanut Butter", 250, "g", name_only=True) assert not check("Egg Whites", 374, "g", name_only=True) assert check("Egg whites", 25, "g", name_only=True) assert not check("Fat-free skim milk", 348, "ml", off_tags={"en:milks", "en:skimmed-milks"}) assert not check("Hummus tahini", 115, "g", off_tags={"en:spreads"}) assert not check("Whole wheat bread", 60, "g", name_only=True) # bread is not name-only assert not check("Oliviina with Olive Oil", 600, "g", off_tags={"en:fats", "en:margarines"}) assert check("Sharp Cheddar Cheese Slices", 36, "g", usda_category="Cheese") assert check("Ice Cream, Vegan Mint Chip", 13, "g", usda_category="Other Frozen Desserts") assert check("ACT II Butter Lovers Popcorn", 4, "g", usda_category="Popcorn, Peanuts, Seeds & Related Snacks") assert check("14\" Supreme Pizza", 21, "g", usda_category="Pizza") print("plausibility self-test ok")