#!/usr/bin/env python3 """Normalize ANSES CIQUAL 2025 (French food composition table, English names) -> FoodIndex/normalized/extra_C.jsonl Source: Anses (2025). Table de composition nutritionnelle des aliments Ciqual 2025. https://doi.org/10.57745/RDMHWY (Recherche Data Gouv, released 2025-11-19; files alim/alim_grp/compo_2025_11_03.xml) Licence: Etalab Open Licence 2.0 (Licence Ouverte 2.0), as stated on the dataset record. Rules: * Name = alim_nom_eng (official English name), whitespace collapsed. Names over 120 characters first lose parenthesised text, then are cut at the last comma before 120 characters. * Energy = constituent 328 "Energy, Regulation EU No 1169/2011 (kcal/100g)" (the label-style energy); per 100 g for every food, beverages included -> unit "g". * "-" (missing) -> dropped. "traces" -> 0.0 (two mineral waters). * If two foods end up with the same English name: identical energy -> keep the lower code; different energy -> both dropped (ambiguous for the user). * Category = CIQUAL sub-group English name. No portions in CIQUAL -> []. Usage: python3 -I normalize_extra_C.py """ import json import os import re import sys import unicodedata import xml.etree.ElementTree as ET from collections import Counter, defaultdict SRC = "C" DATASET_DATE = "2025-11-19" ENERGY_CONST = "328" def iter_records(path: str, tag: str): for _, el in ET.iterparse(path): if el.tag == tag: yield {c.tag: (c.text or "").strip() for c in el} el.clear() def balance_parens(s: str) -> str: """Removes unmatched '(' or ')' (a few CIQUAL English names have typos like 'garlic and herbs)').""" out, stack = list(s), [] for i, ch in enumerate(s): if ch == "(": stack.append(i) elif ch == ")": if stack: stack.pop() else: out[i] = "" for i in stack: out[i] = "" return re.sub(r"\s+", " ", "".join(out)).strip() def clean_name(s: str) -> str: s = unicodedata.normalize("NFC", s or "") s = re.sub(r"\s+", " ", s).strip() s = re.sub(r",(?=\S)", ", ", s) # "mature,flesh" -> "mature, flesh" s = re.sub(r",\s*\)", ")", s) # "(pulp or mixed fruits,)" -> "(pulp or mixed fruits)" s = balance_parens(s) if len(s) > 120: s2 = re.sub(r"\s*\([^()]*\)", "", s) s2 = re.sub(r"\s+", " ", s2).strip() s = s2 if len(s) > 120: cut = s[:120] if ", " in cut: cut = cut.rsplit(", ", 1)[0] s = cut.rstrip(",; ") return s def cap_first(s: str) -> str: return s[:1].upper() + s[1:] if s else s def main() -> None: d, out_path = sys.argv[1], sys.argv[2] files = {f.split("_2025")[0]: os.path.join(d, f) for f in os.listdir(d) if f.endswith(".xml")} groups = {} for g in iter_records(files["alim_grp"], "ALIM_GRP"): groups.setdefault(g["alim_ssgrp_code"], g["alim_ssgrp_nom_eng"] if g["alim_ssgrp_nom_eng"] not in ("", "-") else g["alim_grp_nom_eng"]) groups.setdefault(g["alim_grp_code"], g["alim_grp_nom_eng"]) foods = {a["alim_code"]: a for a in iter_records(files["alim"], "ALIM")} energy = {} for c in iter_records(files["compo"], "COMPO"): if c["const_code"] == ENERGY_CONST: energy[c["alim_code"]] = c["teneur"] drops = Counter() recs = [] for code, a in foods.items(): t = energy.get(code) if t is None or t in ("", "-"): drops["energy_missing"] += 1 continue if t == "traces": kcal = 0.0 drops["(kept) energy_traces_as_0"] += 1 else: try: kcal = float(t.replace(",", ".")) except ValueError: drops["energy_not_numeric"] += 1 continue if not (0.0 <= kcal <= 900.0): drops["energy_out_of_range"] += 1 continue name = clean_name(a.get("alim_nom_eng", "")) if len(name) < 2: drops["no_english_name"] += 1 continue if len(a.get("alim_nom_eng", "")) > 120: drops["(kept) long_name_shortened"] += 1 ss = a.get("alim_ssgrp_code", "") cat = groups.get(ss) if ss and ss != "0000" else None cat = cat or groups.get(a.get("alim_grp_code", ""), "") recs.append({ "id": f"{SRC}:{code}", "name": name, "brand": "", "kcal": round(kcal, 1), "unit": "g", "portions": [], "barcode": "", "src": SRC, "generic": True, "pop": None, "date": DATASET_DATE, "countries": ["fr"], "lang": "en", "category": cap_first(cat), }) by_name = defaultdict(list) for r in recs: by_name[r["name"].lower()].append(r) out = [] for n, rs in by_name.items(): if len(rs) == 1: out.append(rs[0]) continue if len({r["kcal"] for r in rs}) == 1: rs.sort(key=lambda r: int(r["id"].split(":")[1])) out.append(rs[0]) drops["duplicate_name_same_energy"] += len(rs) - 1 else: drops["duplicate_name_different_energy(ambiguous)"] += len(rs) out.sort(key=lambda r: int(r["id"].split(":")[1])) with open(out_path, "w", encoding="utf-8") as f: for r in out: f.write(json.dumps(r, ensure_ascii=False, separators=(",", ":")) + "\n") print("foods:", len(foods), "energy rows:", len(energy)) print("written:", len(out), "->", out_path) print("dropped/notes:", dict(drops)) if __name__ == "__main__": main()