#!/usr/bin/env python3
"""Normalize ANSES CIQUAL 2025 (French food composition table, English names) ->
FoodIndex/normalized/extra_C.jsonl
Source: Anses (2025). Table de composition nutritionnelle des aliments Ciqual 2025. https://doi.org/10.57745/RDMHWY
(Recherche Data Gouv, released 2025-11-19; files alim/alim_grp/compo_2025_11_03.xml)
Licence: Etalab Open Licence 2.0 (Licence Ouverte 2.0), as stated on the dataset record.
Rules:
* Name = alim_nom_eng (official English name), whitespace collapsed. Names over 120 characters first lose
parenthesised text, then are cut at the last comma before 120 characters.
* Energy = constituent 328 "Energy, Regulation EU No 1169/2011 (kcal/100g)" (the label-style energy); per 100 g
for every food, beverages included -> unit "g".
* "-" (missing) -> dropped. "traces" -> 0.0 (two mineral waters).
* If two foods end up with the same English name: identical energy -> keep the lower code; different energy ->
both dropped (ambiguous for the user).
* Category = CIQUAL sub-group English name. No portions in CIQUAL -> [].
Usage: python3 -I normalize_extra_C.py
"""
import json
import os
import re
import sys
import unicodedata
import xml.etree.ElementTree as ET
from collections import Counter, defaultdict
SRC = "C"
DATASET_DATE = "2025-11-19"
ENERGY_CONST = "328"
def iter_records(path: str, tag: str):
for _, el in ET.iterparse(path):
if el.tag == tag:
yield {c.tag: (c.text or "").strip() for c in el}
el.clear()
def balance_parens(s: str) -> str:
"""Removes unmatched '(' or ')' (a few CIQUAL English names have typos like 'garlic and herbs)')."""
out, stack = list(s), []
for i, ch in enumerate(s):
if ch == "(":
stack.append(i)
elif ch == ")":
if stack:
stack.pop()
else:
out[i] = ""
for i in stack:
out[i] = ""
return re.sub(r"\s+", " ", "".join(out)).strip()
def clean_name(s: str) -> str:
s = unicodedata.normalize("NFC", s or "")
s = re.sub(r"\s+", " ", s).strip()
s = re.sub(r",(?=\S)", ", ", s) # "mature,flesh" -> "mature, flesh"
s = re.sub(r",\s*\)", ")", s) # "(pulp or mixed fruits,)" -> "(pulp or mixed fruits)"
s = balance_parens(s)
if len(s) > 120:
s2 = re.sub(r"\s*\([^()]*\)", "", s)
s2 = re.sub(r"\s+", " ", s2).strip()
s = s2
if len(s) > 120:
cut = s[:120]
if ", " in cut:
cut = cut.rsplit(", ", 1)[0]
s = cut.rstrip(",; ")
return s
def cap_first(s: str) -> str:
return s[:1].upper() + s[1:] if s else s
def main() -> None:
d, out_path = sys.argv[1], sys.argv[2]
files = {f.split("_2025")[0]: os.path.join(d, f) for f in os.listdir(d) if f.endswith(".xml")}
groups = {}
for g in iter_records(files["alim_grp"], "ALIM_GRP"):
groups.setdefault(g["alim_ssgrp_code"], g["alim_ssgrp_nom_eng"] if g["alim_ssgrp_nom_eng"] not in ("", "-") else g["alim_grp_nom_eng"])
groups.setdefault(g["alim_grp_code"], g["alim_grp_nom_eng"])
foods = {a["alim_code"]: a for a in iter_records(files["alim"], "ALIM")}
energy = {}
for c in iter_records(files["compo"], "COMPO"):
if c["const_code"] == ENERGY_CONST:
energy[c["alim_code"]] = c["teneur"]
drops = Counter()
recs = []
for code, a in foods.items():
t = energy.get(code)
if t is None or t in ("", "-"):
drops["energy_missing"] += 1
continue
if t == "traces":
kcal = 0.0
drops["(kept) energy_traces_as_0"] += 1
else:
try:
kcal = float(t.replace(",", "."))
except ValueError:
drops["energy_not_numeric"] += 1
continue
if not (0.0 <= kcal <= 900.0):
drops["energy_out_of_range"] += 1
continue
name = clean_name(a.get("alim_nom_eng", ""))
if len(name) < 2:
drops["no_english_name"] += 1
continue
if len(a.get("alim_nom_eng", "")) > 120:
drops["(kept) long_name_shortened"] += 1
ss = a.get("alim_ssgrp_code", "")
cat = groups.get(ss) if ss and ss != "0000" else None
cat = cat or groups.get(a.get("alim_grp_code", ""), "")
recs.append({
"id": f"{SRC}:{code}",
"name": name,
"brand": "",
"kcal": round(kcal, 1),
"unit": "g",
"portions": [],
"barcode": "",
"src": SRC,
"generic": True,
"pop": None,
"date": DATASET_DATE,
"countries": ["fr"],
"lang": "en",
"category": cap_first(cat),
})
by_name = defaultdict(list)
for r in recs:
by_name[r["name"].lower()].append(r)
out = []
for n, rs in by_name.items():
if len(rs) == 1:
out.append(rs[0])
continue
if len({r["kcal"] for r in rs}) == 1:
rs.sort(key=lambda r: int(r["id"].split(":")[1]))
out.append(rs[0])
drops["duplicate_name_same_energy"] += len(rs) - 1
else:
drops["duplicate_name_different_energy(ambiguous)"] += len(rs)
out.sort(key=lambda r: int(r["id"].split(":")[1]))
with open(out_path, "w", encoding="utf-8") as f:
for r in out:
f.write(json.dumps(r, ensure_ascii=False, separators=(",", ":")) + "\n")
print("foods:", len(foods), "energy rows:", len(energy))
print("written:", len(out), "->", out_path)
print("dropped/notes:", dict(drops))
if __name__ == "__main__":
main()