"""More exploration: names, brands, odd units (stdlib only).""" import csv, sys, collections, random csv.field_size_limit(1 << 30) BR = sys.argv[1] random.seed(1) desc = {} for r in csv.DictReader(open(BR + "/food.csv", encoding="utf-8", newline="")): desc[r["fdc_id"]] = r["description"] caps = 0; mixed = 0; n = 0 samples = []; odd = collections.defaultdict(list); brands = [] for r in csv.DictReader(open(BR + "/branded_food.csv", encoding="utf-8", newline="")): n += 1 d = desc.get(r["fdc_id"], "") letters = [c for c in d if c.isalpha()] if letters and all(c.isupper() for c in letters): caps += 1 else: mixed += 1 if random.random() < 0.00004: samples.append((d, r["brand_name"], r["brand_owner"], r["serving_size"], r["serving_size_unit"], r["household_serving_fulltext"], r["branded_food_category"])) u = r["serving_size_unit"] if u not in ("g", "ml", "GRM", "MLT") and len(odd[u]) < 6: odd[u].append((r["fdc_id"], d, r["serving_size"], r["household_serving_fulltext"], r["branded_food_category"], r["data_source"])) print("caps", caps, "mixed", mixed) for s in samples: print(s) for u, L in odd.items(): print("UNIT", repr(u)) for x in L: print(" ", x)