"""Exploration stats for USDA branded_food.csv / food.csv (stdlib only).""" import csv, sys, collections csv.field_size_limit(1 << 30) BR = sys.argv[1] c_unit = collections.Counter(); c_disc = 0; c_country = collections.Counter(); c_src = collections.Counter() c_hh = collections.Counter(); gt = collections.Counter(); n = 0; c_prep = collections.Counter(); c_trade = collections.Counter() glen = collections.Counter(); ex_disc = [] for r in csv.DictReader(open(BR + "/branded_food.csv", encoding="utf-8", newline="")): n += 1 c_unit[r["serving_size_unit"]] += 1 if r["discontinued_date"].strip(): c_disc += 1 if len(ex_disc) < 5: ex_disc.append((r["fdc_id"], r["discontinued_date"], r["available_date"], r["gtin_upc"])) c_country[r["market_country"]] += 1 c_src[r["data_source"]] += 1 c_prep[r["preparation_state_code"]] += 1 c_trade[r["trade_channel"]] += 1 g = "".join(ch for ch in r["gtin_upc"] if ch.isdigit()) glen[len(g)] += 1 gt[g] += 1 if n % 50 == 0: c_hh[r["household_serving_fulltext"]] += 1 print("rows", n) print("units", c_unit.most_common(30)) print("discontinued", c_disc, ex_disc) print("country", c_country.most_common(20)) print("source", c_src.most_common()) print("prep", c_prep.most_common(20)) print("trade", c_trade.most_common(10)) print("gtin digit lengths", sorted(glen.items())) print("distinct gtin", len(gt), "dups", sum(1 for v in gt.values() if v > 1), "max", gt.most_common(3)) print("household sample", c_hh.most_common(60)) dt = collections.Counter(); dates = collections.Counter() for r in csv.DictReader(open(BR + "/food.csv", encoding="utf-8", newline="")): dt[r["data_type"]] += 1; dates[r["publication_date"][:4]] += 1 print("food data_type", dt, "pubyears", sorted(dates.items()))