"""Frequency of household serving unit words in branded_food.csv (stdlib only).""" import csv, sys, collections, re csv.field_size_limit(1 << 30) c = collections.Counter(); first = collections.Counter(); n = 0 for r in csv.DictReader(open(sys.argv[1] + "/branded_food.csv", encoding="utf-8", newline="")): t = r["household_serving_fulltext"].strip() m = re.match(r"^\s*([\d./\s]+)\s*(.*)$", t) if m: rest = re.split(r"[(,]", m.group(2))[0].strip() c[rest.upper()] += 1 else: first[t[:30].upper()] += 1 print("with number:", sum(c.values()), "without:", sum(first.values())) print(c.most_common(250)) print(first.most_common(60))