"""Category / zero-kcal / checksum statistics for the OFF export (analysis only). Usage: python3 -I off_inspect_cats.py """ import gzip, sys, collections, re, json def gs1_ok(d): s = 0 for i, ch in enumerate(reversed(d[:-1])): s += int(ch) * (3 if i % 2 == 0 else 1) return (10 - s % 10) % 10 == int(d[-1]) C = collections.Counter cats = C(); zero_cats = C(); zero_names = []; pnns1 = C(); pnns2 = C(); chk = C(); lead = C() volcats = C() VOL = re.compile(r'\d\s*(ml|cl|dl|l|litre|liter|litres|liters|ltr|fl\.?\s*oz)\b', re.I) with gzip.open(sys.argv[1], 'rb') as f: header = f.readline().decode('utf-8').rstrip('\n').split('\t') H = {c: i for i, c in enumerate(header)} for raw in f: p = raw.decode('utf-8', 'replace').rstrip('\n').split('\t') if len(p) != len(header): continue tags = [t for t in p[H['categories_tags']].split(',') if t] for t in tags: cats[t] += 1 pnns1[p[H['pnns_groups_1']]] += 1 pnns2[p[H['pnns_groups_2']]] += 1 k = p[H['energy-kcal_100g']] try: kv = float(k) if k else None except ValueError: kv = None if kv == 0: for t in tags: zero_cats[t] += 1 if len(zero_names) < 400 and len(zero_names) < 400 and (hash(p[0]) % 50 == 0): zero_names.append((p[H['product_name']], p[H['main_category']], p[H['fat_100g']], p[H['carbohydrates_100g']], p[H['proteins_100g']], p[H['countries_tags']][:30])) if VOL.search(p[H['quantity']]): for t in tags: volcats[t] += 1 d = p[0] L = len(d) if 6 <= L <= 14: chk[(L, gs1_ok(d))] += 1 lead[(L, len(d.lstrip('0')))] += 1 print('CATS', json.dumps(cats.most_common(400))) print('ZEROCATS', json.dumps(zero_cats.most_common(200))) print('VOLCATS', json.dumps(volcats.most_common(200))) print('PNNS1', json.dumps(pnns1.most_common())) print('PNNS2', json.dumps(pnns2.most_common())) print('CHK', sorted(chk.items())) print('LEAD', sorted(lead.items())) for z in zero_names: print('ZERO', json.dumps(z, ensure_ascii=False))