"""Inspect the raw OFF TSV export: field counts, quoting, value distributions. Usage: python3 -I off_inspect.py [max_lines] Reads raw lines (split on '\t', no CSV quoting) so we can see what the file really looks like. """ import gzip, sys, collections, re path = sys.argv[1] max_lines = int(sys.argv[2]) if len(sys.argv) > 2 else 10**12 nf = collections.Counter() quoted_fields = 0 quote_examples = [] dq = collections.Counter() url_lc = collections.Counter() states_c = collections.Counter() ndp = collections.Counter() qty_units = collections.Counter() serving_units = collections.Counter() carb_vs_total = collections.Counter() cr_in_line = 0 n = 0 bad_examples = [] try: with gzip.open(path, 'rb') as f: header = f.readline().decode('utf-8').rstrip('\r\n').split('\t') H = {c: i for i, c in enumerate(header)} for raw in f: n += 1 if n > max_lines: break line = raw.decode('utf-8', errors='replace') if line.endswith('\n'): line = line[:-1] if '\r' in line: cr_in_line += 1 line = line.rstrip('\r') parts = line.split('\t') nf[len(parts)] += 1 if len(parts) != len(header): if len(bad_examples) < 5: bad_examples.append((n, len(parts), line[:300])) continue for p in parts: if len(p) >= 2 and p[0] == '"' and p[-1] == '"': quoted_fields += 1 if len(quote_examples) < 8: quote_examples.append(p[:120]) for t in parts[H['data_quality_errors_tags']].split(','): if t: dq[t] += 1 m = re.match(r'https?://world-([a-z]{2,3})\.', parts[H['url']]) url_lc[m.group(1) if m else ('NOMATCH:' + parts[H['url']][:40])] += 1 for t in parts[H['states_tags']].split(','): if t: states_c[t] += 1 q = parts[H['quantity']].lower() m = re.search(r'(\d)\s*([a-z]+\.?(?:\s?oz)?)\b', q) if m: qty_units[m.group(2)] += 1 s = parts[H['serving_size']].lower() m = re.search(r'(\d)\s*([a-z]+\.?(?:\s?oz)?)\b', s) if m: serving_units[m.group(2)] += 1 c = parts[H['carbohydrates_100g']]; ct = parts[H['carbohydrates-total_100g']] carb_vs_total[(bool(c), bool(ct))] += 1 except EOFError: print('EOF (partial file)') print('rows', n) print('field counts', nf.most_common(10)) print('lines with CR', cr_in_line) print('quoted-looking fields', quoted_fields, quote_examples) print('bad examples', bad_examples) print('url lc', url_lc.most_common(40)) print('dq tags', len(dq)) for t, c in dq.most_common(200): print(' DQ', c, t) print('states', [x for x in states_c.most_common(60)]) print('qty units', qty_units.most_common(40)) print('serving units', serving_units.most_common(40)) print('carb vs total', carb_vs_total)