"""Full-file inspection of the OFF TSV export (raw tab split). Prints JSON stats + writes samples. Usage: python3 -I off_inspect_full.py """ import gzip, sys, collections, re, json path, sample_out = sys.argv[1], sys.argv[2] C = collections.Counter st = collections.defaultdict(C) n = 0 KEYS = ['code','url','product_name','abbreviated_product_name','generic_name','quantity','brands','categories_tags', 'countries_tags','serving_size','serving_quantity','no_nutrition_data','states_tags','product_quantity', 'data_quality_errors_tags','unique_scans_n','completeness','last_modified_t','main_category', 'energy-kj_100g','energy-kcal_100g','energy_100g','fat_100g','carbohydrates_100g','carbohydrates-total_100g', 'proteins_100g','fiber_100g','alcohol_100g','pnns_groups_1','pnns_groups_2','food_groups_tags'] samples = open(sample_out, 'w', encoding='utf-8') with gzip.open(path, 'rb') as f: header = f.readline().decode('utf-8').rstrip('\n').split('\t') H = {c: i for i, c in enumerate(header)} for raw in f: n += 1 try: line = raw.decode('utf-8') except UnicodeDecodeError: st['decode']['bad'] += 1 line = raw.decode('utf-8', errors='replace') line = line.rstrip('\n') if line.endswith('\r'): st['cr']['trailing'] += 1 line = line[:-1] parts = line.split('\t') st['nfields'][len(parts)] += 1 if len(parts) != len(header): if st['badrow_examples']['n'] < 5: st['badrow_examples']['n'] += 1 print('BADROW', n, len(parts), line[:400], file=sys.stderr) continue for i, p in enumerate(parts): if '"' in p: if len(p) >= 2 and p[0] == '"' and p[-1] == '"': inner = p[1:-1] # inner must have only doubled quotes if re.search(r'(?