#!/usr/bin/env python3 """QA one-off: scan every token shard for markup / placeholder tokens (amp, null, nan, x27, ...).""" import json, os, sys, collections d='/FoodIndex/out/site/d/20261115/t' sus={'amp','quot','039','x27','nbsp','lt','gt','reg','trade','tm','apos','ndash','mdash','x2019','u2019','null','none','nan','undefined','unknown','na'} found={} lens=collections.Counter() nonascii=0; total=0; digits=0 for fn in os.listdir(d): w=json.load(open(os.path.join(d,fn),encoding='utf-8'))['w'] for t,l in w.items(): total+=1 if t in sus: found[t]=len(l) if not t.isascii(): nonascii+=1 if t.isdigit(): digits+=1 print('tokens',total,'non-ascii',nonascii,'all-digit',digits) print('suspicious tokens (list length, capped at 2000):',found)