SME Token Optimization Study

import json, csv
BATCHES = ['maybank.csv','cimb.csv','bni.csv']  # segmented by bank
ROUTES = {'rules': {'tokens': 2000, 'pct': 72},
          'ai':    {'tokens': 6500, 'pct': 23},
          'human': {'tokens': 12000,'pct': 5}}

def route_statement(stmt):
    if stmt['format'] in ('standard','known'):
        return 'rules'     # ~1.2-2.8k tokens
    if stmt['anomaly_score'] > 0.7:
        return 'human'     # ~12k tokens
    return 'ai'            # ~4-9k tokens

def process_batch(batch):
    total = 0
    for stmt in batch:
        route = route_statement(stmt)
        total += ROUTES[route]['tokens']
        stmt['route'] = route
    return total

all_statements = [s for b in BATCHES for s in csv.DictReader(open(b))]
total = process_batch(all_statements)
print(f"Total tokens: {total} (vs 7.5M baseline)")
print(f"Efficiency: {round((1 - total/7500000)*100)}% saved")