import os, re, json, sys, time
from pathlib import Path

ROOT = Path('C:/Solutions/michel/knowledge/foster')
OUT = Path('C:/MCSJAgent/foster-workflow-catalog')
OUT.mkdir(parents=True, exist_ok=True)

MODULE_RULES = [
 ('Payroll', r'payroll|salary|attendance|time clock|w-?2|aca'),
 ('Utility', r'utility|meter|billing cycle|cutoff|high.low|water|sewer'),
 ('Finance/AP', r'purchase order|\bpo\b|requisition|vendor|check batch|accounts payable|\bap\b'),
 ('Finance/GL', r'journal|general ledger|\bgl\b|budget|cash receipt|bank reconciliation|cash flow|expenditure|revenue'),
 ('Billing/Collections', r'payment|collection|pay code|cash receipt'),
 ('Misc AR', r'misc ar|miscellaneous ar|invoice|customer|service code|overpayment|write.off'),
 ('Tax', r'property tax|nj tax|tax sale|pilot|lien|assessment'),
 ('CPCE', r'permit|inspection|code enforcement|violation|rental|opa|cpce'),
 ('Animal Licensing', r'animal|dog license|breed'),
 ('System', r'user|security|parameter|year.end|eoy'),
]

def module_for(text):
    s=text.lower()
    for module,pat in MODULE_RULES:
        if re.search(pat,s,re.I): return module
    return 'Other'

def clean_title(stem):
    s=re.sub(r'[_-]+',' ',stem)
    s=re.sub(r'\s+',' ',s).strip()
    return s[:1].upper()+s[1:] if s else s

def read_text(path, cap=100000):
    try:
        return path.read_text(encoding='utf-8',errors='ignore')[:cap]
    except Exception:
        return ''

records=[]
for p in ROOT.rglob('*'):
    if not p.is_file(): continue
    ext=p.suffix.lower()
    rel=str(p.relative_to(ROOT)).replace('\\','/')
    lower=rel.lower()
    kind=None
    if '/tip_sheets/' in '/'+lower: kind='tip_sheet'
    elif '/sops/' in '/'+lower: kind='sop'
    elif '/camps/projects/mcsj/' in '/'+lower and ext=='.md': kind='workflow_reference'
    elif '/skills/training/' in '/'+lower and ext=='.md': kind='skill_reference'
    elif '/media/video/approved/' in '/'+lower and p.name.lower()=='transcript.md': kind='approved_transcript'
    if not kind: continue
    if ext not in {'.md','.json','.html','.pdf','.docx','.png','.jpg','.jpeg'}: continue
    # Deduplicate rendered siblings by preferring JSON/MD, but retain evidence links.
    title=clean_title(p.stem)
    nav=''; steps=[]; text=''
    if ext in {'.md','.html','.json'}:
        text=read_text(p)
        if ext=='.json':
            try:
                d=json.loads(text); title=d.get('title',title)
                ns=d.get('nav_step') or {}; nav=ns.get('nav_path','')
                for x in d.get('card_steps',[]):
                    steps.append(x.get('heading',''))
                    steps.extend(x.get('checks',[]))
            except Exception: pass
        else:
            m=re.search(r'^#\s+(.+)$',text,re.M); title=m.group(1).strip() if m else title
            paths=re.findall(r'(?:\*\*Path:\*\*|Navigate to|Navigation:?)[^\n]*\n?([^\n]{0,180})',text,re.I)
            nav=' | '.join(x.strip(' *`:-') for x in paths[:2])
            steps=[m.group(1).strip() for m in re.finditer(r'^#{2,4}\s+(.+)$',text,re.M)][:12]
    operational=bool(re.search(r'add|create|process|update|post|print|import|export|revers|void|adjust|maint|batch|billing|payment|reconcil|report|setup|calculate',title,re.I))
    exactness=0
    if nav or re.search(r'\b(navigate|path|menu)\b',text,re.I): exactness+=1
    if len(steps)>=2 or re.search(r'^\s*\d+[.)]\s+',text,re.M): exactness+=1
    if ext in {'.json','.md'}: exactness+=1
    if kind in {'tip_sheet','sop'}: exactness+=1
    if re.search(r'watch.?out|warning|verify|proof|update batch',text,re.I): exactness+=1
    records.append({'title':title,'module':module_for(rel+' '+title),'kind':kind,'source':str(p).replace('\\','/'),'extension':ext,'navigation':nav,'sections':steps,'operational':operational,'recipe_score':exactness})

# Collapse same-folder/stem rendered variants into logical sources.
groups={}
for r in records:
    key=re.sub(r'\.(json|html|pdf|docx|png|jpg|jpeg|md)$','',r['source'],flags=re.I)
    g=groups.setdefault(key,{'title':r['title'],'module':r['module'],'kind':r['kind'],'sources':[],'navigation':'','sections':[],'operational':False,'recipe_score':0})
    g['sources'].append(r['source']); g['operational']|=r['operational']; g['recipe_score']=max(g['recipe_score'],r['recipe_score'])
    if r['extension'] in {'.json','.md'}:
        g['title']=r['title']; g['module']=r['module']; g['navigation']=r['navigation'] or g['navigation']; g['sections']=r['sections'] or g['sections']
logical=sorted(groups.values(),key=lambda x:(-x['recipe_score'],x['module'],x['title']))
oper=[x for x in logical if x['operational']]

(OUT/'catalog.json').write_text(json.dumps({'generated_epoch':time.time(),'root':str(ROOT),'logical_count':len(logical),'operational_count':len(oper),'items':logical},indent=2),encoding='utf-8')
lines=['# Foster MCSJ Workflow Catalog','',f'- Logical source groups: **{len(logical)}**',f'- Operational candidates: **{len(oper)}**','', '| Score | Module | Workflow | Best source |','|---:|---|---|---|']
for x in oper[:300]:
    preferred=next((s for s in x['sources'] if s.lower().endswith(('.json','.md'))),x['sources'][0])
    lines.append(f"| {x['recipe_score']} | {x['module']} | {x['title'].replace('|','/')} | `{preferred}` |")
(OUT/'CATALOG.md').write_text('\n'.join(lines)+'\n',encoding='utf-8')
summary={'logical_sources':len(logical),'operational_candidates':len(oper),'by_module':{},'top_candidates':[{'title':x['title'],'module':x['module'],'score':x['recipe_score'],'source':next((s for s in x['sources'] if s.lower().endswith(('.json','.md'))),x['sources'][0])} for x in oper[:25]]}
for x in oper: summary['by_module'][x['module']]=summary['by_module'].get(x['module'],0)+1
print(json.dumps(summary,indent=2))