#!/usr/bin/env python3
"""Download public ALFRED vintage CSVs; verify vintage headers and record gaps."""
import argparse
import concurrent.futures
import csv
from datetime import date,datetime,timedelta,timezone
from decimal import Decimal,InvalidOperation
import hashlib
import io
import json
from pathlib import Path
import urllib.parse
import urllib.request

ROOT=Path(__file__).resolve().parent
SERIES=('INDPRO','CPIAUCSL','UNRATE','FEDFUNDS','GS10','PAYEMS','GDPC1','T10Y3M')
VINTAGES=('2000-01-01','2009-01-01','2020-01-01','2025-01-01','2025-12-31')

def download(item):
    series,vintage,folder=item
    params={'id':series,'cosd':'1926-01-01','coed':'2025-12-31'}
    if vintage:params['vintage_date']=vintage
    endpoint='https://alfred.stlouisfed.org/graph/alfredgraph.csv' if vintage else 'https://fred.stlouisfed.org/graph/fredgraph.csv'
    url=endpoint+'?'+urllib.parse.urlencode(params)
    result={'series':series,'vintage_date':vintage,'source_url':url,'retrieved_at':datetime.now(timezone.utc).isoformat()}
    try:
        with urllib.request.urlopen(url,timeout=25) as response:
            raw=response.read(5_000_001)
            if len(raw)>5_000_000:raise ValueError('download size')
        rows=list(csv.reader(io.StringIO(raw.decode('utf-8-sig'))))
        expected=series+'_'+vintage.replace('-','') if vintage else series
        if not rows or rows[0]!=['observation_date',expected]:raise ValueError('unexpected or mismatched vintage header')
        count=0;first=None;last=None
        for row in rows[1:]:
            if len(row)!=2:raise ValueError('row shape')
            date.fromisoformat(row[0])
            if row[1] not in ('','.'): 
                value=Decimal(row[1])
                if not value.is_finite():raise ValueError('nonfinite')
                first=first or row[0];last=row[0];count+=1
        if not count:raise ValueError('no observations')
        filename=series+('_'+vintage if vintage else '')+'.csv'
        path=folder/filename;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw)
        result.update(status='DOWNLOADED',file=str(path.relative_to(ROOT)),sha256=hashlib.sha256(raw).hexdigest(),observations=count,first_observation=first,last_observation=last,
            available_at=(date.fromisoformat(vintage)+timedelta(days=2)).isoformat()+'T00:00:00+00:00' if vintage else None,
            usage='MACRO_SNAPSHOT_NOT_TRADABLE_PRICE' if vintage else 'DAILY_REFERENCE_ONLY_NOT_INTRADAY_EXECUTION',
            timing_policy='Snapshot withheld until vintage date + 2 days UTC; never backdate to observation month.' if vintage else 'Not independently vintage-validated; excluded from execution replay.')
    except Exception as exc:result.update(status='UNAVAILABLE',error_type=type(exc).__name__,http_status=getattr(exc,'code',None))
    return result

def main():
    p=argparse.ArgumentParser(description=__doc__);p.add_argument('--vintages',nargs='+',default=VINTAGES);p.add_argument('--series',nargs='+',default=SERIES)
    p.add_argument('--with-reference-prices',action='store_true');a=p.parse_args()
    for v in a.vintages:date.fromisoformat(v)
    if any(not s.isalnum() for s in a.series):p.error('invalid series ID')
    jobs=[(s,v,ROOT/'data'/'alfred') for v in a.vintages for s in a.series]
    if a.with_reference_prices:jobs +=[(s,None,ROOT/'data'/'reference_prices') for s in ('CBBTCUSD','CBETHUSD')]
    results=[]
    with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
        for r in pool.map(download,jobs):
            results.append(r);print(json.dumps({k:r.get(k) for k in ('series','vintage_date','status','observations','error_type')}),flush=True)
    (ROOT/'data'/'download_manifest.json').write_text(json.dumps({'retrieved_at':datetime.now(timezone.utc).isoformat(),'records':results,'historical_execution_dataset_complete':False},indent=2))
    snapshots=[]
    for item in results:
        if item['status']!='DOWNLOADED' or not item['vintage_date']:continue
        with (ROOT/item['file']).open() as stream:
            for row in csv.DictReader(stream):
                key=item['series']+'_'+item['vintage_date'].replace('-','')
                if row[key] in ('','.'):continue
                snapshots.append({'series':item['series'],'observation_date':row['observation_date'],'value':row[key],
                    'available_at':item['available_at'],'vintage_date':item['vintage_date'],'url':item['source_url'],'source_sha256':item['sha256']})
    (ROOT/'data'/'macro_observations.jsonl').write_text(''.join(json.dumps(r)+'\n' for r in snapshots))
    print(json.dumps({'downloaded':sum(r['status']=='DOWNLOADED' for r in results),'unavailable':sum(r['status']!='DOWNLOADED' for r in results),'macro_observations':len(snapshots)}))
if __name__=='__main__':main()