#!/usr/bin/env python3
"""THE CHANNEL — measure one charitable money flow across three jurisdictions.

Reads only primary public releases and writes measure.json, which is the single
source for every figure on /channel/. Nothing is imputed. Where a route carries
no joinable key, the script counts the route and records the break rather than
estimating a share.

    python3 measure.py --cra ../../../tmp/chan-src/csv --us ../../../tmp/chan-src/us \
                       --out ../measure.json --extracts ../data

Inputs
  --cra   annual "List of charities" T3010 CSVs from open.canada.ca
          dataset 80c00cdb-1358-415c-bb8b-0de7f12675b8 (and prior years)
  --us    IRS Business Master File extracts and Form 990 Schedule I rows

The CRA country column differs by table, and that difference is the finding:
  SCHEDULE_2_COUNTRIES   Country  len 2   ISO 3166-1 alpha-2, validated
  SCHEDULE_2_RESOURCES   Country  len 2   ISO 3166-1 alpha-2, validated
  NONQD (T1441)          Country  len 125 free text, unvalidated
"""
import argparse, csv, json, os, re, sys
from collections import defaultdict

csv.field_size_limit(10_000_000)

AP = argparse.ArgumentParser()
AP.add_argument('--cra', required=True, help='directory of CRA List of charities CSVs')
AP.add_argument('--us', required=True, help='directory of staged IRS extracts')
AP.add_argument('--out', required=True, help='measure.json to write')
AP.add_argument('--extracts', help='directory to write published row-level extracts')
AP.add_argument('--code', default='IL', help='ISO alpha-2 of the destination studied')
ARGS = AP.parse_args()

HERE = os.path.dirname(os.path.abspath(__file__))
CODES = json.load(open(os.path.join(HERE, 'country_codes.json'), encoding='utf-8'))
CODE = ARGS.code.upper()
COUNTRY = CODES.get(CODE, CODE)
YEARS = ['2020', '2021', '2022', '2023', '2024']


def money(v):
    v = (v or '').strip().replace(',', '').replace('$', '')
    if not v:
        return 0.0
    try:
        return float(v)
    except ValueError:
        return 0.0


def rows(year, stem):
    p = os.path.join(ARGS.cra, f'{year}__{stem}.csv')
    if not os.path.exists(p) or os.path.getsize(p) < 100:
        return []
    with open(p, newline='', encoding='utf-8-sig', errors='replace') as fh:
        return list(csv.DictReader(fh))


# --------------------------------------------------------------------------
# The free-text resolver. Deliberately generous: it is used to show how much a
# reader must guess, so it must not understate what a careful reader recovers.
# Every rule is published; anything it cannot resolve is counted, never binned.
# --------------------------------------------------------------------------
NAME_TO_CODE = {}
for c, n in CODES.items():
    NAME_TO_CODE[n.upper()] = c
    NAME_TO_CODE[re.sub(r'[^A-Z ]', ' ', n.upper()).strip()] = c
EXTRA_NAMES = {
    'USA': 'US', 'U S A': 'US', 'UNITED STATES OF AMERICA': 'US', 'AMERICA': 'US',
    'ETATS UNIS': 'US', 'ÉTATS UNIS': 'US', 'UK': 'GB', 'ENGLAND': 'GB',
    'GREAT BRITAIN': 'GB', 'SOUTH KOREA': 'KR', 'IVORY COAST': 'CI',
    'TANZANIA': 'TZ', 'BOLIVIA': 'BO', 'VENEZUELA': 'VE', 'VIETNAM': 'VN',
    'RUSSIA': 'RU', 'SYRIA': 'SY', 'IRAN': 'IR', 'MOLDOVA': 'MD',
    'DRC': 'CD', 'CONGO': 'CD', 'BRÉSIL': 'BR', 'ISRAËL': 'IL',
}
NAME_TO_CODE.update(EXTRA_NAMES)
NOISE = re.compile(r'^\s*(N/?A|NONE|NIL|N\.?A\.?|-+|0|VARIOUS|MULTIPLE|WORLDWIDE|GLOBAL)\s*$', re.I)
SPLIT = re.compile(r'[;,/|]|\s{2,}|\band\b|\bet\b', re.I)


def resolve(raw):
    """Return (set_of_codes, status). status in resolved|partial|unresolved|blank|noise."""
    s = (raw or '').strip()
    if not s:
        return set(), 'blank'
    if NOISE.match(s):
        return set(), 'noise'
    found, leftover = set(), []
    for part in [p for p in SPLIT.split(s) if p and p.strip()]:
        p = part.strip().strip('.-–—:=')
        if not p:
            continue
        up = re.sub(r'\s+', ' ', p.upper())
        m = re.match(r'^([A-Z]{2})\b[\s\-–—=:.]*(.*)$', up)
        if m and m.group(1) in CODES:
            found.add(m.group(1))
            continue
        key = re.sub(r'[^A-Z ]', ' ', up)
        key = re.sub(r'\s+', ' ', key).strip()
        if key in NAME_TO_CODE:
            found.add(NAME_TO_CODE[key])
            continue
        hit = None
        for nm, c in NAME_TO_CODE.items():
            if len(nm) > 3 and nm in key:
                hit = c
                break
        if hit:
            found.add(hit)
        else:
            leftover.append(p)
    if not found:
        return set(), 'unresolved'
    return found, ('partial' if leftover else 'resolved')


M = {
    'generatedAt': __import__('datetime').datetime.now(
        __import__('datetime').timezone.utc).isoformat(timespec='seconds'),
    'destination': {'code': CODE, 'name': COUNTRY},
    'canada': {}, 'united_states': {}, 'israel': {}, 'sources': {},
}

# ==========================================================================
# CANADA — four routes out, only two of them country-coded
# ==========================================================================
per_year, extract_rows = {}, []
for y in YEARS:
    countries = rows(y, 'Activities_outside_Canada_-_Countries_where_program_was_carried')
    details = rows(y, 'Activities_outside_Canada_-_Details_on_financial')
    resources = rows(y, 'Activities_outside_Canada_-_Financial_resources_used')
    nqd = rows(y, 'Non-Qualified_donees')
    qd = rows(y, 'Qualified_donees')
    genr = rows(y, 'General_information')
    if not countries and not nqd:
        continue

    # ---- Route A: own programs abroad. Countries listed; money not split.
    listed = defaultdict(set)
    for r in countries:
        c = (r.get('Country') or '').strip().upper()
        if c:
            listed[(r['BN'], r['FPE'])].add(c)
    a_filers = {k for k, v in listed.items() if CODE in v}
    a_sole = {k for k in a_filers if listed[k] == {CODE}}
    line200 = {(r['BN'], r['FPE']): money(r.get('200')) for r in details}

    # ---- Route B: line 210 intermediaries. Name + amount + validated code.
    b = [r for r in resources if (r.get('Country') or '').strip().upper() == CODE]
    # ---- Route C: T1441 grants. Name + purpose + amount + FREE TEXT country.
    c_rows, c_status, c_strings = [], defaultdict(int), defaultdict(int)
    for r in nqd:
        raw = r.get('Country') or ''
        c_strings[raw.strip()] += 1
        codes, status = resolve(raw)
        c_status[status] += 1
        if CODE in codes:
            c_rows.append(r)
    c_exact = [r for r in nqd if (r.get('Country') or '').strip().upper() == CODE]
    blank = [r for r in nqd if not (r.get('Country') or '').strip()]

    # ---- Route D: T1236 gifts to qualified donees. No country column at all,
    # but Donee BN is a real key: the next hop inside Canada IS joinable.
    d_total = sum(money(r.get('Total Gifts')) + money(r.get('Gifts in Kind')) for r in qd)
    d_keyed = sum(1 for r in qd if (r.get('Donee BN') or '').strip())

    # ---- The sub-threshold pool: grants of $5,000 or less are never itemised.
    sub_n = sum(money(r.get('5842')) for r in genr)
    sub_amt = sum(money(r.get('5843')) for r in genr)

    # ---- Internal contradiction. The T4033 instruction makes the line 210
    # table a subset of line 200: it reports "the total reported on line 200
    # that was transferred". Any filer whose 210 rows exceed its own line 200
    # has filed two numbers that cannot both be true.
    handed = defaultdict(float)
    for r in resources:
        handed[(r['BN'], r['FPE'])] += money(r.get('Amount'))
    over = [(k, v, line200.get(k, 0.0)) for k, v in handed.items()
            if v > line200.get(k, 0.0) + 1]

    per_year[y] = {
        'route_a': {
            'filers_listing_destination': len(a_filers),
            'filers_destination_only': len(a_sole),
            'filers_mixed_lists': len(a_filers) - len(a_sole),
            'line200_where_destination_is_sole_country': sum(line200.get(k, 0.0) for k in a_sole),
            'line200_mixed_lists_including_destination': sum(
                line200.get(k, 0.0) for k in a_filers - a_sole),
            'amount_attributable_to_destination': None,
        },
        'route_b': {
            'rows': len(b),
            'total': sum(money(r.get('Amount')) for r in b),
            'distinct_recipient_names': len({(r.get('Indiv/Org Name') or '').strip() for r in b}),
            'filers': len({(r['BN'], r['FPE']) for r in b}),
        },
        'route_c': {
            'rows_all_destinations': len(nqd),
            'cash_all_destinations': sum(money(r.get('Cash amount')) for r in nqd),
            'noncash_all_destinations': sum(money(r.get('Non-cash amount')) for r in nqd),
            'rows': len(c_rows),
            'rows_exact_code_match': len(c_exact),
            'total': sum(money(r.get('Cash amount')) + money(r.get('Non-cash amount'))
                         for r in c_rows),
            'total_exact_code_match': sum(money(r.get('Cash amount')) + money(r.get('Non-cash amount'))
                                          for r in c_exact),
            'distinct_recipient_names': len({(r.get('Recipient name') or '').strip() for r in c_rows}),
            'filers': len({(r['BN'], r['FPE']) for r in c_rows}),
            'country_field': {
                'distinct_strings': len([s for s in c_strings if s]),
                'status': dict(c_status),
                'blank_rows': len(blank),
                'blank_cash': sum(money(r.get('Cash amount')) for r in blank),
            },
        },
        'route_d': {
            'rows': len(qd),
            'total_gifts_all_donees': d_total,
            'rows_carrying_donee_bn': d_keyed,
            'country_column_exists': False,
        },
        'sub_threshold': {'grantees': sub_n, 'amount': sub_amt},
        'schedule_2_contradiction': {
            'filers_with_a_line_210_table': len(handed),
            'filers_exceeding_their_own_line_200': len(over),
            'excess': sum(v - l for _, v, l in over),
        },
    }

    if y == '2024':
        for r in c_rows:
            extract_rows.append({
                'bn': r['BN'], 'fpe': r['FPE'],
                'recipient': (r.get('Recipient name') or '').strip(),
                'purpose': (r.get('Purpose') or '').strip(),
                'cash': money(r.get('Cash amount')),
                'noncash': money(r.get('Non-cash amount')),
                'country_as_filed': (r.get('Country') or '').strip(),
            })
        for r in b:
            extract_rows.append({
                'bn': r['BN'], 'fpe': r['FPE'],
                'recipient': (r.get('Indiv/Org Name') or '').strip(),
                'purpose': '(Schedule 2 intermediary — no purpose field on this form)',
                'cash': money(r.get('Amount')), 'noncash': 0.0,
                'country_as_filed': (r.get('Country') or '').strip(),
            })

# ---- who the blank-destination money actually belongs to (2024) -----------
blank_top = []
nqd24 = rows('2024', 'Non-Qualified_donees')
ident = {r['BN']: r['Legal Name'] for r in rows('2024', 'Identification')}
agg, cnt = defaultdict(float), defaultdict(int)
for r in nqd24:
    if (r.get('Country') or '').strip():
        continue
    agg[r['BN']] += money(r.get('Cash amount'))
    cnt[r['BN']] += 1
blank_cash_total = sum(agg.values())
for bn, v in sorted(agg.items(), key=lambda x: -x[1])[:15]:
    blank_top.append({'bn': bn, 'name': ident.get(bn, ''), 'rows': cnt[bn], 'cash': v})

M['canada'] = {
    'per_year': per_year,
    'blank_destination_2024': {
        'cash': blank_cash_total,
        'share_of_all_grant_cash': None,
        'top_filers': blank_top,
        'top15_share': sum(t['cash'] for t in blank_top) / blank_cash_total if blank_cash_total else 0,
    },
}
y24 = per_year.get('2024', {})
if y24:
    M['canada']['blank_destination_2024']['share_of_all_grant_cash'] = (
        blank_cash_total / y24['route_c']['cash_all_destinations'])

# ==========================================================================
# UNITED STATES — the readable counterparty
# ==========================================================================
def load(fn):
    p = os.path.join(ARGS.us, fn)
    if not os.path.exists(p):
        return []
    if fn.endswith('.json'):
        d = json.load(open(p, encoding='utf-8'))
        return d if isinstance(d, list) else []
    with open(p, newline='', encoding='utf-8-sig', errors='replace') as fh:
        return list(csv.DictReader(fh))


def census(recs, rev='REVENUE_AMT', ast='ASSET_AMT'):
    return {'entities': len(recs),
            'revenue': sum(money(r.get(rev, r.get(rev.lower()))) for r in recs),
            'assets': sum(money(r.get(ast, r.get(ast.lower()))) for r in recs)}


sched_i = load('federation-scheduleI.csv')
by_recipient = defaultdict(float)
for r in sched_i:
    by_recipient[(r['federation'], r['recipient'].strip())] += money(r.get('cash_grant_amt'))
jfna = sorted([(k[1], v) for k, v in by_recipient.items() if k[0] == 'JFNA'],
              key=lambda x: -x[1])[:5]
sched_total = sum(money(r.get('cash_grant_amt')) for r in sched_i)

M['united_states'] = {
    'campus_federation': census(load('hillel-entities.csv'), 'revenue_amt', 'asset_amt'),
    'community_federations': census(load('federation-entities.csv'), 'revenue_amt', 'asset_amt'),
    'bridge_entities': census(load('bmf_bridge.json')),
    'schedule_i': {
        'federations_parsed': len({r['federation'] for r in sched_i}),
        'rows': len(sched_i),
        'total': sched_total,
        'top_national_counterparties': [{'recipient': n, 'amount': v} for n, v in jfna],
        'top_two_share_of_national': (sum(v for _, v in jfna[:2]) /
                                      sum(v for k, v in by_recipient.items() if k[0] == 'JFNA'))
        if any(k[0] == 'JFNA' for k in by_recipient) else None,
    },
}

# ==========================================================================
# ISRAEL — the ledger that cannot be joined. Recorded, not computed.
# ==========================================================================
M['israel'] = {
    'registrar_open_data': 'gated behind an OAuth redirect; metadata API open, files are not',
    'guidestar': 'per-organisation pages are public; filings are scanned PDFs, not structured rows',
    'donor_country_field': False,
    'statutory_donor_naming': 'applies to donations from foreign state entities, not private foreign donors',
    'registry_operator_note': ('GuideStar Israel is operated by NPTech, established by Yad Hanadiv '
                               'and JDC-Israel; JDC is itself a major counterparty in the flows the '
                               'registry discloses'),
}

M['sources'] = {
    'cra_open_data': 'https://open.canada.ca/data/en/dataset/80c00cdb-1358-415c-bb8b-0de7f12675b8',
    'cra_data_dictionary': 'https://open.canada.ca/data/dataset/05b3abd0-e70f-4b3b-a9c5-acc436bd15b6',
    'ita_149_1': 'https://laws-lois.justice.gc.ca/eng/acts/i-3.3/section-149.1.html',
    't3010': 'https://www.canada.ca/en/revenue-agency/services/forms-publications/forms/t3010.html',
    't4033': 'https://www.canada.ca/en/revenue-agency/services/forms-publications/publications/t4033/t4033-completing-registered-charity-information-return.html',
    'charities_report': 'https://www.canada.ca/en/revenue-agency/services/charities-giving/charities/about-charities-directorate/report-on-charities-program/report-on-charities-program-2024-2025.html',
    'irs_bmf': 'https://www.irs.gov/pub/irs-soi/eo1.csv',
    'irs_990_xml': 'https://apps.irs.gov/pub/epostcard/990/xml/2024/index_2024.csv',
    'guidestar_israel': 'https://www.guidestar.org.il/',
}

os.makedirs(os.path.dirname(os.path.abspath(ARGS.out)), exist_ok=True)
with open(ARGS.out, 'w', encoding='utf-8') as fh:
    json.dump(M, fh, indent=1, ensure_ascii=False)
    fh.write('\n')

if ARGS.extracts:
    os.makedirs(ARGS.extracts, exist_ok=True)
    extract_rows.sort(key=lambda r: -(r['cash'] + r['noncash']))
    p = os.path.join(ARGS.extracts, f'canada-{CODE.lower()}-coded-2024.csv')
    with open(p, 'w', newline='', encoding='utf-8') as fh:
        w = csv.DictWriter(fh, fieldnames=['bn', 'fpe', 'recipient', 'purpose',
                                           'cash', 'noncash', 'country_as_filed'])
        w.writeheader()
        w.writerows(extract_rows)
    sys.stderr.write(f'wrote {p} — {len(extract_rows)} rows\n')

sys.stderr.write(f'wrote {ARGS.out}\n')