"""Aggregate supplied fixture labels. Does NOT grade arbitrary answer text."""
import json
from pathlib import Path
x = json.loads(Path(__file__).with_name('rag-fixtures.json').read_text())
cases = x['cases']
answerable = [c for c in cases if c['required_sources']]
coverage = sum(set(c['required_sources']) <= set(c['retrieved_sources']) for c in answerable)
passes = sum(c['reviewed_task_pass'] for c in cases)
docs = {d['id']: d for d in x['documents']}
leaks = sum(any(docs[s]['tenant'] is not None and docs[s]['tenant'] != c['requesting_tenant'] for s in c['retrieved_sources']) for c in cases)
print(json.dumps({'scope':'Counts over constructed outcomes and explicit supplied labels; no model/retriever tested.','end_to_end_passes':passes,'cases':len(cases),'end_to_end_fraction':passes/len(cases),'complete_source_coverage_passes':coverage,'answerable_cases':len(answerable),'complete_source_coverage_fraction':coverage/len(answerable),'cross_tenant_exposure_cases':leaks,'proposed_release_gate_pass':leaks==0},indent=2))