OBLITERATUS/scripts/aggregate_contributions.py

#!/usr/bin/env python3
"""Aggregate community contributions into paper-ready tables.

Usage:
    python scripts/aggregate_contributions.py [--dir community_results] [--format latex|csv|json]

Reads all contribution JSON files from the specified directory, aggregates
them by model and method, and outputs summary tables suitable for inclusion
in the paper.
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

# Add project root to path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from obliteratus.community import (
    aggregate_results,
    generate_latex_table,
    load_contributions,
)


def main():
    parser = argparse.ArgumentParser(
        description="Aggregate community contributions into paper tables."
    )
    parser.add_argument(
        "--dir",
        default="community_results",
        help="Directory containing contribution JSON files (default: community_results)",
    )
    parser.add_argument(
        "--format",
        choices=["latex", "csv", "json", "summary"],
        default="summary",
        help="Output format (default: summary)",
    )
    parser.add_argument(
        "--metric",
        default="refusal_rate",
        help="Metric to display in tables (default: refusal_rate)",
    )
    parser.add_argument(
        "--methods",
        nargs="*",
        help="Methods to include (default: all)",
    )
    parser.add_argument(
        "--min-runs",
        type=int,
        default=1,
        help="Minimum runs per (model, method) to include (default: 1)",
    )
    args = parser.parse_args()

    # Load all contributions
    records = load_contributions(args.dir)
    if not records:
        print(f"No contributions found in {args.dir}/", file=sys.stderr)
        sys.exit(1)

    print(f"Loaded {len(records)} contribution(s) from {args.dir}/", file=sys.stderr)

    # Aggregate
    aggregated = aggregate_results(records)

    # Filter by minimum runs
    if args.min_runs > 1:
        for model in list(aggregated.keys()):
            for method in list(aggregated[model].keys()):
                if aggregated[model][method]["n_runs"] < args.min_runs:
                    del aggregated[model][method]
            if not aggregated[model]:
                del aggregated[model]

    if not aggregated:
        print("No results meet the minimum run threshold.", file=sys.stderr)
        sys.exit(1)

    # Output
    if args.format == "summary":
        _print_summary(aggregated, args.metric)
    elif args.format == "latex":
        print(generate_latex_table(aggregated, methods=args.methods, metric=args.metric))
    elif args.format == "json":
        print(json.dumps(aggregated, indent=2))
    elif args.format == "csv":
        _print_csv(aggregated, args.metric)


def _print_summary(aggregated: dict, metric: str):
    """Print a human-readable summary of aggregated results."""
    total_runs = sum(
        data["n_runs"]
        for model_data in aggregated.values()
        for data in model_data.values()
    )
    n_models = len(aggregated)
    n_methods = len(set(
        method
        for model_data in aggregated.values()
        for method in model_data
    ))

    print(f"\n{'=' * 70}")
    print("Community Contribution Summary")
    print(f"{'=' * 70}")
    print(f"  Total runs:    {total_runs}")
    print(f"  Models:        {n_models}")
    print(f"  Methods:       {n_methods}")
    print()

    for model in sorted(aggregated.keys()):
        model_data = aggregated[model]
        short = model.split("/")[-1] if "/" in model else model
        print(f"  {short}:")
        for method in sorted(model_data.keys()):
            data = model_data[method]
            n = data["n_runs"]
            if metric in data:
                stats = data[metric]
                mean = stats["mean"]
                std = stats["std"]
                if std > 0 and n > 1:
                    print(f"    {method:20s}  {metric}={mean:.2f} ± {std:.2f}  (n={n})")
                else:
                    print(f"    {method:20s}  {metric}={mean:.2f}  (n={n})")
            else:
                print(f"    {method:20s}  (no {metric} data, n={n})")
        print()

    print(f"{'=' * 70}")
    print(f"To generate LaTeX: python {sys.argv[0]} --format latex")
    print(f"To generate CSV:   python {sys.argv[0]} --format csv")


def _print_csv(aggregated: dict, metric: str):
    """Print results as CSV."""
    print("model,method,n_runs,mean,std,min,max")
    for model in sorted(aggregated.keys()):
        for method in sorted(aggregated[model].keys()):
            data = aggregated[model][method]
            n = data["n_runs"]
            if metric in data:
                stats = data[metric]
                print(
                    f"{model},{method},{n},"
                    f"{stats['mean']:.4f},{stats['std']:.4f},"
                    f"{stats['min']:.4f},{stats['max']:.4f}"
                )


if __name__ == "__main__":
    main()