remove-ai-watermarks/tests/test_synthid_research_manifest.py

"""Tests for the private SynthID research-manifest auditor."""

from __future__ import annotations

import csv
import hashlib
import sys
from pathlib import Path

from PIL import Image

sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))

import synthid_research_manifest as manifest


def _write_image(path: Path, color: tuple[int, int, int]) -> tuple[str, str]:
    image = Image.new("RGB", (8, 6), color)
    image.save(path)
    artifact_sha = hashlib.sha256(path.read_bytes()).hexdigest()
    pixel_sha = hashlib.sha256(image.tobytes()).hexdigest()
    return artifact_sha, pixel_sha


def _row(artifact_sha: str, pixel_sha: str, *, path: str = "image.png", **updates: str) -> dict[str, str]:
    row = {
        "artifact_sha256": artifact_sha,
        "pixel_sha256": pixel_sha,
        "artifact_path": path,
        "parent_sha256": "",
        "group_id": "group-1",
        "target_provider": "openai",
        "source_provider": "openai",
        "surface": "api",
        "model_epoch": "gpt-image-2026-08",
        "generation_session": "session-1",
        "content_stratum": "flat-graphic",
        "width": "8",
        "height": "6",
        "format": "png",
        "transform": "original",
        "split": "train",
        "c2pa_outcome": "detected",
        "synthid_outcome": "detected",
        "verified_via": "openai-api",
        "evidence_reference": "",
        "oracle_session": "oracle-1",
        "oracle_role": "ordinary",
        "captured_at": "2026-08-08T12:00:00Z",
        "oracle_checked_at": "2026-08-08T12:05:00Z",
        "notes": "synthetic test row",
    }
    row.update(updates)
    return row


def _write_manifest(path: Path, rows: list[dict[str, str]]) -> None:
    with path.open("w", newline="", encoding="utf-8") as stream:
        writer = csv.DictWriter(stream, fieldnames=manifest.FIELDNAMES)
        writer.writeheader()
        writer.writerows(rows)


def test_valid_manifest_and_files_pass(tmp_path: Path):
    artifact = tmp_path / "image.png"
    artifact_sha, pixel_sha = _write_image(artifact, (10, 20, 30))
    path = tmp_path / "manifest.csv"
    _write_manifest(path, [_row(artifact_sha, pixel_sha)])

    assert manifest.audit_manifest(path, verify_files=True) == []


def test_rejects_same_provider_negative_without_matching_oracle(tmp_path: Path):
    artifact_sha = "a" * 64
    pixel_sha = "b" * 64
    path = tmp_path / "manifest.csv"
    row = _row(
        artifact_sha,
        pixel_sha,
        synthid_outcome="not_detected",
        verified_via="source-evidence",
        oracle_session="",
    )
    _write_manifest(path, [row])

    errors = manifest.audit_manifest(path)

    assert any("same-provider negative" in error for error in errors)


def test_rejects_external_source_evidence_without_reference(tmp_path: Path):
    path = tmp_path / "manifest.csv"
    row = _row(
        "a" * 64,
        "b" * 64,
        source_provider="camera",
        synthid_outcome="not_detected",
        verified_via="source-evidence",
        oracle_session="",
        c2pa_outcome="not_present",
    )
    _write_manifest(path, [row])

    errors = manifest.audit_manifest(path)

    assert any("source-evidence requires evidence_reference" in error for error in errors)


def test_accepts_external_source_evidence_with_reference(tmp_path: Path):
    path = tmp_path / "manifest.csv"
    row = _row(
        "a" * 64,
        "b" * 64,
        source_provider="camera",
        synthid_outcome="not_detected",
        verified_via="source-evidence",
        evidence_reference="https://example.test/original-record",
        oracle_session="",
        c2pa_outcome="not_present",
    )
    _write_manifest(path, [row])

    assert manifest.audit_manifest(path) == []


def test_rejects_indeterminate_training_label(tmp_path: Path):
    path = tmp_path / "manifest.csv"
    row = _row(
        "a" * 64,
        "b" * 64,
        synthid_outcome="indeterminate",
        verified_via="openai-api",
        oracle_checked_at="",
    )
    _write_manifest(path, [row])

    errors = manifest.audit_manifest(path)

    assert any("requires a detected or not_detected" in error for error in errors)


def test_rejects_group_leakage_across_splits(tmp_path: Path):
    first = _row("a" * 64, "b" * 64)
    second = _row(
        "c" * 64,
        "d" * 64,
        artifact_path="other.png",
        split="test",
        generation_session="session-2",
    )
    path = tmp_path / "manifest.csv"
    _write_manifest(path, [first, second])

    errors = manifest.audit_manifest(path)

    assert any("leaks across splits" in error for error in errors)


def test_rejects_derivative_with_missing_parent(tmp_path: Path):
    path = tmp_path / "manifest.csv"
    row = _row("a" * 64, "b" * 64, transform="jpeg-q90", parent_sha256="c" * 64)
    _write_manifest(path, [row])

    errors = manifest.audit_manifest(path)

    assert any("parent_sha256 is not present" in error for error in errors)


def test_rejects_identical_pixels_in_different_groups(tmp_path: Path):
    first = _row("a" * 64, "b" * 64)
    second = _row(
        "c" * 64,
        "b" * 64,
        artifact_path="other.png",
        group_id="group-2",
        generation_session="session-2",
    )
    path = tmp_path / "manifest.csv"
    _write_manifest(path, [first, second])

    errors = manifest.audit_manifest(path)

    assert any("appears in multiple groups" in error for error in errors)


def test_verify_files_detects_changed_artifact(tmp_path: Path):
    artifact = tmp_path / "image.png"
    artifact_sha, pixel_sha = _write_image(artifact, (10, 20, 30))
    path = tmp_path / "manifest.csv"
    _write_manifest(path, [_row(artifact_sha, pixel_sha)])
    _write_image(artifact, (11, 21, 31))

    errors = manifest.audit_manifest(path, verify_files=True)

    assert any("artifact_sha256 does not match" in error for error in errors)
    assert any("pixel_sha256 does not match" in error for error in errors)


def test_rejects_path_traversal_during_file_verification(tmp_path: Path):
    path = tmp_path / "manifest.csv"
    _write_manifest(path, [_row("a" * 64, "b" * 64, path="../outside.png")])

    errors = manifest.audit_manifest(path, verify_files=True)

    assert any("safe manifest-relative path" in error for error in errors)


def test_accepts_candidate_negative_with_detected_session_control(tmp_path: Path):
    control = _row("a" * 64, "b" * 64, oracle_role="source_control")
    candidate = _row(
        "c" * 64,
        "d" * 64,
        artifact_path="candidate.png",
        parent_sha256="a" * 64,
        transform="carrier-subtract",
        synthid_outcome="not_detected",
        oracle_role="candidate",
    )
    path = tmp_path / "manifest.csv"
    _write_manifest(path, [control, candidate])

    assert manifest.audit_manifest(path) == []


def test_rejects_candidate_negative_without_detected_session_control(tmp_path: Path):
    candidate = _row(
        "a" * 64,
        "b" * 64,
        synthid_outcome="not_detected",
        oracle_role="candidate",
    )
    path = tmp_path / "manifest.csv"
    _write_manifest(path, [candidate])

    errors = manifest.audit_manifest(path)

    assert any("requires a detected source_control" in error for error in errors)


def test_rejects_failed_source_control(tmp_path: Path):
    control = _row(
        "a" * 64,
        "b" * 64,
        synthid_outcome="not_detected",
        oracle_role="source_control",
    )
    path = tmp_path / "manifest.csv"
    _write_manifest(path, [control])

    errors = manifest.audit_manifest(path)

    assert any("source_control must have a detected" in error for error in errors)