mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-08-19 12:07:13 +02:00
244 lines
8.5 KiB
Python
244 lines
8.5 KiB
Python
"""Tests for the manifest-driven SynthID D1 confound challenge."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
from PIL import Image
|
|
from PIL.PngImagePlugin import PngInfo
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
|
|
|
import synthid_confound_probe as probe
|
|
import synthid_research_manifest as manifest
|
|
|
|
|
|
def _write_image(path: Path, color: tuple[int, int, int], *, note: str | None = None) -> tuple[str, str]:
|
|
image = Image.new("RGB", (12, 10), color)
|
|
pnginfo = None
|
|
if note is not None:
|
|
pnginfo = PngInfo()
|
|
pnginfo.add_text("note", note)
|
|
image.save(path, pnginfo=pnginfo)
|
|
return hashlib.sha256(path.read_bytes()).hexdigest(), hashlib.sha256(image.tobytes()).hexdigest()
|
|
|
|
|
|
def _row(
|
|
artifact_sha: str,
|
|
pixel_sha: str,
|
|
artifact_path: str,
|
|
*,
|
|
group_id: str,
|
|
split: str,
|
|
outcome: str,
|
|
source_provider: str,
|
|
oracle_role: str = "ordinary",
|
|
) -> dict[str, str]:
|
|
matching_oracle = source_provider == "openai"
|
|
return {
|
|
"artifact_sha256": artifact_sha,
|
|
"pixel_sha256": pixel_sha,
|
|
"artifact_path": artifact_path,
|
|
"parent_sha256": "",
|
|
"group_id": group_id,
|
|
"target_provider": "openai",
|
|
"source_provider": source_provider,
|
|
"surface": "synthetic-test",
|
|
"model_epoch": "test-epoch",
|
|
"generation_session": f"session-{group_id}",
|
|
"content_stratum": "flat-graphic",
|
|
"width": "12",
|
|
"height": "10",
|
|
"format": "png",
|
|
"transform": "original",
|
|
"split": split,
|
|
"c2pa_outcome": "detected" if outcome == "detected" else "not_present",
|
|
"synthid_outcome": outcome,
|
|
"verified_via": "openai-api" if matching_oracle else "source-evidence",
|
|
"evidence_reference": "" if matching_oracle else f"https://example.test/{group_id}",
|
|
"oracle_session": f"oracle-{group_id}" if matching_oracle else "",
|
|
"oracle_role": oracle_role,
|
|
"captured_at": "2026-08-09T10:00:00Z",
|
|
"oracle_checked_at": "2026-08-09T10:05:00Z",
|
|
"notes": "synthetic confound fixture",
|
|
}
|
|
|
|
|
|
def _write_manifest(root: Path, rows: list[dict[str, str]]) -> Path:
|
|
path = root / "manifest.csv"
|
|
with path.open("w", newline="", encoding="utf-8") as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=manifest.FIELDNAMES)
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
return path
|
|
|
|
|
|
def _corpus(root: Path, *, include_temporal: bool = True) -> Path:
|
|
rows: list[dict[str, str]] = []
|
|
index = 0
|
|
split_sizes = {"train": 4, "validation": 2, "test": 2}
|
|
if include_temporal:
|
|
split_sizes["temporal"] = 2
|
|
for split, per_class in split_sizes.items():
|
|
for label in (1, 0):
|
|
for offset in range(per_class):
|
|
index += 1
|
|
color = (230, 10 + index, 20 + offset) if label else (20 + offset, 10 + index, 230)
|
|
path = root / f"image-{index}.png"
|
|
artifact_sha, pixel_sha = _write_image(path, color)
|
|
source_provider = "openai" if label or offset == 0 else "camera"
|
|
rows.append(
|
|
_row(
|
|
artifact_sha,
|
|
pixel_sha,
|
|
path.name,
|
|
group_id=f"group-{index}",
|
|
split=split,
|
|
outcome="detected" if label else "not_detected",
|
|
source_provider=source_provider,
|
|
)
|
|
)
|
|
return _write_manifest(root, rows)
|
|
|
|
|
|
def test_canonical_features_ignore_container_metadata(tmp_path: Path):
|
|
first = tmp_path / "first.png"
|
|
second = tmp_path / "second.png"
|
|
first_sha, _ = _write_image(first, (10, 20, 30), note="short")
|
|
second_sha, _ = _write_image(second, (10, 20, 30), note="a much longer metadata value")
|
|
first_example = probe.Example(first, first_sha, "one", "train", 1, None)
|
|
second_example = probe.Example(second, second_sha, "two", "train", 1, None)
|
|
|
|
assert not np.array_equal(
|
|
probe.extract_features(first_example, "container"),
|
|
probe.extract_features(second_example, "container"),
|
|
)
|
|
np.testing.assert_array_equal(
|
|
probe.extract_features(first_example, "canonical"),
|
|
probe.extract_features(second_example, "canonical"),
|
|
)
|
|
|
|
|
|
def test_feature_matrices_decode_each_artifact_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
|
first = tmp_path / "first.png"
|
|
second = tmp_path / "second.png"
|
|
first_sha, _ = _write_image(first, (10, 20, 30))
|
|
second_sha, _ = _write_image(second, (30, 20, 10))
|
|
examples = [
|
|
probe.Example(first, first_sha, "one", "train", 1, None),
|
|
probe.Example(second, second_sha, "two", "train", 0, "external"),
|
|
]
|
|
original = probe._decoded_rgb
|
|
decode_count = 0
|
|
|
|
def counting_decode(path: Path):
|
|
nonlocal decode_count
|
|
decode_count += 1
|
|
return original(path)
|
|
|
|
monkeypatch.setattr(probe, "_decoded_rgb", counting_decode)
|
|
|
|
matrices = probe.feature_matrices(examples)
|
|
|
|
assert decode_count == len(examples)
|
|
assert set(matrices) == set(probe.FEATURE_FAMILIES)
|
|
assert all(matrix.shape[0] == len(examples) for matrix in matrices.values())
|
|
|
|
|
|
def test_logistic_model_separates_simple_content_confound():
|
|
features = np.asarray([[0.0], [0.1], [0.9], [1.0]], dtype=np.float64)
|
|
labels = np.asarray([0, 0, 1, 1], dtype=np.int64)
|
|
|
|
model = probe.fit_logistic(features, labels)
|
|
scores = probe.predict_scores(model, features)
|
|
|
|
assert max(scores[:2]) < min(scores[2:])
|
|
|
|
|
|
def test_threshold_respects_validation_false_positive_limit():
|
|
labels = np.asarray([1, 1, 0, 0], dtype=np.int64)
|
|
scores = np.asarray([0.9, 0.8, 0.7, 0.1], dtype=np.float64)
|
|
|
|
threshold = probe.select_threshold(labels, scores, max_fpr=0.0)
|
|
metrics = probe.calculate_metrics(labels, scores, threshold)
|
|
|
|
assert threshold == pytest.approx(0.8)
|
|
assert metrics.tpr == 1.0
|
|
assert metrics.fpr == 0.0
|
|
|
|
|
|
def test_run_experiment_reports_same_provider_and_temporal_gate(tmp_path: Path):
|
|
path = _corpus(tmp_path)
|
|
|
|
report = probe.run_experiment(path, "openai")
|
|
|
|
assert report["evidence_ready"] is True
|
|
assert report["evidence_missing"] == []
|
|
canonical = report["families"]["canonical"]
|
|
assert canonical["metrics"]["test"]["auc"] == 1.0
|
|
assert canonical["negative_cohorts"]["test"]["same_provider"]["count"] == 1
|
|
assert canonical["metrics"]["temporal"]["positives"] == 2
|
|
assert canonical["metrics"]["temporal"]["negatives"] == 2
|
|
assert str(tmp_path) not in json.dumps(report)
|
|
|
|
|
|
def test_report_stays_explicit_when_temporal_holdout_is_missing(tmp_path: Path):
|
|
path = _corpus(tmp_path, include_temporal=False)
|
|
|
|
report = probe.run_experiment(path, "openai")
|
|
|
|
assert report["evidence_ready"] is False
|
|
assert report["evidence_missing"] == ["temporal split does not contain both labels"]
|
|
assert report["families"]["container"]["metrics"]["temporal"] is None
|
|
|
|
|
|
def test_candidate_rows_are_not_detector_examples(tmp_path: Path):
|
|
path = _corpus(tmp_path)
|
|
rows = probe._read_manifest(path)
|
|
candidate_path = tmp_path / "candidate.png"
|
|
artifact_sha, pixel_sha = _write_image(candidate_path, (90, 90, 90))
|
|
rows.append(
|
|
_row(
|
|
artifact_sha,
|
|
pixel_sha,
|
|
candidate_path.name,
|
|
group_id="candidate-group",
|
|
split="train",
|
|
outcome="detected",
|
|
source_provider="openai",
|
|
oracle_role="candidate",
|
|
)
|
|
)
|
|
_write_manifest(tmp_path, rows)
|
|
|
|
examples = probe.load_examples(path, "openai")
|
|
|
|
assert all(example.artifact_path.name != candidate_path.name for example in examples)
|
|
|
|
|
|
def test_manifest_audit_blocks_group_leakage_before_training(tmp_path: Path):
|
|
path = _corpus(tmp_path)
|
|
rows = probe._read_manifest(path)
|
|
rows[-1]["group_id"] = rows[0]["group_id"]
|
|
_write_manifest(tmp_path, rows)
|
|
|
|
with pytest.raises(ValueError, match="manifest audit failed"):
|
|
probe.run_experiment(path, "openai")
|
|
|
|
|
|
def test_positive_source_provider_must_match_target(tmp_path: Path):
|
|
path = _corpus(tmp_path)
|
|
rows = probe._read_manifest(path)
|
|
positive = next(row for row in rows if row["synthid_outcome"] == "detected")
|
|
positive["source_provider"] = "google"
|
|
_write_manifest(tmp_path, rows)
|
|
|
|
with pytest.raises(ValueError, match="declares source_provider"):
|
|
probe.load_examples(path, "openai")
|