diff --git a/.github/workflows/publish-source-classify-hf.yml b/.github/workflows/publish-source-classify-hf.yml new file mode 100644 index 0000000..4445a46 --- /dev/null +++ b/.github/workflows/publish-source-classify-hf.yml @@ -0,0 +1,64 @@ +name: Publish source classifier to Hugging Face + +# Manual only. The derivative model weights live in a GitHub model-freeze +# release; the private training corpus and caches never enter this repository. + +on: + workflow_dispatch: + inputs: + confirm: + description: "Type publish" + required: true + weights_tag: + description: "GitHub Release tag holding source-pipeline-mlp.npz" + required: true + default: source-classify-freeze-2026-09-16 + +jobs: + publish: + if: ${{ github.event.inputs.confirm == 'publish' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + + - uses: astral-sh/setup-uv@v7 + with: + python-version: "3.12" + + - name: Download derivative model weights + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.event.inputs.weights_tag }} + run: | + set -euo pipefail + mkdir -p weights + gh release download "$TAG" \ + --repo "${{ github.repository }}" \ + --pattern source-pipeline-mlp.npz \ + --dir weights + test -s weights/source-pipeline-mlp.npz + + - name: Verify the frozen artifact + run: | + set -euo pipefail + expected="$(jq -r .artifact_sha256 docs/source-classify-hf/metrics.json)" + test "${#expected}" -eq 64 + echo "$expected weights/source-pipeline-mlp.npz" | sha256sum --check + + - name: Publish to Hugging Face + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + SHA: ${{ github.sha }} + run: | + set -euo pipefail + if [ -z "${HF_TOKEN:-}" ]; then + echo "HF_TOKEN secret is not set" >&2 + exit 1 + fi + uv run --no-project --with huggingface-hub \ + python scripts/publish_source_classify_hf.py \ + --src weights \ + --message "Source-classify freeze (GitHub ${SHA:0:7})" diff --git a/docs/index.md b/docs/index.md index 962784d..63d03d2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -42,6 +42,7 @@ The current behavior is defined by the code, tests, README, and user guides. - [AI-generated image classifiers](ai-generated-image-classifiers.md) (photo AI-versus-camera freeze consumed by `classify_pixels`, not `identify`) - [Photo classifier Hugging Face card](photo-classify-hf/README.md) (Hub publication text of the 2026-08-31 freeze) - [SynthID source classifiers](synthid-classifiers.md) (metadata-free OpenAI/Gemini source finding and provider-lineage experiments) +- [Source-pipeline classifier model card](source-classify-hf/README.md) (published derivative weights, aggregate evaluation, and intended-use boundary) - [SynthID local detector research](synthid-detector-research.md) - [SynthID mark removal research](synthid-removal-research.md) - [SynthID identity research](synthid-robust-identity-research.md) diff --git a/docs/release-and-distribution.md b/docs/release-and-distribution.md index 3c9e6fe..8d85335 100644 --- a/docs/release-and-distribution.md +++ b/docs/release-and-distribution.md @@ -92,7 +92,15 @@ alone does not upload new heads -- bump the pinned Hub revision when a new head must reach installed clients. Pin a Hub revision in production so a later freeze cannot silently change installed clients. -The workflow can also be started manually with an optional version input. +The separate `wiltodelta/raiw-source-classify` model is published with +`.github/workflows/publish-source-classify-hf.yml`. The workflow downloads the +derivative `source-pipeline-mlp.npz` artifact from the declared GitHub model +freeze, verifies its SHA-256, and uploads only that file plus the tracked card +and aggregate metrics under `docs/source-classify-hf/`. The private corpus, +catalog, paths, hashes, embeddings, and feature caches are outside this flow. + +The workflow is manual and requires the model-freeze release tag plus an +explicit `publish` confirmation. If a distribution job fails because a repository or Hugging Face credential is invalid, rotate the corresponding GitHub secret and rerun the failed job. A diff --git a/docs/source-classify-hf/README.md b/docs/source-classify-hf/README.md new file mode 100644 index 0000000..64c77fd --- /dev/null +++ b/docs/source-classify-hf/README.md @@ -0,0 +1,64 @@ +--- +license: apache-2.0 +library_name: numpy +pipeline_tag: image-classification +tags: + - image-forensics + - source-attribution + - open-set-classification +--- + +# RAIW source-pipeline classifier + +This is a compact, abstaining pixel-forensics classifier for three labels: +`openai`, `google`, and `unknown`. It is intended for original-export-looking +images whose metadata may have been removed without changing the pixels. + +This model is **not a SynthID detector**. It does not decode or verify a +watermark payload. Its labels describe complete generation and export +pipelines, so renderer, geometry, encoding, post-processing, and watermark +state may be confounded. `unknown` does not mean that SynthID is absent, and a +provider label does not prove that SynthID is present. + +## Model + +- Input: a 124-dimensional residual/spectral forensic feature vector plus a + 768-dimensional phase-free spectral vector. +- Head: 892 -> 128 -> 64 -> 3 MLP with GELU activations. +- Output: `openai`, `google`, or abstain as `unknown` using independent + provider margins. +- Artifact: `source-pipeline-mlp.npz` +- SHA-256: `0df4c644d02616265d083dbd3204e4dc617f7133b94af0eb0dc59d5362189290` + +The artifact contains only learned parameters, normalization constants, +labels, and thresholds. It does not contain training images, paths, image +hashes, embeddings, or a training catalog. + +## Evaluation + +The operating point was selected on an internal calibration split. The locked +internal test and the public `bigdatamark/synthid-research` dataset were not +used for architecture, checkpoint, or threshold selection. + +| Evaluation | Class | Recall | Precision | +|---|---|---:|---:| +| Locked internal test (5,369 images) | OpenAI | 60.4% | 84.9% | +| Locked internal test (5,369 images) | Google | 57.6% | 87.9% | +| Locked internal test (5,369 images) | Unknown | 98.6% | 93.7% | +| Public external originals (900 images) | OpenAI | 68.7% | 100.0% | +| Public external originals (900 images) | Google | 81.3% | 98.8% | +| Public external originals (900 images) | Unknown/Flux | 99.0% | 66.4% | + +On the public external set, a 95% resize reduced Google recall to 1.0%, and a +JPEG quality-75 round trip caused the model to abstain on every image. These +are expected abstentions, not evidence that a watermark was removed. + +## Intended use + +Use this model as one probabilistic signal in a broader forensic report or to +suggest an appropriate processing path. Do not use it to claim watermark +presence, authorship, policy compliance, fraud, or provenance. Applications +should expose the abstention and the model's narrow source-pipeline claim. + +Feature extraction and inference are versioned in the +`remove-ai-watermarks` Python project rather than in this model repository. diff --git a/docs/source-classify-hf/metrics.json b/docs/source-classify-hf/metrics.json new file mode 100644 index 0000000..77be858 --- /dev/null +++ b/docs/source-classify-hf/metrics.json @@ -0,0 +1,22 @@ +{ + "claim": "abstaining OpenAI/Google source-pipeline classification; not SynthID detection", + "artifact_sha256": "0df4c644d02616265d083dbd3204e4dc617f7133b94af0eb0dc59d5362189290", + "classes": ["openai", "google", "unknown"], + "locked_internal_test": { + "n": 5369, + "confusion_actual_by_predicted": [[230, 6, 145], [0, 217, 160], [41, 24, 4546]], + "recall": {"openai": 0.6036745407, "google": 0.5755968170, "unknown": 0.9859032748}, + "precision": {"openai": 0.8487084871, "google": 0.8785425101, "unknown": 0.9371263657} + }, + "public_external_originals": { + "dataset": "bigdatamark/synthid-research", + "n": 900, + "confusion_actual_by_predicted": [[206, 0, 94], [0, 244, 56], [0, 3, 297]], + "recall": {"openai": 0.6866666667, "google": 0.8133333333, "unknown": 0.99}, + "precision": {"openai": 1.0, "google": 0.9878542510, "unknown": 0.6644295302} + }, + "selection": { + "minimum_calibration_precision": {"openai": 0.99, "google": 0.95}, + "maximum_calibration_unknown_false_positive_rate": 0.01 + } +} diff --git a/docs/synthid-classifiers.md b/docs/synthid-classifiers.md index 2f671cf..2469ade 100644 --- a/docs/synthid-classifiers.md +++ b/docs/synthid-classifiers.md @@ -112,6 +112,40 @@ and a causal marked/unmarked contrast. Mark's same-prompt triples remain an external source/export test, not evidence that the fitted features read the SynthID payload. +### Full-corpus source-pipeline model, 2026-09-16 + +The next experiment trained a compact nonlinear head on the complete available +private corpus, including target-provider exports, photographs, and foreign +generator negatives. The corpus itself is not published. The released +`source-pipeline-mlp.npz` is a derivative artifact containing only learned +parameters, normalization constants, labels, and abstention thresholds; it +contains no images, paths, image hashes, embeddings, or training catalog. + +The model joins the 124-dimensional forensic descriptor with the +768-dimensional phase-free spectral descriptor and applies a +892 -> 128 -> 64 -> 3 GELU MLP. Checkpoint and thresholds were selected on a +separate calibration partition. The 5,369-image hash-disjoint locked test and +Mark's public 900-image originals were not used for architecture, checkpoint, +or threshold selection. + +| Evaluation | Class | Recall | Precision | +| --- | --- | ---: | ---: | +| Locked internal test (5,369 images) | OpenAI | 60.4% | 84.9% | +| Locked internal test (5,369 images) | Google | 57.6% | 87.9% | +| Locked internal test (5,369 images) | Unknown | 98.6% | 93.7% | +| Mark public originals (900 images) | OpenAI | 68.7% | 100.0% | +| Mark public originals (900 images) | Google | 81.3% | 98.8% | +| Mark public originals (900 images) | Unknown/FLUX | 99.0% | 66.4% | + +A 95% resize reduced Google recall on the public set to 1%, and a JPEG +quality-75 round trip made every row abstain. This behavior matches the narrow +claim: the model classifies original-looking source/export pipelines after +metadata removal. It neither decodes nor detects SynthID, and its `unknown` +label does not mean that SynthID is absent. The model card and aggregate +metrics are tracked under +[`docs/source-classify-hf/`](source-classify-hf/README.md). Runtime integration +belongs in the library rather than in the model repository. + Krawetz's Gemini-chat TPR critique is a verifier-quality claim, not a feature we can ship. [Lead Stories, 2026-07](https://leadstories.com/analysis/2026/07/google-gemini-synthid-detector-confuses-results-within-same-chat.html) documented Gemini repeating the first file's SynthID verdict inside a diff --git a/scripts/README.md b/scripts/README.md index 125e49e..ffdde43 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -55,6 +55,7 @@ explicitly names a tracked canonical result. | `watermark_benchmark_real_cohort.py` | Build a publication-cleared, provider-paired and content-stratified real-image cohort. | | `retrain_photo_classify.py` | CPU-retrain the 2026-08-31 photo heads from a sha256-keyed cache pack, no images. | | `publish_photo_classify_hf.py` | Upload the photo-classify card and freeze weights to Hugging Face `wiltodelta/raiw-photo-classify`. Manual; the Action `publish-photo-classify-hf.yml` is the write-token path. | +| `publish_source_classify_hf.py` | Upload the derivative source-pipeline model and its public card to Hugging Face `wiltodelta/raiw-source-classify`. Manual; the Action `publish-source-classify-hf.yml` is the write-token path. | | `verify_engine_selection_fixtures.py` | Verify hashes, dimensions, and prompt pairing in the tracked auto-engine content matrix. | ## Research and diagnostic prototypes @@ -87,5 +88,7 @@ running the sync command without `--check` is the explicit snapshot update. `_plain_console.py` provides plain-text fallbacks for Rich output, and `_text_eval.py` contains normalization helpers shared by text-evaluation scripts. +`hf_publish.py` owns the common authenticated, allowlisted Hugging Face model +repository upload used by the two model publishers. `engine_selection_manifest.py` owns the content-matrix schema and validation shared by its integrity check and the real-image benchmark builder. diff --git a/scripts/hf_publish.py b/scripts/hf_publish.py new file mode 100644 index 0000000..bcfa80b --- /dev/null +++ b/scripts/hf_publish.py @@ -0,0 +1,31 @@ +"""Shared Hugging Face model-repository publication helper.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + + +def publish_model_folder( + stage: Path, + *, + repo_id: str, + token: str, + message: str, + allow_patterns: Iterable[str], +) -> str: + """Upload an explicit file allowlist and return the Hub revision.""" + from huggingface_hub import HfApi, create_repo + + create_repo(repo_id, repo_type="model", exist_ok=True, private=False, token=token) + commit = HfApi(token=token).upload_folder( + folder_path=str(stage), + repo_id=repo_id, + repo_type="model", + commit_message=message, + allow_patterns=list(allow_patterns), + ) + return getattr(commit, "oid", "") or "" diff --git a/scripts/publish_photo_classify_hf.py b/scripts/publish_photo_classify_hf.py index cf17e08..e23ca12 100644 --- a/scripts/publish_photo_classify_hf.py +++ b/scripts/publish_photo_classify_hf.py @@ -24,6 +24,8 @@ import shutil import tempfile from pathlib import Path +from hf_publish import publish_model_folder + log = logging.getLogger(__name__) HUB_REPO = "wiltodelta/raiw-photo-classify" @@ -92,18 +94,13 @@ def stage_weights(dest: Path, src: Path) -> None: def publish(stage: Path, *, token: str, message: str) -> str: - from huggingface_hub import HfApi, create_repo - - create_repo(HUB_REPO, repo_type="model", exist_ok=True, private=False, token=token) - api = HfApi(token=token) - commit = api.upload_folder( - folder_path=str(stage), + return publish_model_folder( + stage, repo_id=HUB_REPO, - repo_type="model", - commit_message=message, + token=token, + message=message, allow_patterns=["README.md", "operating-point.json", *WEIGHT_FILES, GATE_FILE, GATE_STABLE_FILE], ) - return getattr(commit, "oid", "") or "" def main() -> int: diff --git a/scripts/publish_source_classify_hf.py b/scripts/publish_source_classify_hf.py new file mode 100644 index 0000000..55f73e7 --- /dev/null +++ b/scripts/publish_source_classify_hf.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Publish the source-pipeline model to Hugging Face. + +The derivative weight file is downloaded by the GitHub Action from a model +freeze release. The private corpus, feature caches, and training catalog never +enter this repository or the Hub staging directory. +""" + +from __future__ import annotations + +import argparse +import logging +import os +import shutil +import tempfile +from pathlib import Path + +from hf_publish import publish_model_folder + +log = logging.getLogger(__name__) + +HUB_REPO = "wiltodelta/raiw-source-classify" +MODEL_FILE = "source-pipeline-mlp.npz" +REPO_ROOT = Path(__file__).resolve().parents[1] +CARD_DIR = REPO_ROOT / "docs" / "source-classify-hf" +PUBLIC_FILES = ("README.md", "metrics.json", MODEL_FILE) + + +def stage_release(dest: Path, src: Path) -> None: + """Stage the two tracked documents and one derivative model artifact.""" + sources = { + "README.md": CARD_DIR / "README.md", + "metrics.json": CARD_DIR / "metrics.json", + MODEL_FILE: src.expanduser().resolve() / MODEL_FILE, + } + missing = [str(path) for path in sources.values() if not path.is_file()] + if missing: + raise SystemExit(f"missing publication files: {', '.join(missing)}") + dest.mkdir(parents=True, exist_ok=True) + for name, path in sources.items(): + shutil.copy2(path, dest / name) + + +def publish(stage: Path, *, token: str, message: str) -> str: + """Upload the exact staged allowlist and return the Hub revision.""" + return publish_model_folder( + stage, + repo_id=HUB_REPO, + token=token, + message=message, + allow_patterns=PUBLIC_FILES, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--src", type=Path, required=True) + parser.add_argument("--message", default="Publish source-pipeline classifier") + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(message)s") + token = os.environ.get("HF_TOKEN", "").strip() + if not token: + raise SystemExit("HF_TOKEN is not set (write role required)") + with tempfile.TemporaryDirectory(prefix="raiw-source-hf-") as tmp: + stage = Path(tmp) + stage_release(stage, args.src) + revision = publish(stage, token=token, message=args.message) + log.info("published %s revision=%s", HUB_REPO, revision or "(unknown)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_publish_photo_classify_hf.py b/tests/test_publish_photo_classify_hf.py index a0b9d44..060ab39 100644 --- a/tests/test_publish_photo_classify_hf.py +++ b/tests/test_publish_photo_classify_hf.py @@ -2,29 +2,21 @@ from __future__ import annotations -import importlib.util +import sys from pathlib import Path -from typing import Any import pytest - -def _publish() -> Any: - path = Path(__file__).resolve().parents[1] / "scripts" / "publish_photo_classify_hf.py" - spec = importlib.util.spec_from_file_location("publish_photo_classify_hf", path) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module +SCRIPTS = Path(__file__).resolve().parents[1] / "scripts" +sys.path.insert(0, str(SCRIPTS)) +import publish_photo_classify_hf as publish # noqa: E402 def test_hub_id_is_the_photo_classify_repo() -> None: - assert _publish().HUB_REPO == "wiltodelta/raiw-photo-classify" + assert publish.HUB_REPO == "wiltodelta/raiw-photo-classify" def test_publish_freeze_includes_both_clip_runtimes() -> None: - publish = _publish() from remove_ai_watermarks.classify import _WEIGHT_FILES, CLIP_ONNX_FILE, WEIGHTS_ALLOW_PATTERNS assert publish.CLIP_FILE in publish.WEIGHT_FILES @@ -41,7 +33,7 @@ def test_weight_paths_accepts_run1_layout(tmp_path: Path) -> None: run1.mkdir() (run1 / "detector.pt").write_bytes(b"det") (run1 / "provider.pt").write_bytes(b"prov") - found = _publish().weight_paths(tmp_path) + found = publish.weight_paths(tmp_path) assert found["detector.pt"] == run1 / "detector.pt" assert found["clip-l-ft.pt"] == tmp_path / "clip-l-ft.pt" @@ -50,12 +42,12 @@ def test_weight_paths_requires_every_file(tmp_path: Path) -> None: (tmp_path / "clip-l-ft.pt").write_bytes(b"clip") (tmp_path / "clip-l-ft-vision-fp32.onnx").write_bytes(b"onnx") with pytest.raises(SystemExit, match=r"missing probe-weights-clip-l-ft\.npz"): - _publish().weight_paths(tmp_path) + publish.weight_paths(tmp_path) def test_stage_card_copies_tracked_files(tmp_path: Path) -> None: dest = tmp_path / "stage" - _publish().stage_card(dest) + publish.stage_card(dest) assert (dest / "README.md").is_file() assert (dest / "operating-point.json").is_file() text = (dest / "operating-point.json").read_text() diff --git a/tests/test_publish_source_classify_hf.py b/tests/test_publish_source_classify_hf.py new file mode 100644 index 0000000..6c5597c --- /dev/null +++ b/tests/test_publish_source_classify_hf.py @@ -0,0 +1,37 @@ +"""Local staging checks for the source-classifier Hub publisher.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +SCRIPTS = Path(__file__).resolve().parents[1] / "scripts" +sys.path.insert(0, str(SCRIPTS)) +import publish_source_classify_hf as publish # noqa: E402 + + +def test_hub_id_is_the_source_classify_repo() -> None: + assert publish.HUB_REPO == "wiltodelta/raiw-source-classify" + + +def test_stage_release_copies_only_public_artifacts(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "source-pipeline-mlp.npz").write_bytes(b"weights") + destination = tmp_path / "stage" + publish.stage_release(destination, source) + assert sorted(path.name for path in destination.iterdir()) == sorted(publish.PUBLIC_FILES) + + +def test_stage_release_requires_the_model(tmp_path: Path) -> None: + with pytest.raises(SystemExit, match=r"source-pipeline-mlp\.npz"): + publish.stage_release(tmp_path / "stage", tmp_path) + + +def test_card_sha_matches_the_metrics_manifest() -> None: + metrics = json.loads((publish.CARD_DIR / "metrics.json").read_text()) + card = (publish.CARD_DIR / "README.md").read_text() + assert metrics["artifact_sha256"] in card