mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-14 13:30:55 +02:00
Publish source classifier model card
This commit is contained in:
12 files changed
+352
-26
No files matched your search
@@ -55,6 +55,7 @@ explicitly names a tracked canonical result.
|
||||
| `watermark_benchmark_real_cohort.py` | Build a publication-cleared, provider-paired and content-stratified real-image cohort. |
|
||||
| `retrain_photo_classify.py` | CPU-retrain the 2026-08-31 photo heads from a sha256-keyed cache pack, no images. |
|
||||
| `publish_photo_classify_hf.py` | Upload the photo-classify card and freeze weights to Hugging Face `wiltodelta/raiw-photo-classify`. Manual; the Action `publish-photo-classify-hf.yml` is the write-token path. |
|
||||
| `publish_source_classify_hf.py` | Upload the derivative source-pipeline model and its public card to Hugging Face `wiltodelta/raiw-source-classify`. Manual; the Action `publish-source-classify-hf.yml` is the write-token path. |
|
||||
| `verify_engine_selection_fixtures.py` | Verify hashes, dimensions, and prompt pairing in the tracked auto-engine content matrix. |
|
||||
|
||||
## Research and diagnostic prototypes
|
||||
@@ -87,5 +88,7 @@ running the sync command without `--check` is the explicit snapshot update.
|
||||
|
||||
`_plain_console.py` provides plain-text fallbacks for Rich output, and
|
||||
`_text_eval.py` contains normalization helpers shared by text-evaluation scripts.
|
||||
`hf_publish.py` owns the common authenticated, allowlisted Hugging Face model
|
||||
repository upload used by the two model publishers.
|
||||
`engine_selection_manifest.py` owns the content-matrix schema and validation
|
||||
shared by its integrity check and the real-image benchmark builder.
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Shared Hugging Face model-repository publication helper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def publish_model_folder(
|
||||
stage: Path,
|
||||
*,
|
||||
repo_id: str,
|
||||
token: str,
|
||||
message: str,
|
||||
allow_patterns: Iterable[str],
|
||||
) -> str:
|
||||
"""Upload an explicit file allowlist and return the Hub revision."""
|
||||
from huggingface_hub import HfApi, create_repo
|
||||
|
||||
create_repo(repo_id, repo_type="model", exist_ok=True, private=False, token=token)
|
||||
commit = HfApi(token=token).upload_folder(
|
||||
folder_path=str(stage),
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
commit_message=message,
|
||||
allow_patterns=list(allow_patterns),
|
||||
)
|
||||
return getattr(commit, "oid", "") or ""
|
||||
@@ -24,6 +24,8 @@ import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from hf_publish import publish_model_folder
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
HUB_REPO = "wiltodelta/raiw-photo-classify"
|
||||
@@ -92,18 +94,13 @@ def stage_weights(dest: Path, src: Path) -> None:
|
||||
|
||||
|
||||
def publish(stage: Path, *, token: str, message: str) -> str:
|
||||
from huggingface_hub import HfApi, create_repo
|
||||
|
||||
create_repo(HUB_REPO, repo_type="model", exist_ok=True, private=False, token=token)
|
||||
api = HfApi(token=token)
|
||||
commit = api.upload_folder(
|
||||
folder_path=str(stage),
|
||||
return publish_model_folder(
|
||||
stage,
|
||||
repo_id=HUB_REPO,
|
||||
repo_type="model",
|
||||
commit_message=message,
|
||||
token=token,
|
||||
message=message,
|
||||
allow_patterns=["README.md", "operating-point.json", *WEIGHT_FILES, GATE_FILE, GATE_STABLE_FILE],
|
||||
)
|
||||
return getattr(commit, "oid", "") or ""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Publish the source-pipeline model to Hugging Face.
|
||||
|
||||
The derivative weight file is downloaded by the GitHub Action from a model
|
||||
freeze release. The private corpus, feature caches, and training catalog never
|
||||
enter this repository or the Hub staging directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from hf_publish import publish_model_folder
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
HUB_REPO = "wiltodelta/raiw-source-classify"
|
||||
MODEL_FILE = "source-pipeline-mlp.npz"
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
CARD_DIR = REPO_ROOT / "docs" / "source-classify-hf"
|
||||
PUBLIC_FILES = ("README.md", "metrics.json", MODEL_FILE)
|
||||
|
||||
|
||||
def stage_release(dest: Path, src: Path) -> None:
|
||||
"""Stage the two tracked documents and one derivative model artifact."""
|
||||
sources = {
|
||||
"README.md": CARD_DIR / "README.md",
|
||||
"metrics.json": CARD_DIR / "metrics.json",
|
||||
MODEL_FILE: src.expanduser().resolve() / MODEL_FILE,
|
||||
}
|
||||
missing = [str(path) for path in sources.values() if not path.is_file()]
|
||||
if missing:
|
||||
raise SystemExit(f"missing publication files: {', '.join(missing)}")
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for name, path in sources.items():
|
||||
shutil.copy2(path, dest / name)
|
||||
|
||||
|
||||
def publish(stage: Path, *, token: str, message: str) -> str:
|
||||
"""Upload the exact staged allowlist and return the Hub revision."""
|
||||
return publish_model_folder(
|
||||
stage,
|
||||
repo_id=HUB_REPO,
|
||||
token=token,
|
||||
message=message,
|
||||
allow_patterns=PUBLIC_FILES,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--src", type=Path, required=True)
|
||||
parser.add_argument("--message", default="Publish source-pipeline classifier")
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
token = os.environ.get("HF_TOKEN", "").strip()
|
||||
if not token:
|
||||
raise SystemExit("HF_TOKEN is not set (write role required)")
|
||||
with tempfile.TemporaryDirectory(prefix="raiw-source-hf-") as tmp:
|
||||
stage = Path(tmp)
|
||||
stage_release(stage, args.src)
|
||||
revision = publish(stage, token=token, message=args.message)
|
||||
log.info("published %s revision=%s", HUB_REPO, revision or "(unknown)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user