Publish source classifier model card

This commit is contained in:
Victor Kuznetsov committed 2026-09-15 22:36:34 -07:00
1 parent cf30b4f95a
commit 4795149d34
12 files changed
+352 -26

No files matched your search

+3
View File
@@ -55,6 +55,7 @@ explicitly names a tracked canonical result.
| `watermark_benchmark_real_cohort.py` | Build a publication-cleared, provider-paired and content-stratified real-image cohort. |
| `retrain_photo_classify.py` | CPU-retrain the 2026-08-31 photo heads from a sha256-keyed cache pack, no images. |
| `publish_photo_classify_hf.py` | Upload the photo-classify card and freeze weights to Hugging Face `wiltodelta/raiw-photo-classify`. Manual; the Action `publish-photo-classify-hf.yml` is the write-token path. |
| `publish_source_classify_hf.py` | Upload the derivative source-pipeline model and its public card to Hugging Face `wiltodelta/raiw-source-classify`. Manual; the Action `publish-source-classify-hf.yml` is the write-token path. |
| `verify_engine_selection_fixtures.py` | Verify hashes, dimensions, and prompt pairing in the tracked auto-engine content matrix. |
## Research and diagnostic prototypes
@@ -87,5 +88,7 @@ running the sync command without `--check` is the explicit snapshot update.
`_plain_console.py` provides plain-text fallbacks for Rich output, and
`_text_eval.py` contains normalization helpers shared by text-evaluation scripts.
`hf_publish.py` owns the common authenticated, allowlisted Hugging Face model
repository upload used by the two model publishers.
`engine_selection_manifest.py` owns the content-matrix schema and validation
shared by its integrity check and the real-image benchmark builder.
+31
View File
@@ -0,0 +1,31 @@
"""Shared Hugging Face model-repository publication helper."""
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Iterable
from pathlib import Path
def publish_model_folder(
stage: Path,
*,
repo_id: str,
token: str,
message: str,
allow_patterns: Iterable[str],
) -> str:
"""Upload an explicit file allowlist and return the Hub revision."""
from huggingface_hub import HfApi, create_repo
create_repo(repo_id, repo_type="model", exist_ok=True, private=False, token=token)
commit = HfApi(token=token).upload_folder(
folder_path=str(stage),
repo_id=repo_id,
repo_type="model",
commit_message=message,
allow_patterns=list(allow_patterns),
)
return getattr(commit, "oid", "") or ""
+6 -9
View File
@@ -24,6 +24,8 @@ import shutil
import tempfile
from pathlib import Path
from hf_publish import publish_model_folder
log = logging.getLogger(__name__)
HUB_REPO = "wiltodelta/raiw-photo-classify"
@@ -92,18 +94,13 @@ def stage_weights(dest: Path, src: Path) -> None:
def publish(stage: Path, *, token: str, message: str) -> str:
from huggingface_hub import HfApi, create_repo
create_repo(HUB_REPO, repo_type="model", exist_ok=True, private=False, token=token)
api = HfApi(token=token)
commit = api.upload_folder(
folder_path=str(stage),
return publish_model_folder(
stage,
repo_id=HUB_REPO,
repo_type="model",
commit_message=message,
token=token,
message=message,
allow_patterns=["README.md", "operating-point.json", *WEIGHT_FILES, GATE_FILE, GATE_STABLE_FILE],
)
return getattr(commit, "oid", "") or ""
def main() -> int:
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""Publish the source-pipeline model to Hugging Face.
The derivative weight file is downloaded by the GitHub Action from a model
freeze release. The private corpus, feature caches, and training catalog never
enter this repository or the Hub staging directory.
"""
from __future__ import annotations
import argparse
import logging
import os
import shutil
import tempfile
from pathlib import Path
from hf_publish import publish_model_folder
log = logging.getLogger(__name__)
HUB_REPO = "wiltodelta/raiw-source-classify"
MODEL_FILE = "source-pipeline-mlp.npz"
REPO_ROOT = Path(__file__).resolve().parents[1]
CARD_DIR = REPO_ROOT / "docs" / "source-classify-hf"
PUBLIC_FILES = ("README.md", "metrics.json", MODEL_FILE)
def stage_release(dest: Path, src: Path) -> None:
"""Stage the two tracked documents and one derivative model artifact."""
sources = {
"README.md": CARD_DIR / "README.md",
"metrics.json": CARD_DIR / "metrics.json",
MODEL_FILE: src.expanduser().resolve() / MODEL_FILE,
}
missing = [str(path) for path in sources.values() if not path.is_file()]
if missing:
raise SystemExit(f"missing publication files: {', '.join(missing)}")
dest.mkdir(parents=True, exist_ok=True)
for name, path in sources.items():
shutil.copy2(path, dest / name)
def publish(stage: Path, *, token: str, message: str) -> str:
"""Upload the exact staged allowlist and return the Hub revision."""
return publish_model_folder(
stage,
repo_id=HUB_REPO,
token=token,
message=message,
allow_patterns=PUBLIC_FILES,
)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--src", type=Path, required=True)
parser.add_argument("--message", default="Publish source-pipeline classifier")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(message)s")
token = os.environ.get("HF_TOKEN", "").strip()
if not token:
raise SystemExit("HF_TOKEN is not set (write role required)")
with tempfile.TemporaryDirectory(prefix="raiw-source-hf-") as tmp:
stage = Path(tmp)
stage_release(stage, args.src)
revision = publish(stage, token=token, message=args.message)
log.info("published %s revision=%s", HUB_REPO, revision or "(unknown)")
return 0
if __name__ == "__main__":
raise SystemExit(main())