mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-14 13:30:55 +02:00
Fix output preservation, research validation, and documentation drift
Repair image and metadata publication contracts, detection routing, research artifact validation, release verification, and the published agent skill. Refresh the dependency lock and reconcile documentation with verified behavior. Add regression coverage and record the completed audit and corrected VideoSeal measurements. Preserve the existing Accelerate and Lightning versions. Validation: Ruff and Pyright pass; 2358 tests pass with 3 skipped. The maintenance gate exits 2 on the documented Accelerate and Lightning security findings.
This commit is contained in:
101 files changed
+3747
-1202
No files matched your search
@@ -13,8 +13,8 @@ CRASH TOLERANCE IS NOT OPTIONAL AT THIS SCALE
|
||||
because results were only written at the end). So this script:
|
||||
* writes every result to JSONL as it arrives -- a kill never costs more than a batch;
|
||||
* is resumable -- an interrupted run skips what it already recorded;
|
||||
* runs a FRESH pool per batch with a timeout, so one poisoned file costs one batch,
|
||||
and that batch is retried serially to find and record the offender.
|
||||
* runs each image in a fresh interpreter with a per-image timeout and bounded
|
||||
concurrency; a poisoned file is recorded without retrying it in the parent.
|
||||
|
||||
Treat input datasets as sensitive and read-only, and keep output gitignored.
|
||||
|
||||
@@ -29,12 +29,12 @@ import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from concurrent.futures import TimeoutError as FutureTimeout
|
||||
from concurrent.futures.process import BrokenProcessPool
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from _isolated_image_workers import run_batch
|
||||
|
||||
# The package's own format set. An inlined copy here silently skipped .heif, which
|
||||
# CLAUDE.md documents as supported.
|
||||
@@ -60,20 +60,8 @@ def _one(path: str) -> dict[str, object]:
|
||||
|
||||
|
||||
def _run_batch(batch: list[str], jobs: int, timeout: int) -> list[dict[str, object]]:
|
||||
"""One batch in a fresh pool. On a native worker crash or timeout, retry serially so
|
||||
the poisoned file is identified and recorded instead of stalling the whole sweep."""
|
||||
try:
|
||||
with ProcessPoolExecutor(max_workers=jobs) as ex:
|
||||
futs = {ex.submit(_one, p): p for p in batch}
|
||||
return [f.result() for f in as_completed(futs, timeout=timeout)]
|
||||
except (FutureTimeout, BrokenProcessPool, OSError, RuntimeError):
|
||||
out: list[dict[str, object]] = []
|
||||
for p in batch:
|
||||
try:
|
||||
out.append(_one(p))
|
||||
except BaseException: # a native crash here kills only this file, not the sweep
|
||||
out.append({"path": p, "keys": [], "status": "crashed"})
|
||||
return out
|
||||
"""Bound each image in an independent interpreter, including native failures."""
|
||||
return run_batch(Path(__file__), batch, jobs=jobs, timeout=timeout)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -81,7 +69,7 @@ def main() -> None:
|
||||
ap.add_argument("--jobs", type=int, default=max(1, (os.cpu_count() or 4) - 2))
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
ap.add_argument("--batch", type=int, default=200)
|
||||
ap.add_argument("--timeout", type=int, default=600, help="seconds per batch before falling back to serial")
|
||||
ap.add_argument("--timeout", type=int, default=600, help="seconds per image before terminating its isolated worker")
|
||||
ap.add_argument("--out", type=Path, default=OUT)
|
||||
ap.add_argument("--restart", action="store_true")
|
||||
a = ap.parse_args()
|
||||
|
||||
Reference in new issue
Block a user