From 4428aedce6d4da03aca658acd8a16456680139c8 Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Tue, 8 Sep 2026 14:02:25 -0700 Subject: [PATCH] Prefer provider APIs before web oracles --- .claude/rules/development.md | 6 ++++ .claude/skills/provider-oracles/SKILL.md | 13 ++++++++ docs/provider-oracles.md | 26 +++++++++++++-- docs/verification-plan.md | 3 +- scripts/README.md | 2 +- scripts/provider_oracles.py | 40 +++++++++++++++++++++++- scripts/qwen_scrub_prototype.py | 8 +++-- tests/test_provider_oracles.py | 28 +++++++++++++++-- 8 files changed, 115 insertions(+), 11 deletions(-) diff --git a/.claude/rules/development.md b/.claude/rules/development.md index 0ba8bab..22e9ba0 100644 --- a/.claude/rules/development.md +++ b/.claude/rules/development.md @@ -34,6 +34,12 @@ Do not add account or IP rotation, rate-limit failover, or cross-session result Every manual result stays bound to the exact sanitized upload hash and preserves the provider response verbatim. +Provider surface selection is API-first. OpenAI and Microsoft use their official API +adapters before their Web surfaces when the requested media and local configuration are +supported. Google and Meta currently use Web because this tooling has no usable API route +for them. Web remains an explicit second surface, never an automatic retry after an API +refusal, quota response, transport error, or indeterminate verdict. + Google Web oracle automation uses only the user's existing authenticated real Chrome session and explicit `/u/N/` account index. OpenAI, Microsoft, and Meta Web automation uses isolated Playwright contexts; it must never attach to or copy state from the user's diff --git a/.claude/skills/provider-oracles/SKILL.md b/.claude/skills/provider-oracles/SKILL.md index a484ec6..41f2fba 100644 --- a/.claude/skills/provider-oracles/SKILL.md +++ b/.claude/skills/provider-oracles/SKILL.md @@ -18,6 +18,19 @@ Do not optimize candidates adaptively against an external oracle. A retry, different account, or different network is a new explicit operator decision, not automatic fallback. +## Default surface priority + +Prefer an official API adapter whenever that provider, media type, and configured +slot support it. Use Web only when no usable API route exists or the operator +explicitly requests the Web surface as a second check. For OpenAI and Microsoft, +the default plans are API then Web. Google and Meta currently have only usable +Web routes in this tooling. Inspect the order with `plan PROVIDER`. + +The priority is a selection policy, not automatic failover. An API refusal, +quota response, transport error, or indeterminate verdict must be recorded and +reported. Do not submit the same artifact to Web unless the operator separately +authorizes that next surface. + ## Route by surface ### Google Gemini diff --git a/docs/provider-oracles.md b/docs/provider-oracles.md index c76942a..a22f85e 100644 --- a/docs/provider-oracles.md +++ b/docs/provider-oracles.md @@ -11,12 +11,32 @@ through the installed CLI or public Python API. | Surface key | Provider signal | Access | Project evidence | | --- | --- | --- | --- | -| `gemini-web` | Google SynthID | Signed-in Gemini web account | Images and videos | | `openai-api` | OpenAI SynthID | Official Content Provenance API | Images | -| `openai-web` | OpenAI SynthID | Public web verifier | Images | | `microsoft-api` | Microsoft InvisMark | Azure Content Provenance Detection API | Images | -| `microsoft-web` | Microsoft InvisMark | Public Azure AI validation page | Images | +| `gemini-web` | Google SynthID | Signed-in Gemini web account | Images and videos | | `meta-web` | Meta Content Seal | Anonymous Meta identification page | Images | +| `microsoft-web` | Microsoft InvisMark | Public Azure AI validation page | Images | +| `openai-web` | OpenAI SynthID | Public web verifier | Images | + +## Surface priority + +The default selection order is API first, then Web. OpenAI therefore plans +`openai-api` before `openai-web`, and Microsoft plans `microsoft-api` before +`microsoft-web`. Google and Meta currently have no usable API adapter in this +tooling, so their Web surfaces are primary rather than fallback routes. + +Inspect the machine-readable order before preparing a batch: + +```bash +uv run python scripts/provider_oracles.py plan openai --json +uv run python scripts/provider_oracles.py plan microsoft +``` + +API-first is a selection policy, not automatic failover. Use Web only when the +provider, requested media, or local configuration has no usable API route, or +when the operator explicitly requests a second surface. An API refusal, quota +response, transport failure, or indeterminate verdict is recorded as-is and +never silently triggers a Web upload. The old Google SynthID Detector portal does not work. On 2026-09-07, the official `labs.google/synthid` entry point redirected a signed-in browser to a diff --git a/docs/verification-plan.md b/docs/verification-plan.md index 7605396..b466fbe 100644 --- a/docs/verification-plan.md +++ b/docs/verification-plan.md @@ -258,7 +258,8 @@ and implemented by `scripts/provider_oracles.py`. Proprietary watermark removal cannot be verified locally by design -- no public decoder exists. Each vendor has its own oracle and it covers only that vendor's content: -`openai.com/verify` for OpenAI, the Gemini app for Google, and Microsoft's +the Content Provenance API first and `openai.com/verify` as an explicit fallback for +OpenAI, the Gemini app for Google, and Microsoft's [Content Provenance Detection API](https://learn.microsoft.com/en-us/azure/ai-services/content-safety/how-to/how-to-provenance-detection) for InvisMark. The Microsoft API reports `Watermark` and `C2PA` separately. It therefore needs a pixel-identical metadata-stripped control: the control must lose `C2PA` while diff --git a/scripts/README.md b/scripts/README.md index 72f0e48..2c85870 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -18,7 +18,7 @@ explicitly names a tracked canonical result. | `pill_gate_audit.py` | Measure the Jimeng pill detector on the product path. | | `provider_oracle_web.py` | Implement isolated Playwright submissions for the unified provider-oracle runner. | | `provider_oracle_microsoft.py` | Call Microsoft's Content Provenance API with hash-bound public media and Azure CLI authentication. | -| `provider_oracles.py` | Drive and validate hash-bound Google, OpenAI, Microsoft, and Meta oracle checks. | +| `provider_oracles.py` | Plan API-first and drive hash-bound Google, OpenAI, Microsoft, and Meta oracle checks. | | `real_examples_e2e.py` | Run end-to-end confidence checks over local examples. | | `record_parity_audit.py` | Compare record-based and file-based identification. | | `resource_ceilings.py` | Measure peak RSS and runtime for fill backends. | diff --git a/scripts/provider_oracles.py b/scripts/provider_oracles.py index 56f8685..4c998c1 100644 --- a/scripts/provider_oracles.py +++ b/scripts/provider_oracles.py @@ -257,6 +257,24 @@ SURFACES: dict[str, OracleSurface] = { PROVIDER_KEYS = frozenset(surface.provider_key for surface in SURFACES.values()) +def _surface_order(surface: OracleSurface) -> tuple[bool, str]: + """Sort API surfaces before Web surfaces, then stabilize by key.""" + return surface.automation != "api", surface.key + + +def provider_plan(provider_key: str) -> list[OracleSurface]: + """Return one provider's surfaces in the API-first operator order.""" + if provider_key not in PROVIDER_KEYS: + raise ValueError(f"unknown provider oracle: {provider_key}") + surfaces = [surface for surface in SURFACES.values() if surface.provider_key == provider_key] + return sorted(surfaces, key=_surface_order) + + +def ordered_surfaces() -> list[OracleSurface]: + """Return the catalog with every API surface before every Web surface.""" + return sorted(SURFACES.values(), key=_surface_order) + + def _inside(path: Path, parent: Path) -> bool: """Return whether resolved PATH is inside resolved PARENT.""" try: @@ -820,7 +838,7 @@ def cli() -> None: @click.option("--json", "as_json", is_flag=True, help="Emit machine-readable JSON.") def list_oracles(as_json: bool) -> None: """List provider scope, supported media, and verification surfaces.""" - definitions = [SURFACES[key] for key in sorted(SURFACES)] + definitions = ordered_surfaces() if as_json: click.echo(json.dumps([definition.to_dict() for definition in definitions], indent=2, sort_keys=True)) return @@ -829,6 +847,26 @@ def list_oracles(as_json: bool) -> None: click.echo(f"{definition.key}\t{definition.automation}\t{media}\t{definition.surface}\t{definition.url}") +@cli.command() +@click.argument("provider", type=click.Choice(sorted(PROVIDER_KEYS))) +@click.option("--json", "as_json", is_flag=True, help="Emit machine-readable JSON.") +def plan(provider: str, as_json: bool) -> None: + """Show the API-first surface order for one provider.""" + definitions = provider_plan(provider) + rows = [] + for index, definition in enumerate(definitions): + row = definition.to_dict() + row["preference"] = "primary" if index == 0 else "fallback" + rows.append(row) + if as_json: + click.echo(json.dumps(rows, indent=2, sort_keys=True)) + return + for row in rows: + click.echo(f"{row['preference']}\t{row['key']}\t{row['automation']}\t{row['surface']}") + if len(rows) > 1: + click.echo("Web fallback requires a separate operator decision; failures do not trigger it automatically.") + + @cli.command() @click.argument("surface", type=click.Choice(sorted(SURFACES))) @click.option("--slots", "slots_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)) diff --git a/scripts/qwen_scrub_prototype.py b/scripts/qwen_scrub_prototype.py index ecae2b5..c4faa15 100644 --- a/scripts/qwen_scrub_prototype.py +++ b/scripts/qwen_scrub_prototype.py @@ -23,7 +23,8 @@ invisible pipeline onto Qwen-Image-Edit). It is DELIBERATELY standalone: * Qwen-Image is ~20B, so it needs a real GPU (CUDA) -- it will not fit on MPS. Run (on a GPU box / Modal), then eyeball the outputs AND submit them to the -matching oracle (openai.com/verify for OpenAI, the Gemini app for Google): +matching oracle (the Content Provenance API first for OpenAI, the Gemini app for +Google; inspect the current order with ``provider_oracles.py plan PROVIDER``): uv run scripts/qwen_scrub_prototype.py INPUT.png -o out/ --strengths 0.1,0.2,0.3,0.4 @@ -119,8 +120,9 @@ def main( log.info( "\nDone. Eyeball text/face fidelity, then submit each output to the matching oracle " - "(openai.com/verify / Gemini app). The smallest strength that clears the oracle while " - "keeping fidelity is the number to compare against the SDXL floors (OpenAI 0.10 / Google 0.15)." + "(OpenAI Content Provenance API first / Gemini app). The smallest strength that clears " + "the oracle while keeping fidelity is the number to compare against the SDXL floors " + "(OpenAI 0.10 / Google 0.15)." ) diff --git a/tests/test_provider_oracles.py b/tests/test_provider_oracles.py index d4203bf..2ff47cc 100644 --- a/tests/test_provider_oracles.py +++ b/tests/test_provider_oracles.py @@ -42,16 +42,40 @@ def test_list_json_exposes_scope_and_surface() -> None: assert result.exit_code == 0, result.output payload = json.loads(result.output) assert [row["key"] for row in payload] == [ + "microsoft-api", + "openai-api", "gemini-web", "meta-web", - "microsoft-api", "microsoft-web", - "openai-api", "openai-web", ] assert next(row for row in payload if row["key"] == "meta-web")["signal_family"] == "content_seal" +@pytest.mark.parametrize( + ("provider", "expected"), + [ + ("google", ["gemini-web"]), + ("meta", ["meta-web"]), + ("microsoft", ["microsoft-api", "microsoft-web"]), + ("openai", ["openai-api", "openai-web"]), + ], +) +def test_provider_plan_prefers_api_before_web(provider: str, expected: list[str]) -> None: + assert [surface.key for surface in oracles.provider_plan(provider)] == expected + + +def test_plan_json_marks_web_as_an_explicit_fallback_when_api_exists() -> None: + result = CliRunner().invoke(oracles.cli, ["plan", "openai", "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert [(row["key"], row["preference"]) for row in payload] == [ + ("openai-api", "primary"), + ("openai-web", "fallback"), + ] + + def test_slots_support_multiple_accounts_and_networks(tmp_path: Path) -> None: config = tmp_path / "slots.json" config.write_text(