From e3bbe88155809866042c245aba2ea573e61b1088 Mon Sep 17 00:00:00 2001 From: Victor Kuznetsov Date: Tue, 22 Sep 2026 08:11:48 -0700 Subject: [PATCH] Authenticate C2PA registry-sync GitHub API calls to fix CI rate-limiting scripts/sync_c2pa_soft_bindings.py --check hits api.github.com to resolve the latest upstream commit SHA, unauthenticated. That endpoint's 60/hour rate limit is per source IP, which GitHub Actions runners exhaust fast since many workflows share the runner IP pool -- this was failing the lint job with "403: rate limit exceeded" on every push regardless of whether the change touched anything related. Send an Authorization: Bearer header built from GITHUB_TOKEN/GH_TOKEN when set, scoped to api.github.com only (raw.githubusercontent.com is a separate, much higher-limit CDN and doesn't need it). Wire the workflow's own GITHUB_TOKEN secret into that step's env so CI actually has one to send, raising the limit to 5000/hour. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/test.yml | 2 ++ scripts/sync_c2pa_soft_bindings.py | 14 +++++++++++- tests/test_c2pa_registry_sync.py | 34 ++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c03e85a..d370fe6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,8 @@ jobs: - name: Sync dev environment run: uv sync --frozen --extra dev - name: Check C2PA soft-binding registry snapshot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: uv run python scripts/sync_c2pa_soft_bindings.py --check - name: Ruff lint run: uv run ruff check diff --git a/scripts/sync_c2pa_soft_bindings.py b/scripts/sync_c2pa_soft_bindings.py index 046cd11..f5152c0 100644 --- a/scripts/sync_c2pa_soft_bindings.py +++ b/scripts/sync_c2pa_soft_bindings.py @@ -7,6 +7,7 @@ import argparse import hashlib import json import logging +import os import re import sys from dataclasses import dataclass @@ -54,7 +55,18 @@ class RegistryRow: def _read_bytes(source: str) -> bytes: parsed = urlparse(source) if parsed.scheme in {"http", "https"}: - request = Request(source, headers={"User-Agent": "remove-ai-watermarks-c2pa-registry-sync"}) + headers = {"User-Agent": "remove-ai-watermarks-c2pa-registry-sync"} + # api.github.com rate-limits unauthenticated requests to 60/hour PER SOURCE IP, + # which CI runners exhaust fast since many workflows across many repos share + # GitHub's runner IP pool -- a 403 "rate limit exceeded" here is that, not an + # outage. A GITHUB_TOKEN (Actions injects one into every run; export it as an + # env var to reach this process) raises the same endpoint to 5000/hour. + # raw.githubusercontent.com is a separate, much higher-limit CDN and does not + # need this. + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token and parsed.hostname == "api.github.com": + headers["Authorization"] = f"Bearer {token}" + request = Request(source, headers=headers) with urlopen(request, timeout=30) as response: return response.read() return Path(source).read_bytes() diff --git a/tests/test_c2pa_registry_sync.py b/tests/test_c2pa_registry_sync.py index 1b586e6..74635ac 100644 --- a/tests/test_c2pa_registry_sync.py +++ b/tests/test_c2pa_registry_sync.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +from unittest.mock import MagicMock, patch import pytest @@ -112,3 +113,36 @@ def test_packaged_snapshot_preserves_published_resolution_apis(): f"upstream softbinding-algorithm-list resolution-API set changed: {sorted(with_apis)}; " "revisit docs/c2pa-resolution-research.md before regenerating" ) + + +class TestReadBytesAuth: + """api.github.com rate-limits unauthenticated requests to 60/hour PER SOURCE IP, + which CI runners exhaust fast since many workflows share GitHub's runner IP pool + (observed: --check failing with 403 on every platform in one push). A GITHUB_TOKEN + raises that to 5000/hour; these pin that the header is sent to api.github.com when + a token is present, withheld when it is not, and never sent to the unrelated raw + CDN host even with a token set.""" + + def _urlopen_headers(self, source: str) -> dict[str, str]: + response = MagicMock() + response.__enter__.return_value.read.return_value = b"{}" + with patch("scripts.sync_c2pa_soft_bindings.urlopen", return_value=response) as mock_urlopen: + sync._read_bytes(source) + request = mock_urlopen.call_args[0][0] + return dict(request.headers) + + def test_sends_bearer_token_to_github_api(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("GITHUB_TOKEN", "test-token") + headers = self._urlopen_headers("https://api.github.com/repos/example/example/commits") + assert headers.get("Authorization") == "Bearer test-token" + + def test_omits_authorization_without_a_token(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + headers = self._urlopen_headers("https://api.github.com/repos/example/example/commits") + assert "Authorization" not in headers + + def test_never_sends_the_token_to_the_raw_cdn_host(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("GITHUB_TOKEN", "test-token") + headers = self._urlopen_headers("https://raw.githubusercontent.com/example/example/main/x.json") + assert "Authorization" not in headers