mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-10-14 05:20:46 +02:00
Authenticate C2PA registry-sync GitHub API calls to fix CI rate-limiting
scripts/sync_c2pa_soft_bindings.py --check hits api.github.com to resolve the latest upstream commit SHA, unauthenticated. That endpoint's 60/hour rate limit is per source IP, which GitHub Actions runners exhaust fast since many workflows share the runner IP pool -- this was failing the lint job with "403: rate limit exceeded" on every push regardless of whether the change touched anything related. Send an Authorization: Bearer header built from GITHUB_TOKEN/GH_TOKEN when set, scoped to api.github.com only (raw.githubusercontent.com is a separate, much higher-limit CDN and doesn't need it). Wire the workflow's own GITHUB_TOKEN secret into that step's env so CI actually has one to send, raising the limit to 5000/hour. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
be8f24c7cb
commit
e3bbe88155
3 files changed
+49
-1
No files matched your search
@@ -21,6 +21,8 @@ jobs:
|
||||
- name: Sync dev environment
|
||||
run: uv sync --frozen --extra dev
|
||||
- name: Check C2PA soft-binding registry snapshot
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: uv run python scripts/sync_c2pa_soft_bindings.py --check
|
||||
- name: Ruff lint
|
||||
run: uv run ruff check
|
||||
|
||||
@@ -7,6 +7,7 @@ import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
@@ -54,7 +55,18 @@ class RegistryRow:
|
||||
def _read_bytes(source: str) -> bytes:
|
||||
parsed = urlparse(source)
|
||||
if parsed.scheme in {"http", "https"}:
|
||||
request = Request(source, headers={"User-Agent": "remove-ai-watermarks-c2pa-registry-sync"})
|
||||
headers = {"User-Agent": "remove-ai-watermarks-c2pa-registry-sync"}
|
||||
# api.github.com rate-limits unauthenticated requests to 60/hour PER SOURCE IP,
|
||||
# which CI runners exhaust fast since many workflows across many repos share
|
||||
# GitHub's runner IP pool -- a 403 "rate limit exceeded" here is that, not an
|
||||
# outage. A GITHUB_TOKEN (Actions injects one into every run; export it as an
|
||||
# env var to reach this process) raises the same endpoint to 5000/hour.
|
||||
# raw.githubusercontent.com is a separate, much higher-limit CDN and does not
|
||||
# need this.
|
||||
token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
|
||||
if token and parsed.hostname == "api.github.com":
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
request = Request(source, headers=headers)
|
||||
with urlopen(request, timeout=30) as response:
|
||||
return response.read()
|
||||
return Path(source).read_bytes()
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -112,3 +113,36 @@ def test_packaged_snapshot_preserves_published_resolution_apis():
|
||||
f"upstream softbinding-algorithm-list resolution-API set changed: {sorted(with_apis)}; "
|
||||
"revisit docs/c2pa-resolution-research.md before regenerating"
|
||||
)
|
||||
|
||||
|
||||
class TestReadBytesAuth:
|
||||
"""api.github.com rate-limits unauthenticated requests to 60/hour PER SOURCE IP,
|
||||
which CI runners exhaust fast since many workflows share GitHub's runner IP pool
|
||||
(observed: --check failing with 403 on every platform in one push). A GITHUB_TOKEN
|
||||
raises that to 5000/hour; these pin that the header is sent to api.github.com when
|
||||
a token is present, withheld when it is not, and never sent to the unrelated raw
|
||||
CDN host even with a token set."""
|
||||
|
||||
def _urlopen_headers(self, source: str) -> dict[str, str]:
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.read.return_value = b"{}"
|
||||
with patch("scripts.sync_c2pa_soft_bindings.urlopen", return_value=response) as mock_urlopen:
|
||||
sync._read_bytes(source)
|
||||
request = mock_urlopen.call_args[0][0]
|
||||
return dict(request.headers)
|
||||
|
||||
def test_sends_bearer_token_to_github_api(self, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("GITHUB_TOKEN", "test-token")
|
||||
headers = self._urlopen_headers("https://api.github.com/repos/example/example/commits")
|
||||
assert headers.get("Authorization") == "Bearer test-token"
|
||||
|
||||
def test_omits_authorization_without_a_token(self, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
|
||||
monkeypatch.delenv("GH_TOKEN", raising=False)
|
||||
headers = self._urlopen_headers("https://api.github.com/repos/example/example/commits")
|
||||
assert "Authorization" not in headers
|
||||
|
||||
def test_never_sends_the_token_to_the_raw_cdn_host(self, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("GITHUB_TOKEN", "test-token")
|
||||
headers = self._urlopen_headers("https://raw.githubusercontent.com/example/example/main/x.json")
|
||||
assert "Authorization" not in headers
|
||||
Reference in new issue
Block a user