mirror of
https://github.com/wiltodelta/remove-ai-watermarks.git
synced 2026-08-19 12:07:13 +02:00
Cover QuickTime TC260 placements, Luma AI tags, and MiniMax video provenance
This commit is contained in:
@@ -107,7 +107,9 @@ The `video metadata` command does not transcode video or audio streams. Unlike
|
||||
the image command above, when `-o` is omitted it writes `<source>_clean` and
|
||||
preserves the original. MP4 and MOV
|
||||
inspection includes the native TC260 `AIGC` tag in
|
||||
`moov.udta.meta.keys/ilst`, including a `moov` placed after the media payload.
|
||||
`moov.udta.meta.keys/ilst`, including a `moov` placed after the media payload,
|
||||
plus the QuickTime-form `meta` variants Doubao's iOS export writes (a bare
|
||||
`meta` box as a direct `moov` child, and a keyless `hdlr=mdir` metadata list).
|
||||
MKV and WebM inspection reads the normative
|
||||
`Segment.Tags.Tag.SimpleTag` placement. AVI uses `LIST/INFO/AIGC`, while FLV
|
||||
uses `script.onMetaData.AIGC`. The non-ISOBMFF formats are remuxed with stream
|
||||
|
||||
+3
-1
@@ -229,7 +229,9 @@ Supported containers are MP4, MOV, M4V, WebM, MKV, AVI, and FLV. The operation
|
||||
delegates to the same verified metadata scanner and stripper as the generic
|
||||
`metadata` command, so detection and removal stay in parity. Video and audio
|
||||
streams are not transcoded. For MP4 and MOV, this includes the native TC260
|
||||
`AIGC` key and JSON value stored in `moov.udta.meta.keys/ilst`. The inspector
|
||||
`AIGC` key and JSON value stored in `moov.udta.meta.keys/ilst`, plus the
|
||||
QuickTime-form `meta` variants Doubao's iOS export writes (a bare `meta` box
|
||||
as a direct `moov` child, and a keyless `hdlr=mdir` metadata list). The inspector
|
||||
seeks past a large `mdat` to find a tail `moov`. Removal stream-copies the
|
||||
container in bounded chunks, converts supported top-level provenance boxes to
|
||||
same-size `free` boxes, and blanks the TC260 key/value in place. Box sizes,
|
||||
|
||||
@@ -12,6 +12,10 @@ Read this reference for environment setup, dependency recovery, CI behavior, and
|
||||
|
||||
The optional TrustMark decoder downloads weights into its installed package directory. After pruning that extra, a leftover weights directory can make availability checks see an empty namespace package. If Pyright reports an unknown `TrustMark` import and `find_spec("trustmark")` returns a loader-less spec, remove that regenerable remnant from the active virtual environment and resync.
|
||||
|
||||
### Known security-gate blocks
|
||||
|
||||
`maintain.sh` fails while PyPI ships no fixed release for a transitive CVE. Current case (triaged 2026-08-17): `lightning` 2.6.5, pulled only by the optional `trustmark` extra, carries PYSEC-2026-3624 (RCE via `load_from_checkpoint` on an attacker-crafted checkpoint). The vulnerable API is unreachable here, because the TrustMark decoder loads only its own pinned weights downloaded from the TrustMark release, never a user-supplied checkpoint. The upstream fix is merged but unreleased, and ignores are never added, so run and report the remaining core checks (Ruff, Pyright, tests) separately until a fixed `lightning` release lands; bump it with `uv lock --upgrade-package lightning` as soon as one does.
|
||||
|
||||
## CI
|
||||
|
||||
`.github/workflows/test.yml` runs Ruff, a test matrix over every supported Python
|
||||
|
||||
@@ -158,7 +158,14 @@ Native MP4/MOV TC260 labels follow TC260-PG-20257A:
|
||||
`moov.udta.meta.keys` maps an `AIGC` key to a raw JSON value in `ilst`.
|
||||
[`_internal/isobmff.py`](../src/remove_ai_watermarks/_internal/isobmff.py) walks those
|
||||
nested boxes by seeking, so detection reaches a tail `moov` without reading the
|
||||
preceding `mdat`. The MP4/MOV/M4V/M4A removal path first validates the top-level
|
||||
preceding `mdat`. Two Doubao iOS variants sit outside that normative placement
|
||||
and are covered by the same walker (2026-08-17 corpus findings, both previously
|
||||
undetected): a QuickTime-form `meta` box as a *direct* `moov` child (no FullBox
|
||||
header, disambiguated by probing the child-box offset), and a QuickTime
|
||||
`hdlr=mdir` metadata list under `udta.meta` whose `ilst` data items carry the
|
||||
validated JSON with no `keys` box at all (content-validated, so only genuine
|
||||
TC260 JSON matches; the keyless entry has no key name to blank, so its value
|
||||
alone is spaced out). The MP4/MOV/M4V/M4A removal path first validates the top-level
|
||||
box walk, then copies the source to a sibling temporary file in bounded chunks.
|
||||
Supported C2PA/JUMBF/AI-label boxes become same-size `free` boxes with blank
|
||||
payloads; TC260 removal changes the four-byte key to `free` and blanks only the
|
||||
|
||||
+2
-1
@@ -431,7 +431,8 @@ must use the same container extension as the source.
|
||||
The returned `VideoMetadataResult` records the source, output, metadata detected
|
||||
before removal, and any markers remaining after the verified strip. MP4/MOV
|
||||
inspection recognizes the native TC260 `AIGC` entry in
|
||||
`moov.udta.meta.keys/ilst`; its removal preserves container size and encoded
|
||||
`moov.udta.meta.keys/ilst` and the QuickTime-form `meta` variants Doubao's iOS
|
||||
export writes; its removal preserves container size and encoded
|
||||
stream bytes. MP4/MOV/M4V are copied in bounded chunks, so a large `mdat` is not
|
||||
loaded into memory; publication is atomic. MKV/WebM inspection recognizes the corresponding
|
||||
`Segment.Tags.Tag.SimpleTag` representation; its removal requires ffmpeg for a
|
||||
|
||||
@@ -37,7 +37,7 @@ when you can select the affected area yourself.
|
||||
| `veo` | Current four-point diamond and legacy `Veo` text | Fixed bottom-right corner | Uses separate silhouettes and requires a recurring match; learned fill is preferable on structured backgrounds. |
|
||||
| `seedance` | Boxed `AI` label | Fixed bottom-right corner | Requires an anchored recurring match; the full localized box is filled because a thinner synthetic shape mask leaves the real translucent rim behind. |
|
||||
| `dola` | `Dola AI` text | Fixed bottom-right corner | Requires an anchored recurring match; ByteDance or BytePlus provenance can relax only an existing visual run. |
|
||||
| `hailuo` | `MINIMAX \| hailuo AI` composite label | Fixed lower edge | Uses a synthetic waveform, text, separator, and ring silhouette; the complete recurring label box is filled. |
|
||||
| `hailuo` | `MINIMAX \| hailuo AI` composite label | Fixed lower edge | Uses a synthetic waveform, text, separator, and ring silhouette; the complete recurring label box is filled. A TC260 label naming MiniMax as producer can relax only an existing stable run. |
|
||||
| `kling` | Kling swirl, `KLING AI`, version, and optional `PRO` suffix | Fixed bottom-right edge | Combines a synthetic logo rescue with font variants, an edge gate, a white-label gate, and anchored temporal recurrence. |
|
||||
|
||||
`video identify`, `video visible`, and `video all` share this registry and the
|
||||
@@ -162,6 +162,7 @@ not a universal clean verdict.
|
||||
| FLUX | None registered | Diffusion regeneration; optional open decoder | C2PA for supported sources |
|
||||
| Adobe Firefly | None registered | Optional TrustMark Variant P decoder | C2PA |
|
||||
| Midjourney | None registered | No registered pixel decoder | EXIF, XMP, and IPTC signals |
|
||||
| Luma AI | None registered | No registered pixel decoder | PNG text generator tags (Uni-1) |
|
||||
| ByteDance generators | Doubao and Jimeng marks | No registered pixel decoder | TC260 AIGC, supported C2PA, and exact app-export AIGC disclosures |
|
||||
| Qwen | Qwen mark | No registered pixel decoder | TC260 AIGC |
|
||||
| Kling | Kling image and video marks | No registered pixel decoder | TC260 AIGC |
|
||||
|
||||
@@ -45,7 +45,7 @@ pair. On the ISOBMFF path, `blank_ai_exif_tokens` provides the corresponding
|
||||
in-place scrub for supported EXIF values, TC260 AIGC blocks, and the xAI pair.
|
||||
- **China TC260 AIGC label (caught by `AIGC_MARKERS` / `metadata.aigc_label`, surfaced by `identify` as the `aigc` signal):** China-served generators embed an XMP `<TC260:AIGC>{"Label":"1","ContentProducer":...}` block — China's mandatory AI-content labeling (TC260 namespace `tc260.org.cn/ns/AIGC`). The label says only "this is AI", but its `ContentProducer` names the signing entity — `001` + `1` + an 18-char Unified Social Credit Code + a 5-digit product suffix, normalized by `metadata.uscc_of`, or for a few generators a bare product name. `KnownMark.tc260_producer_codes` maps the codes settled per vendor by `scripts/vendor_cohort_harvest.py` to registry mark keys, so an AIGC image relaxes the detector of the vendor it actually carries rather than a guessed pair; an unmapped or absent producer falls back to ByteDance's two products. A code identifies a legal entity, not necessarily one brand, so a hosting or aggregating platform that signs for several apps is a recall bet rather than a proof.
|
||||
|
||||
**Doubao** (ByteDance) uses it (verified on a public issue sample; `ContentProducer` `001191110102MACQD9K64010000`, no C2PA/SynthID/imwatermark — the XMP block is the only signal; GitHub attachment upload did NOT strip it). The same standard is mandatory for Jimeng/Kling/Qwen/Ernie etc., so the one marker covers the whole China-AIGC-labeled ecosystem. `aigc_label` reads **four image serializations** through a shared `_parse` helper: the HTML-entity-encoded XMP `TC260:AIGC` block in **either RDF form** — the nested element `<TC260:AIGC>{...}</TC260:AIGC>` (Doubao) or the attribute `TC260:AIGC="{...}"` (**PicWish**, `ContentProducer="picwish"`, verified on compatible samples) — via a container-agnostic raw-byte scan (any JSON object accepted), a raw-JSON PNG `AIGC` tEXt chunk (Doubao also writes the label this way, no namespaced marker at all — confirmed on compatible samples, `ContentProducer="doubao"`), a bare raw-JSON `{"AIGC":{...}}` object embedded in **JPEG EXIF (UserComment)** by some China-served generators, brace-matched from the scan head with `json.JSONDecoder().raw_decode` (no namespaced marker, no PNG chunk — confirmed on compatible samples, `ContentProducer="001191440300708461136T1308L"`), **and** a bare `AIGC{...}` blob (the label glued straight to its JSON, no `"AIGC":` key wrapper) embedded in a **JPEG APP segment near the JFIF header** — confirmed on compatible samples. The two raw-JSON forms are scanned in one loop (`'"AIGC"'` then `AIGC{`) that **falls through on a non-TC260 / undecodable hit instead of returning** — a quoted `"AIGC"` can appear later in an XMP packet while the real label is a bare `AIGC{...}` earlier in the file, so an unconditional early return on the quoted form would shadow the bare form (the exact bug behind the 06-10 misses). Native MP4/MOV is a fifth serialization: TC260-PG-20257A stores an `AIGC` key in `moov.udta.meta.keys` and the raw JSON in the matching `ilst` item. The seeking parser reaches a tail `moov` without reading `mdat`; removal replaces the key with `free` and blanks the validated value at the same length so every box size and stream offset stays fixed. All generic forms are gated on at least one TC260 field (`TC260_AIGC_FIELDS`) so a generic `AIGC` key cannot false-positive; the namespaced XMP element is unambiguous and needs no gate. `TC260_AIGC_FIELDS` covers **two schemas**: the producer-side one (`Label` / `ContentProducer` / `ProduceID` / `ContentPropagator` / `PropagateID`, Doubao and most China gens) and the **service-provider** one (`ServiceProvider` / `ServiceUser`, plus generic `Time` / `ContentId` which are NOT gated on) — **Tencent Cloud's** AIGC variant (`ServiceProvider` = `腾讯云`), embedded in **EXIF `ImageDescription`**, verified on compatible samples. In `identify`, `aigc` fires on the parsed label **or** the `AIGC_MARKERS` byte scan (the latter preserves the laundering-tell case where the JSON payload is truncated).
|
||||
**Doubao** (ByteDance) uses it (verified on a public issue sample; `ContentProducer` `001191110102MACQD9K64010000`, no C2PA/SynthID/imwatermark — the XMP block is the only signal; GitHub attachment upload did NOT strip it). The same standard is mandatory for Jimeng/Kling/Qwen/Ernie etc., so the one marker covers the whole China-AIGC-labeled ecosystem. `aigc_label` reads **four image serializations** through a shared `_parse` helper: the HTML-entity-encoded XMP `TC260:AIGC` block in **either RDF form** — the nested element `<TC260:AIGC>{...}</TC260:AIGC>` (Doubao) or the attribute `TC260:AIGC="{...}"` (**PicWish**, `ContentProducer="picwish"`, verified on compatible samples) — via a container-agnostic raw-byte scan (any JSON object accepted), a raw-JSON PNG `AIGC` tEXt chunk (Doubao also writes the label this way, no namespaced marker at all — confirmed on compatible samples, `ContentProducer="doubao"`), a bare raw-JSON `{"AIGC":{...}}` object embedded in **JPEG EXIF (UserComment)** by some China-served generators, brace-matched from the scan head with `json.JSONDecoder().raw_decode` (no namespaced marker, no PNG chunk — confirmed on compatible samples, `ContentProducer="001191440300708461136T1308L"`), **and** a bare `AIGC{...}` blob (the label glued straight to its JSON, no `"AIGC":` key wrapper) embedded in a **JPEG APP segment near the JFIF header** — confirmed on compatible samples. The two raw-JSON forms are scanned in one loop (`'"AIGC"'` then `AIGC{`) that **falls through on a non-TC260 / undecodable hit instead of returning** — a quoted `"AIGC"` can appear later in an XMP packet while the real label is a bare `AIGC{...}` earlier in the file, so an unconditional early return on the quoted form would shadow the bare form (the exact bug behind the 06-10 misses). Native MP4/MOV is a fifth serialization: TC260-PG-20257A stores an `AIGC` key in `moov.udta.meta.keys` and the raw JSON in the matching `ilst` item; Doubao's iOS export additionally writes two QuickTime-form variants (a bare `meta` box as a direct `moov` child, and a keyless `hdlr=mdir` metadata list whose `ilst` data items carry the JSON with no key name) — both verified on retained carriers (2026-08-17). The seeking parser reaches a tail `moov` without reading `mdat`; removal replaces the key with `free` and blanks the validated value at the same length so every box size and stream offset stays fixed. All generic forms are gated on at least one TC260 field (`TC260_AIGC_FIELDS`) so a generic `AIGC` key cannot false-positive; the namespaced XMP element is unambiguous and needs no gate. `TC260_AIGC_FIELDS` covers **two schemas**: the producer-side one (`Label` / `ContentProducer` / `ProduceID` / `ContentPropagator` / `PropagateID`, Doubao and most China gens) and the **service-provider** one (`ServiceProvider` / `ServiceUser`, plus generic `Time` / `ContentId` which are NOT gated on) — **Tencent Cloud's** AIGC variant (`ServiceProvider` = `腾讯云`), embedded in **EXIF `ImageDescription`**, verified on compatible samples. In `identify`, `aigc` fires on the parsed label **or** the `AIGC_MARKERS` byte scan (the latter preserves the laundering-tell case where the JSON payload is truncated).
|
||||
|
||||
Native MKV/WebM is a sixth serialization. TC260-PG-20257A stores
|
||||
`TagName=AIGC` and the raw JSON `TagString` in
|
||||
|
||||
@@ -171,6 +171,11 @@ AI_GENERATOR_TOKENS = frozenset(
|
||||
"dreamstudio",
|
||||
"novelai",
|
||||
"reve.com",
|
||||
# Luma AI stamps PNG tEXt Source="Luma AI" / Comment="Generated by
|
||||
# Luma AI's Uni-1 model (https://lumalabs.ai)"; the space-bearing token
|
||||
# avoids matching incidental "luma" runs (luma/chroma key names etc.).
|
||||
"luma ai",
|
||||
"lumalabs",
|
||||
"aphrodite ai",
|
||||
"apple photos clean up",
|
||||
"fal-ai",
|
||||
|
||||
@@ -12,7 +12,10 @@ so pixel, video, and audio data is preserved bit-for-bit.
|
||||
TC260-PG-20257A video metadata is nested instead:
|
||||
``moov.udta.meta.keys/ilst``. Its detector seeks through those boxes without
|
||||
reading media payloads, and its stripper blanks the validated key/value in
|
||||
place so fast-start media offsets remain valid.
|
||||
place so fast-start media offsets remain valid. Two serialization variants are
|
||||
covered: the ISO form (``meta`` as a FullBox under ``udta``) and the QuickTime
|
||||
form Doubao's iOS export writes (a bare ``meta`` box as a direct ``moov`` child,
|
||||
no FullBox header).
|
||||
|
||||
This file intentionally avoids dependencies on format-specific libraries
|
||||
(pillow-heif, pillow-jxl, pymp4) so it works on systems where they aren't
|
||||
@@ -184,68 +187,116 @@ def _tc260_key_indices(
|
||||
return found
|
||||
|
||||
|
||||
# The box types a TC260-bearing ``meta`` box opens with or contains: ISO files
|
||||
# have ``hdlr`` then ``keys``/``ilst``; QuickTime metadata lists have ``hdlr``
|
||||
# and ``ilst`` only. Which payload offset (0 vs 4) yields such children is what
|
||||
# disambiguates the QuickTime form (no FullBox header) from the ISO one.
|
||||
_META_CHILD_TYPES = frozenset({b"hdlr", b"keys", b"ilst"})
|
||||
|
||||
|
||||
def _meta_child_boxes(
|
||||
stream: BinaryIO,
|
||||
meta_payload: int,
|
||||
meta_end: int,
|
||||
) -> Iterator[tuple[int, int, bytes, int]]:
|
||||
"""Yield the child boxes of one ``meta`` box in either serialized form.
|
||||
|
||||
ISO serializes ``meta`` as a FullBox, so its children start 4 bytes into
|
||||
the payload; QuickTime writes a bare box, so they start at 0. Both real
|
||||
forms open with a recognized child (``hdlr``), so the form is picked by
|
||||
which offset's first box type is one of ``_META_CHILD_TYPES``; a wrong
|
||||
probe reads garbage header bytes that match no known type.
|
||||
"""
|
||||
for offset in (0, 4):
|
||||
boxes = iter_file_boxes(stream, meta_payload + offset, meta_end)
|
||||
first = next(boxes, None)
|
||||
if first is not None and first[2] in _META_CHILD_TYPES:
|
||||
yield first
|
||||
yield from boxes
|
||||
return
|
||||
|
||||
|
||||
def _iter_tc260_meta_boxes(
|
||||
stream: BinaryIO,
|
||||
moov_payload: int,
|
||||
moov_end: int,
|
||||
) -> Iterator[tuple[int, int]]:
|
||||
"""Yield ``(payload, end)`` of every ``meta`` box that may hold a TC260 label.
|
||||
|
||||
The normative ISO placement is ``moov.udta.meta``; Doubao's iOS MOV export
|
||||
instead stores the label in a QuickTime-form ``meta`` box that hangs
|
||||
directly off ``moov``. Both are yielded so one consumer covers them.
|
||||
"""
|
||||
for _start, end, box_type, payload in iter_file_boxes(stream, moov_payload, moov_end):
|
||||
if box_type == b"udta":
|
||||
for _udta_start, udta_end, udta_type, udta_payload in iter_file_boxes(stream, payload, end):
|
||||
if udta_type == b"meta":
|
||||
yield udta_payload, udta_end
|
||||
elif box_type == b"meta":
|
||||
yield payload, end
|
||||
|
||||
|
||||
def _tc260_aigc_regions(
|
||||
stream: BinaryIO,
|
||||
file_size: int,
|
||||
) -> list[tuple[int, int, int, int, bytes]]:
|
||||
) -> list[tuple[tuple[int, int] | None, int, int, bytes]]:
|
||||
"""Locate validated native TC260 entries without reading media payloads.
|
||||
|
||||
Each tuple is ``(key_start, key_end, value_start, value_end, value)``.
|
||||
Each tuple is ``(key_span, value_start, value_end, value)``; ``key_span`` is
|
||||
the byte span of the ``AIGC`` key when the normative ``keys`` box maps the
|
||||
item, or None for the QuickTime metadata-list form (``hdlr=mdir``) Doubao's
|
||||
iOS export writes, where the JSON sits in a bare ``ilst`` data item with no
|
||||
key name to blank.
|
||||
"""
|
||||
regions: list[tuple[int, int, int, int, bytes]] = []
|
||||
regions: list[tuple[tuple[int, int] | None, int, int, bytes]] = []
|
||||
for _moov_start, moov_end, moov_type, moov_payload in iter_file_boxes(stream, 0, file_size):
|
||||
if moov_type != b"moov":
|
||||
continue
|
||||
for _udta_start, udta_end, udta_type, udta_payload in iter_file_boxes(
|
||||
stream,
|
||||
moov_payload,
|
||||
moov_end,
|
||||
):
|
||||
if udta_type != b"udta":
|
||||
continue
|
||||
for _meta_start, meta_end, meta_type, meta_payload in iter_file_boxes(
|
||||
for meta_payload, meta_end in _iter_tc260_meta_boxes(stream, moov_payload, moov_end):
|
||||
keys: dict[int, tuple[int, int]] = {}
|
||||
ilst_boxes: list[tuple[int, int]] = []
|
||||
keyed = False
|
||||
for _child_start, child_end, child_type, child_payload in _meta_child_boxes(
|
||||
stream,
|
||||
udta_payload,
|
||||
udta_end,
|
||||
meta_payload,
|
||||
meta_end,
|
||||
):
|
||||
if meta_type != b"meta" or meta_payload + 4 > meta_end:
|
||||
continue
|
||||
keys: dict[int, tuple[int, int]] = {}
|
||||
ilst_boxes: list[tuple[int, int]] = []
|
||||
for _child_start, child_end, child_type, child_payload in iter_file_boxes(
|
||||
if child_type == b"keys":
|
||||
keyed = True
|
||||
keys.update(_tc260_key_indices(stream, child_payload, child_end))
|
||||
elif child_type == b"ilst":
|
||||
ilst_boxes.append((child_payload, child_end))
|
||||
if not ilst_boxes:
|
||||
continue
|
||||
for ilst_payload, ilst_end in ilst_boxes:
|
||||
for _item_start, item_end, item_type, item_payload in iter_file_boxes(
|
||||
stream,
|
||||
meta_payload + 4,
|
||||
meta_end,
|
||||
ilst_payload,
|
||||
ilst_end,
|
||||
):
|
||||
if child_type == b"keys":
|
||||
keys.update(_tc260_key_indices(stream, child_payload, child_end))
|
||||
elif child_type == b"ilst":
|
||||
ilst_boxes.append((child_payload, child_end))
|
||||
if not keys:
|
||||
continue
|
||||
for ilst_payload, ilst_end in ilst_boxes:
|
||||
for _item_start, item_end, item_type, item_payload in iter_file_boxes(
|
||||
index = int.from_bytes(item_type, "big")
|
||||
key_span = keys.get(index)
|
||||
if keyed and key_span is None:
|
||||
# A keyed (ISO) meta box maps items through ``keys``;
|
||||
# an unmapped index is not an AIGC entry, and reading
|
||||
# its value would pull arbitrary metadata (e.g. cover
|
||||
# art) through the JSON parser on every scan. Only the
|
||||
# keyless QuickTime list falls through to content
|
||||
# validation below.
|
||||
continue
|
||||
for _data_start, data_end, data_type, data_payload in iter_file_boxes(
|
||||
stream,
|
||||
ilst_payload,
|
||||
ilst_end,
|
||||
item_payload,
|
||||
item_end,
|
||||
):
|
||||
index = int.from_bytes(item_type, "big")
|
||||
key_span = keys.get(index)
|
||||
if key_span is None:
|
||||
value_start = data_payload + 8
|
||||
value_size = data_end - value_start
|
||||
if data_type != b"data" or value_size < 0 or value_size > MAX_TC260_VALUE_BYTES:
|
||||
continue
|
||||
for _data_start, data_end, data_type, data_payload in iter_file_boxes(
|
||||
stream,
|
||||
item_payload,
|
||||
item_end,
|
||||
):
|
||||
value_start = data_payload + 8
|
||||
value_size = data_end - value_start
|
||||
if data_type != b"data" or value_size < 0 or value_size > MAX_TC260_VALUE_BYTES:
|
||||
continue
|
||||
stream.seek(value_start)
|
||||
value = stream.read(value_size)
|
||||
if len(value) == value_size and parse_tc260_aigc_json(value) is not None:
|
||||
regions.append((*key_span, value_start, data_end, value))
|
||||
stream.seek(value_start)
|
||||
value = stream.read(value_size)
|
||||
if len(value) == value_size and parse_tc260_aigc_json(value) is not None:
|
||||
regions.append((key_span, value_start, data_end, value))
|
||||
return regions
|
||||
|
||||
|
||||
@@ -257,7 +308,7 @@ def tc260_aigc_payloads(path: str | Path) -> tuple[bytes, ...]:
|
||||
return ()
|
||||
stream.seek(0, 2)
|
||||
file_size = stream.tell()
|
||||
return tuple(region[4] for region in _tc260_aigc_regions(stream, file_size))
|
||||
return tuple(region[3] for region in _tc260_aigc_regions(stream, file_size))
|
||||
except OSError:
|
||||
return ()
|
||||
|
||||
@@ -268,6 +319,8 @@ def blank_tc260_aigc_tags(data: bytes) -> tuple[bytes, int]:
|
||||
Removing a nested ``ilst`` item would shift ``mdat`` in a fast-start MP4 and
|
||||
invalidate its chunk offsets. Replacing the four-byte key with ``free`` and
|
||||
the JSON value with spaces keeps every box size and media offset unchanged.
|
||||
A keyless QuickTime metadata-list entry has no key name, so only its value
|
||||
is blanked.
|
||||
"""
|
||||
if not is_isobmff(data):
|
||||
return data, 0
|
||||
@@ -276,9 +329,10 @@ def blank_tc260_aigc_tags(data: bytes) -> tuple[bytes, int]:
|
||||
return data, 0
|
||||
out = bytearray(data)
|
||||
key_spans: set[tuple[int, int]] = set()
|
||||
for key_start, key_end, value_start, value_end, _value in regions:
|
||||
key_spans.add((key_start, key_end))
|
||||
out[key_start:key_end] = b"free"
|
||||
for key_span, value_start, value_end, _value in regions:
|
||||
if key_span is not None:
|
||||
key_spans.add(key_span)
|
||||
out[key_span[0] : key_span[1]] = b"free"
|
||||
out[value_start:value_end] = b" " * (value_end - value_start)
|
||||
return bytes(out), len(key_spans)
|
||||
|
||||
@@ -456,7 +510,7 @@ def strip_isobmff_media_file(
|
||||
max_scan=max_box_scan,
|
||||
)
|
||||
tc260_regions = _tc260_aigc_regions(stream, file_size) if targets is not None else []
|
||||
tc260_key_spans = {(region[0], region[1]) for region in tc260_regions}
|
||||
tc260_key_spans = {region[0] for region in tc260_regions if region[0] is not None}
|
||||
|
||||
with atomic_video_output(output_path) as temporary_path:
|
||||
with source_path.open("rb") as source_stream, temporary_path.open("r+b") as temporary:
|
||||
@@ -466,9 +520,10 @@ def strip_isobmff_media_file(
|
||||
temporary.seek(box_start + 4)
|
||||
temporary.write(b"free")
|
||||
_overwrite_range(temporary, payload_start, box_end, byte=b"\x00")
|
||||
for key_start, _key_end, value_start, value_end, _value in tc260_regions:
|
||||
temporary.seek(key_start)
|
||||
temporary.write(b"free")
|
||||
for key_span, value_start, value_end, _value in tc260_regions:
|
||||
if key_span is not None:
|
||||
temporary.seek(key_span[0])
|
||||
temporary.write(b"free")
|
||||
_overwrite_range(temporary, value_start, value_end, byte=b" ")
|
||||
temporary.flush()
|
||||
os.fsync(temporary.fileno())
|
||||
|
||||
@@ -796,6 +796,8 @@ _AI_VENDOR_TOKENS: tuple[tuple[str, str], ...] = (
|
||||
("fal-ai", "fal.ai"),
|
||||
("bria", "Bria"),
|
||||
("apple photos clean up", "Apple"),
|
||||
("luma ai", "Luma AI"),
|
||||
("lumalabs", "Luma AI"),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1184,6 +1184,11 @@ def get_ai_metadata(image_path: Path) -> dict[str, str]:
|
||||
if (aigc := aigc_label(image_path)) is not None:
|
||||
producer = aigc.get("ContentProducer", "")
|
||||
result["aigc_label"] = f"China AIGC label (TC260){f'; producer {producer}' if producer else ''}"
|
||||
# The structural producer beside its display rendering: a machine
|
||||
# consumer (video-visible provenance) must not parse the formatted
|
||||
# sentence above, whose wording can change without notice.
|
||||
if producer:
|
||||
result["aigc_producer"] = producer
|
||||
|
||||
app_scan = scan_head(image_path)
|
||||
app_provenance, app_generator = _app_metadata_evidence(app_scan)
|
||||
|
||||
@@ -239,6 +239,7 @@ def _visible_removal_plan(
|
||||
from remove_ai_watermarks.video_visible import (
|
||||
VISIBLE_MARK_POLICIES,
|
||||
has_bytedance_video_provenance,
|
||||
has_hailuo_video_provenance,
|
||||
has_sora_provenance,
|
||||
has_veo_provenance,
|
||||
stabilize_localizations,
|
||||
@@ -249,6 +250,7 @@ def _visible_removal_plan(
|
||||
"veo": has_veo_provenance,
|
||||
"seedance": has_bytedance_video_provenance,
|
||||
"dola": has_bytedance_video_provenance,
|
||||
"hailuo": has_hailuo_video_provenance,
|
||||
}.get(selected_mark)
|
||||
policy = VISIBLE_MARK_POLICIES[selected_mark]
|
||||
regions = stabilize_localizations(
|
||||
|
||||
@@ -890,11 +890,14 @@ class VisibleMarkPolicy:
|
||||
Every value here is MEASURED per provider; the arbiter itself
|
||||
(:func:`_stabilize_localizations`) is shared and knows nothing about providers.
|
||||
|
||||
``accepts_provenance`` is load-bearing rather than cosmetic. Hailuo and Kling have
|
||||
no metadata that could confirm them, so their rows force ``provenance=False``; that
|
||||
used to be guaranteed structurally by wrappers that took no ``provenance``
|
||||
parameter at all, and this flag is what preserves the guarantee now that one entry
|
||||
point serves every mark.
|
||||
``accepts_provenance`` is load-bearing rather than cosmetic. A vendor with no
|
||||
metadata that could confirm it forces ``provenance=False``; that used to be
|
||||
guaranteed structurally by wrappers that took no ``provenance`` parameter at
|
||||
all, and this flag is what preserves the guarantee now that one entry point
|
||||
serves every mark. Kling keeps the flag off (its TC260 producer code is image
|
||||
evidence; no kling video corpus row ties a producer to the moving mark), while
|
||||
Hailuo accepts it through the MiniMax TC260 label
|
||||
(:func:`has_hailuo_video_provenance`).
|
||||
|
||||
``padding_fraction`` and ``mask_style`` belong to the removal plan rather than the
|
||||
arbiter, but they are per-provider constants like the rest, so they live on the same
|
||||
@@ -969,7 +972,9 @@ VISIBLE_MARK_POLICIES: dict[str, VisibleMarkPolicy] = {
|
||||
anchor_iou=0.80,
|
||||
padding_fraction=0.12,
|
||||
mask_style="box",
|
||||
accepts_provenance=False,
|
||||
# No provenance_weak_floor: the measured weak floor stays the entry bar;
|
||||
# a MiniMax TC260 label only drops the strong-frame requirement for an
|
||||
# already-stable run (see has_hailuo_video_provenance).
|
||||
),
|
||||
"kling": VisibleMarkPolicy(
|
||||
weak_floor=_KLING_WEAK_CONFIDENCE,
|
||||
@@ -1458,3 +1463,15 @@ def has_bytedance_video_provenance(markers: dict[str, str]) -> bool:
|
||||
).lower()
|
||||
source_type = markers.get("source_type", "").lower()
|
||||
return ("bytedance" in identity or "byteplus" in identity) and "trainedalgorithmicmedia" in source_type
|
||||
|
||||
|
||||
def has_hailuo_video_provenance(markers: dict[str, str]) -> bool:
|
||||
"""Whether a TC260 label names MiniMax, Hailuo's maker, as the producer.
|
||||
|
||||
Hailuo video exports carry no C2PA; their TC260 ``ContentProducer`` is the
|
||||
bare name ``MiniMax`` (verified on retained MiniMax-hailuo clips). The
|
||||
producer travels as its own structural marker (``aigc_producer``, set in
|
||||
``metadata.get_ai_metadata``), so this matches the field exactly rather
|
||||
than parsing the human-readable ``aigc_label`` sentence.
|
||||
"""
|
||||
return markers.get("aigc_producer", "").strip().lower() == "minimax"
|
||||
|
||||
@@ -823,6 +823,42 @@ class TestExifGenerator:
|
||||
path = _img_with_software(tmp_path, "jpg", "Forever Editor 2.0")
|
||||
assert exif_generator(path) is None
|
||||
|
||||
def test_luma_ai_png_text_chunks_detected(self, tmp_path: Path):
|
||||
# Luma AI stamps tEXt Software="Uni-1" (model name, not a token) plus
|
||||
# Source/Comment values carrying "Luma AI"; the Source value must match.
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
info = PngInfo()
|
||||
info.add_text("Software", "Uni-1")
|
||||
info.add_text("Source", "Luma AI")
|
||||
info.add_text("Comment", "Generated by Luma AI's Uni-1 model (https://lumalabs.ai)")
|
||||
path = tmp_path / "luma.png"
|
||||
Image.new("RGB", (64, 64)).save(path, pnginfo=info)
|
||||
assert exif_generator(path) == "Luma AI"
|
||||
|
||||
def test_luma_token_not_overmatched(self, tmp_path: Path):
|
||||
# The token is "luma ai" with the space: a bare "luma" (e.g. a luma
|
||||
# chart tool) must not fire.
|
||||
path = _img_with_software(tmp_path, "jpg", "Luma Chart Export 2.0")
|
||||
assert exif_generator(path) is None
|
||||
|
||||
def test_luma_removal_parity(self, tmp_path: Path):
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from remove_ai_watermarks.metadata import remove_ai_metadata
|
||||
|
||||
info = PngInfo()
|
||||
info.add_text("Software", "Uni-1")
|
||||
info.add_text("Source", "Luma AI")
|
||||
info.add_text("Comment", "Generated by Luma AI's Uni-1 model (https://lumalabs.ai)")
|
||||
src = tmp_path / "luma.png"
|
||||
Image.new("RGB", (64, 64)).save(src, pnginfo=info)
|
||||
assert exif_generator(src) == "Luma AI"
|
||||
|
||||
out = tmp_path / "clean.png"
|
||||
remove_ai_metadata(src, out)
|
||||
assert exif_generator(out) is None
|
||||
|
||||
def test_aphrodite_make_detected(self, tmp_path: Path):
|
||||
# Aphrodite AI writes EXIF Make="Aphrodite AI".
|
||||
exif = piexif.dump({"0th": {piexif.ImageIFD.Make: b"Aphrodite AI"}, "Exif": {}, "GPS": {}, "1st": {}})
|
||||
|
||||
@@ -65,6 +65,47 @@ def _video_with_tc260(path: Path, *, media_payload: bytes = _VIDEO_PAYLOAD) -> P
|
||||
return path
|
||||
|
||||
|
||||
def _hdlr(handler: bytes) -> bytes:
|
||||
# version/flags + pre_defined + handler_type + reserved[3]; the TC260 walker
|
||||
# never parses it, but a real-shaped hdlr keeps the fixture honest.
|
||||
return _box(b"hdlr", b"\x00\x00\x00\x00" + b"\x00\x00\x00\x00" + handler + b"\x00" * 12)
|
||||
|
||||
|
||||
def _quicktime_meta(*children: bytes) -> bytes:
|
||||
# QuickTime form: no 4-byte FullBox header before the children.
|
||||
return _box(b"meta", b"".join(children))
|
||||
|
||||
|
||||
def _video_with_tc260_moov_meta(path: Path, *, media_payload: bytes = _VIDEO_PAYLOAD) -> Path:
|
||||
"""Doubao's iOS export form: a QuickTime ``meta`` box as a direct ``moov`` child."""
|
||||
keys = _box(
|
||||
b"keys",
|
||||
b"\x00\x00\x00\x00"
|
||||
+ (2).to_bytes(4, "big")
|
||||
+ _metadata_key(b"com.apple.quicktime.artwork")
|
||||
+ _metadata_key(b"AIGC"),
|
||||
)
|
||||
ilst = _box(
|
||||
b"ilst",
|
||||
_metadata_value(1, b'{"source_type":"","data":{"product":"doubao"}}') + _metadata_value(2, _TC260_AIGC),
|
||||
)
|
||||
meta = _quicktime_meta(_hdlr(b"mdta"), keys, ilst)
|
||||
path.write_bytes(_MP4_FTYP + _box(b"mdat", media_payload) + _box(b"moov", meta))
|
||||
return path
|
||||
|
||||
|
||||
def _video_with_tc260_mdir_list(path: Path, *, media_payload: bytes = _VIDEO_PAYLOAD) -> Path:
|
||||
"""Doubao's iOS QuickTime metadata list: ``udta.meta(hdlr=mdir)/ilst`` data
|
||||
items with numeric indices and no ``keys`` box at all."""
|
||||
ilst = _box(
|
||||
b"ilst",
|
||||
_metadata_value(0, b"vid:standard-video-id") + _metadata_value(0, _TC260_AIGC),
|
||||
)
|
||||
meta = _quicktime_meta(_hdlr(b"mdir"), ilst)
|
||||
path.write_bytes(_MP4_FTYP + _box(b"mdat", media_payload) + _box(b"moov", _box(b"udta", meta)))
|
||||
return path
|
||||
|
||||
|
||||
def _ebml_size(value: int) -> bytes:
|
||||
for length in range(1, 9):
|
||||
if value < (1 << (7 * length)) - 1:
|
||||
@@ -612,6 +653,67 @@ class TestVideoMetadataApi:
|
||||
assert b"AIGC" not in cleaned
|
||||
assert _TC260_AIGC not in cleaned
|
||||
|
||||
@pytest.mark.parametrize("suffix", [".mp4", ".mov"])
|
||||
def test_inspects_native_tc260_in_quicktime_moov_meta(self, tmp_path: Path, suffix: str):
|
||||
# Doubao's iOS export stores the label in a QuickTime-form meta box
|
||||
# (no FullBox header) hanging directly off moov, not under udta.
|
||||
from remove_ai_watermarks.video import inspect_video_metadata
|
||||
|
||||
source = _video_with_tc260_moov_meta(tmp_path / f"source{suffix}")
|
||||
|
||||
report = inspect_video_metadata(source)
|
||||
|
||||
assert report.has_ai_metadata is True
|
||||
assert report.markers["aigc_label"].endswith("producer 00119144030008867405X210002")
|
||||
|
||||
def test_removes_native_tc260_in_quicktime_moov_meta(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.video import remove_video_metadata
|
||||
|
||||
source = _video_with_tc260_moov_meta(tmp_path / "source.mov")
|
||||
output = tmp_path / "clean.mov"
|
||||
|
||||
result = remove_video_metadata(source, output)
|
||||
cleaned = output.read_bytes()
|
||||
|
||||
assert result.detected["aigc_label"].startswith("China AIGC label")
|
||||
# The artwork item's Doubao product JSON is app-export provenance, a
|
||||
# separate signal the TC260 blanker does not touch -- as on the real
|
||||
# Doubao iOS export this fixture mirrors.
|
||||
assert result.remaining == {"app_provenance": "App export provenance (ByteDance Doubao)"}
|
||||
assert len(cleaned) == source.stat().st_size
|
||||
assert _VIDEO_PAYLOAD in cleaned
|
||||
assert _TC260_AIGC not in cleaned
|
||||
|
||||
def test_inspects_native_tc260_in_quicktime_mdir_metadata_list(self, tmp_path: Path):
|
||||
# The second Doubao iOS variant: a QuickTime metadata list under
|
||||
# udta.meta with hdlr=mdir and ilst data items but no keys box.
|
||||
from remove_ai_watermarks.video import inspect_video_metadata
|
||||
|
||||
source = _video_with_tc260_mdir_list(tmp_path / "source.mov")
|
||||
|
||||
report = inspect_video_metadata(source)
|
||||
|
||||
assert report.has_ai_metadata is True
|
||||
assert "aigc_label" in report.markers
|
||||
|
||||
def test_removes_native_tc260_in_quicktime_mdir_metadata_list(self, tmp_path: Path):
|
||||
from remove_ai_watermarks.video import remove_video_metadata
|
||||
|
||||
source = _video_with_tc260_mdir_list(tmp_path / "source.mov")
|
||||
output = tmp_path / "clean.mov"
|
||||
|
||||
result = remove_video_metadata(source, output)
|
||||
cleaned = output.read_bytes()
|
||||
|
||||
assert result.detected["aigc_label"].startswith("China AIGC label")
|
||||
assert result.remaining == {}
|
||||
assert len(cleaned) == source.stat().st_size
|
||||
assert _VIDEO_PAYLOAD in cleaned
|
||||
# The keyless entry has no key name to blank; only the JSON is spaced
|
||||
# out, and the neighboring standard ilst item survives untouched.
|
||||
assert _TC260_AIGC not in cleaned
|
||||
assert b"vid:standard-video-id" in cleaned
|
||||
|
||||
def test_streams_large_isobmff_without_full_file_read(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
@@ -1894,6 +1996,25 @@ class TestAdditionalProviderTemporalArbiter:
|
||||
assert stabilize(weak) == [None] * 12
|
||||
assert stabilize(strong) == [box] * 12
|
||||
|
||||
def test_hailuo_provenance_accepts_sub_strong_stable_run(self):
|
||||
# A TC260 label naming MiniMax relaxes only the strong-frame requirement:
|
||||
# the entry bar stays the measured weak floor (0.30), so a stable run
|
||||
# between weak and strong (0.34) passes with provenance, fails without.
|
||||
from remove_ai_watermarks.video_visible import FrameLocalization, stabilize_localizations
|
||||
|
||||
detections = [FrameLocalization(index, 0.31, self._HAILUO_BOX) for index in range(12)]
|
||||
|
||||
assert stabilize_localizations("hailuo", detections, provenance=False) == [None] * 12
|
||||
assert stabilize_localizations("hailuo", detections, provenance=True) == [self._HAILUO_BOX] * 12
|
||||
|
||||
def test_hailuo_provenance_requires_a_minimax_producer_label(self):
|
||||
from remove_ai_watermarks.video_visible import has_hailuo_video_provenance
|
||||
|
||||
assert has_hailuo_video_provenance({"aigc_producer": "MiniMax"})
|
||||
assert not has_hailuo_video_provenance({"aigc_producer": "001191110102MACQD9K64010000"})
|
||||
assert not has_hailuo_video_provenance({"aigc_label": "China AIGC label (TC260); producer MiniMax"})
|
||||
assert not has_hailuo_video_provenance({"issuer": "Some other vendor"})
|
||||
|
||||
|
||||
class TestVideoVisibleScan:
|
||||
def test_auto_prepares_each_frame_once_for_every_detector(
|
||||
|
||||
Reference in New Issue
Block a user