mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 00:03:15 +01:00
fix(brain-sync): hold case-colliding paths instead of halting the sync (#3220)
Builds on Kkkkkuro's staged-index and casefold checks (#3245, from #3223). A queued path whose spelling differs only by letter case from another path in the index or the batch is held before staging: it stays queued, every other file keeps syncing, and the status is "collision" rather than "ok". An established spelling keeps syncing; only the new alias waits. Skill start prints one ARTIFACTS_SYNC attention line naming the colliding names and the repair anchor (docs/troubleshooting.md#artifact-case-collision); gstack-doctor warns the same way. The outgoing check is cached by HEAD so the empty-queue drain at every skill boundary stays near main's cost. gstack-slug and skill start's slug path are unchanged. Co-authored-by: Kkkkkuro <180382840+Kkkkkuro@users.noreply.github.com> Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
2670a02ecd
commit
a95fb09956
10 files changed
+289
-30
No files matched your search
+76
-15
@@ -56,6 +56,9 @@
|
||||
# stale-push trigger: no push for 24 hours while the drain's drainable count is above zero
|
||||
# line: ARTIFACTS_SYNC: attention: stale-push. Files are ready to sync but nothing has been pushed for over 24 hours. See why: <bin> --status
|
||||
# fix: the waiting reason <bin> --status names (a coupled group waiting for a consistent generation, a git add failure)
|
||||
# case-collision trigger: case_names is set: artifact paths differ only by letter case, so the drain holds queued files under them
|
||||
# line: ARTIFACTS_SYNC: attention: case-collision held=<N> names=<names>. Artifact paths differ only by letter case, which a Mac or Windows disk stores as one file; sync holds <N> queued file(s) under them and syncs everything else. Repair: https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#artifact-case-collision. See the paths: <bin> --status
|
||||
# fix: keep one spelling: the manual repair in docs/troubleshooting.md#artifact-case-collision
|
||||
# stale-drain trigger: last_drain_at older than 24 hours while the queue directory holds records
|
||||
# line: ARTIFACTS_SYNC: attention: stale-drain. The sync has not run for over 24 hours while files wait in the queue. See why: <bin> --status
|
||||
# fix: if no gstack process is running, remove ~/.gstack/.brain-sync.lock.d, then run <bin> --once
|
||||
@@ -115,6 +118,8 @@ REMOVAL_VALVE=20
|
||||
ST_HELD='[]'
|
||||
ST_DRAINABLE=0
|
||||
ST_NOTE=""
|
||||
ST_CASE_HELD=0
|
||||
ST_CASE_NAMES=""
|
||||
|
||||
# origin host for receipt records (github.com etc).
|
||||
remote_host() {
|
||||
@@ -136,8 +141,13 @@ remote_auth_hint() {
|
||||
}
|
||||
|
||||
# Status file schema (one JSON object; readers: --status, --attention, doctor):
|
||||
# status ok | idle | held | blocked | push_failed | error
|
||||
# status ok | idle | held | collision | blocked | push_failed | error
|
||||
# (collision: files were held only for case-colliding paths)
|
||||
# held_count files the secret scan held back this drain (hits + dependents)
|
||||
# case_held queued files held because their path differs only by letter
|
||||
# case from another path in the repo or the batch
|
||||
# case_names the colliding path component names, comma-separated and
|
||||
# clamped to [A-Za-z0-9._?-]; empty when the tree has none
|
||||
# drainable queued files a drain could publish now: excludes scanner-held
|
||||
# and privacy-held records
|
||||
# last_drain_at UTC time this drain ran (every write happens inside a drain)
|
||||
@@ -152,9 +162,9 @@ write_status() {
|
||||
local code="$1" msg="$2${ST_NOTE}" ts last_push=""
|
||||
ts=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "")
|
||||
[ -f "$LAST_PUSH_FILE" ] && last_push=$(head -1 "$LAST_PUSH_FILE" 2>/dev/null || echo "")
|
||||
python3 - "$STATUS_FILE" "$code" "$msg" "$ts" "$last_push" "$ST_DRAINABLE" "$ST_HELD" <<'PYEOF' 2>/dev/null || true
|
||||
python3 - "$STATUS_FILE" "$code" "$msg" "$ts" "$last_push" "$ST_DRAINABLE" "$ST_HELD" "$ST_CASE_HELD" "$ST_CASE_NAMES" <<'PYEOF' 2>/dev/null || true
|
||||
import json, re, sys
|
||||
path, code, msg, ts, last_push, drainable, held = sys.argv[1:8]
|
||||
path, code, msg, ts, last_push, drainable, held, case_held, case_names = sys.argv[1:10]
|
||||
try:
|
||||
held = [h for h in json.loads(held) if isinstance(h, dict)]
|
||||
except Exception:
|
||||
@@ -171,6 +181,8 @@ data = {
|
||||
"last_drain_at": ts,
|
||||
"last_push_at": last_push if re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ", last_push) else None,
|
||||
"ts": ts,
|
||||
"case_held": int(case_held) if case_held.isdigit() else 0,
|
||||
"case_names": case_names if re.fullmatch(r"[A-Za-z0-9._?,-]{0,300}", case_names) else "",
|
||||
"held": held,
|
||||
"message": msg,
|
||||
}
|
||||
@@ -769,13 +781,48 @@ stage_paths() {
|
||||
|
||||
# Check outgoing trees too: a retry merge can combine two individually safe
|
||||
# spellings, and a retained local commit can be pushed with an empty queue.
|
||||
# Also records the tree's case-colliding names for skill start. Without
|
||||
# python3 no drain can stage anything, so there is no new tree to check. A
|
||||
# passing verdict is cached by HEAD's commit id, so the empty-queue drain at
|
||||
# every skill boundary skips the tree walk until HEAD moves. A cached HEAD
|
||||
# that falls behind origin is only pushed after a merge, which moves HEAD.
|
||||
validate_outgoing_paths() {
|
||||
local path_error
|
||||
if ! path_error=$(python3 "$SCRIPT_DIR/gstack-brain-sync-index.py" --outgoing "$GSTACK_HOME" 2>&1); then
|
||||
write_status "error" "outgoing artifact path validation refused; local commits and queue preserved; inspect path spellings"
|
||||
printf '%s\n' "$path_error" >&2
|
||||
local out err key cached cache="$GSTACK_HOME/.brain-case-check"
|
||||
command -v python3 >/dev/null 2>&1 || return 0
|
||||
key=$(git -C "$GSTACK_HOME" rev-parse --verify -q HEAD 2>/dev/null)
|
||||
cached=$(head -1 "$cache" 2>/dev/null || true)
|
||||
if [ "${cached%% *}" = "$key" ]; then
|
||||
[ -n "$ST_CASE_NAMES" ] || ST_CASE_NAMES=${cached#* }
|
||||
return 0
|
||||
fi
|
||||
err=$(mktemp "${TMPDIR:-/tmp}/brain-sync-outgoing.XXXXXX") || return 0
|
||||
if ! out=$(python3 "$SCRIPT_DIR/gstack-brain-sync-index.py" --outgoing "$GSTACK_HOME" 2>"$err"); then
|
||||
ST_CASE_NAMES=$out
|
||||
write_status "error" "outgoing artifact path validation refused: a local commit adds a path that differs only by letter case from a path on the remote; local commits and queue preserved. Repair: https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#artifact-case-collision"
|
||||
cat "$err" >&2
|
||||
rm -f "$err"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$err"
|
||||
printf '%s\t%s\n' "$key" "$out" > "$cache" 2>/dev/null || true
|
||||
[ -n "$ST_CASE_NAMES" ] || ST_CASE_NAMES=$out
|
||||
}
|
||||
|
||||
# Hold the queued paths in $1 whose spelling collides by letter case with a
|
||||
# path in the index or the batch (see gstack-brain-sync-index.py hold). They
|
||||
# leave $1 before staging, so their records stay queued. Sets ST_CASE_HELD.
|
||||
hold_case_collisions() {
|
||||
local paths_file="$1" out
|
||||
tr -d '\r' < "$paths_file" > "$paths_file.lf" && mv -f "$paths_file.lf" "$paths_file" || return 1
|
||||
out=$(python3 "$SCRIPT_DIR/gstack-brain-sync-index.py" --hold "$GSTACK_HOME" "$paths_file" "$paths_file.case") || return 1
|
||||
ST_CASE_HELD=$(grep -c . "$paths_file.case" 2>/dev/null || true)
|
||||
ST_CASE_HELD=${ST_CASE_HELD:-0}
|
||||
[ "$ST_CASE_HELD" -gt 0 ] || return 0
|
||||
ST_CASE_NAMES=$out
|
||||
{ grep -vxF -f "$paths_file.case" "$paths_file" || true; } > "$paths_file.kept"
|
||||
mv -f "$paths_file.kept" "$paths_file"
|
||||
ST_NOTE="$ST_NOTE; $ST_CASE_HELD file(s) held because their path differs only by letter case from another path ($out), NOT synced and still queued: $(head -3 "$paths_file.case" | tr '\n' ' ')(repair: https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#artifact-case-collision)"
|
||||
echo "BRAIN_SYNC: holding $ST_CASE_HELD file(s) whose path differs only by letter case ($out); see $SELF_BIN --status" >&2
|
||||
}
|
||||
|
||||
# Plan the holds for one drain (one Python pass):
|
||||
@@ -1196,13 +1243,18 @@ subcmd_once() {
|
||||
write_status "error" "classification failed; queue preserved (next run retries)"
|
||||
exit 0
|
||||
fi
|
||||
if [ -s "$paths_file" ] && ! hold_case_collisions "$paths_file"; then
|
||||
write_status "error" "could not check queued paths for letter-case collisions; queue preserved (next run retries)"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -s "$paths_file" ]; then
|
||||
# Nothing stageable. Finalize the snapshot (retained entries survive;
|
||||
# classified drops removed; records created after the snapshot untouched).
|
||||
finalize_queue "$snapshot_file" "$class_file" "$paths_file"
|
||||
local summary
|
||||
local summary idle_code=idle
|
||||
summary=$(queue_summary "$class_file")
|
||||
write_status "idle" "no stageable changes${summary:+ ($summary)}"
|
||||
[ "$ST_CASE_HELD" -gt 0 ] && idle_code=collision
|
||||
write_status "$idle_code" "no stageable changes${summary:+ ($summary)}"
|
||||
exit 0
|
||||
fi
|
||||
local n_stageable
|
||||
@@ -1246,6 +1298,8 @@ subcmd_once() {
|
||||
ok_code="held"
|
||||
ST_NOTE="$ST_NOTE; the secret scan held back $held_n flagged file(s) (and any coupled files), NOT synced; see held[] for each file's fixes"
|
||||
echo "BRAIN_SYNC: held $held_n flagged file(s); see $SELF_BIN --status" >&2
|
||||
elif [ "$ST_CASE_HELD" -gt 0 ]; then
|
||||
ok_code="collision"
|
||||
fi
|
||||
|
||||
# A verified empty staged diff is the only "nothing to commit"; anything
|
||||
@@ -1254,8 +1308,8 @@ subcmd_once() {
|
||||
git -C "$GSTACK_HOME" diff --cached --quiet >/dev/null 2>&1 || cached_rc=$?
|
||||
if [ "$cached_rc" -eq 0 ]; then
|
||||
finalize_queue "$snapshot_file" "$class_file" "$paths_file"
|
||||
if [ "$ok_code" = "held" ]; then
|
||||
write_status "held" "nothing else to sync"
|
||||
if [ "$ok_code" != "ok" ]; then
|
||||
write_status "$ok_code" "nothing else to sync"
|
||||
else
|
||||
write_status "idle" "queue drained but no new changes to commit"
|
||||
fi
|
||||
@@ -1413,11 +1467,12 @@ subcmd_unskip_file() {
|
||||
# Skill start's attention lines: one per row of the sync status table in this
|
||||
# script's header that applies. Every value read from the status file is
|
||||
# clamped (status code to the table's closed set, counts to digits,
|
||||
# timestamps to their shape); the message, held paths and scanner output never
|
||||
# reach this output, only fixed text and this script's absolute path.
|
||||
# timestamps to their shape, case-colliding names to [A-Za-z0-9._?,-]); the
|
||||
# message, held paths and scanner output never reach this output, only fixed
|
||||
# text, those clamped values and this script's absolute path.
|
||||
subcmd_attention() {
|
||||
sync_active || exit 0
|
||||
local line="" prefix code="" held="" drainable="" last_drain="" last_push="" bin="$SELF_BIN"
|
||||
local line="" prefix code="" held="" drainable="" last_drain="" last_push="" case_held="" case_names="" bin="$SELF_BIN"
|
||||
if [ -f "$STATUS_FILE" ]; then
|
||||
line=$(head -c 65536 "$STATUS_FILE" 2>/dev/null | tr -d '\r\n')
|
||||
# The counts and timestamps precede every free-text field.
|
||||
@@ -1427,10 +1482,13 @@ subcmd_attention() {
|
||||
drainable=$(printf '%s' "$prefix" | sed -n 's/.*"drainable": *\([0-9][0-9]*\).*/\1/p')
|
||||
last_drain=$(printf '%s' "$prefix" | sed -n 's/.*"last_drain_at": *"\([^"]*\)".*/\1/p')
|
||||
last_push=$(printf '%s' "$prefix" | sed -n 's/.*"last_push_at": *"\([^"]*\)".*/\1/p')
|
||||
case "$code" in ok|idle|held|blocked|push_failed|error) ;; *) code="unknown" ;; esac
|
||||
case_held=$(printf '%s' "$prefix" | sed -n 's/.*"case_held": *\([0-9][0-9]*\).*/\1/p')
|
||||
case_names=$(printf '%s' "$prefix" | sed -n 's/.*"case_names": *"\([A-Za-z0-9._?,-]*\)".*/\1/p' | head -c 300)
|
||||
case "$code" in ok|idle|held|collision|blocked|push_failed|error) ;; *) code="unknown" ;; esac
|
||||
fi
|
||||
case "$held" in ''|*[!0-9]*) held=0 ;; esac
|
||||
case "$drainable" in ''|*[!0-9]*) drainable=0 ;; esac
|
||||
case "$case_held" in ''|*[!0-9]*) case_held=0 ;; esac
|
||||
local iso='[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]Z'
|
||||
case "$last_drain" in $iso) ;; *) last_drain="" ;; esac
|
||||
case "$last_push" in $iso) ;; *) last_push="" ;; esac
|
||||
@@ -1442,6 +1500,9 @@ subcmd_attention() {
|
||||
error) echo "ARTIFACTS_SYNC: attention: status=error. The last sync stopped before committing; the queue is kept and retried. See the cause: $bin --status" ;;
|
||||
unknown) echo "ARTIFACTS_SYNC: attention: status=unknown. The sync status file is unreadable. See it: $bin --status" ;;
|
||||
esac
|
||||
if [ -n "$case_names" ] || [ "$case_held" -gt 0 ]; then
|
||||
echo "ARTIFACTS_SYNC: attention: case-collision held=$case_held names=$case_names. Artifact paths differ only by letter case, which a Mac or Windows disk stores as one file; sync holds $case_held queued file(s) under them and syncs everything else. Repair: https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#artifact-case-collision. See the paths: $bin --status"
|
||||
fi
|
||||
|
||||
local now cutoff oldest="" oldest_epoch
|
||||
now=$(date +%s 2>/dev/null || echo "")
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -10,7 +11,9 @@ import unicodedata
|
||||
|
||||
|
||||
class Refusal(Exception):
|
||||
pass
|
||||
def __init__(self, message, spellings=()):
|
||||
super().__init__(message)
|
||||
self.spellings = spellings
|
||||
|
||||
|
||||
def git(root, *args, data=None, allow_missing=False):
|
||||
@@ -38,15 +41,57 @@ def prefixes(paths):
|
||||
return groups
|
||||
|
||||
|
||||
def names(spellings):
|
||||
# Only the colliding component names, clamped to a slug's charset: skill
|
||||
# start prints them, and the artifacts repo's paths are not trusted text.
|
||||
leaves = sorted({re.sub(r'[^A-Za-z0-9._-]', '?', s.rsplit('/', 1)[-1])[:64] for s in spellings})
|
||||
return ','.join(leaves[:4])
|
||||
|
||||
|
||||
def first_collision(groups):
|
||||
found = sorted((key.count('/'), key) for key, spellings in groups.items() if len(spellings) > 1)
|
||||
return groups[found[0][1]] if found else ()
|
||||
|
||||
|
||||
def tree_paths(root, *args):
|
||||
return {p.decode('utf-8') for p in git(root, *args).split(b'\0') if p}
|
||||
|
||||
|
||||
def refuse_new_collisions(root, paths, baseline='HEAD'):
|
||||
# Compare complete component prefixes: Foo/a and foo/b collide too.
|
||||
# Existing aliases are historical data; this check does not migrate them
|
||||
# or block unrelated writes, but it refuses any additional spelling.
|
||||
head = git(root, 'ls-tree', '-rz', '--name-only', baseline) if baseline else b''
|
||||
previous = prefixes(p.decode('utf-8') for p in head.split(b'\0') if p)
|
||||
for key, spellings in prefixes(paths).items():
|
||||
groups = prefixes(paths)
|
||||
for key, spellings in sorted(groups.items(), key=lambda item: (item[0].count('/'), item[0])):
|
||||
if len(spellings) > 1 and not spellings <= previous.get(key, set()):
|
||||
raise Refusal('new casefold path collision; inspect path spellings before retrying')
|
||||
raise Refusal('new casefold path collision; inspect path spellings before retrying', spellings)
|
||||
return first_collision(groups)
|
||||
|
||||
|
||||
def hold(root, requested_file, held_file):
|
||||
# A requested path is held when one of its prefixes has another spelling
|
||||
# in the index or in this batch, unless its own spelling is the only one
|
||||
# the index already tracks (an established project keeps syncing; the new
|
||||
# alias waits). Held paths are never staged, so git cannot record them
|
||||
# under the other spelling, and their queue records stay.
|
||||
requested = {p for p in Path(requested_file).read_text(encoding='utf-8').replace('\r', '').split('\n') if p}
|
||||
tracked_paths = tree_paths(root, 'ls-files', '-z')
|
||||
tracked = prefixes(tracked_paths)
|
||||
groups = prefixes(tracked_paths | requested)
|
||||
held, colliding = set(), []
|
||||
for path in requested:
|
||||
parts = path.split('/')
|
||||
for i in range(1, len(parts) + 1):
|
||||
spelling = '/'.join(parts[:i])
|
||||
key = unicodedata.normalize('NFC', spelling).casefold()
|
||||
if len(groups[key]) > 1 and tracked.get(key) != {spelling}:
|
||||
held.add(path)
|
||||
colliding.append((key.count('/'), key))
|
||||
break
|
||||
Path(held_file).write_text(''.join(p + '\n' for p in sorted(held)), encoding='utf-8')
|
||||
return groups[min(colliding)[1]] if colliding else ()
|
||||
|
||||
|
||||
def verify(root, requested_file, names_file):
|
||||
@@ -90,19 +135,23 @@ def verify(root, requested_file, names_file):
|
||||
|
||||
|
||||
def outgoing(root):
|
||||
if not git(root, 'rev-parse', '--verify', '--quiet', 'HEAD', allow_missing=True):
|
||||
return ()
|
||||
branch = git(root, 'rev-parse', '--abbrev-ref', 'HEAD').decode('utf-8').strip()
|
||||
remote = 'refs/remotes/origin/' + branch
|
||||
baseline = remote if git(root, 'rev-parse', '--verify', '--quiet', remote, allow_missing=True) else None
|
||||
head = git(root, 'ls-tree', '-rz', '--name-only', 'HEAD')
|
||||
refuse_new_collisions(root, (p.decode('utf-8') for p in head.split(b'\0') if p), baseline)
|
||||
return refuse_new_collisions(root, tree_paths(root, 'ls-tree', '-rz', '--name-only', 'HEAD'), baseline)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
if sys.argv[1] == '--outgoing':
|
||||
outgoing(Path(sys.argv[2]))
|
||||
print(names(outgoing(Path(sys.argv[2]))))
|
||||
elif sys.argv[1] == '--hold':
|
||||
print(names(hold(Path(sys.argv[2]), sys.argv[3], sys.argv[4])))
|
||||
else:
|
||||
verify(Path(sys.argv[1]), sys.argv[2], sys.argv[3])
|
||||
except (Refusal, OSError, UnicodeError, ValueError, IndexError) as error:
|
||||
print(names(getattr(error, 'spellings', ())))
|
||||
print('BRAIN_SYNC: staged artifact validation refused: ' + str(error), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -27,6 +27,7 @@
|
||||
# .brain-skip.txt — user-maintained skip list
|
||||
# .brain-sync.lock.d/ — lock dir (if present)
|
||||
# .brain-sync-status.json — health status
|
||||
# .brain-case-check — cached case-collision verdict for HEAD
|
||||
# consumers.json — consumer/reader registry
|
||||
#
|
||||
# What it clears (via gstack-config):
|
||||
@@ -131,6 +132,7 @@ rm -f "$GSTACK_HOME/.brain-last-pull" 2>/dev/null || true
|
||||
rm -f "$GSTACK_HOME/.brain-worktree-last-advance" 2>/dev/null || true
|
||||
rm -f "$GSTACK_HOME/.brain-skip.txt" 2>/dev/null || true
|
||||
rm -f "$GSTACK_HOME/.brain-sync-status.json" 2>/dev/null || true
|
||||
rm -f "$GSTACK_HOME/.brain-case-check" 2>/dev/null || true
|
||||
rm -rf "$GSTACK_HOME/.brain-sync.lock.d" 2>/dev/null || true
|
||||
|
||||
# ---- unregister gbrain federated source + remove worktree (best-effort) ----
|
||||
|
||||
+7
-2
@@ -267,7 +267,7 @@ check_codex_probe() {
|
||||
}
|
||||
|
||||
check_sync() {
|
||||
local mode file json status held drainable drain push queue=0 now f detail fix="$BIN/gstack-brain-sync --status"
|
||||
local mode file json status held drainable drain push queue=0 now f detail case_names fix="$BIN/gstack-brain-sync --status"
|
||||
mode=$("$BIN/gstack-config" get artifacts_sync_mode 2>/dev/null || echo off)
|
||||
if [ ! -d "$STATE_ROOT/.git" ]; then
|
||||
row "not configured" "artifacts sync" "off; cross-machine artifacts sync is unavailable" "$BIN/gstack-artifacts-init"
|
||||
@@ -289,8 +289,9 @@ check_sync() {
|
||||
fi
|
||||
json=$(head -c 16384 "$file" 2>/dev/null | tr '\n\r' ' ')
|
||||
status=$(json_field status "$json")
|
||||
case "$status" in ok|idle|held|blocked|push_failed|error) ;; *) status=unknown ;; esac
|
||||
case "$status" in ok|idle|held|collision|blocked|push_failed|error) ;; *) status=unknown ;; esac
|
||||
held=$(json_field held_count "$json" | tr -cd '0-9')
|
||||
case_names=$(json_field case_names "$json" | tr -cd 'A-Za-z0-9._?,-' | head -c 300)
|
||||
drainable=$(json_field drainable "$json" | tr -cd '0-9')
|
||||
drain=$(epoch_of "$(json_field last_drain_at "$json")")
|
||||
push=$(epoch_of "$(json_field last_push_at "$json")")
|
||||
@@ -301,6 +302,10 @@ check_sync() {
|
||||
blocked) row warn "artifacts sync" "status blocked (a re-scan still flags staged content)" "$fix"; return ;;
|
||||
push_failed|error|unknown) row warn "artifacts sync" "status $status" "$fix"; return ;;
|
||||
esac
|
||||
if [ -n "$case_names" ]; then
|
||||
row warn "artifacts sync" "status $status; artifact paths differ only by letter case ($case_names), see docs/troubleshooting.md#artifact-case-collision" "$fix"
|
||||
return
|
||||
fi
|
||||
if [ "${drainable:-0}" -gt 0 ] && { [ -z "$push" ] || [ $((now - push)) -gt 86400 ]; }; then
|
||||
row warn "artifacts sync" "status $status; no push for 24h while $drainable record(s) wait" "$fix"
|
||||
elif [ "$queue" -gt 0 ] && [ -n "$drain" ] && [ $((now - drain)) -gt 86400 ]; then
|
||||
|
||||
@@ -44,6 +44,7 @@ cause, the held files and the fix.
|
||||
| `error` | The drain stopped before committing and kept the queue | `status=error. The last sync stopped before committing; the queue is kept and retried. See the cause: <bin> --status` | The cause `--status` names: a git index lock, a commit hook, a `git add` failure |
|
||||
| `unknown` | The status file holds a status code outside this table | `status=unknown. The sync status file is unreadable. See it: <bin> --status` | Run `<bin> --once` to rewrite it |
|
||||
| stale-push | No push for 24 hours while the drain's `drainable` count is above zero | `stale-push. Files are ready to sync but nothing has been pushed for over 24 hours. See why: <bin> --status` | The waiting reason `--status` names (a coupled group waiting for a consistent generation, a `git add` failure) |
|
||||
| case-collision | `case_names` is set: artifact paths differ only by letter case, so the drain holds queued files under them | `case-collision held=<N> names=<names>. Artifact paths differ only by letter case, which a Mac or Windows disk stores as one file; sync holds <N> queued file(s) under them and syncs everything else. Repair: https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#artifact-case-collision. See the paths: <bin> --status` | Keep one spelling: the manual repair in [troubleshooting](troubleshooting.md#artifact-case-collision) |
|
||||
| stale-drain | `last_drain_at` older than 24 hours while the queue holds records | `stale-drain. The sync has not run for over 24 hours while files wait in the queue. See why: <bin> --status` | If no gstack process is running, remove `~/.gstack/.brain-sync.lock.d`, then run `<bin> --once` |
|
||||
|
||||
`<bin>` is the absolute path, for example
|
||||
|
||||
+5
-2
@@ -90,8 +90,11 @@ On machine B:
|
||||
gstack-brain-sync --status
|
||||
```
|
||||
|
||||
Shows: the status (`ok`, `idle`, `held`, `blocked`, `push_failed`,
|
||||
`error`), files held back by the secret scan with their fixes, the
|
||||
Shows: the status (`ok`, `idle`, `held`, `collision`, `blocked`,
|
||||
`push_failed`, `error`), files held back by the secret scan with their
|
||||
fixes, files held because their path differs only by letter case from
|
||||
another path (`case_held`, `case_names`; see
|
||||
[troubleshooting](troubleshooting.md#artifact-case-collision)), the
|
||||
`drainable` count, the last sync and push times, pending queue depth, and
|
||||
the current privacy mode.
|
||||
|
||||
|
||||
@@ -630,6 +630,64 @@ has nothing to calibrate from. It no longer says "calibrated".
|
||||
|
||||
**Fix.** Answer more registered questions, then `gstack-developer-profile --derive`.
|
||||
|
||||
<a id="artifact-case-collision"></a>
|
||||
### `ARTIFACTS_SYNC: attention: case-collision held=<N> names=<names>. Artifact paths differ only by letter case, ...`
|
||||
|
||||
**Meaning.** The artifacts repo holds paths that differ only by letter case,
|
||||
such as `projects/MyApp/` and `projects/myapp/`. A Mac or Windows disk stores
|
||||
them as one file, so git there would stage a write under the other spelling,
|
||||
and sync would report success while one spelling stays modified and the other
|
||||
machines read stale data (#3220). Sync now holds every queued file under such a
|
||||
path instead of staging it: the file stays queued, everything else keeps
|
||||
syncing, and the status is `collision` rather than `ok`. When only one
|
||||
spelling is tracked, that spelling keeps syncing and only the new alias is
|
||||
held. `<names>` are the colliding folder or file names; `gstack-brain-sync
|
||||
--status` lists the held paths in `message`. A local commit that would push a
|
||||
new alias to the remote (two machines created both spellings at once) is
|
||||
refused with status `error` and kept locally.
|
||||
|
||||
Nothing is repaired automatically. Slug resolution and skill start are
|
||||
unaffected; only the sync holds files.
|
||||
|
||||
**Fix.** Keep one spelling, by hand, on one machine:
|
||||
|
||||
1. Choose the spelling to keep: the one `gstack-slug` prints in that project
|
||||
today. If the project resolves to different spellings (a remote URL typed
|
||||
two ways, a folder reached as `~/code/MyApp` and `~/code/myapp`), make it
|
||||
stable first: fix the remote URL's case, or set `GSTACK_PROJECT_SLUG`.
|
||||
2. Fold the other spelling's tracked files into the kept one. Read both
|
||||
spellings from git, because a Mac's working tree shows only one of them,
|
||||
and the working files too, because they hold the held, unsynced writes.
|
||||
For JSONL logs, concatenate and drop duplicate lines:
|
||||
|
||||
```bash
|
||||
cd ~/.gstack
|
||||
keep=projects/MyApp; drop=projects/myapp
|
||||
git -c core.ignorecase=false ls-files -- "$drop" | while IFS= read -r f; do
|
||||
rel=${f#"$drop"/}
|
||||
{ git show ":$keep/$rel"; git show ":$f"; cat "$keep/$rel" "$f"; } 2>/dev/null | awk '!seen[$0]++' > "$HOME/.gstack-fold.tmp"
|
||||
mkdir -p "$keep/$(dirname "$rel")" && mv "$HOME/.gstack-fold.tmp" "$keep/$rel"
|
||||
git -c core.ignorecase=false add -f -- "$keep/$rel"
|
||||
git -c core.ignorecase=false rm -q --cached -- "$f"
|
||||
done
|
||||
git -c core.ignorecase=false commit -m "repair: fold $drop into $keep"
|
||||
git push origin HEAD
|
||||
```
|
||||
|
||||
For Markdown and other non-log files that exist under both spellings,
|
||||
compare `git show ":$keep/<file>"` with `git show ":$drop/<file>"` and keep
|
||||
the right content instead of concatenating. On Linux the two folders are
|
||||
separate on disk: remove the dropped folder after the commit
|
||||
(`rm -r "$drop"`). On a Mac or Windows they are the same folder, so do not
|
||||
remove anything.
|
||||
3. Clear the held records of the dropped spelling and queue the kept files:
|
||||
`gstack-brain-sync --skip-file <dropped path>` for each held path that
|
||||
`--status` lists, then `gstack-brain-enqueue <kept path>` and
|
||||
`gstack-brain-sync --once`.
|
||||
4. On every other machine, let the next skill start pull the repair (or run
|
||||
`git -C ~/.gstack pull`). The line disappears once no tracked path
|
||||
differs only by case.
|
||||
|
||||
<a id="skill-start-former-bucket"></a>
|
||||
### `gstack: earlier data for this project is in projects/<slug> (a worktree's old bucket). Merge it: ...`
|
||||
|
||||
|
||||
+68
-3
@@ -678,7 +678,7 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
['projects/p/ceo-plans/Foo/old.md', 'projects/p/ceo-plans/foo/new.md'],
|
||||
['projects/Straße/ceo-plans/old.md', 'projects/STRASSE/ceo-plans/new.md'],
|
||||
]) {
|
||||
test(`new casefold prefix collision is refused: ${oldPath}`, () => {
|
||||
test(`a new casefold prefix alias is held, never staged: ${oldPath}`, () => {
|
||||
init();
|
||||
// Create the historical tree directly so this works on either a case-
|
||||
// sensitive or insensitive host, without materializing conflicting dirs.
|
||||
@@ -687,11 +687,13 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
write(newPath, '# queued plan\n');
|
||||
const before = git(['rev-parse', 'HEAD']).stdout;
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('error');
|
||||
const s = statusJson();
|
||||
expect(s.status).toBe('collision');
|
||||
expect(s.case_held).toBe(1);
|
||||
expect(spoolText()).toContain(newPath);
|
||||
expect(git(['rev-parse', 'HEAD']).stdout).toBe(before);
|
||||
expect(remoteFiles()).not.toContain(newPath);
|
||||
expect(staged()).toContain(newPath);
|
||||
expect(staged()).toBe('');
|
||||
});
|
||||
}
|
||||
|
||||
@@ -701,10 +703,72 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
write(CLEAN, '# independent plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('ok');
|
||||
expect(statusJson().case_held).toBe(0);
|
||||
expect(statusJson().case_names).toBe('OLD,old');
|
||||
expect(remoteFiles()).toContain(CLEAN);
|
||||
expect(spoolFiles()).toHaveLength(0);
|
||||
});
|
||||
|
||||
// #3220: two spellings of one project are both tracked (a case-sensitive
|
||||
// producer committed both). A write to either is held instead of staged, so
|
||||
// a case-insensitive checkout cannot record it under the other spelling and
|
||||
// report ok; every other file still syncs.
|
||||
const WS_UPPER = 'projects/Example_Workspace/learnings.jsonl';
|
||||
const WS_LOWER = 'projects/Example_workspace/learnings.jsonl';
|
||||
test('#3220: a write into a tracked case-colliding project is held; the rest syncs and nothing reports success', () => {
|
||||
init();
|
||||
seedIndexedPaths([WS_UPPER, WS_LOWER]);
|
||||
write(WS_LOWER, '{"key":"c","insight":"new on mac"}\n');
|
||||
write(CLEAN, '# independent plan\n');
|
||||
const result = drain();
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stderr).toContain('differs only by letter case (Example_Workspace,Example_workspace)');
|
||||
expect(remoteFiles()).toContain(CLEAN);
|
||||
expect(remoteShow(WS_LOWER).stdout).toBe('# historical artifact\n');
|
||||
expect(staged()).toBe('');
|
||||
expect(spoolText()).toContain(WS_LOWER);
|
||||
expect(spoolText()).not.toContain(CLEAN);
|
||||
const s = statusJson();
|
||||
expect(s.status).toBe('collision');
|
||||
expect(s.case_held).toBe(1);
|
||||
expect(s.case_names).toBe('Example_Workspace,Example_workspace');
|
||||
expect(s.message).toContain(WS_LOWER);
|
||||
expect(s.message).toContain('docs/troubleshooting.md#artifact-case-collision');
|
||||
expect(run(['gstack-brain-sync', '--attention']).stdout.trim()).toBe(
|
||||
`ARTIFACTS_SYNC: attention: case-collision held=1 names=Example_Workspace,Example_workspace. Artifact paths differ only by letter case, which a Mac or Windows disk stores as one file; sync holds 1 queued file(s) under them and syncs everything else. Repair: https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#artifact-case-collision. See the paths: ${SELF} --status`,
|
||||
);
|
||||
// Held, not dropped: the next drain holds it again and still never says ok.
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('collision');
|
||||
expect(spoolText()).toContain(WS_LOWER);
|
||||
});
|
||||
|
||||
test('a tracked spelling keeps syncing while only the new case alias is held', () => {
|
||||
init();
|
||||
seedIndexedPaths(['projects/MyApp/ceo-plans/a.md']);
|
||||
write('projects/MyApp/ceo-plans/a.md', '# updated\n');
|
||||
write('projects/myapp/ceo-plans/b.md', '# other project\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(remoteShow('projects/MyApp/ceo-plans/a.md').stdout).toBe('# updated\n');
|
||||
expect(remoteFiles()).not.toContain('projects/myapp/ceo-plans/b.md');
|
||||
const s = statusJson();
|
||||
expect(s.status).toBe('collision');
|
||||
expect(s.case_held).toBe(1);
|
||||
expect(s.case_names).toBe('MyApp,myapp');
|
||||
expect(spoolText()).toContain('projects/myapp/ceo-plans/b.md');
|
||||
});
|
||||
|
||||
test('without a case collision the status carries no case fields and skill start no case line', () => {
|
||||
init();
|
||||
write(CLEAN, '# a plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
const s = statusJson();
|
||||
expect(s.status).toBe('ok');
|
||||
expect(s.case_held).toBe(0);
|
||||
expect(s.case_names).toBe('');
|
||||
expect(run(['gstack-brain-sync', '--attention']).stdout).toBe('');
|
||||
});
|
||||
|
||||
test('a retry merge introducing casefold aliases stays local and keeps the queue', () => {
|
||||
init();
|
||||
const peer = fs.mkdtempSync(path.join(tmpHome, 'peer-'));
|
||||
@@ -719,6 +783,7 @@ describe('gstack-brain-sync per-file hold', () => {
|
||||
write(CLEAN, '# local plan\n');
|
||||
expect(drain().status).toBe(0);
|
||||
expect(statusJson().status).toBe('error');
|
||||
expect(statusJson().case_names).toBe('P,p');
|
||||
expect(spoolText()).toContain(CLEAN);
|
||||
expect(remoteFiles()).toContain(peerPath);
|
||||
expect(remoteFiles()).not.toContain(CLEAN);
|
||||
|
||||
@@ -263,6 +263,20 @@ describe('gstack-doctor', () => {
|
||||
expect(doctor(f).row('artifacts sync').state).toBe('ok');
|
||||
});
|
||||
|
||||
test('artifacts sync: case-colliding paths warn with the clamped names, held or not', () => {
|
||||
const f = makeFixture();
|
||||
syncOn(f, { status: 'collision', message: 'held projects/myapp/x.md', case_held: 1, case_names: 'MyApp,myapp', drainable: 0, last_drain_at: iso(60), last_push_at: iso(60) });
|
||||
const held = doctor(f);
|
||||
expect(held.row('artifacts sync').state).toBe('warn');
|
||||
expect(held.row('artifacts sync').detail).toBe('status collision; artifact paths differ only by letter case (MyApp,myapp), see docs/troubleshooting.md#artifact-case-collision');
|
||||
expect(held.out).not.toContain('projects/myapp');
|
||||
|
||||
syncOn(f, { status: 'ok', message: 'pushed 1 file(s)', case_held: 0, case_names: 'OLD,old', drainable: 0, last_drain_at: iso(60), last_push_at: iso(60) });
|
||||
expect(doctor(f).row('artifacts sync').state).toBe('warn');
|
||||
syncOn(f, { status: 'ok', message: 'pushed 1 file(s)', case_held: 0, case_names: '', drainable: 0, last_drain_at: iso(60), last_push_at: iso(60) });
|
||||
expect(doctor(f).row('artifacts sync').state).toBe('ok');
|
||||
});
|
||||
|
||||
test('artifacts sync: stale push and stale drain warn, a fresh push clears them', () => {
|
||||
const f = makeFixture();
|
||||
syncOn(f, { status: 'ok', message: 'x', drainable: 4, last_drain_at: iso(60), last_push_at: iso(90_000) });
|
||||
|
||||
@@ -611,8 +611,9 @@ describe('ARTIFACTS_SYNC attention lines (sync status table)', () => {
|
||||
status: code, held_count: 0, drainable: 0, last_drain_at: NOW(), last_push_at: NOW(), ts: NOW(), held: [], message: 'm', ...extra,
|
||||
});
|
||||
|
||||
const FIXTURES: Record<string, { status: () => Record<string, unknown>; queued?: number; n?: number }> = {
|
||||
const FIXTURES: Record<string, { status: () => Record<string, unknown>; queued?: number; n?: number; names?: string }> = {
|
||||
held: { status: () => status('held', { held_count: 2 }), n: 2 },
|
||||
'case-collision': { status: () => status('collision', { case_held: 2, case_names: 'MyApp,myapp' }), n: 2, names: 'MyApp,myapp' },
|
||||
blocked: { status: () => status('blocked') },
|
||||
push_failed: { status: () => status('push_failed') },
|
||||
error: { status: () => status('error') },
|
||||
@@ -640,7 +641,7 @@ describe('ARTIFACTS_SYNC attention lines (sync status table)', () => {
|
||||
expect(fx).toBeDefined();
|
||||
const gh = syncHome(fx.status(), fx.queued ?? 0);
|
||||
try {
|
||||
const expected = row.line.replaceAll('<bin>', SYNC).replaceAll('<N>', String(fx.n ?? 0));
|
||||
const expected = row.line.replaceAll('<bin>', SYNC).replaceAll('<N>', String(fx.n ?? 0)).replaceAll('<names>', fx.names ?? '');
|
||||
expect(attention(runStart([], { GSTACK_HOME: gh }))).toEqual([expected]);
|
||||
} finally {
|
||||
fs.rmSync(gh, { recursive: true, force: true });
|
||||
|
||||
Reference in new issue
Block a user