diff --git a/changelog.d/gst-935-ubi-pack-subdir.md b/changelog.d/gst-935-ubi-pack-subdir.md new file mode 100644 index 000000000..a62923bbc --- /dev/null +++ b/changelog.d/gst-935-ubi-pack-subdir.md @@ -0,0 +1,8 @@ +**`gstack-ubi sync` and `pack` work on a subdirectory of a git checkout (GST-935).** Packing any folder that sat inside a checkout but was not its root died with "does not appear to be a git repository", after the VM was already up, because pack fetched the checkout's refs from the subdirectory and git only fetches from a repository's root. The gstack-evals launcher syncs `quick/laneb/runner`, so every launch from a git checkout failed at its first sync. + +### Fixed +- **A subdirectory packs its own files.** Pack ships the files git lists under that folder (tracked, edited and untracked, not ignored) with no `.git`. A whole-repository `.git` would show the VM every file outside the folder as deleted. Packing a checkout's root and packing a non-git folder work as before. +- **A subdirectory git lists no files under is refused.** When the checkout ignores the folder, pack exits with "pack: git lists no files under " instead of uploading an empty folder, which a run could otherwise pass on. + +### Tests +- `test/ubicloud-runner.test.ts`: packing a subdirectory of a checkout and of a linked worktree ships its tracked, edited and untracked files without a `.git` and leaves out ignored ones, and an ignored subdirectory is refused. This test fails on 1.92.58.0. diff --git a/scripts/ubicloud/ubi-runner.sh b/scripts/ubicloud/ubi-runner.sh index 81468387d..fcac57beb 100755 --- a/scripts/ubicloud/ubi-runner.sh +++ b/scripts/ubicloud/ubi-runner.sh @@ -1442,11 +1442,31 @@ cmd_down() { # options at the first operand and resolves -T names after a -C, so no single # tar command line works on both it and GNU tar (#3680). COPYFILE_DISABLE=1 # keeps Apple's tar from adding ._* AppleDouble entries for files with -# extended attributes, which the VM would see as untracked files. +# extended attributes, which the VM would see as untracked files. A +# subdirectory of a work tree ships the same file set without a .git: git's +# local transport fetches only from a repository's root, and the whole +# repository's .git would show the VM every file outside the subdirectory as +# deleted (GST-935). A subdirectory git lists no files under (one the parent +# ignores) is refused rather than shipped empty: cmd_run keys the run on the +# local files and skips the tree check for a subdirectory, so an empty upload +# would pass on content that never ran. cmd_pack() { local src tmp gitdir branch head entry src=$(cd "${1:-.}" && pwd) - if git -C "$src" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + if git -C "$src" rev-parse --is-inside-work-tree >/dev/null 2>&1 && [ -n "$(git -C "$src" rev-parse --show-prefix)" ]; then + [ -n "$(git -C "$src" ls-files -co --exclude-standard | head -n 1)" ] \ + || die "pack: git lists no files under $src (ignored by its checkout, or empty); nothing to ship" + ( + cd "$src" + export COPYFILE_DISABLE=1 + git ls-files -z -co --exclude-standard \ + | while IFS= read -r -d '' f; do + if [ -e "$f" ] || [ -L "$f" ]; then printf '%s\0' "$f"; fi + done \ + | tar --null -T - -cf - \ + | gzip -c + ) + elif git -C "$src" rev-parse --is-inside-work-tree >/dev/null 2>&1; then head=$(git -C "$src" rev-parse --verify -q HEAD) || die "pack: $src has no commits; the VM needs a HEAD to fingerprint" branch=$(git -C "$src" symbolic-ref -q --short HEAD || true) tmp=$(mktemp -d) @@ -1767,8 +1787,10 @@ cmd_run() { for n in ${passes[@]+"${passes[@]}"}; do inherited "$n"; printf 'export %s=%q\n' "$n" "$PASS_VALUE"; done; } \ | remote "$name" 'umask 077 && cat > ~/.ubirun-env' cmd_sync "$name" "$src" "$dest" - # A subdirectory of a checkout uploads the whole .git with only its own - # files, so its tree is compared VM to VM (tiers) but not to this side. + # A subdirectory of a checkout ships without a .git, and its local tree + # (a scratch repository, blind to the parent's ignore rules) can count + # files pack leaves out, so it is compared VM to VM (tiers) but not to + # this side. if [ -z "$(git -C "$src" rev-parse --show-prefix 2>/dev/null)" ]; then vmt=$(vm_tree "$name" "$qdest") || vmt=unavailable if [ "$vmt" != "$TREE" ]; then diff --git a/test/ubicloud-runner.test.ts b/test/ubicloud-runner.test.ts index 66f6605bf..bd9fe4844 100644 --- a/test/ubicloud-runner.test.ts +++ b/test/ubicloud-runner.test.ts @@ -380,6 +380,29 @@ test('pack ships a linked worktree with a self-contained .git on the checked-out } finally { rmSync(d, { recursive: true, force: true }); } }, 60_000); + // Value: protects=sync and pack of a subdirectory of a git checkout (gstack-evals' launch.sh syncs quick/laneb/runner) ship that subdirectory's working files; fails_when=pack fetches refs from a path below the work tree's root, which git's local transport refuses, or ships the whole repository's .git so the VM sees every file outside the subdirectory as deleted; why_new=every pack test packs a work tree's root or a plain directory; seam=none + test('pack of a subdirectory of a checkout or a linked worktree ships its tracked, edited and untracked files without a .git', () => { + const d = mkdtempSync(join(tmpdir(), 'ubi-pack-subdir-')); + try { + const sh = (cmd: string, cwd: string) => { const r = Bun.spawnSync(['bash', '-c', cmd], { cwd, timeout: 30_000, env: { ...process.env, GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@t', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@t' } }); expect(r.exitCode, `${cmd}\n${r.stderr.toString()}`).toBe(0); return r.stdout.toString(); }; + sh('git init -q repo && cd repo && echo top > top && mkdir -p sub/nested && echo a > sub/a && echo b > sub/nested/b && echo g > sub/gone && echo "*.log" > sub/.gitignore && git add -A && git commit -qm one && git worktree add -q ../wt', d); + const tree = (dir: string) => sh('find . -type f | sort', dir).split('\n').filter(Boolean); + for (const checkout of ['repo', 'wt']) { + const sub = join(d, checkout, 'sub'); + sh('echo edited > a && rm gone && echo n > new && echo x > x.log', sub); + const vm = join(d, `vm-${checkout}`); + sh(`mkdir ${JSON.stringify(vm)} && bash ${JSON.stringify(join(DIR, 'ubi-runner.sh'))} pack sub | tar -xzf - -C ${JSON.stringify(vm)}`, join(d, checkout)); + expect(tree(vm)).toEqual(['./.gitignore', './a', './nested/b', './new']); + expect(readFileSync(join(vm, 'a'), 'utf8')).toBe('edited\n'); + } + // A subdirectory the checkout ignores lists no files; pack refuses it instead of shipping an empty archive. + sh('echo scratch/ > .gitignore && mkdir -p scratch/job && echo s > scratch/job/f', join(d, 'repo')); + const ignored = Bun.spawnSync(['bash', join(DIR, 'ubi-runner.sh'), 'pack', 'scratch/job'], { cwd: join(d, 'repo'), timeout: 30_000 }); + expect(ignored.exitCode).toBe(1); + expect(ignored.stderr.toString()).toContain('pack: git lists no files under'); + } finally { rmSync(d, { recursive: true, force: true }); } + }, 60_000); + test('pack snapshots a plain checkout without reading its live .git: remotes resolve, a detached HEAD gets a branch, and git running beside it cannot fail the tar', () => { const d = mkdtempSync(join(tmpdir(), 'ubi-pack-live-')); try { diff --git a/ubicloud/SKILL.md b/ubicloud/SKILL.md index 00e06d577..5bc7ab60b 100644 --- a/ubicloud/SKILL.md +++ b/ubicloud/SKILL.md @@ -184,8 +184,9 @@ UBI_TTL_HOURS=3 $U run --setup scripts/ubicloud/bootstrap.sh -- 'make test' ``` `run` creates the VM (`-s SIZE`, default `standard-16`; `-l LOCATION`, default -`eu-central-h1`), streams the checkout (tracked files, untracked files that -are not ignored, and `.git`, so uncommitted edits are tested), runs the +`eu-central-h1`), streams the checkout (tracked and non-ignored untracked +files, and `.git`, so uncommitted edits are tested; a subdirectory skips +`.git`), runs the bootstrap once, runs the command in a login shell with `umask 022`, copies `--pull REMOTE_GLOB:LOCAL_DIR` artifacts back and destroys the VM. It exits with the command's status. `--env K=V` passes a literal; `--pass NAME` forwards diff --git a/ubicloud/SKILL.md.tmpl b/ubicloud/SKILL.md.tmpl index 601d650c2..2b8aa447a 100644 --- a/ubicloud/SKILL.md.tmpl +++ b/ubicloud/SKILL.md.tmpl @@ -110,8 +110,9 @@ UBI_TTL_HOURS=3 $U run --setup scripts/ubicloud/bootstrap.sh -- 'make test' ``` `run` creates the VM (`-s SIZE`, default `standard-16`; `-l LOCATION`, default -`eu-central-h1`), streams the checkout (tracked files, untracked files that -are not ignored, and `.git`, so uncommitted edits are tested), runs the +`eu-central-h1`), streams the checkout (tracked and non-ignored untracked +files, and `.git`, so uncommitted edits are tested; a subdirectory skips +`.git`), runs the bootstrap once, runs the command in a login shell with `umask 022`, copies `--pull REMOTE_GLOB:LOCAL_DIR` artifacts back and destroys the VM. It exits with the command's status. `--env K=V` passes a literal; `--pass NAME` forwards