Report free-suite results to Trunk Flaky Tests for detection only (GST-1859) (#4187)

* Report free-suite results to Trunk Flaky Tests for detection only (GST-1859)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Say that test files that time out or crash do not reach Trunk

Review-Finding: p1/red-team#2 reader=red-team
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Keep a failed JUnit upload from failing a free-suite shard, and correct what the docs say about timed-out children

Review-Finding: p11/outside-voice#1 reader=outside-voice
Review-Finding: p1fc/fix-check#1 reader=fix-check
Review-Finding: p1fc/fix-check#2 reader=fix-check
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Say the Trunk uploader job also sees a secret in Free Tests

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

.github/workflows/free-tests.yml

@@ -8,8 +8,9 @@ name: Free Tests
 # provider keys at all — least privilege, and fork PRs get real test signal
 # here (the eval matrix skips fork PRs because repository secrets can't reach
 # them). test/free-tests-workflow-wiring.test.ts fails CI if a secret sneaks in.
-# The one exception is TRUNK_API_TOKEN in the trunk-dynamic-ci observer, which
-# runs no pull-request code and which nothing depends on (see that job).
+# The exceptions are TRUNK_API_TOKEN in the trunk-dynamic-ci observer and
+# TRUNK_ORG_API_TOKEN in the trunk-flaky-tests uploader. Each runs no
+# pull-request code and nothing depends on it (see those jobs).
 #
 # This is a REQUIRED check from day one (branch protection lists it). If it's
 # red, fix or quarantine-with-issue — don't make it advisory; an advisory lane
@@ -705,7 +706,7 @@ jobs:
       # if a future edit drops the gate build (or poppler), the lane FAILS
       # instead of the gates silently self-skipping back to false green.
       - name: Run free suite
-        run: xvfb-run -a bun run test:free --ci-run "$RUNNER_TEMP/free-plan.json" --shard ${{ matrix.shard }} --result "$RUNNER_TEMP/free-results/shard-${{ matrix.shard }}.json"
+        run: xvfb-run -a bun run test:free --ci-run "$RUNNER_TEMP/free-plan.json" --shard ${{ matrix.shard }} --result "$RUNNER_TEMP/free-results/shard-${{ matrix.shard }}.json" --junit-dir "$RUNNER_TEMP/free-junit"
         env:
           GSTACK_EXPECT_BINARIES: "1"
           # WS1 flake telemetry: a single timing flake must not red the only
@@ -733,6 +734,19 @@ jobs:
           path: ${{ runner.temp }}/free-results/*.json
           if-no-files-found: error
 
+      # bun's JUnit report from every child of this shard, the flaky retry
+      # included. Only trunk-flaky-tests reads it; no verdict does.
+      - name: Upload shard JUnit reports
+        if: always()
+        timeout-minutes: 3
+        continue-on-error: true
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
+        with:
+          name: free-junit-${{ matrix.shard }}
+          path: ${{ runner.temp }}/free-junit/*.xml
+          if-no-files-found: ignore
+          retention-days: 7
+
       # Uploaded unconditionally (not just on failure): a flaky-pass run is
       # GREEN — that's the point — so its evidence must survive green runs.
       - name: Upload flake ledger
@@ -763,6 +777,60 @@ jobs:
           include-hidden-files: true
           if-no-files-found: ignore
 
+  # Trunk Flaky Tests, DETECTION ONLY (GST-1859). Uploads every free-suite
+  # shard's JUnit reports, failures included, to the Trunk test collection
+  # 7yDVXXE3 so Trunk can flag tests that fail and then pass on the same code.
+  # Nothing is quarantined or skipped: the uploader runs with quarantine off,
+  # the collection has quarantining off, and no job needs this one or reads it,
+  # so it cannot change which tests gate or turn free-tests red. It is the only
+  # job that sees TRUNK_ORG_API_TOKEN (Trunk's v1 organization token; the v2 API
+  # key in TRUNK_API_TOKEN does not work with the uploader), so the token never
+  # shares a job with pull-request code: it checks out nothing, downloads only
+  # the XML artifacts and runs one pinned action with no GITHUB_TOKEN scopes.
+  # The CLI version is pinned and use-cache is off, so the binary always comes
+  # from Trunk's checksummed release, never from an Actions cache entry that a
+  # pull-request job could have written.
+  # Same-repository pull requests and main pushes only; a missing token skips
+  # both steps. Fail-open at step and job level. Review:
+  # docs/TESTING_INTERNALS.md.
+  trunk-flaky-tests:
+    name: Trunk Flaky Tests (detection only)
+    needs: free-suite
+    if: always() && needs.free-suite.result != 'skipped' && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository))
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    continue-on-error: true
+    permissions: {}
+    env:
+      TRUNK_ORG_API_TOKEN: ${{ secrets.TRUNK_ORG_API_TOKEN }}
+    steps:
+      - name: Download every shard's JUnit reports
+        if: env.TRUNK_ORG_API_TOKEN != ''
+        timeout-minutes: 3
+        continue-on-error: true
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
+        with:
+          pattern: free-junit-*
+          path: ${{ runner.temp }}/free-junit
+      - name: Upload them to Trunk Flaky Tests (detection only, quarantine off)
+        if: env.TRUNK_ORG_API_TOKEN != ''
+        timeout-minutes: 5
+        continue-on-error: true
+        uses: trunk-io/analytics-uploader@a86313c7029d22e6ea2d5c346060a224d90053dc # v2.1.6
+        with:
+          junit-paths: ${{ runner.temp }}/free-junit/**/*.xml
+          org-slug: garry-tan
+          token: ${{ env.TRUNK_ORG_API_TOKEN }}
+          test-collection-id: 7yDVXXE3
+          cli-version: 0.16.2
+          use-cache: 'false'
+          quarantine: 'false'
+          use-uncloned-repo: 'true'
+          github-token: ''
+          gh-repo-url: ${{ github.server_url }}/${{ github.repository }}
+          gh-repo-head-sha: ${{ github.event.pull_request.head.sha || github.sha }}
+          gh-repo-head-branch: ${{ github.event.pull_request.head.ref || github.ref_name }}
+
   # Branch protection already requires the `free-tests` context. Keep that
   # stable name as an always-running aggregate so every platform-specific CSO
   # gate is merge-blocking without a separate branch-protection migration.

changelog.d/report-free-suite-results-to-trunk-flaky-tests.md

@@ -0,0 +1 @@
+- **Free Tests now reports every free-suite result to Trunk Flaky Tests, for detection only.** Each shard keeps bun's JUnit report from every child, the flaky retry included (`--junit-dir`), and a new `trunk-flaky-tests` job uploads them to Trunk's "gstack free suite" collection on same-repository pull requests and main pushes, so Trunk can flag tests that fail and then pass on the same code. Nothing is quarantined or skipped, no job depends on the upload, and an upload failure never turns a check red. A test child that times out or crashes writes no report, so Trunk sees none of that child's results; it also skips the flaky retry, so the flake ledger has no entry for it and the red shard result is the only record. The token lives only in that job, which checks out and runs nothing from the pull request. The flake ledger and the daily flaky pass are unchanged.

docs/TESTING_INTERNALS.md

@@ -1738,8 +1738,9 @@ or commit disappears), the observer's own check reports failure, and
 `gstack-ship-queue ready` counts every check at the head. Rerun it once;
 if it fails again, delete the job. Fork pull requests, pushes and dispatches
 skip the job. A missing token skips only its step, so the job still fetches
-the action. Its only secret is `TRUNK_API_TOKEN` (in free-tests.yml it is
-the only job that sees any secret; evals.yml's other jobs hold their own), so
+the action. Its only secret is `TRUNK_API_TOKEN` (in free-tests.yml only it
+and the trunk-flaky-tests uploader, with `TRUNK_ORG_API_TOKEN`, see a secret;
+evals.yml's other jobs hold their own), so
 it checks out nothing and runs none of the pull request's code. It holds no GITHUB_TOKEN scopes (`permissions: {}`). At the
 pinned commit (`src/config.ts`, `src/changed-files.ts`), the action makes one
 network call: a request for verdicts to
@@ -1767,6 +1768,27 @@ receipt, so discount that vote when you judge their would-skip rate. Any skip
 added there later also has to pass the aggregate's receipt check. Turning on
 skipping is a separate decision for Garry, made after that review.
 
+### Trunk Flaky Tests (detection only)
+
+Since GST-1859, every free-suite shard keeps bun's JUnit report from each of
+its children, the flaky retry included (`--junit-dir`), and the
+`trunk-flaky-tests` job uploads them, failures included, to the Trunk test
+collection `7yDVXXE3` ("gstack free suite") on same-repository pull requests
+and main pushes. Trunk flags a test as flaky when it fails and then passes on
+the same code, or keeps failing on main, and lists it under Flaky Tests at
+app.trunk.io/garry-tan. That job is detection only: quarantine is off in the
+uploader and in the collection, nothing skips a test, no job needs or reads it,
+and an upload failure can't turn a check red. A child killed at its wall
+timeout, or one that crashes, writes no report, so Trunk sees none of that
+child's results, its passing files included. Such a child also skips the
+flaky retry, so the flake ledger has no entry for it; the red shard result is
+the only record. The uploader job is the only one that sees
+`TRUNK_ORG_API_TOKEN` (Trunk's v1 organization token, which the uploader
+needs), and it checks out and runs nothing of the pull request's. The [flake
+ledger](#flake-ledger), its one-rerun rule and the daily flaky pass are
+unchanged and stay the source of record.
+`test/free-tests-workflow-wiring.test.ts` locks the job's shape.
+
 ### Windows free lane
 
 Windows has minimal support ([support tiers](support-tiers.md)). The required

scripts/test-free-shards.ts

@@ -86,6 +86,9 @@
  *       # --ci-run and overrides the plan: a positive whole number up to the shard's file count,
  *       # else FREE_JOBS_INVALID. --ci-verify requires every result to have run at the plan's k.
  *       # GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run.
+ *   bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json --junit-dir junit/
+ *       # also keep each child's bun JUnit report (every sub-shard and the flaky retry) in junit/;
+ *       # CI uploads them to Trunk Flaky Tests for detection only. Verdicts never read them.
  *   bun run scripts/test-free-shards.ts --ci-verify plan.json --results results/
  */
 
@@ -600,6 +603,8 @@ type CliOptions = {
   result: string | null;
   results: string | null;
   filesFrom: string | null;
+  /** Set only by --junit-dir (with --ci-run): where each child's JUnit report is kept. */
+  junitDir?: string;
   /** Set only by --jobs (with --ci-run); absent means the plan's per-shard k. */
   jobs?: number;
   /** Set only by --shard-jobs (with --ci-plan); absent means k = 1 in every shard. */
@@ -623,6 +628,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
   let jobs: number | undefined;
   let shardJobs: number | undefined;
   let jobSetupSeconds: number | undefined;
+  let junitDir: string | undefined;
   const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = {
     ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null,
   };
@@ -646,6 +652,11 @@ export function parseCliOptions(argv: string[]): CliOptions {
     '--result': pathFlag('--result', 'result'),
     '--results': pathFlag('--results', 'results'),
     '--files-from': pathFlag('--files-from', 'filesFrom'),
+    '--junit-dir': (next) => {
+      const value = next();
+      if (!value || value.startsWith('--')) throw new Error('Missing path for --junit-dir');
+      junitDir = value;
+    },
     '--jobs': (next) => { jobs = parseJobsFlag(next()); },
     '--shard-jobs': (next) => { shardJobs = parseJobsFlag(next(), '--shard-jobs'); },
     '--job-setup-seconds': (next) => {
@@ -676,11 +687,12 @@ export function parseCliOptions(argv: string[]): CliOptions {
   if (ciModes > 1 || (ciModes && (quick || listOnly || dryRun || recordDurations || attributeHome))) throw new Error('CI modes cannot be combined with other selection modes');
   if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result');
   if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results');
+  if (junitDir !== undefined && !paths.ciRun) throw new Error('--junit-dir applies only to --ci-run');
   if (jobs !== undefined && !paths.ciRun) rejectJobsWithoutCiRun();
   if (shardJobs !== undefined && !paths.ciPlan) rejectShardJobsWithoutCiPlan();
   if (jobSetupSeconds !== undefined && !paths.ciPlan) throw new Error('--job-setup-seconds applies only to --ci-plan');
   if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection');
-  return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }), ...(shardJobs === undefined ? {} : { shardJobs }), ...(jobSetupSeconds === undefined ? {} : { jobSetupSeconds }) };
+  return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }), ...(shardJobs === undefined ? {} : { shardJobs }), ...(jobSetupSeconds === undefined ? {} : { jobSetupSeconds }), ...(junitDir === undefined ? {} : { junitDir }) };
 }
 
 const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`;
@@ -1172,6 +1184,8 @@ export interface RunFreeShardOptions {
   commandFor?: (files: string[], junitPath?: string) => ShardCommand;
   /** Ask bun for a JUnit report in the shard sandbox and record per-file counts on the outcome (--ci-run). */
   testCounts?: boolean;
+  /** Also keep the child's JUnit report in this directory, one uniquely named file per child (--junit-dir). Never read by a verdict. */
+  junitDir?: string;
   /** Suppress ALL child output from the console (tests). The classifier and the log file still see every byte. */
   quiet?: boolean;
   /** Forward the full child stream to the console (legacy firehose). Default: the quiet filtered console. */
@@ -1721,7 +1735,7 @@ export async function runFreeShard(
   // realpath: the browser tracker refuses a state dir whose path resolves elsewhere.
   const { stateDir, env } = createShardSandbox('gstack-free-shard-', options.env ?? process.env, { realpath: true });
   // The JUnit reporter leaves the console stream byte-identical (Bun 1.4.2); it only adds per-file counts.
-  const junitPath = options.testCounts ? path.join(stateDir, 'junit.xml') : undefined;
+  const junitPath = options.testCounts || options.junitDir ? path.join(stateDir, 'junit.xml') : undefined;
   const { command, args } = options.commandFor
     ? options.commandFor(files, junitPath)
     : { command: process.execPath, args: [...buildShardArgs(files, { rootDir }), ...(junitPath ? ['--reporter=junit', `--reporter-outfile=${junitPath}`] : [])] };
@@ -1791,6 +1805,14 @@ export async function runFreeShard(
     }
     await new Promise<void>((resolve) => shardLog.stream.end(() => resolve()));
     if (junitPath) junitXml = readJunitReport(junitPath);
+    if (options.junitDir && junitXml !== null) {
+      try {
+        fs.mkdirSync(options.junitDir, { recursive: true });
+        fs.writeFileSync(path.join(options.junitDir, `junit-${shardNumber}-of-${totalShards}-${env.GSTACK_FREE_SHARD_ID}.xml`), junitXml);
+      } catch (error) {
+        log(`${label} could not keep its JUnit report: ${error instanceof Error ? error.message : String(error)}; the verdict is unchanged`);
+      }
+    }
     try {
       if (!cleanupError) fs.rmSync(stateDir, { recursive: true, force: true });
     } catch {
@@ -1833,7 +1855,7 @@ export async function runFreeShard(
     shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles, unattributedFailures,
     ...(status === 'passed' ? {} : { failures: report.failures }),
     summary: { testsRan: report.testsRan, filesRan: report.filesRan, sawTerminalSummary: report.sawTerminalSummary },
-    ...(junitPath ? { counts: parseJunitCounts(junitXml, files, report.testsRan, rootDir) } : {}),
+    ...(options.testCounts ? { counts: parseJunitCounts(junitXml, files, report.testsRan, rootDir) } : {}),
     ...(status === 'timed-out' && report.inFlight.length ? { inFlight: report.inFlight } : {}),
   };
   log(shardEpilogue(outcome, totalShards));
@@ -1915,7 +1937,7 @@ async function recordFreeTestDurations(files: string[], jobs: number, wallTimeou
 
 async function retryFailedFreeFiles(
   outcomes: FreeShardOutcome[], totalShards: number,
-  options: Pick<CliOptions, 'wallTimeoutExplicit' | 'wallTimeoutMs' | 'verbose'>,
+  options: Pick<CliOptions, 'wallTimeoutExplicit' | 'wallTimeoutMs' | 'verbose' | 'junitDir'>,
 ): Promise<{ exitCode: number; retry: FreeShardOutcome | null }> {
   let worst = Math.max(...outcomes.map(outcome => exitCodeFor(outcome.status)));
   let retry: FreeShardOutcome | null = null;
@@ -1948,6 +1970,7 @@ async function retryFailedFreeFiles(
       const retryOutcome = await runFreeShard(flakyFiles, totalShards + 1, totalShards + 1, {
         wallTimeoutMs: shardTimeout(flakyFiles.length),
         verbose: options.verbose,
+        ...(options.junitDir ? { junitDir: options.junitDir } : {}),
       });
       retry = retryOutcome;
       if (retryOutcome.status === 'passed') {
@@ -2048,7 +2071,7 @@ async function main(): Promise<number> {
       wallTimeoutMs: options.wallTimeoutExplicit ? options.wallTimeoutMs
         : wallTimeoutForPackedShard(shard.predictedMs, options.wallTimeoutMs, shard.files.length),
     }, {
-      runShard: (files, shardNumber, totalShards, runOptions) => runFreeShard(files, shardNumber, totalShards, runOptions),
+      runShard: (files, shardNumber, totalShards, runOptions) => runFreeShard(files, shardNumber, totalShards, { ...runOptions, ...(options.junitDir ? { junitDir: options.junitDir } : {}) }),
       retry: outcomes => retryFailedFreeFiles(outcomes, plan.shards.length, options),
       pack: (files, count) => packShardsByDuration(files, count, durations).shards,
       isSerial: runsSerialUnderJobs,

test/ci-reuse-workflow-wiring.test.ts

@@ -19,8 +19,8 @@ const ROOT = path.resolve(import.meta.dir, '..');
 const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8');
 const MAIN_PUSH = "github.event_name == 'push' && github.ref == 'refs/heads/main'";
 const FILES = {
-  'free-tests.yml': { aggregate: 'free-tests', profile: 'ci-reuse/free-tests', plan: 'free-plan', exempt: ['trunk-dynamic-ci', 'docs-scope', 'docs-lane'], observer: 'trunk-dynamic-ci', manual: false, reuseIf: `(${MAIN_PUSH}) || github.event_name == 'pull_request'`, pullRequest: true, docsScope: true, skip: "steps.route.outputs.skip != 'true'" },
-  'windows-free-tests.yml': { aggregate: 'windows-free-tests', profile: 'ci-reuse/windows-free-tests', plan: 'windows-plan', exempt: ['windows-record-durations'], observer: null, manual: true, reuseIf: MAIN_PUSH, pullRequest: false, docsScope: false, skip: "steps.route.outputs.reused != 'true'" },
+  'free-tests.yml': { aggregate: 'free-tests', profile: 'ci-reuse/free-tests', plan: 'free-plan', exempt: ['trunk-dynamic-ci', 'trunk-flaky-tests', 'docs-scope', 'docs-lane'], observer: 'trunk-dynamic-ci', uploader: 'trunk-flaky-tests', manual: false, reuseIf: `(${MAIN_PUSH}) || github.event_name == 'pull_request'`, pullRequest: true, docsScope: true, skip: "steps.route.outputs.skip != 'true'" },
+  'windows-free-tests.yml': { aggregate: 'windows-free-tests', profile: 'ci-reuse/windows-free-tests', plan: 'windows-plan', exempt: ['windows-record-durations'], observer: null, uploader: null, manual: true, reuseIf: MAIN_PUSH, pullRequest: false, docsScope: false, skip: "steps.route.outputs.reused != 'true'" },
 };
 const PROFILES = JSON.parse(read('.gstack/pregate.json')).run_profiles as Record<string, { workflow: string; jobs: string[]; artifact: string; events: string[]; os: string }>;
 
@@ -47,9 +47,10 @@ for (const [file, spec] of Object.entries(FILES)) {
       expect(jobs.reuse.if).toBe(spec.reuseIf);
       expect(jobs.reuse.permissions).toEqual({ contents: 'read', actions: 'read' });
       expect(wf.permissions).toEqual({ contents: 'read' });
-      for (const name of Object.keys(jobs).filter(n => n !== 'reuse' && n !== spec.observer)) expect([name, jobs[name].permissions]).toEqual([name, undefined]);
+      for (const name of Object.keys(jobs).filter(n => n !== 'reuse' && n !== spec.observer && n !== spec.uploader)) expect([name, jobs[name].permissions]).toEqual([name, undefined]);
       if (spec.observer) expect([jobs[spec.observer].permissions, jobs[spec.observer].needs, jobs[spec.observer].steps.some((s: any) => s.uses?.startsWith('actions/checkout@'))]).toEqual([{}, undefined, false]);
-      expect(source.match(/secrets\s*(?:\.|\[)\s*['"]?[A-Za-z0-9_-]*|toJSON\(\s*secrets\b/gi) ?? []).toEqual(spec.observer ? ['secrets.TRUNK_API_TOKEN'] : []);
+      if (spec.uploader) expect([jobs[spec.uploader].permissions, jobs[spec.uploader].needs, jobs[spec.uploader].steps.some((s: any) => s.uses?.startsWith('actions/checkout@'))]).toEqual([{}, 'free-suite', false]);
+      expect(source.match(/secrets\s*(?:\.|\[)\s*['"]?[A-Za-z0-9_-]*|toJSON\(\s*secrets\b/gi) ?? []).toEqual(spec.observer ? ['secrets.TRUNK_API_TOKEN', 'secrets.TRUNK_ORG_API_TOKEN'] : []);
       expect(source).not.toContain('pull_request_target');
       const find = jobs.reuse.steps.find((s: any) => s.id === 'find');
       expect(find.env).toEqual({ GH_TOKEN: '${{ github.token }}' });

test/cso-distribution.test.ts

@@ -403,7 +403,7 @@ describe('CSO runtime staging gates', () => {
     const gate = job.steps.find((step: any) => step.run?.trim() === 'bun test --max-concurrency 1 test/cso-docker-integration.test.ts');
     expect(gate.env.GSTACK_CSO_DOCKER_TESTS).toBe('1');
     expect(JSON.stringify(job)).not.toContain('secrets.');
-    expect(raw.match(/secrets\s*(?:\.|\[)\s*['"]?[A-Za-z0-9_-]*|toJSON\(\s*secrets\b/gi)).toEqual(['secrets.TRUNK_API_TOKEN']);
+    expect(raw.match(/secrets\s*(?:\.|\[)\s*['"]?[A-Za-z0-9_-]*|toJSON\(\s*secrets\b/gi)).toEqual(['secrets.TRUNK_API_TOKEN', 'secrets.TRUNK_ORG_API_TOKEN']);
     expect(job.steps.map((s: any) => s.run ?? '').join('\n')).toContain('docker --host unix:///var/run/docker.sock info');
     expect(job['continue-on-error']).not.toBe(true);
     expect(gate['continue-on-error']).not.toBe(true);

test/free-tests-workflow-wiring.test.ts

@@ -6,10 +6,11 @@
  *      `bun test <dirs>` glob — the runner owns TEST_ROOTS and strict-output
  *      classification, so a truncated run can't report green.
  *   2. It is SECRETLESS where code runs: free tests make no API calls, and
- *      keeping keys out means fork PRs get real signal here. The one allowed
- *      `secrets.` reference is TRUNK_API_TOKEN in the learning-only
- *      trunk-dynamic-ci observer, which checks out and runs nothing of the PR's
- *      and which no job depends on; any other reference is a regression.
+ *      keeping keys out means fork PRs get real signal here. The two allowed
+ *      `secrets.` references are TRUNK_API_TOKEN in the learning-only
+ *      trunk-dynamic-ci observer and TRUNK_ORG_API_TOKEN in the detection-only
+ *      trunk-flaky-tests uploader. Neither checks out or runs anything of the
+ *      PR's and no job depends on either; any other reference is a regression.
  *   3. It triggers on `pull_request` (never `pull_request_target`, which
  *      would hand a fork PR the base repo's context).
  *
@@ -38,8 +39,9 @@ describe('free-tests workflow wiring', () => {
 
   test('secretless: no secrets reach the free lane', () => {
     const jobs = (Bun.YAML.parse(source) as any).jobs;
-    expect(source.match(/secrets\s*(?:\.|\[)\s*['"]?[A-Za-z0-9_-]*|toJSON\(\s*secrets\b/gi)).toEqual(['secrets.TRUNK_API_TOKEN']);
+    expect(source.match(/secrets\s*(?:\.|\[)\s*['"]?[A-Za-z0-9_-]*|toJSON\(\s*secrets\b/gi)).toEqual(['secrets.TRUNK_API_TOKEN', 'secrets.TRUNK_ORG_API_TOKEN']);
     expect(jobs['trunk-dynamic-ci'].env).toEqual({ TRUNK_API_TOKEN: '${{ secrets.TRUNK_API_TOKEN }}' });
+    expect(jobs['trunk-flaky-tests'].env).toEqual({ TRUNK_ORG_API_TOKEN: '${{ secrets.TRUNK_ORG_API_TOKEN }}' });
     expect(source).not.toContain('ANTHROPIC_API_KEY');
     expect(source).not.toContain('OPENAI_API_KEY');
   });
@@ -66,10 +68,59 @@ describe('free-tests workflow wiring', () => {
     for (const key of step.with['job-keys'].split(',')) expect(jobs[key], key).toBeDefined();
     for (const name of Object.keys(jobs).filter(n => n !== 'trunk-dynamic-ci')) {
       expect([name, [jobs[name].needs ?? []].flat()]).not.toEqual([name, expect.arrayContaining(['trunk-dynamic-ci'])]);
+      if (name === 'trunk-flaky-tests') continue;
       expect([name, JSON.stringify(jobs[name])]).not.toEqual([name, expect.stringMatching(/trunk|dynamic-ci/i)]);
     }
   });
 
+  // Value: protects=GST-1859's detection-only contract: Trunk Flaky Tests receives every free-suite shard's JUnit, failures included, and can never quarantine, skip, gate or fail anything, while the token never shares a job with pull-request code; fails_when=the suite stops writing or uploading JUnit, the uploader job checks out or runs code, gains token scopes, is needed or read by any job, loses fail-open or its token skip, turns quarantine on, runs on forks, or either action is unpinned; seam=.github/workflows/free-tests.yml; no_claim=what Trunk flags
+  test('trunk-flaky-tests is a detection-only uploader: pinned, scope-less, fail-open, quarantine off, read by no job', () => {
+    const jobs = (Bun.YAML.parse(source) as any).jobs;
+    const suite = jobs['free-suite'];
+    expect(suite.steps.find((step: any) => step.name === 'Run free suite').run).toContain('--junit-dir "$RUNNER_TEMP/free-junit"');
+    const keep = suite.steps.find((step: any) => step.name === 'Upload shard JUnit reports');
+    expect(keep.if).toBe('always()');
+    expect(keep.uses).toMatch(/^actions\/upload-artifact@[0-9a-f]{40}$/);
+    expect(keep.with).toEqual({ name: 'free-junit-${{ matrix.shard }}', path: '${{ runner.temp }}/free-junit/*.xml', 'if-no-files-found': 'ignore', 'retention-days': 7 });
+    const flaky = jobs['trunk-flaky-tests'];
+    expect(flaky.needs).toBe('free-suite');
+    expect(flaky.if).toBe("always() && needs.free-suite.result != 'skipped' && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository))");
+    expect(flaky['runs-on']).toBe('ubuntu-24.04');
+    expect(flaky.permissions).toEqual({});
+    expect(flaky['continue-on-error']).toBe(true);
+    expect(flaky['timeout-minutes']).toBe(10);
+    expect(flaky.outputs).toBeUndefined();
+    expect(flaky.steps).toHaveLength(2);
+    const [download, upload] = flaky.steps;
+    for (const step of flaky.steps) {
+      expect(step.run).toBeUndefined();
+      expect(step.if).toBe("env.TRUNK_ORG_API_TOKEN != ''");
+      expect(step['continue-on-error']).toBe(true);
+    }
+    expect(download.uses).toMatch(/^actions\/download-artifact@[0-9a-f]{40}$/);
+    expect(download.with).toEqual({ pattern: 'free-junit-*', path: '${{ runner.temp }}/free-junit' });
+    expect(upload.uses).toBe('trunk-io/analytics-uploader@a86313c7029d22e6ea2d5c346060a224d90053dc');
+    expect(upload['timeout-minutes']).toBe(5);
+    expect(upload.with).toEqual({
+      'junit-paths': '${{ runner.temp }}/free-junit/**/*.xml',
+      'org-slug': 'garry-tan',
+      token: '${{ env.TRUNK_ORG_API_TOKEN }}',
+      'test-collection-id': '7yDVXXE3',
+      'cli-version': '0.16.2',
+      'use-cache': 'false',
+      quarantine: 'false',
+      'use-uncloned-repo': 'true',
+      'github-token': '',
+      'gh-repo-url': '${{ github.server_url }}/${{ github.repository }}',
+      'gh-repo-head-sha': '${{ github.event.pull_request.head.sha || github.sha }}',
+      'gh-repo-head-branch': '${{ github.event.pull_request.head.ref || github.ref_name }}',
+    });
+    for (const name of Object.keys(jobs).filter(n => n !== 'trunk-flaky-tests')) {
+      expect([name, [jobs[name].needs ?? []].flat().includes('trunk-flaky-tests')]).toEqual([name, false]);
+      expect([name, /TRUNK_ORG_API_TOKEN|analytics-uploader|free-junit-\*/.test(JSON.stringify(jobs[name]))]).toEqual([name, false]);
+    }
+  });
+
   test('pull_request trigger, never pull_request_target', () => {
     expect(source).toContain('pull_request:');
     expect(source).not.toContain('pull_request_target');
@@ -186,7 +237,8 @@ describe('free-tests workflow wiring', () => {
         expect(step.run, name).toBe('npx playwright install --with-deps chromium');
       }
       expect(steps[install]['continue-on-error'], name).toBe(true);
-      expect(steps.filter((step: any) => step['continue-on-error']), name).toEqual([steps[install]]);
+      const failOpen = name === 'free-suite' ? [steps[install], steps.find((step: any) => step.name === 'Upload shard JUnit reports')] : [steps[install]];
+      expect(steps.filter((step: any) => step['continue-on-error']), name).toEqual(failOpen);
     }
   });
 

test/shard-jobs.test.ts

@@ -442,6 +442,49 @@ describe('per-file test and expect() counts from bun JUnit', () => {
     } finally { fs.rmSync(root, { recursive: true, force: true }); }
   }, 60_000);
 
+  // Value: protects=the Trunk Flaky Tests feed (GST-1859): --junit-dir keeps every child's bun JUnit report, failures included, without touching verdicts, counts or the stream; fails_when=the report is dropped with the sandbox, a child overwrites another's file, a retry-style child (no testCounts) gains counts, a failing write changes the outcome, or --junit-dir is accepted outside --ci-run; seam=scripts/test-free-shards.ts runFreeShard; no_claim=what Trunk does with the files
+  test('--junit-dir keeps each real child\'s JUnit report, failures included, and changes no verdict', async () => {
+    const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-shard-junit-')));
+    const errors = spyOn(console, 'error').mockImplementation(() => {});
+    try {
+      const fixtures: Record<string, string> = {
+        't/pass.test.ts': "import { test, expect } from 'bun:test';\ntest('p1', () => { expect(1).toBe(1); });\n",
+        't/fail.test.ts': "import { test, expect } from 'bun:test';\ntest('f1', () => { expect(1).toBe(2); });\n",
+      };
+      for (const [file, body] of Object.entries(fixtures)) {
+        fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
+        fs.writeFileSync(path.join(root, file), body);
+      }
+      const junitDir = path.join(root, 'junit');
+      const run = (runFiles: string[], n: number, extra: Parameters<typeof runFreeShard>[3]) =>
+        runFreeShard(runFiles, n, 3, { rootDir: root, quiet: true, log: () => {}, logFilePath: path.join(root, `stream-${n}.log`), ...extra });
+      const counted = await run(['t/pass.test.ts'], 1, { testCounts: true, junitDir });
+      const retryLike = await run(['t/fail.test.ts'], 2, { junitDir });
+      const bare = await run(['t/fail.test.ts'], 3, {});
+      expect(counted.status).toBe('passed');
+      expect(counted.counts?.complete).toBe(true);
+      expect(retryLike.counts).toBeUndefined();
+      expect([retryLike.status, retryLike.failingFiles]).toEqual([bare.status, bare.failingFiles]);
+      const kept = fs.readdirSync(junitDir).sort();
+      expect(kept).toHaveLength(2);
+      expect(kept[0]).toMatch(/^junit-1-of-3-[0-9a-f-]{36}\.xml$/);
+      expect(kept[1]).toMatch(/^junit-2-of-3-[0-9a-f-]{36}\.xml$/);
+      expect(fs.readFileSync(path.join(junitDir, kept[0]), 'utf8')).toContain('name="p1"');
+      expect(fs.readFileSync(path.join(junitDir, kept[1]), 'utf8')).toMatch(/name="f1"[\s\S]*<failure/);
+      const blocked = path.join(root, 'not-a-dir');
+      fs.writeFileSync(blocked, '');
+      const lines: string[] = [];
+      const unwritable = await runFreeShard(['t/pass.test.ts'], 1, 1, { rootDir: root, quiet: true, log: line => lines.push(line), logFilePath: path.join(root, 'blocked.log'), testCounts: true, junitDir: blocked });
+      expect([unwritable.status, unwritable.counts?.complete]).toEqual(['passed', true]);
+      expect(lines.some(line => line.includes('could not keep its JUnit report') && line.includes('the verdict is unchanged'))).toBe(true);
+      const base = ['--ci-run', 'plan.json', '--shard', '1', '--result', 'r.json'];
+      expect(parseCliOptions([...base, '--junit-dir', 'junit']).junitDir).toBe('junit');
+      expect('junitDir' in parseCliOptions(base)).toBe(false);
+      expect(() => parseCliOptions(['--junit-dir', 'junit'])).toThrow('--junit-dir applies only to --ci-run');
+      expect(() => parseCliOptions([...base, '--junit-dir'])).toThrow('Missing path for --junit-dir');
+    } finally { errors.mockRestore(); fs.rmSync(root, { recursive: true, force: true }); }
+  }, 60_000);
+
   test('a hung sub-shard is killed at its wall while its sibling finishes, and no child outlives the parent', async () => {
     const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-shard-jobs-')));
     const errors = spyOn(console, 'error').mockImplementation(() => {});

test/trunk-dynamic-ci-observers.test.ts

@@ -51,7 +51,9 @@ describe('Trunk Dynamic CI observers', () => {
       for (const key of keys.split(',')) expect([key, wf.jobs[key] !== undefined]).toEqual([key, true]);
       for (const [name, job] of Object.entries(wf.jobs as Record<string, any>).filter(([n]) => n !== 'trunk-dynamic-ci')) {
         expect([name, [job.needs ?? []].flat().includes('trunk-dynamic-ci')]).toEqual([name, false]);
-        expect([name, /trunk|dynamic-ci|TRUNK_API_TOKEN/i.test(JSON.stringify(job))]).toEqual([name, false]);
+        const text = JSON.stringify(job);
+        const mentions = name === 'trunk-flaky-tests' && file === 'free-tests.yml' ? /dynamic-ci|TRUNK_API_TOKEN/i.test(text) : /trunk|dynamic-ci|TRUNK_API_TOKEN/i.test(text);
+        expect([name, mentions]).toEqual([name, false]);
       }
       const trunkRefs = (JSON.stringify(wf.jobs).match(SECRET_REF) ?? []).filter(ref => ref === 'secrets.TRUNK_API_TOKEN');
       expect(trunkRefs).toEqual(['secrets.TRUNK_API_TOKEN']);