fix(cli): keep shannon logs tailing through a Temporal blip

- End the interactive tail on the log's own terminal marker or Ctrl-C, so a
  transient Temporal outage no longer aborts the command with exit 1.
- Rebuild the memoized Temporal client after a failed poll: a wedged gRPC
  channel was cached forever, so "retrying…" could never reconnect.
- Keep start --follow (CI) bounded — a genuinely dead Temporal still fails
  the run instead of hanging.
This commit is contained in:
ajmallesh
2026-08-30 10:48:45 -07:00
parent 0fe0c67ca5
commit 602bc27bbc
2 changed files with 53 additions and 18 deletions
+22 -1
View File
@@ -70,13 +70,31 @@ let clientPromise: Promise<Client> | null = null;
function getClient(): Promise<Client> {
if (!clientPromise) {
clientPromise = Connection.connect({ address: ADDRESS }).then(
const pending = Connection.connect({ address: ADDRESS }).then(
(connection) => new Client({ connection, namespace: NAMESPACE }),
);
// A rejected connect must not be cached forever: clear the memo so the next call rebuilds
// instead of replaying the same failure. Scoped to `pending` so a later successful reconnect
// that replaced the memo is left untouched.
pending.catch(() => resetClient(pending));
clientPromise = pending;
}
return clientPromise;
}
/**
* Drop the memoized client so the next {@link getClient} builds a fresh Connection. The underlying
* gRPC channel can wedge such that every reused call fails identically ("Unexpected error while
* making gRPC request"), and only a new Connection recovers. Best-effort closes the old channel.
* When `only` is given, the memo is cleared only if it still holds that exact promise.
*/
function resetClient(only?: Promise<Client>): void {
if (only !== undefined && clientPromise !== only) return;
const previous = clientPromise;
clientPromise = null;
previous?.then((client) => client.connection.close()).catch(() => {});
}
/** Describe a scan: status, timing, and the agents currently running (from pendingActivities). Null if not found. */
export async function describeScan(workflowId: string): Promise<ScanDescription | null> {
const client = await getClient();
@@ -201,6 +219,9 @@ export async function waitForWorkflowClose(workflowId: string, opts: WatchOption
if (err instanceof WorkflowNotFoundError) {
return { reason: 'closed' };
}
// Drop the wedged channel so the next poll dials a fresh one; a cached dead channel would
// otherwise fail every retry identically and never recover.
resetClient();
connectFailures++;
lastError = err instanceof Error ? err.message : String(err);
if (!warned && connectFailures >= warnAfterFailures) {