diff --git a/apps/p2/src/hosts/session-workers.ts b/apps/p2/src/hosts/session-workers.ts index c44dbbed0..c6ce3f244 100644 --- a/apps/p2/src/hosts/session-workers.ts +++ b/apps/p2/src/hosts/session-workers.ts @@ -332,7 +332,8 @@ export class SessionWorkers implements SessionWorkerControl { /** * A worker's start, crash or failure: an owner notice (redacted), with the stderr tail as Pi Mobile; a crash is - * "Worker exited (signal): tail" in the notice and the status error too, as in v1. + * "Worker exited (signal): tail" in the notice and the status error too, as in v1. A lost link Chord recovers from + * is only a log event. */ #failed(error: WorkersError): void { if (error.kind === "worker_exited") { @@ -342,6 +343,15 @@ export class SessionWorkers implements SessionWorkerControl { this.#crashed(session); return; } + if (error.kind === "worker_link_lost") { + // Chord stopped the worker and its holders start a new one: an event, no notice. + const session = error.address.slice("session|".length); + const reason = processLogs.redactor.redact( + error.error instanceof Error ? error.error.message : String(error.error), + ); + processLogs.append(`${SESSION_PREFIX}${session}`, "event", `Worker lost its link (${reason}); restarting it`); + return; + } if (error.kind !== "worker_failed") { console.error("p2 workers:", error); return; diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index c3e3641fb..c6a64e36b 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -28,6 +28,8 @@ - Chord v2's workers on macOS and the BSDs: a worker killed by a signal (the grace's or the start timer's SIGKILL) left its lock's socket file, and every later start of its address failed with `... is a stale lock file`, `retry` too, until the file was removed by hand. The workers component now removes a lock socket file nobody accepts on before each start and after a child's exit by a signal. - Chord v2's `processLock` on macOS spun until its context ended when a regular file sat at the lock's path (`ENOTSOCK`); it now fails closed naming the file, as for a socket file nobody listens on. +- Chord v2's heartbeat closed every link after the process had slept (a laptop's sleep) or blocked its event loop, before reading what the peer had sent meanwhile. Time the process did not run no longer counts as silence: after such a pause each open link pings at once and closes only if its peer stays silent for `close - ping` (15 s by default). +- Chord v2's workers no longer leave an address failed when a running worker loses its link (the heartbeat, a protocol error, the worker closing its own link): the worker is stopped, `worker_link_lost { address, error }` is reported, and its holders get a new worker. A second lost link within a minute of the last recovery still fails the address (public error `worker lost its link`). Crashes, `fatal`s and failed starts still stay failed until `retry` or a rollout. ## [1.5.0] - 2026-11-02 diff --git a/packages/chord/docs/decisions.md b/packages/chord/docs/decisions.md index 489105045..3468dbcfd 100644 --- a/packages/chord/docs/decisions.md +++ b/packages/chord/docs/decisions.md @@ -2093,6 +2093,21 @@ validation and copies the v2 kernel added around Delta. address's queue, so never two clears of one name at once. A live holder (an older worker still exiting) accepts and is kept. One connect per start on macOS and the BSDs, nothing on Linux, Android and Windows. A taker also treats `ENOTSOCK` (a non-socket file at the path on macOS) as stale; before it retried until its context ended. +- **A pause of the process is no silence (owner, 2026-11-03; refines "Slice 4a spec decisions"):** after a laptop + slept, the host's heartbeat timer fires before any I/O is read and found every link silent past `close`: the server + closed its worker links (and the worker its parent link) though both peers were fine, and p2's session workers + stayed failed ("worker exited (code 0)") until Retry. A tick at least one interval late is time this process did + not run (a sleep, a blocked event loop); that time does not count as silence: each link's silence is what it was + before the pause, at least `ping`, so an open link pings at once and a dead peer (a TCP connection the sleep + dropped) closes `close - ping` later (15 s by default). One subtraction per link on a late tick, nothing otherwise. +- **A lost worker link is recovered from (owner, 2026-11-03: "should we not auto reconnect?"; refines R2 of "Slice + 10's points"):** R2 counted a link lost while its child lives (the heartbeat, a protocol error, the child closing its + own link) as a crash. A lost link of a `running` worker is now a stop instead: the child is stopped (grace 5 s, then + killed), `worker_link_lost { address, error }` is reported, the address does not fail, and its held route resolves + again (the kernel resolves once after a held link drops): a new worker starts after the exit. No restart loop: a + second lost link within a minute of the address's last recovery fails it as before, with the public error `worker + lost its link` (`worker_exited` and `worker_failed` as for a crash). A link lost before `ready` is still a failed + start. Crashes, `fatal`s and failed starts stay failed until `retry` or a rollout (R2 unchanged for them). - **Slice 10: a route's overhead accepted; no header accessor (owner, 2026-10-31: "1 yes, 2 no"):** after the `fetch` rework and the Node fast path a route costs +20-21 µs over a plain Node handler on the standard bench, +12-13 µs warmed (before the rework +4.5 µs): the request's scope (2.1 µs), URL parsing, admission and routing, the baseline's headers, diff --git a/packages/chord/docs/design.md b/packages/chord/docs/design.md index e8de3a77e..437c13651 100644 --- a/packages/chord/docs/design.md +++ b/packages/chord/docs/design.md @@ -364,7 +364,10 @@ host.route(Session, { behind the window. A link silent for 30 s, the handshake included, is closed (its references go `loading`). Both numbers are host options (`heartbeat`). The heartbeat detects a silent transport (TCP does not notice a sleeping laptop), not a peer that answers pings but withholds acks (bounded by the window and the pending cap instead). A - handler that blocks the event loop longer than 30 s gets its links closed (documented). + handler that blocks the event loop longer than 30 s gets its links closed (documented). Time this process did not + run (a tick at least one interval late: a sleep, a blocked event loop) is no silence: each link's silence stays + what it was before, at least `ping`, so an open link pings at once (decision "A pause of the process is no + silence"). - **Flow control: an ack-clocked window per origin** (owner, 2026-10-23; per origin 2026-10-24). Per (link, origin), at most **K = 2** of the provider's own `bound` and `state` messages are unacknowledged; only these two kinds count (everything else is control, never queued behind them). The consumer sends `ack` (with the message's origin) after @@ -2289,10 +2292,11 @@ decision "Stale lock files are cleared by the starter"; a start waits for a drai - **Keep-alive** (D9, R3, Q5): the child reports when `host.activity`'s references, jobs and lingering objects and the app's `busy` counts flip between all zero and not (local calls do not count; the first state with `ready`, which also carries the load report); the server stops a worker idle for `idleMs` (5 min). -- **Never restarted by itself** (R2): a crash, a `fatal`, a link lost while its child lives (killed after a 5 s grace) - or a failed start leaves the address failed; its holders see `failed { waiting }` and stay on the path (Q1); - `retry(address)` or a rollout reaching workers clears it and starts the held ones (Q2). `stop` suspends the address - while it has holders (R1, Q3). +- **Never restarted by itself** (R2): a crash, a `fatal` or a failed start leaves the address failed; its holders see + `failed { waiting }` and stay on the path (Q1); `retry(address)` or a rollout reaching workers clears it and starts + the held ones (Q2). `stop` suspends the address while it has holders (R1, Q3). A running worker's lost link (killed + after a 5 s grace) is a stop its holders start a new worker after (`worker_link_lost`); a second within a minute of + the last recovery fails the address (`worker lost its link`; decision "A lost worker link is recovered from"). - **Errors in public and private** (decision "worker errors in public"): `state` and holders get a fixed message (`worker failed to start`, `worker exited (SIGSEGV)`) or the child's redacted `fatal`; the full error and the stderr tail go only to `onError` (`worker_failed`). Every app hook of a start (`start`, `principal`, `lock`, the fork) is diff --git a/packages/chord/docs/slice-10.md b/packages/chord/docs/slice-10.md index acf01afb2..de6291a1c 100644 --- a/packages/chord/docs/slice-10.md +++ b/packages/chord/docs/slice-10.md @@ -144,11 +144,13 @@ await workerMain(async ({ address, start, parent, plugins, onClose }, ctx) => { run during its `load`. The start timer (60 s from the fork) covers fork, lock and connect only (Q4): past it the tree is killed and the start fails. `ready` (after the app's function returned, carrying the idle state and its load report) sets `state` to `running` and lets idle stops begin; a `load` that never ends shows `starting` until `stop`. -- **Workers never restart by themselves** (R2): a crash (an exit nobody asked for), a `fatal`, a link lost while its - child lives, or a failed start (`start` threw, the start timer, a lock not taken in time) leaves the address `failed` - with its error (`worker_failed` reported once). A link lost while its child lives (the heartbeat after a blocked - event loop, a protocol error) starts the stop: the grace (5 s), then the tree is killed; its exit is the crash ("link - lost"). `resolve` of a failed address rejects at once with that error; its holders and new holds see `failed { +- **Workers never restart by themselves** (R2): a crash (an exit nobody asked for), a `fatal`, or a failed start + (`start` threw, the start timer, a lock not taken in time) leaves the address `failed` with its error + (`worker_failed` reported once). A link lost while its child lives (the heartbeat after a blocked event loop, a + protocol error, the child closing its own link) starts the stop: the grace (5 s), then the tree is killed. After + `ready` that exit is no failure (decision "A lost worker link is recovered from", 2026-11-03): `worker_link_lost` + is reported and the held route resolves again, a new worker; a second within a minute of the address's last + recovery fails it (`worker lost its link`), as does one before `ready`. `resolve` of a failed address rejects at once with that error; its holders and new holds see `failed { waiting }` with it as cause (Q1) and stay on the address's path, nothing forks. `workers.retry(address)` (`RouteHandle.retry`) or a rollout reaching workers clears it (Q2, §4). An idle stop is no failure: the next hold starts the worker on demand. @@ -186,7 +188,8 @@ await workerMain(async ({ address, start, parent, plugins, onClose }, ctx) => { message, code?, reason? }`. - **Errors** (`onError`, `WorkersError`, which includes `BatchError` for a `state` watch listener that throws): `worker_failed { address, error, stderrTail? }` (private), `authorize_failed { address, error }` (a throwing `authorize`: the hold is refused `forbidden`), `worker_exited { - address, code, signal }` (only for an exit nobody asked for: not after a stop or a start the component failed). + address, code, signal }` (only for an exit nobody asked for: not after a stop or a start the component failed), + `worker_link_lost { address, error }` (a running worker's lost link recovered from). ### Plugins in workers (D11) diff --git a/packages/chord/src/host/README.md b/packages/chord/src/host/README.md index bee7208b2..377ace2ea 100644 --- a/packages/chord/src/host/README.md +++ b/packages/chord/src/host/README.md @@ -135,7 +135,8 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: is the default deadline of a `get` and of `host.registry`. `heartbeat: { ping?, close? }`: a link silent for `ping` ms (15 s) sends a `ping`, one silent for `close` ms (30 s), its handshake included, is closed (its references go `loading`); one interval timer per host (`min(ping, close) / 10`), running only while it has links. A handler that blocks the - event loop longer than `close` gets its links closed. + event loop longer than `close` gets its links closed by its peers; its own late tick counts the time it did not run + (a block, a sleeping laptop) as no silence, so its links ping at once instead of closing. - `revokedBy(principal) => R[]`: what revokes a principal; its revocation keys are the canonical JSON of each value, computed once per principal object (cached weakly). `host.revoke(r, ctx)`, synchronously: every remote object whose principal has `r` among its keys ends at once, whatever references remain (R11): each holder gets `forbidden` with diff --git a/packages/chord/src/host/host.ts b/packages/chord/src/host/host.ts index 044c00f49..cd86dce47 100644 --- a/packages/chord/src/host/host.ts +++ b/packages/chord/src/host/host.ts @@ -1358,10 +1358,16 @@ export class ChordHost

{ const now = Date.now(); - for (const each of [...this.#links]) each.tick(now, ping, close); - }, Math.min(ping, close) / 10); + // A tick a whole interval late is time this process did not run (a sleeping laptop, a blocked event loop). + const late = now - last - every; + const paused = late >= every ? late : 0; + last = now; + for (const each of [...this.#links]) each.tick(now, paused, ping, close); + }, every); } return made; } diff --git a/packages/chord/src/link/README.md b/packages/chord/src/link/README.md index f6d36105c..f67ebf55c 100644 --- a/packages/chord/src/link/README.md +++ b/packages/chord/src/link/README.md @@ -120,8 +120,10 @@ tab.connect(webSocketLink(new WebSocket(url)), { principal: { kind: "server" } } until it settles; they are then handled in order, and dropped when the link closes meanwhile; a rejection is a protocol error. - The heartbeat, in `Channel`: every received message stamps `lastReceived` (sends do not count); the host's one - interval timer calls `tick(now, ping, close)` on every link: a link silent for `close` ms (30 s by default), the - handshake included, closes; an open one silent for `ping` ms (15 s) sends one `ping`. A `ping` is answered with + interval timer calls `tick(now, paused, ping, close)` on every link: a link silent for `close` ms (30 s by default), + the handshake included, closes; an open one silent for `ping` ms (15 s) sends one `ping`. `paused` is the time this + process did not run before a tick at least one interval late (a sleep, a blocked event loop): it is no silence, the + silence stays what it was before, at least `ping`, so an open link pings at once. A `ping` is answered with `pong` at once, never queued behind the window; neither reaches the handler. It detects a silent transport, not a peer that answers pings but withholds acks (the window and the pending cap bound that). - Who checks what (layers 1-3 failing close the link): diff --git a/packages/chord/src/link/channel.ts b/packages/chord/src/link/channel.ts index bc6e8e94d..d66695914 100644 --- a/packages/chord/src/link/channel.ts +++ b/packages/chord/src/link/channel.ts @@ -132,9 +132,12 @@ export class Channel { /** * The heartbeat, at `now`: a link silent for `close` ms (the handshake included) closes; an open one silent for - * `ping` ms sends one `ping`. + * `ping` ms sends one `ping`. `paused` ms this process did not run before it are no silence: what the peer sent + * meanwhile is still unread (timers run before I/O). The silence stays what it was before the pause, at least `ping` + * ms, so a dead peer (a dropped TCP connection) is found `close - ping` ms later. */ - tick(now: number, ping: number, close: number): void { + tick(now: number, paused: number, ping: number, close: number): void { + if (paused > 0) this.#lastReceived = Math.min(this.#lastReceived + paused, now - ping); const silent = now - this.#lastReceived; if (silent >= close) this.close(); else if (silent >= ping && !this.#pinged && this.#state.kind === "open") { diff --git a/packages/chord/src/node/workers/README.md b/packages/chord/src/node/workers/README.md index 9ba953aa2..fee223bb1 100644 --- a/packages/chord/src/node/workers/README.md +++ b/packages/chord/src/node/workers/README.md @@ -52,9 +52,13 @@ workerMain(async ({ address, start, parent, plugins, onClose }, ctx) => { lock (waiting for an old holder without polling), makes its parent link and sends `started` when the app's host connects; `resolve` returns the link then. The start timer (60 s) covers fork, lock and connect only. `ready` (after the app's function returned, with the idle state and the load report) makes the worker `running`. -- **Failures stay** (R2): a crash (an exit nobody asked for), a `fatal`, a link closed by this side (the heartbeat, a - protocol error: the child is stopped, grace 5 s, then killed) or a failed start leaves the address `failed` +- **Failures stay** (R2): a crash (an exit nobody asked for), a `fatal` or a failed start leaves the address `failed` (`worker_failed` once, `worker_exited` for the exit) until `retry` or a rollout reaching workers. +- **A lost link is recovered from** (decision "A lost worker link is recovered from"): a link closed by this side (the + heartbeat, a protocol error: the child is stopped, grace 5 s, then killed) or by the child while it lives on (killed + after the grace). After `ready` its exit is a stop: `worker_link_lost { address, error }`, and the held route + resolves again, a new worker. A second within a minute of the address's last recovery fails it (public error + `worker lost its link`), as does one before `ready`. - **Keep-alive:** the child reports when its idleness flips: `host.activity`'s `references`, `jobs` and `lingering` and the app's `busy` (a JSON watchable of a record of numbers) all zero, no reload running; local calls send nothing. The server stops a worker idle since `idleMs()` (checked every second, from `ready`). A stop: `shutdown`; the child diff --git a/packages/chord/src/node/workers/workers.ts b/packages/chord/src/node/workers/workers.ts index 66c3eef04..f08af6199 100644 --- a/packages/chord/src/node/workers/workers.ts +++ b/packages/chord/src/node/workers/workers.ts @@ -1,6 +1,7 @@ // The workers component (slice 10 §3, §4; decision "Slice 10's points"): a forked worker process per address of one // host kind, started on demand by the route it owns, stopped when idle, never restarted by itself (R2): a crash, a -// `fatal`, a link lost while its child lives or a failed start leaves the address failed until `retry` or a rollout. +// `fatal` or a failed start leaves the address failed until `retry` or a rollout. A running worker's lost link is a +// stop its holders start a new worker after, once a minute (decision "A lost worker link is recovered from"). // Per address, a child's events are recorded on that child as they happen; starts and stops run one after another. import type { Context } from "../../context/index.ts"; @@ -71,6 +72,12 @@ export type WorkersError = readonly code: number | null; readonly signal: string | null; } + | { + /** A running worker's link was lost (the heartbeat, a protocol error): it was stopped, its holders start anew. */ + readonly kind: "worker_link_lost"; + readonly address: string; + readonly error: unknown; + } | BatchError; /** The workers component. */ @@ -138,6 +145,8 @@ type Cause = { readonly public?: CrossedError; /** The component ended the child itself (a failed start): its exit is no `worker_exited`. */ readonly killed?: true; + /** A running child's link was lost: its address may recover (`#recovers`) instead of failing. */ + readonly lost?: true; }; /** How a child's reloads end once it is gone: stopped cleanly (a stop's exit) or with an error. */ @@ -181,6 +190,8 @@ type Routed = { failure: Failure | undefined; /** A start was queued and has not settled. */ starting: Promise | undefined; + /** When it last recovered from a lost link (`#recovers`). */ + recovered: number | undefined; }; /** Starts a worker may take until `started` (fork, lock, connect): 60 s. */ @@ -195,6 +206,9 @@ const IDLE_CHECK = 1_000; /** How long a worker idles before it is stopped unless `idleMs` says otherwise: 5 minutes. */ const IDLE_MS = 5 * 60_000; +/** How long after a recovered lost link another one fails the address: 1 minute. */ +const RECOVERY_WINDOW = 60_000; + /** The public error of a start that failed. */ const START_FAILED: CrossedError = { message: "worker failed to start", code: "unavailable" }; @@ -298,6 +312,7 @@ class WorkersComponent implements Workers { child: undefined, failure: undefined, starting: undefined, + recovered: undefined, }; this.#addresses.set(address.key, routed); } @@ -463,31 +478,34 @@ class WorkersComponent implements Workers { } } - /** This side closed `child`'s link (the heartbeat, a protocol error): its child is stopped, its exit a crash. */ + /** This side closed `child`'s link (the heartbeat, a protocol error): its child is stopped, its exit a lost link. */ #linkLost(child: Child): void { this.#ended(child); if (child.stopping) return; - child.cause ??= { error: new Error("worker: link lost") }; + child.cause ??= lostLink(child); this.#shutdown(child); this.#state.refresh(); } /** * The child's side closed its link (it is exiting, or it closed its IPC and lives on): its reloads end; one that does - * not exit within the grace is killed, its exit a crash (link lost). + * not exit within the grace is killed, its exit a lost link. */ #linkClosed(child: Child): void { this.#ended(child); if (child.stopping || child.grace !== undefined) return; + const lost = lostLink(child); child.grace = setTimeout(() => { - child.cause ??= { error: new Error("worker: link lost") }; + child.cause ??= lost; child.process.kill(); }, GRACE); } /** - * `child` exited: a stop's exit forgets it; any other is a crash that fails the address (`worker_exited` and - * `worker_failed` once, the full error and stderr tail for the app's logs only), except once the component closes. + * `child` exited: a stop's exit forgets it, as does a lost link the address recovers from (`worker_link_lost`: its + * holders' route resolves again, a start after this exit); any other is a crash that fails the address + * (`worker_exited` and `worker_failed` once, the full error and stderr tail for the app's logs only), except once the + * component closes. */ #exited(routed: Routed, child: Child, exit: Exit): void { clearTimeout(child.grace); @@ -506,17 +524,32 @@ class WorkersComponent implements Workers { if (routed.child !== child) return; routed.child = undefined; if (!clean && !this.#closing.signal.aborted) { - const { code, signal } = exit; - // A failed start's kill is the component's own act (Opus's second recheck): only `worker_failed`. - if (cause.killed !== true) { - this.#options.onError({ kind: "worker_exited", address: routed.address.key, code, signal }); + if (cause.lost === true && this.#recovers(routed)) { + this.#options.onError({ kind: "worker_link_lost", address: routed.address.key, error: cause.error }); + } else { + const { code, signal } = exit; + // A failed start's kill is the component's own act (Opus's second recheck): only `worker_failed`. + if (cause.killed !== true) { + this.#options.onError({ kind: "worker_exited", address: routed.address.key, code, signal }); + } + this.#fail(routed, { ...cause, public: error }, child.process.stderrTail); } - this.#fail(routed, { ...cause, public: error }, child.process.stderrTail); } this.#state.refresh(); this.#forget(routed); } + /** + * Whether `routed` recovers from its running child's lost link, recorded when it does: not within a minute of the + * last recovery, so a worker that loses every link fails instead of restarting in a loop. + */ + #recovers(routed: Routed): boolean { + const now = Date.now(); + if (routed.recovered !== undefined && now - routed.recovered < RECOVERY_WINDOW) return false; + routed.recovered = now; + return true; + } + /** * `child`'s link is gone or it exited (the first of these): its outstanding reload and every one queued behind it * settle, `stopped` when a stop asked for it, else failed. @@ -687,6 +720,18 @@ function closed(): ChordError { return new ChordError({ code: "released", reason: "closed", message: "workers: closed" }); } +/** The public error of a running worker's lost link that fails its address (a second one within the window). */ +const LINK_LOST: CrossedError = { message: "worker lost its link", code: "unavailable" }; + +/** + * The cause of `child`'s lost link (the heartbeat, a protocol error, its own close of its link) while it lives: after + * `ready`, one its address may recover from; before it, a failed start. + */ +function lostLink(child: Child): Cause { + const error = new Error("worker: link lost"); + return child.status === "running" ? { error, public: LINK_LOST, lost: true } : { error }; +} + /** The full error of an exit nobody asked for (the app's logs). */ function exitError(address: Address, { code, signal }: Exit, child: Child): Error { const how = signal === null ? `code ${code}` : `signal ${signal}`; diff --git a/packages/chord/src/remote/host-link.ts b/packages/chord/src/remote/host-link.ts index bc7e4d23e..200841626 100644 --- a/packages/chord/src/remote/host-link.ts +++ b/packages/chord/src/remote/host-link.ts @@ -109,8 +109,8 @@ export class HostLink implements ChannelHandler { } /** The heartbeat (see `Channel.tick`). */ - tick(now: number, ping: number, close: number): void { - this.#channel.tick(now, ping, close); + tick(now: number, paused: number, ping: number, close: number): void { + this.#channel.tick(now, paused, ping, close); } opened(): void { diff --git a/packages/chord/test/link/channel.test.ts b/packages/chord/test/link/channel.test.ts index dee0e3e72..1797b26d4 100644 --- a/packages/chord/test/link/channel.test.ts +++ b/packages/chord/test/link/channel.test.ts @@ -162,6 +162,30 @@ describe("the heartbeat", () => { expect(notes.state.value).toEqual({ kind: "loading" }); }); + it("counts no pause of this process as silence (a sleeping laptop): pings at once, closes 15 s later unanswered", async () => { + const { peer } = await rawLinked(() => {}); + await elapse(3_000); + // The process sleeps 10 minutes: no timer runs meanwhile, the clock jumps. + vi.setSystemTime(Date.now() + 600_000); + await elapse(1_500); + expect(peer.closed).toBe(false); + expect(peer.kinds()).toEqual(["hello", "ping"]); + await elapse(13_500); + expect(peer.closed).toBe(false); + await elapse(1_500); + expect(peer.closed).toBe(true); + }); + + it("keeps a link through a pause when its peer answers the ping", async () => { + const { peer } = await rawLinked(() => {}); + await elapse(20_000); + vi.setSystemTime(Date.now() + 600_000); + await elapse(1_500); + peer.send({ kind: "pong" }); + await elapse(25_000); + expect(peer.closed).toBe(false); + }); + it("sends no pings on a busy link", async () => { const { peer } = await rawLinked(() => {}); for (let second = 0; second < 60; second += 5) { diff --git a/packages/chord/test/node/workers/workers.test.ts b/packages/chord/test/node/workers/workers.test.ts index fec07c969..638d1bec3 100644 --- a/packages/chord/test/node/workers/workers.test.ts +++ b/packages/chord/test/node/workers/workers.test.ts @@ -1,8 +1,8 @@ // Tests of the workers component (slice 10 §3, §4): starts on demand (the link made at the fork, `started` on -// `connect`, the start timer covering only the start), failures that stay (a crash, a `fatal`, a lost link, a failed -// start: no fork for a new hold, holders `failed { waiting }`) until `retry` or a rollout, idle stops and `busy`, `stop` -// and its suspension, the per-address queue, `close`, the rollout step, stale lock files cleared, and nothing kept of -// 1,000 addresses. +// `connect`, the start timer covering only the start), failures that stay (a crash, a `fatal`, a failed start: no fork +// for a new hold, holders `failed { waiting }`) until `retry` or a rollout, a running worker's lost link recovered from +// once a minute, idle stops and `busy`, `stop` and its suspension, the per-address queue, `close`, the rollout step, +// stale lock files cleared, and nothing kept of 1,000 addresses. import { spawnSync } from "node:child_process"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; @@ -118,27 +118,32 @@ describe("failures stay until retry or a rollout (R2)", () => { notes.release(); }); - it("a link lost while its child lives (a blocked event loop past the heartbeat): killed after the grace, failed; retry starts it past the lock file the kill left", async () => { + it("a running worker's lost link (a blocked event loop past the heartbeat): killed after the grace, its holders get a new worker past the lock file the kill left; a second within a minute fails the address", async () => { const { server, workers, starts, errors } = await fleet({ heartbeat: { ping: 100, close: 300 } }); const notes = await server.get(Notes, { id: "S1" }, context); const control = await server.get(Control, { id: "S1" }, context); await control.block(20_000, context); await until(() => stateOf(workers.state.value, "S1")?.status === "stopping", 5_000); - await until(() => stateOf(workers.state.value, "S1")?.status === "failed", 10_000); - expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker exited (SIGKILL)"); + // SIGKILL skipped the child's exit hooks: on macOS and the BSDs its lock's socket file was left, and the next start + // cleared it (else the new child would fail closed on it). + await until(() => starts.length === 2 && notes.state.value.kind === "live", 15_000); + await until(() => stateOf(workers.state.value, "S1")?.status === "running"); + expect(errors).toMatchObject([ + { kind: "worker_link_lost", address: "session|S1", error: { message: "worker: link lost" } }, + ]); + // The new worker loses its link too: no restart loop, the address fails until `retry`. + await until(() => control.state.value.kind === "live"); + await control.block(20_000, context); + await until(() => stateOf(workers.state.value, "S1")?.status === "failed", 15_000); + expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker lost its link"); expect(errors.find(({ kind }) => kind === "worker_failed")).toMatchObject({ error: { message: "worker: link lost" }, }); - expect(starts).toEqual(["session|S1"]); - // SIGKILL skipped the child's exit hooks: on macOS and the BSDs its lock's socket file was left, and the component - // cleared it after the exit (else the new child would fail closed on it). - await workers.retry(Session({ id: "S1" }), context); - await until(() => notes.state.value.kind === "live", 10_000); - expect(stateOf(workers.state.value, "S1")?.status).not.toBe("failed"); + await until(() => notes.state.value.kind === "failed"); expect(starts).toEqual(["session|S1", "session|S1"]); notes.release(); control.release(); - }, 30_000); + }, 45_000); it("an exit before started fails the start; a worker that never connects is killed at the start timer", async () => { const { server, workers, behaviours, errors } = await fleet(); @@ -599,15 +604,16 @@ describe("fix round 1", () => { notes.release(); }); - it("reaps a worker that closed its own IPC channel and lives on: killed after the grace, failed (Sol)", async () => { - const { server, workers, behaviours } = await fleet(); + it("reaps a worker that closed its own IPC channel and lives on: killed after the grace, a lost link its holder gets a new worker for (Sol)", async () => { + const { server, workers, behaviours, starts, errors } = await fleet(); behaviours.set("S1", { disconnectAndBlock: true }); const notes = await server.get(Notes, { id: "S1" }, context); - await until(() => stateOf(workers.state.value, "S1")?.status === "failed", 10_000); - expect(stateOf(workers.state.value, "S1")?.error?.message).toBe("worker exited (SIGKILL)"); - await until(() => notes.state.value.kind === "failed"); + behaviours.delete("S1"); + await until(() => errors.some(({ kind }) => kind === "worker_link_lost"), 10_000); + await until(() => starts.length === 2 && notes.state.value.kind === "live", 10_000); + expect(stateOf(workers.state.value, "S1")?.status).not.toBe("failed"); notes.release(); - }, 15_000); + }, 25_000); it("keeps no reload report once consumed: the reload queue settles to nothing (Sol)", async () => { const env = await fleet();