Merge pull request 'feat: master/agent phase 2 — jobs and manual update (#7)' (#20) from feat/master-agent-jobs-7 into main

Reviewed-on: #20

.dockerignore

@@ -7,4 +7,3 @@ README.md
 .env
 .env.*
 gitea-action/
-docker-compose.example.yml

CLAUDE.md

@@ -1,104 +1,111 @@
 # package-updater — Claude context
 
-Webhook-driven Docker Compose updater. Go service, single binary, deployed as
-a container with `/var/run/docker.sock` mounted.
+Master/agent fleet updater. A master (UI behind OIDC, Postgres) shows every Compose
+stack, Swarm service and container on every machine and starts updates; one agent per
+machine (`/var/run/docker.sock` mounted) reports its inventory and runs the jobs.
+Both are in one image: `updater-master` and `updater-agent`.
 
 See `README.md` for user-facing docs. This file is for working ON the code.
 
 ## Commands
 
 ```bash
-go test ./...                          # all tests, fast (no docker required)
-go test -run TestFoo ./internal/api    # single test
-go build ./cmd/server                  # produces ./server
-go build ./cmd/master ./cmd/agent      # new master/agent binaries (phase 1: inventory)
-DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/   # Postgres tests
+go test ./...                          # all tests, fast (no docker, no postgres)
+go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/
+go test -run TestFoo ./internal/master/hub    # single test
+go build ./cmd/master ./cmd/agent      # produce ./master and ./agent
 docker build -t package-updater:dev .  # multi-stage, builds golang:1.26-alpine
 
-# Local smoke test (builds from Dockerfile, runs against /var/run/docker.sock):
-cp env.sample .env && $EDITOR .env     # fill UPDATER_API_KEY
-docker compose -f docker-compose.local.yml up --build
-curl -sH "Authorization: Bearer $UPDATER_API_KEY" \
-  -d '{"image":"foo"}' http://localhost:8080/update | jq
+# Postgres tests (store), against any throwaway database:
+DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/
 ```
 
 ## Architecture (one-line per package)
 
-- `cmd/server`        — wiring only: config → docker client → discovery → queue → http
-- `internal/api`      — HTTP handlers, bearer-token auth, request-id + access-log middleware
-- `internal/config`   — env-var loading; fails fast if `UPDATER_API_KEY` missing
-- `internal/discovery`— given an image, return Compose `Job`s to run (label parsing, path check, dedup)
-- `internal/discovery/swarm.go`  — SwarmDiscovery: same FindJobs signature, lists `docker service ls`, gates on service-level opt-in label instead of STACKS_ROOT path-check
-- `internal/updater`  — FIFO queue + single worker + `docker compose` subprocess executor
-- `internal/updater/swarm_executor.go` — SwarmExecutor: `docker service update --image` via the Docker API instead of a `docker compose` subprocess
-- `internal/selfupdate` — flush-then-exec wrapper for updating ourselves (NOT wired in live, see gotcha)
-- `internal/metrics`  — Prometheus collectors
+- `cmd/master`        — wiring only: config → store → hub → jobs → web, two listeners (UI, agents over pinned TLS)
+- `cmd/agent`         — wiring only: config → docker client → inventory → job runner → master connection
+- `internal/protocol` — the WebSocket messages both sides speak (`Version`, `JobStart`, `JobResult`, ...)
+- `internal/pin`      — master key pair, certificate and fingerprint pinning
+- `internal/agenttoken` — agent token generation and hashing
+- `internal/imageref` — image reference parsing (repo, tag, digest)
 - `internal/logging`  — slog JSON, request-id context propagation
-- `cmd/master`, `cmd/agent` — master/agent architecture (spec `2026-10-12-master-agent-design.md`), phase 1 = inventory only. `cmd/server` keeps doing updates until phase 3.
-- `internal/protocol`  — agent↔master WebSocket messages
-- `internal/pin`       — master key pair, self-signed cert, fingerprint-pinned TLS client
-- `internal/agenttoken`— per-agent tokens (`pua_` prefix, only the SHA-256 hash is stored)
-- `internal/imageref`  — split `repo:tag@digest`
-- `internal/agent/...` — inventory collection, Docker event debounce, connection loop
-- `internal/master/...`— Postgres store, agent hub, change broker, OIDC auth, htmx UI
+- `internal/agent/config`    — env-var loading; fails fast on missing `MASTER_URL`/`MASTER_KEY`/`AGENT_TOKEN`
+- `internal/agent/inventory` — reads Compose, Swarm and plain containers from the Docker daemon
+- `internal/agent/conn`      — dial, hello, snapshot, job messages, reconnect with backoff; one writer goroutine per session
+- `internal/agent/updater`   — job `Runner` (single FIFO worker, outbox until acked), Compose and Swarm executors, override file, mount-root check, registry auth
+- `internal/master/config`   — env-var loading
+- `internal/master/store`    — Postgres access and migrations
+- `internal/master/hub`      — agent connections: auth, per-connection writer, routing, job reconcile on hello
+- `internal/master/jobs`     — decides whether a service may be updated and starts a job on the right agent
+- `internal/master/web`, `webauth` — HTMX UI behind OIDC login, SSE signals
+- `internal/master/events`   — in-process publish/subscribe for UI refresh signals
+- `internal/master/model`    — domain types shared by the above
 
 ## Design intent (do not break without discussion)
 
-- **Single FIFO worker by design** (`internal/updater/queue.go`). Spec §5.7. Never
-  parallelise the queue — two `docker compose` calls against the same stack race.
-- **Defense in depth: token AND opt-in label AND STACKS_ROOT prefix** must all
-  hold before a container is touched. Weakening any of these breaks the security
-  model — discuss before changing.
-- **Auth on `/update` only**. `/healthz`, `/metrics`, `/version` are intentionally
-  unauthenticated (internal network, scraper/healthcheck need them). See
-  `cmd/server/main.go` `routeAuth`.
-- **`metrics *Metrics` parameters may be nil**; constructors and call sites check.
-  Tests rely on this — don't drop the nil-check.
-- **Stateless**: no DB, no config file, no on-disk audit log. Docker daemon is
-  the source of truth.
-- **Compose mode and Swarm mode are selected once per deployment via `MODE`**, never
-  mixed at request time. Swarm mode's security gate is opt-in label only — there is
-  no STACKS_ROOT-equivalent path check, since Swarm services have no local compose
-  file. Don't add one; don't weaken Compose mode's three-factor gate to match.
+- **Single FIFO worker per agent by design** (`internal/agent/updater/runner.go`).
+  Two `docker compose` calls against the same stack race.
+- **One queued or running job per service**, enforced by the unique index
+  `update_jobs_one_active` (not by application code).
+- **The agent writes exactly one file**, `<stack dir>/compose.override.yaml`, and only
+  if it is absent or starts with the header line. It refuses the job when any other
+  auto-loaded override name exists.
+- **Every config file and the override path must be inside a read-write bind mount of
+  the agent's own container whose source equals its destination.** Mounts are read per
+  job (`ContainerInspect(SELF_ID)`) and checked after symlink resolution. This replaces
+  the old `STACKS_ROOT`.
+- **Self-protection on the master**: a service is refused when its image repo equals the
+  repo of `AGENT_IMAGE` or the row has the `self` flag. The agent does not re-check.
+- **Unknown message types are logged and ignored; malformed messages close the connection.**
+- **Reconcile before routable**: the hub fails a reconnecting agent's jobs it does not
+  list in `known_jobs`, and only then accepts jobs for that connection.
+- **Auth**: agents by token over pinned TLS; people by OIDC. Every state-changing UI
+  request is a POST with a CSRF token.
 
 ## Gotchas
 
-- **`internal/selfupdate.Wrapped` is implemented and unit-tested but NOT wired
-  into the live queue.** Spec §15 calls this out as a v1 gap. If you "fix" this,
-  read the package doc — flush-before-exec ordering is subtle.
-- **Image matching is case-sensitive and tag/digest-agnostic.** The tricky case
-  is `localhost:5000/foo:v1` where the first colon is a port, not a tag. See
-  `internal/discovery/matching.go` `NormaliseImage`.
+- **Image matching is case-sensitive.** The tricky case is `localhost:5000/foo:v1` where
+  the first colon is a port, not a tag. See `internal/imageref`.
 - **Path safety uses `filepath.Rel` + ".." prefix check**, NOT `strings.HasPrefix`.
-  Prevents the `/foo-evil` vs `/foo` confusion. See `internal/discovery/pathcheck.go`.
-- **`STACKS_ROOT` defaults to `/home/shcizo/self-hosted`** in prod but `env.sample`
-  suggests `/tmp` for smoke tests. The default in `config.go` is the prod value
-  — set explicitly in test/dev envs.
+  Prevents the `/foo-evil` vs `/foo` confusion. See `internal/agent/updater/pathcheck.go`.
 - **Go 1.26.3** (`go.mod`). Dockerfile pins `golang:1.26-alpine`. Bumping one
   without the other has caused a fix commit already.
 - **`<summary>` C# convention from global CLAUDE.md does not apply here** — this
   is Go. Use idiomatic GoDoc (`// FuncName does X.`).
-- **Swarm mode requires manager-node API access.** `docker service update` fails
-  with a permission error against a worker-only node. This is an operator/deployment
-  concern (point `DOCKER_HOST` at a manager, or schedule the updater on a manager),
-  not something the code can detect or work around.
-- **WebSocket read limit is raised to 1 MiB on both sides.** coder/websocket defaults to 32 KiB, which a full snapshot from a busy machine exceeds — symptom is an agent stuck in a reconnect loop.
-- **The master's key lives in Postgres (`master_identity`).** Wiping the database means every agent needs a new `MASTER_KEY`.
-- **SSE responses must flush through every wrapper.** `web.events` uses `http.ResponseController`; a middleware that wraps `ResponseWriter` without `Unwrap()` silently breaks live updates. Nginx needs `X-Accel-Buffering: no` (set by the handler).
-- **The "stateless" and "label gate" design intents above describe `cmd/server` only.** The master is stateful by design and gates updates per service in the UI (spec 2026-10-12).
-- **The master does OIDC discovery once at startup and exits if the issuer is unreachable.** It does not retry; Swarm restarts the container, so a short Authentik outage during deploy resolves itself.
+- **Compose loads only one override file.** Of `compose.override.yml`,
+  `compose.override.yaml`, `docker-compose.override.yml`, `docker-compose.override.yaml`
+  only the first that exists is read. Hence the refusal described above.
+- **Stacks started with explicit `-f` do not load the override on a manual `up`.** The
+  agent's own jobs always pass it explicitly.
+- **Override entries are never pruned.** A service deleted from the stack file keeps its
+  `image:` line in `compose.override.yaml`.
+- **There is no `STACKS_ROOT`.** Stack directories must be mounted read-write at the host
+  path (`/path:/path`), and `SELF_ID` must resolve to the agent's own container. On Docker
+  Desktop/OrbStack bind sources are translated, so Compose jobs fail there.
+- **`job.start` carries no paths.** The agent reads working dir and config files from its
+  own Docker daemon at execution time.
+- **`protocol.Service.ConfigFiles` is the raw comma-joined label string**, so `Service`
+  stays comparable with `==` (`inventory.Diff` depends on it).
+- **Swarm "succeeded" means the daemon accepted the new spec**, not that tasks converged.
+- **Swarm updates need manager-node API access** for the agent doing the update.
+- **The image's default entrypoint is `updater-agent`.** The master overrides it with
+  `updater-master`. There is no `POST /update` until phase 3.
+- **Job output is capped at `protocol.MaxJobOutput` (128 KiB, tail).** NUL bytes are
+  stripped by the hub because Postgres text rejects them.
 
 ## References
 
-- Design spec: `docs/superpowers/specs/2026-05-22-package-updater-design.md` (489 lines, authoritative)
-- Implementation plan: `docs/superpowers/plans/2026-05-22-package-updater-implementation.md`
-- Consumer-side CI integration: `gitea-action/`
-- Master/agent spec: `docs/superpowers/specs/2026-10-12-master-agent-design.md`
+- Master/agent spec: `docs/superpowers/specs/2026-10-12-master-agent-design.md` (authoritative)
 - Phase 1 plan: `docs/superpowers/plans/2026-10-12-master-agent-phase-1-inventory.md`
+- Phase 2 plan: `docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md`
+- Original single-instance design (historical, its webhook is gone until phase 3):
+  `docs/superpowers/specs/2026-05-22-package-updater-design.md`
+- Consumer-side CI integration: `gitea-action/` (calls `POST /update`, which no longer
+  exists until phase 3)
 
 ## Conventions
 
 - Conventional Commits (`feat:`, `fix:`, `docs:`, `ci:`, `build:`, `chore:`).
 - One package = one responsibility; interfaces defined at consumer site
-  (`api.Finder`, `api.Submitter`, `api.Pinger`) for testability.
+  (`jobs.Store`, `jobs.Hub`, `conn.Jobs`, `hub.Store`) for testability.
 - Table-driven tests with `stretchr/testify`. No mocks beyond hand-written fakes.

Dockerfile

@@ -8,7 +8,7 @@ COPY . .
 ARG VERSION=dev
 ARG COMMIT=unknown
 ARG BUILD_TIME=unknown
-RUN for cmd in server:package-updater master:updater-master agent:updater-agent; do \
+RUN for cmd in master:updater-master agent:updater-agent; do \
       CGO_ENABLED=0 go build \
         -ldflags="-s -w \
           -X main.version=${VERSION} \
@@ -20,5 +20,5 @@ RUN for cmd in server:package-updater master:updater-master agent:updater-agent;
 FROM alpine:3.20
 RUN apk add --no-cache docker-cli docker-cli-compose ca-certificates wget
 COPY --from=build /out/ /usr/local/bin/
-EXPOSE 8080
-ENTRYPOINT ["/usr/local/bin/package-updater"]
+EXPOSE 8080 8090
+ENTRYPOINT ["/usr/local/bin/updater-agent"]

README.md

@@ -1,121 +1,132 @@
 # package-updater
 
-Webhook-driven Docker Compose service updater. Fills the gap between Watchtower (polling, no CI integration) and full GitOps (Argo CD, Flux) for a self-hosted, single-host environment.
+Updates self-hosted Docker Compose and Docker Swarm services from one place. A
+**master** (web UI behind OIDC login, state in Postgres) shows every Compose stack,
+Swarm service and container on every machine, and starts updates when someone clicks
+*Update*. One **agent** per machine reports its inventory and runs the jobs. Both are in
+the same image: `updater-master` and `updater-agent` (the default entrypoint).
 
-**Trigger flow:**
+Design: [master/agent spec](docs/superpowers/specs/2026-10-12-master-agent-design.md).
+Phase 2 plan: [jobs and manual update](docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md).
 
-1. Gitea workflow builds and pushes a new image to your registry.
-2. Workflow calls `POST /update` on this service with the image name.
-3. Service finds the matching Compose-managed container(s) on the host via Docker labels.
-4. Runs `docker compose pull` + `up -d` for the relevant service(s).
+> **Webhook:** the old `POST /update` endpoint is gone. It returns in phase 3 on the
+> master; until then the Gitea action in `gitea-action/` has nothing to call.
 
-See [design spec](docs/superpowers/specs/2026-05-22-package-updater-design.md) and [implementation plan](docs/superpowers/plans/2026-05-22-package-updater-implementation.md) for full design and rationale.
+## What an update does
 
-## How it finds the right stack
+Clicking *Update* on a service re-pulls the service's **current tag** (`repo:tag`). The
+agent asks its Docker daemon for the tag's digest and deploys `repo:tag@digest`, so what
+runs is exactly what was pulled. The log streams to the browser while it runs; job
+history is kept in Postgres. A service without a tag (`repo@digest`), a plain container,
+and the updater's own services cannot be updated, and the UI says why. A bare image
+(`image: postgres`, no tag and no digest) counts as `postgres:latest`; only a reference
+with a digest and no tag (`repo@sha256:...`) "follows no tag". Compose services with a `build:` section are shown but never updated
+("built from source (build:)"). One update at a
+time per service, and one job at a time per agent.
 
-The service queries the Docker socket and reads the labels Compose itself attaches to every container:
+## Deploying the master
 
-- `com.docker.compose.project`
-- `com.docker.compose.service`
-- `com.docker.compose.project.working_dir`
-- `com.docker.compose.project.config_files`
+Use [`deploy/master.stack.yml`](deploy/master.stack.yml) as a starting point (Swarm stack;
+needs Postgres and an OIDC provider). The master listens for the UI on `PORT` (default
+`8080`, put it behind your reverse proxy) and for agents on `AGENT_PORT` (default `8090`,
+pinned TLS, reached directly on the internal network).
 
-A container is eligible for update only if it has **both**:
+## Adding agents
 
-- An image name matching the request (tag-agnostic), AND
-- The opt-in label `se.shcizo.auto-update=true`.
-
-Defense in depth: a valid bearer token AND the opt-in label must both be present before any container is touched.
-
-## Swarm mode
-
-Set `MODE=swarm` to update Docker Swarm services instead of Compose stacks. The
-service runs `docker service update --image` via the Docker API instead of
-shelling out to `docker compose`.
-
-Two things to get right when running in Swarm mode:
-
-- **The opt-in label goes on the service, not the container/task.** Swarm mode
-  reads `Service.Spec.Labels`, so add it with
-  `docker service update --label-add se.shcizo.auto-update=true <service>` or
-  set it under `deploy.labels` in the stack file — a plain `labels:` entry on
-  the service (container-level) is not visible to Swarm mode's discovery.
-- **The updater must talk to a Swarm manager.** `docker service update`
-  requires manager API access, so either point `DOCKER_HOST` at a manager node
-  or schedule the updater container itself on a manager with the socket
-  mounted. This is a deployment concern the service cannot detect or work
-  around.
-
-## Master/agent (in progress)
-
-A master with one agent per machine is replacing the single-instance webhook
-model; see the [design spec](docs/superpowers/specs/2026-10-12-master-agent-design.md).
-Phase 1 delivers a live inventory: every Compose stack, Swarm service and
-container on every machine, behind Authentik login. Updates still go through
-the existing `/update` webhook until phase 3.
-
-- Master: `updater-master` in the same image, example stack in `deploy/master.stack.yml`.
-  At startup it does OIDC discovery against `OIDC_ISSUER` and exits if the issuer
-  is unreachable; Swarm then restarts it.
-- Agents: add one per machine on the master's **Agents** page; it shows a
-  ready-to-run compose file with the agent's token and the master's key
-  fingerprint.
-
-## Quick start
-
-1. Build and push the image (e.g. via your own CI).
-2. Copy `docker-compose.example.yml` to `/home/shcizo/self-hosted/package-updater/docker-compose.yml`.
-3. Create `.env` next to it: `UPDATER_API_KEY=$(openssl rand -hex 32)`.
-4. Point your reverse proxy (NPM/Traefik/Caddy) at `package-updater:8080`. NPM should handle TLS.
-5. `docker compose up -d`.
-6. Add the opt-in label `se.shcizo.auto-update: "true"` to each service you want auto-updated.
-7. Use the [Gitea composite action](gitea-action/README.md) in your repos to call `/update` after a build.
-
-## Configuration
-
-All via environment variables.
+Open the master's **Agents** page and add one. It shows a ready-to-run compose file with
+the agent's token (shown once) and the master's key fingerprint; the same layout is in
+[`deploy/agent.compose.yml`](deploy/agent.compose.yml).
 
 | Variable | Required | Default | Purpose |
 |---|---|---|---|
-| `UPDATER_API_KEY` | **yes** | — | Bearer token. Service refuses to start without it. |
-| `STACKS_ROOT` | no | `/home/shcizo/self-hosted` | Required parent for any stack eligible to update. |
-| `PORT` | no | `8080` | HTTP listen port. |
-| `LOG_LEVEL` | no | `info` | `debug` / `info` / `warn` / `error`. |
+| `MASTER_URL` | yes | — | `wss://host:port/agent` |
+| `MASTER_KEY` | yes | — | Master key fingerprint (`sha256:...`) shown in the UI. |
+| `AGENT_TOKEN` | yes | — | Token shown once when the agent was added. |
+| `SELF_ID` | no | hostname | Container ID or name of the agent itself; used to read its mounts. |
 | `UPDATE_TIMEOUT` | no | `5m` | Per-job timeout (Go duration). |
-| `OPT_IN_LABEL` | no | `se.shcizo.auto-update` | Label name to check; value must equal `"true"`. |
-| `MODE` | no | `compose` | `compose` or `swarm`. Selects the update mechanism for the whole deployment; not mixed per-request. |
+| `DOCKER_CONFIG` | no | — | Directory with a `config.json` holding registry credentials. Only inline `auths` entries work; `credsStore`/`credHelpers` are not available in the container. |
+| `LOG_LEVEL` | no | `info` | `debug` / `info` / `warn` / `error`. |
 
-## Endpoints
+`docker compose` runs inside the agent with only `PATH`, `HOME` and the `DOCKER_*`
+variables. Variables a stack uses for interpolation must come from the stack's own `.env`
+file or an `--env-file` (which the agent honours), never from the agent container's
+environment.
 
-| Endpoint | Auth | Purpose |
-|---|---|---|
-| `POST /update` | Bearer token | Trigger pull + restart for matching services |
-| `GET /healthz` | none | Liveness + Docker socket reachability |
-| `GET /version` | none | Build info |
-| `GET /metrics` | none | Prometheus exposition |
+### Registry credentials
 
-`/healthz`, `/version`, and `/metrics` are intentionally unauthenticated — they're internal-network only behind the reverse proxy.
+With `DOCKER_CONFIG` set, Compose jobs pull with the same credentials as `docker compose`.
+For Swarm jobs the agent passes the matching registry's credentials with the service
+update (same as `docker service update --with-registry-auth`), so Swarm stores them in the
+cluster and hands them to the nodes that pull the image. Only the entry for the image's
+registry host is ever sent.
 
-## Observability
+### Mounting stack directories
 
-- **Logs**: JSON to stdout, picked up by Promtail/Alloy → Loki.
-- **Metrics**: Prometheus exposition on `/metrics`. Notable: `package_updater_update_jobs_total{project,service,status}`, `package_updater_last_update_timestamp{project,service}`, `package_updater_docker_ping_up`.
+There is no `STACKS_ROOT`. The agent may only touch Compose stacks in directories that
+are bind-mounted into its own container **read-write at the same path as on the host**
+(Compose labels hold host paths). Add one line per directory:
+
+```yaml
+volumes:
+  - /var/run/docker.sock:/var/run/docker.sock
+  - /home/shcizo/self-hosted:/home/shcizo/self-hosted
+```
+
+Read-only mounts and mounts whose source differs from the destination are refused with a
+clear message. On Docker Desktop and OrbStack bind sources are translated host paths, so
+Compose jobs fail there; that is a development-only limitation.
+
+## How Compose updates work, and their limits
+
+The agent never edits your files. It writes one file next to the stack's base file,
+`compose.override.yaml`, with a header line and an `image:` entry per updated service
+(`repo:tag@digest`), then runs `docker compose pull` and `up -d` for that service with
+the override passed explicitly.
+
+- The agent only writes `compose.override.yaml` if it is absent or starts with its header
+  line, and **refuses the job** if any other auto-loaded override exists
+  (`compose.override.yml`, `docker-compose.override.yml`, `docker-compose.override.yaml`):
+  Compose loads only one of them, so ours would silently disable yours.
+- Stacks you start with explicit `-f` files do not load the override on a manual
+  `docker compose up`; the agent's own jobs always pass it.
+- **The override is never pruned.** If you delete a service from the stack file, its
+  `image:` entry stays in `compose.override.yaml`, and a manual `docker compose up -d`
+  would recreate it. Remove the entry by hand.
+
+**Check before enabling.** List the config files of every running stack to spot existing
+override files and stacks started with explicit `-f`:
+
+```bash
+docker ps --format '{{.Label "com.docker.compose.project.config_files"}}' | sort -u
+```
+
+## Swarm updates
+
+The agent updates the service with `ServiceUpdate` and the pinned image. "Succeeded"
+means the daemon accepted the new spec, not that the tasks converged; the inventory shows
+`degraded` until they do. The agent doing the update must talk to a manager node (run it
+on a manager, or point `DOCKER_HOST` at one).
+
+## Upgrading from the single-instance updater
+
+This release removes `cmd/server` and changes the image's default entrypoint to
+`updater-agent`. A running old-style updater (default entrypoint, `UPDATER_API_KEY`) will
+crash-loop the next time its image is pulled. Anything that pulls `:latest` for the old
+instance triggers this: its own `/update` webhook, Watchtower or Shepherd.
+
+Pinning means two things: pin the old instance to the last pre-phase-2 tag **and** remove
+its `se.shcizo.auto-update` label, so nothing updates it to the new image. Or do not tag a
+release until phase 3 moves `POST /update` to the master.
+
+Master and agents must be on this release together: a phase-1 agent connects but every
+job sent to it fails "agent too old for updates".
 
 ## Development
 
 ```bash
 go test ./...
-go build ./cmd/server
+go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/
+go build ./cmd/master ./cmd/agent
 docker build -t package-updater:dev .
+DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/
 ```
-
-## Known v1 gaps
-
-These are tracked in the spec's section 2 and section 15 as deliberate out-of-scope:
-
-- **Self-update wiring**: `internal/selfupdate.Wrapped` exists and is unit-tested but is not wired into the live queue. The HTTP response flush ordering for self-replacement is a future enhancement; for now, expect to manually rerun `docker compose up -d` on the host if pushing a new image of `package-updater` itself causes a mid-response interruption.
-- **No rollback**: Compose's "keep old container if new fails to start" is the only safety net.
-- **Single host only in Compose mode**. Swarm mode (`MODE=swarm`) is the
-  multi-node path, but only from a manager node's point of view — the updater
-  itself still needs manager API access (see "Swarm mode" above).
-- **No per-repo API keys**: a single shared bearer token is used.

cmd/agent/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/shcizo/package-updater/internal/agent/config"
 	"github.com/shcizo/package-updater/internal/agent/conn"
 	"github.com/shcizo/package-updater/internal/agent/inventory"
+	"github.com/shcizo/package-updater/internal/agent/updater"
 	"github.com/shcizo/package-updater/internal/logging"
 )
 
@@ -42,19 +43,49 @@ func run() error {
 	}
 	defer dockerCli.Close()
 
-	collector := inventory.New(dockerCli, cfg.SelfID)
+	auth, err := updater.LoadRegistryAuth(cfg.DockerConfig, logger)
+	if err != nil {
+		return fmt.Errorf("registry auth: %w", err)
+	}
+	resolver := updater.NewResolver(dockerCli, auth)
+	exec := updater.NewDispatch(
+		updater.NewComposeExecutor(dockerCli, resolver, cfg.SelfID, updater.ExecDocker, updater.ExecDockerOutput),
+		updater.NewSwarmExecutor(dockerCli, resolver, auth),
+	)
+	runner := updater.NewRunner(exec, cfg.UpdateTimeout)
+
+	collector := inventory.New(dockerCli, cfg.SelfID).
+		WithBuildCheck(updater.NewBuildChecker(dockerCli, cfg.SelfID, updater.ExecDockerOutput), logger)
 	agent := conn.New(conn.Config{
 		URL: cfg.MasterURL, Token: cfg.Token, Fingerprint: cfg.MasterKey, Version: version,
-	}, collector, logger)
+	}, collector, runner, logger)
 
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
+	logMounts(ctx, dockerCli, cfg.SelfID, logger)
+	go runner.Run(ctx)
 	go watchLoop(ctx, dockerCli, agent, logger)
-	logger.Info("agent_starting", "version", version, "master_url", cfg.MasterURL)
+	logger.Info("agent_starting", "version", version, "master_url", cfg.MasterURL,
+		"update_timeout", cfg.UpdateTimeout.String(), "registry_auth", auth != nil)
 	return agent.Run(ctx)
 }
 
+// logMounts warns at startup when Compose updates cannot work. Each job reads
+// the mounts again itself; this is only for the operator.
+func logMounts(ctx context.Context, cli updater.SelfInspector, selfID string, log *slog.Logger) {
+	roots, err := updater.MountRoots(ctx, cli, selfID)
+	if err != nil || len(roots) == 0 {
+		attrs := []any{"hint", "Compose updates will fail; mount each stacks directory at the same path, read-write"}
+		if err != nil {
+			attrs = append(attrs, "error", err)
+		}
+		log.Warn("no_stack_mounts", attrs...)
+		return
+	}
+	log.Info("stack_mounts", "roots", roots)
+}
+
 // watchLoop keeps a Docker event subscription alive. When the stream breaks
 // (daemon restart), it resubscribes and asks for a refresh, because events
 // may have been missed in between. It also asks for a full re-inventory every

cmd/master/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/shcizo/package-updater/internal/master/config"
 	"github.com/shcizo/package-updater/internal/master/events"
 	"github.com/shcizo/package-updater/internal/master/hub"
+	"github.com/shcizo/package-updater/internal/master/jobs"
 	"github.com/shcizo/package-updater/internal/master/store"
 	"github.com/shcizo/package-updater/internal/master/web"
 	"github.com/shcizo/package-updater/internal/master/webauth"
@@ -66,6 +67,7 @@ func run() error {
 
 	broker := events.NewBroker()
 	agentHub := hub.New(st, broker, logger)
+	jobService := jobs.New(st, agentHub, broker, cfg.AgentImage, logger)
 
 	provider, err := webauth.NewOIDCProvider(ctx, webauth.OIDCConfig{
 		Issuer: cfg.OIDCIssuer, ClientID: cfg.OIDCClientID, ClientSecret: cfg.OIDCClientSecret,
@@ -76,7 +78,7 @@ func run() error {
 	}
 	auth := webauth.New(provider, st, cfg.AllowedGroup, strings.HasPrefix(cfg.PublicURL, "https://"), logger)
 
-	ui, err := web.New(st, agentHub, broker, web.Config{
+	ui, err := web.New(st, agentHub, jobService, broker, web.Config{
 		AgentURL: cfg.AgentURL, MasterKey: fingerprint, AgentImage: cfg.AgentImage,
 	}, logger)
 	if err != nil {

cmd/server/main.go

@@ -1,144 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-
-	"github.com/docker/docker/client"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/config"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/shcizo/package-updater/internal/metrics"
-	"github.com/shcizo/package-updater/internal/updater"
-)
-
-// Build info is injected at link time via -ldflags="-X main.version=..."
-var (
-	version   = "dev"
-	commit    = "unknown"
-	buildTime = "unknown"
-)
-
-func main() {
-	if err := run(); err != nil {
-		fmt.Fprintln(os.Stderr, "fatal:", err)
-		os.Exit(1)
-	}
-}
-
-func run() error {
-	cfg, err := config.Load()
-	if err != nil {
-		return err
-	}
-
-	logger := logging.New(cfg.LogLevel)
-	logger.Info("starting",
-		"version", version, "commit", commit, "port", cfg.Port,
-		"mode", cfg.Mode, "stacks_root", cfg.StacksRoot, "opt_in_label", cfg.OptInLabel,
-	)
-
-	dockerCli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-	if err != nil {
-		return fmt.Errorf("docker client: %w", err)
-	}
-	defer dockerCli.Close()
-
-	var finder api.Finder
-	var exec updater.Executor
-	switch cfg.Mode {
-	case "swarm":
-		finder = discovery.NewSwarm(dockerCli, cfg.OptInLabel)
-		exec = updater.NewSwarmExecutor(dockerCli)
-	case "compose":
-		finder = discovery.New(dockerCli, cfg.StacksRoot, cfg.OptInLabel)
-		exec = updater.NewComposeExecutor()
-	default:
-		// Unreachable: config.Load already validates Mode, but guard
-		// here too rather than silently falling through to a nil
-		// finder/exec pair.
-		return fmt.Errorf("unknown MODE %q", cfg.Mode)
-	}
-
-	reg := prometheus.NewRegistry()
-	m := metrics.New(reg)
-	m.BuildInfo.WithLabelValues(version, commit).Set(1)
-
-	queue := updater.NewQueue(exec, m)
-	queue.Start(context.Background())
-	defer queue.Stop()
-
-	handlers := api.NewHandlers(finder, &submitterAdapter{queue: queue, timeout: cfg.UpdateTimeout}, dockerCli, version, commit, buildTime, m)
-
-	mux := http.NewServeMux()
-	mux.HandleFunc("POST /update", handlers.Update)
-	mux.HandleFunc("GET /healthz", handlers.Healthz)
-	mux.HandleFunc("GET /version", handlers.Version)
-	mux.Handle("GET /metrics", metrics.Handler(reg))
-
-	authed := api.Auth(cfg.APIKey)
-	handler := api.RequestID(api.RequestLogger(logger, m)(routeAuth(mux, authed)))
-
-	srv := &http.Server{
-		Addr:              ":" + cfg.Port,
-		Handler:           handler,
-		ReadHeaderTimeout: 10 * time.Second,
-	}
-
-	errCh := make(chan error, 1)
-	go func() {
-		logger.Info("listening", "addr", srv.Addr)
-		errCh <- srv.ListenAndServe()
-	}()
-
-	sig := make(chan os.Signal, 1)
-	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
-
-	select {
-	case s := <-sig:
-		logger.Info("shutdown_signal", "signal", s.String())
-	case err := <-errCh:
-		if err != nil && err != http.ErrServerClosed {
-			return err
-		}
-	}
-
-	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	_ = srv.Shutdown(shutdownCtx)
-	return nil
-}
-
-// routeAuth applies the bearer-token middleware to /update only.
-// /healthz, /metrics, /version are unauthenticated by design (internal
-// network only; healthcheck and Prometheus scraper need to reach them
-// without secrets).
-func routeAuth(next *http.ServeMux, mw func(http.Handler) http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/update" {
-			mw(next).ServeHTTP(w, r)
-			return
-		}
-		next.ServeHTTP(w, r)
-	})
-}
-
-// submitterAdapter wraps Queue.Submit to inject the per-request timeout
-// from config, satisfying the api.Submitter interface.
-type submitterAdapter struct {
-	queue   *updater.Queue
-	timeout time.Duration
-}
-
-func (s *submitterAdapter) Submit(ctx context.Context, jobs []discovery.Job) []updater.Result {
-	ctx, cancel := context.WithTimeout(ctx, s.timeout)
-	defer cancel()
-	return s.queue.Submit(ctx, jobs)
-}

deploy/agent.compose.yml

@@ -0,0 +1,19 @@
+# Example agent deployment, one per machine. The master UI generates this
+# snippet with MASTER_URL, MASTER_KEY and AGENT_TOKEN filled in (Agents -> Add
+# agent); the token is shown once.
+services:
+  updater-agent:
+    image: gitea.shcizo.se/shcizo/package-updater:latest
+    entrypoint: ["/usr/local/bin/updater-agent"]
+    restart: unless-stopped
+    environment:
+      MASTER_URL: wss://10.0.0.5:8090/agent
+      MASTER_KEY: sha256:<fingerprint shown in the master UI>
+      AGENT_TOKEN: <token shown once when the agent was added>
+      UPDATE_TIMEOUT: 5m
+      # Optional registry credentials (a config.json written by docker login):
+      # DOCKER_CONFIG: /docker-config
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      # - /path/to/stacks:/path/to/stacks  # mount each stacks directory at the same path, read-write
+      # - /root/.docker/config.json:/docker-config/config.json:ro

deploy/master.stack.yml

@@ -1,4 +1,4 @@
-# Example Swarm stack for the package-updater master (phase 1: inventory only).
+# Example Swarm stack for the package-updater master (phase 2: inventory and manual updates).
 # Deploy on the manager: docker stack deploy -c master.stack.yml package-updater-master
 #
 # Before deploying:

docker-compose.example.yml

@@ -1,32 +0,0 @@
-# Example deployment of package-updater itself.
-# Copy to /home/shcizo/self-hosted/package-updater/docker-compose.yml
-# and create a .env file alongside containing:
-#   UPDATER_API_KEY=<openssl rand -hex 32>
-
-services:
-  package-updater:
-    image: registry.example.com/package-updater:latest
-    container_name: package-updater
-    restart: unless-stopped
-    environment:
-      - UPDATER_API_KEY=${UPDATER_API_KEY}
-      - STACKS_ROOT=/home/shcizo/self-hosted
-      - LOG_LEVEL=info
-      - PORT=8080
-      - UPDATE_TIMEOUT=5m
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - /home/shcizo/self-hosted:/home/shcizo/self-hosted:ro
-    labels:
-      - "se.shcizo.auto-update=true"
-    networks:
-      - proxy
-    healthcheck:
-      test: ["CMD", "wget", "-q", "-O-", "http://localhost:8080/healthz"]
-      interval: 30s
-      timeout: 5s
-      retries: 3
-
-networks:
-  proxy:
-    external: true

docker-compose.local.yml

@@ -1,40 +0,0 @@
-# Local development / smoke-test compose file.
-# Unlike docker-compose.example.yml (which pulls a pre-built image from a registry),
-# this file BUILDS the image from the local Dockerfile so you can iterate without
-# pushing anywhere.
-#
-# Usage:
-#   1. Copy .env.example to .env and fill in UPDATER_API_KEY (e.g. `openssl rand -hex 32`)
-#   2. Optionally adjust STACKS_ROOT below to point at a directory you actually have
-#      compose stacks in (default points at this repo dir, which is fine for smoke tests)
-#   3. docker compose -f docker-compose.local.yml up --build
-#   4. curl -s http://localhost:8080/healthz | jq
-#   5. curl -sH "Authorization: Bearer $UPDATER_API_KEY" -d '{"image":"test"}' \
-#        http://localhost:8080/update | jq
-
-services:
-  package-updater:
-    build:
-      context: .
-      args:
-        VERSION: dev-local
-        COMMIT: ${COMMIT:-unknown}
-        BUILD_TIME: ${BUILD_TIME:-unknown}
-    container_name: package-updater-local
-    restart: unless-stopped
-    environment:
-      - UPDATER_API_KEY=${UPDATER_API_KEY}
-      - STACKS_ROOT=${STACKS_ROOT:-/tmp}
-      - LOG_LEVEL=debug
-      - PORT=8080
-      - UPDATE_TIMEOUT=5m
-    ports:
-      - "8080:8080"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - ${STACKS_ROOT:-/tmp}:${STACKS_ROOT:-/tmp}:ro
-    healthcheck:
-      test: ["CMD", "wget", "-q", "-O-", "http://localhost:8080/healthz"]
-      interval: 30s
-      timeout: 5s
-      retries: 3

env.sample

@@ -1,15 +0,0 @@
-# Copy to .env (which is gitignored) and fill in real values:
-#   cp env.sample .env && $EDITOR .env
-
-# Required: bearer token clients must send as `Authorization: Bearer <value>`.
-# Generate with: openssl rand -hex 32
-UPDATER_API_KEY=
-
-# Optional: root directory the service is allowed to manage stacks under.
-# Defaults to /tmp for local smoke testing. For real use, set to your stacks dir,
-# e.g. /home/shcizo/self-hosted
-# STACKS_ROOT=/home/shcizo/self-hosted
-
-# Optional: build info baked into the binary via -ldflags. Not required.
-# COMMIT=$(git rev-parse --short HEAD)
-# BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)

go.mod

@@ -5,23 +5,22 @@ go 1.26.3
 require (
 	github.com/coder/websocket v1.8.15
 	github.com/coreos/go-oidc/v3 v3.21.0
+	github.com/distribution/reference v0.6.0
 	github.com/docker/docker v28.5.2+incompatible
 	github.com/jackc/pgx/v5 v5.10.0
-	github.com/prometheus/client_golang v1.23.2
-	github.com/prometheus/client_model v0.6.2
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/stretchr/testify v1.11.1
 	golang.org/x/oauth2 v0.36.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/errdefs v1.0.0 // indirect
 	github.com/containerd/errdefs/pkg v0.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/go-connections v0.7.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -35,25 +34,18 @@ require (
 	github.com/moby/sys/atomicwriter v0.1.0 // indirect
 	github.com/moby/term v0.5.2 // indirect
 	github.com/morikuni/aec v1.1.0 // indirect
-	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
 	go.opentelemetry.io/otel v1.43.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect
 	go.opentelemetry.io/otel/metric v1.43.0 // indirect
 	go.opentelemetry.io/otel/trace v1.43.0 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect
 	golang.org/x/text v0.35.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	google.golang.org/protobuf v1.36.11 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.2 // indirect
 )

go.sum

@@ -2,8 +2,6 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK
 github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -52,14 +50,10 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
 github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
@@ -70,8 +64,6 @@ github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
 github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
 github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
 github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
@@ -80,14 +72,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
-github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
-github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
-github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
-github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
-github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
-github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
-github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
@@ -117,10 +101,6 @@ go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09
 go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
 go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
 go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
-go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
 golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=

internal/agent/config/config.go

@@ -2,10 +2,14 @@
 package config
 
 import (
+	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 )
 
 var masterKeyPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
@@ -17,8 +21,17 @@ type Config struct {
 	Token     string // AGENT_TOKEN, shown once when the agent was added
 	SelfID    string // SELF_ID, defaults to the hostname (= container ID prefix)
 	LogLevel  string // LOG_LEVEL, default info
+
+	// UpdateTimeout bounds one update job (UPDATE_TIMEOUT, default 5m).
+	UpdateTimeout time.Duration
+	// DockerConfig is the directory holding config.json with registry
+	// credentials (DOCKER_CONFIG). Empty means anonymous registry access.
+	DockerConfig string
 }
 
+// defaultUpdateTimeout is used when UPDATE_TIMEOUT is unset.
+const defaultUpdateTimeout = 5 * time.Minute
+
 // Load reads and validates the agent configuration. It fails fast so a
 // misconfigured agent never starts dialling.
 func Load() (*Config, error) {
@@ -28,6 +41,9 @@ func Load() (*Config, error) {
 		Token:     os.Getenv("AGENT_TOKEN"),
 		SelfID:    os.Getenv("SELF_ID"),
 		LogLevel:  os.Getenv("LOG_LEVEL"),
+
+		UpdateTimeout: defaultUpdateTimeout,
+		DockerConfig:  os.Getenv("DOCKER_CONFIG"),
 	}
 	switch {
 	case cfg.MasterURL == "":
@@ -41,6 +57,18 @@ func Load() (*Config, error) {
 	case cfg.Token == "":
 		return nil, errors.New("AGENT_TOKEN is required")
 	}
+	if v := os.Getenv("UPDATE_TIMEOUT"); v != "" {
+		d, err := time.ParseDuration(v)
+		if err != nil || d <= 0 {
+			return nil, fmt.Errorf("UPDATE_TIMEOUT must be a positive duration such as 5m, got %q", v)
+		}
+		cfg.UpdateTimeout = d
+	}
+	if cfg.DockerConfig != "" {
+		if err := checkDockerConfig(cfg.DockerConfig); err != nil {
+			return nil, err
+		}
+	}
 	if cfg.SelfID == "" {
 		cfg.SelfID, _ = os.Hostname()
 	}
@@ -49,3 +77,17 @@ func Load() (*Config, error) {
 	}
 	return cfg, nil
 }
+
+// checkDockerConfig fails when DOCKER_CONFIG is set but its config.json is
+// missing or not JSON, so a typo does not silently turn into anonymous pulls.
+func checkDockerConfig(dir string) error {
+	path := filepath.Join(dir, "config.json")
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("DOCKER_CONFIG: read %s: %w", path, err)
+	}
+	if !json.Valid(raw) {
+		return fmt.Errorf("DOCKER_CONFIG: %s is not valid JSON", path)
+	}
+	return nil
+}

internal/agent/config/config_test.go

@@ -2,7 +2,9 @@ package config
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -16,6 +18,8 @@ func setValid(t *testing.T) {
 	t.Setenv("AGENT_TOKEN", "pua_abc")
 	t.Setenv("SELF_ID", "")
 	t.Setenv("LOG_LEVEL", "")
+	t.Setenv("UPDATE_TIMEOUT", "")
+	t.Setenv("DOCKER_CONFIG", "")
 }
 
 func TestLoad_Valid(t *testing.T) {
@@ -25,7 +29,7 @@ func TestLoad_Valid(t *testing.T) {
 	host, _ := os.Hostname()
 	assert.Equal(t, &Config{
 		MasterURL: "wss://10.0.0.5:8090/agent", MasterKey: validKey, Token: "pua_abc",
-		SelfID: host, LogLevel: "info",
+		SelfID: host, LogLevel: "info", UpdateTimeout: 5 * time.Minute,
 	}, cfg)
 }
 
@@ -48,3 +52,45 @@ func TestLoad_Invalid(t *testing.T) {
 		})
 	}
 }
+
+func TestLoad_JobSettings(t *testing.T) {
+	withConfig := func(t *testing.T, body string) string {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(body), 0o600))
+		return dir
+	}
+	cases := []struct {
+		name      string
+		timeout   string
+		dockerCfg func(t *testing.T) string
+		wantTime  time.Duration
+		wantErr   string
+	}{
+		{name: "defaults", wantTime: 5 * time.Minute},
+		{name: "custom timeout", timeout: "90s", wantTime: 90 * time.Second},
+		{name: "zero timeout", timeout: "0", wantErr: "UPDATE_TIMEOUT"},
+		{name: "garbage timeout", timeout: "abc", wantErr: "UPDATE_TIMEOUT"},
+		{name: "docker config without file", dockerCfg: func(t *testing.T) string { return t.TempDir() }, wantErr: "DOCKER_CONFIG"},
+		{name: "docker config not json", dockerCfg: func(t *testing.T) string { return withConfig(t, "nope") }, wantErr: "not valid JSON"},
+		{name: "docker config ok", dockerCfg: func(t *testing.T) string { return withConfig(t, `{"auths":{}}`) }, wantTime: 5 * time.Minute},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			setValid(t)
+			t.Setenv("UPDATE_TIMEOUT", c.timeout)
+			dir := ""
+			if c.dockerCfg != nil {
+				dir = c.dockerCfg(t)
+				t.Setenv("DOCKER_CONFIG", dir)
+			}
+			cfg, err := Load()
+			if c.wantErr != "" {
+				assert.ErrorContains(t, err, c.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, c.wantTime, cfg.UpdateTimeout)
+			assert.Equal(t, dir, cfg.DockerConfig)
+		})
+	}
+}

internal/agent/conn/conn.go

@@ -1,37 +1,64 @@
 // Package conn keeps an agent connected to the master: it dials over pinned
-// TLS, announces itself, sends its inventory, and streams changes until the
-// connection drops, then reconnects with backoff.
+// TLS, announces itself, sends its inventory, runs the jobs the master sends
+// and streams changes and job results until the connection drops, then
+// reconnects with backoff.
 package conn
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/coder/websocket"
 	"github.com/coder/websocket/wsjson"
 
 	"github.com/shcizo/package-updater/internal/agent/inventory"
+	"github.com/shcizo/package-updater/internal/agent/updater"
 	"github.com/shcizo/package-updater/internal/pin"
 	"github.com/shcizo/package-updater/internal/protocol"
 )
 
-// readLimit caps a single incoming message. The library default (32 KiB)
-// is too small for the master's future job messages; matches the master.
+// readLimit caps a single incoming message. Matches the master.
 const readLimit = 1 << 20
 
 // dialTimeout bounds the TLS + WebSocket handshake so a master that accepts
 // TCP but never answers does not hang the agent forever.
 const dialTimeout = 30 * time.Second
 
+// writeTimeout bounds one websocket write so a stalled peer ends the session
+// instead of blocking the writer forever.
+const writeTimeout = 10 * time.Second
+
+// defaultProgressInterval is how often job output is batched to the master.
+const defaultProgressInterval = 500 * time.Millisecond
+
 // Source supplies what the agent reports. *inventory.Collector satisfies it.
 type Source interface {
 	Hello(ctx context.Context) (protocol.Hello, error)
 	Snapshot(ctx context.Context) ([]protocol.Service, error)
 }
 
+// Jobs is the agent's job queue as the connection sees it. *updater.Runner
+// satisfies it.
+type Jobs interface {
+	// Submit queues a job; a job ID already held is ignored.
+	Submit(id string, start protocol.JobStart)
+	// Ack forgets a finished job the master has stored.
+	Ack(id string)
+	// Known lists every job ID held: queued, running, or finished unacked.
+	Known() []string
+	// TakeProgress returns output produced since the previous call.
+	TakeProgress() []updater.Progress
+	// Pending returns results the master has not acknowledged yet.
+	Pending() []updater.Finished
+	// Changed receives when there is new progress or a new result.
+	Changed() <-chan struct{}
+}
+
 // Config holds connection settings. Token is sent as a bearer header and must
 // never be logged.
 type Config struct {
@@ -41,25 +68,30 @@ type Config struct {
 	Version          string
 	MinBackoff       time.Duration // default 1s
 	MaxBackoff       time.Duration // default 30s
+	ProgressInterval time.Duration // default 500ms
 }
 
 // Agent is the agent's side of the master connection.
 type Agent struct {
 	cfg     Config
 	src     Source
+	jobs    Jobs
 	log     *slog.Logger
 	changes chan struct{}
 }
 
 // New returns an Agent. Call Run to connect.
-func New(cfg Config, src Source, log *slog.Logger) *Agent {
+func New(cfg Config, src Source, jobs Jobs, log *slog.Logger) *Agent {
 	if cfg.MinBackoff == 0 {
 		cfg.MinBackoff = time.Second
 	}
 	if cfg.MaxBackoff == 0 {
 		cfg.MaxBackoff = 30 * time.Second
 	}
-	return &Agent{cfg: cfg, src: src, log: log, changes: make(chan struct{}, 1)}
+	if cfg.ProgressInterval == 0 {
+		cfg.ProgressInterval = defaultProgressInterval
+	}
+	return &Agent{cfg: cfg, src: src, jobs: jobs, log: log, changes: make(chan struct{}, 1)}
 }
 
 // Notify tells the agent its inventory may have changed. It never blocks;
@@ -94,6 +126,9 @@ func (a *Agent) Run(ctx context.Context) error {
 
 // session runs one connection until it fails. connected reports whether the
 // hello was delivered, so Run resets its backoff only after real progress.
+//
+// Two goroutines serve the connection: a reader, which handles what the
+// master sends, and a writer, the only goroutine that writes to the socket.
 func (a *Agent) session(parent context.Context) (connected bool, err error) {
 	httpClient := &http.Client{Transport: &http.Transport{TLSClientConfig: pin.ClientConfig(a.cfg.Fingerprint)}}
 	// The dial context only bounds the handshake; coder/websocket does not
@@ -107,59 +142,140 @@ func (a *Agent) session(parent context.Context) (connected bool, err error) {
 	if err != nil {
 		return false, fmt.Errorf("dial master: %w", err)
 	}
-	defer c.CloseNow()
 	c.SetReadLimit(readLimit)
 
-	ctx, cancel := context.WithCancel(parent)
-	defer cancel()
-	// The master sends nothing yet in this phase, but reading is how a
-	// dropped connection is noticed, so a reader runs for the whole session.
-	go func() {
-		defer cancel()
-		for {
-			if _, _, err := c.Read(ctx); err != nil {
-				return
-			}
+	ctx, cancel := context.WithCancelCause(parent)
+	out := make(chan protocol.Envelope, 64)
+	var wg sync.WaitGroup
+	defer func() {
+		cancel(nil)
+		if parent.Err() != nil {
+			_ = c.Close(websocket.StatusNormalClosure, "")
 		}
+		c.CloseNow()
+		wg.Wait()
 	}()
 
+	wg.Add(2)
+	go func() { defer wg.Done(); a.writeLoop(ctx, cancel, c, out) }()
+	go func() { defer wg.Done(); a.readLoop(ctx, cancel, c) }()
+
+	// send queues an envelope for the writer. It gives up when the session ends.
+	send := func(typ string, payload any) error {
+		env, err := protocol.Encode(typ, payload)
+		if err != nil {
+			return err
+		}
+		return enqueue(ctx, out, env)
+	}
+	sendJob := func(typ, id string, payload any) error {
+		env, err := protocol.EncodeJob(typ, id, payload)
+		if err != nil {
+			return err
+		}
+		return enqueue(ctx, out, env)
+	}
+	fail := func(connected bool, err error) (bool, error) {
+		if cause := context.Cause(ctx); cause != nil && !errors.Is(cause, context.Canceled) {
+			return connected, cause
+		}
+		return connected, err
+	}
+
 	hello, err := a.src.Hello(ctx)
 	if err != nil {
-		return false, err
+		return fail(false, err)
 	}
 	hello.AgentVersion = a.cfg.Version
-	if err := send(ctx, c, protocol.TypeHello, hello); err != nil {
-		return false, err
+	hello.ProtocolVersion = protocol.Version
+	hello.KnownJobs = a.jobs.Known()
+	if err := send(protocol.TypeHello, hello); err != nil {
+		return fail(false, err)
 	}
 
 	last, err := a.src.Snapshot(ctx)
 	if err != nil {
-		return true, err
+		return fail(true, err)
 	}
-	if err := send(ctx, c, protocol.TypeSnapshot, protocol.Snapshot{Services: last}); err != nil {
-		return true, err
+	if err := send(protocol.TypeSnapshot, protocol.Snapshot{Services: last}); err != nil {
+		return fail(true, err)
 	}
-	a.log.Info("master_connected", "services", len(last))
+	a.log.Info("master_connected", "services", len(last), "known_jobs", len(hello.KnownJobs))
 
+	sent := make(map[string]bool) // job results already sent on this connection
+	flush := func() error {
+		for _, p := range a.jobs.TakeProgress() {
+			if err := sendJob(protocol.TypeJobProgress, p.ID, protocol.JobProgress{Output: p.Output}); err != nil {
+				return err
+			}
+		}
+		for _, f := range a.jobs.Pending() {
+			if sent[f.ID] {
+				continue
+			}
+			if err := sendJob(protocol.TypeJobResult, f.ID, f.Result); err != nil {
+				return err
+			}
+			sent[f.ID] = true
+		}
+		return nil
+	}
+
+	// flush is throttled to one call per ProgressInterval; a signal that
+	// arrives early arms the timer for the remainder instead of being lost.
+	lastFlush := time.Time{}
+	var timer *time.Timer
+	var timerC <-chan time.Time
+	defer func() {
+		if timer != nil {
+			timer.Stop()
+		}
+	}()
+	maybeFlush := func() error {
+		wait := a.cfg.ProgressInterval - time.Since(lastFlush)
+		if wait > 0 {
+			if timerC == nil {
+				timer = time.NewTimer(wait)
+				timerC = timer.C
+			}
+			return nil
+		}
+		lastFlush = time.Now()
+		return flush()
+	}
+	if err := maybeFlush(); err != nil {
+		return fail(true, err)
+	}
+
+	jobsChanged := a.jobs.Changed()
 	for {
 		select {
 		case <-ctx.Done():
-			_ = c.Close(websocket.StatusNormalClosure, "")
-			return true, ctx.Err()
+			return fail(true, ctx.Err())
+		case <-jobsChanged:
+			if err := maybeFlush(); err != nil {
+				return fail(true, err)
+			}
+		case <-timerC:
+			timerC = nil
+			lastFlush = time.Now()
+			if err := flush(); err != nil {
+				return fail(true, err)
+			}
 		case <-a.changes:
 			next, err := a.src.Snapshot(ctx)
 			if err != nil {
-				return true, err
+				return fail(true, err)
 			}
 			upserts, removes := inventory.Diff(last, next)
 			for _, s := range upserts {
-				if err := send(ctx, c, protocol.TypeUpsert, protocol.Upsert{Service: s}); err != nil {
-					return true, err
+				if err := send(protocol.TypeUpsert, protocol.Upsert{Service: s}); err != nil {
+					return fail(true, err)
 				}
 			}
 			for _, k := range removes {
-				if err := send(ctx, c, protocol.TypeRemove, protocol.Remove{Key: k}); err != nil {
-					return true, err
+				if err := send(protocol.TypeRemove, protocol.Remove{Key: k}); err != nil {
+					return fail(true, err)
 				}
 			}
 			last = next
@@ -167,13 +283,58 @@ func (a *Agent) session(parent context.Context) (connected bool, err error) {
 	}
 }
 
-func send(ctx context.Context, c *websocket.Conn, typ string, payload any) error {
-	env, err := protocol.Encode(typ, payload)
-	if err != nil {
-		return err
-	}
-	if err := wsjson.Write(ctx, c, env); err != nil {
-		return fmt.Errorf("send %s: %w", typ, err)
-	}
+// enqueue hands env to the writer, or gives up when the session ends.
+func enqueue(ctx context.Context, out chan<- protocol.Envelope, env protocol.Envelope) error {
+	select {
+	case out <- env:
 		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// writeLoop is the only goroutine that writes to c. A failed or timed-out
+// write ends the session.
+func (a *Agent) writeLoop(ctx context.Context, cancel context.CancelCauseFunc, c *websocket.Conn, out <-chan protocol.Envelope) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case env := <-out:
+			wctx, wcancel := context.WithTimeout(ctx, writeTimeout)
+			err := wsjson.Write(wctx, c, env)
+			wcancel()
+			if err != nil {
+				cancel(fmt.Errorf("send %s: %w", env.Type, err))
+				return
+			}
+		}
+	}
+}
+
+// readLoop handles what the master sends. A message it cannot parse ends the
+// session; a message type it does not know is logged and ignored, so a newer
+// master can add message types without breaking this agent.
+func (a *Agent) readLoop(ctx context.Context, cancel context.CancelCauseFunc, c *websocket.Conn) {
+	for {
+		var env protocol.Envelope
+		if err := wsjson.Read(ctx, c, &env); err != nil {
+			cancel(fmt.Errorf("read: %w", err))
+			return
+		}
+		switch env.Type {
+		case protocol.TypeJobStart:
+			start, err := protocol.Decode[protocol.JobStart](env)
+			if err != nil || env.ID == "" {
+				cancel(fmt.Errorf("malformed job.start (id %q): %v", env.ID, err))
+				return
+			}
+			a.jobs.Submit(env.ID, start)
+		case protocol.TypeJobAck:
+			// JobAck has no fields, so the payload is not inspected.
+			a.jobs.Ack(env.ID)
+		default:
+			a.log.Warn("unknown_message_type", "type", env.Type)
+		}
+	}
 }

internal/agent/conn/conn_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/shcizo/package-updater/internal/agent/updater"
 	"github.com/shcizo/package-updater/internal/pin"
 	"github.com/shcizo/package-updater/internal/protocol"
 )
@@ -113,12 +114,111 @@ var (
 	svcB = protocol.Service{Kind: protocol.KindContainer, ContainerName: "b", Image: "b:1", Status: "running"}
 )
 
+// fakeJobs records what the connection tells it and returns scripted state.
+type fakeJobs struct {
+	mu       sync.Mutex
+	known    []string
+	pending  []updater.Finished
+	progress []string // output chunks since the last TakeProgress
+	submits  map[string]protocol.JobStart
+	acks     []string
+	changed  chan struct{}
+}
+
+func newFakeJobs() *fakeJobs {
+	return &fakeJobs{submits: map[string]protocol.JobStart{}, changed: make(chan struct{}, 1)}
+}
+
+func (f *fakeJobs) Submit(id string, s protocol.JobStart) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.submits[id] = s
+}
+
+func (f *fakeJobs) Ack(id string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.acks = append(f.acks, id)
+	kept := f.pending[:0:0]
+	for _, p := range f.pending {
+		if p.ID != id {
+			kept = append(kept, p)
+		}
+	}
+	f.pending = kept
+}
+
+func (f *fakeJobs) Known() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.known
+}
+
+func (f *fakeJobs) TakeProgress() []updater.Progress {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if len(f.progress) == 0 {
+		return nil
+	}
+	out := updater.Progress{ID: "1", Output: strings.Join(f.progress, "")}
+	f.progress = nil
+	return []updater.Progress{out}
+}
+
+func (f *fakeJobs) Pending() []updater.Finished {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]updater.Finished(nil), f.pending...)
+}
+
+func (f *fakeJobs) Changed() <-chan struct{} { return f.changed }
+
+func (f *fakeJobs) signal() {
+	select {
+	case f.changed <- struct{}{}:
+	default:
+	}
+}
+
+func (f *fakeJobs) addProgress(s string) {
+	f.mu.Lock()
+	f.progress = append(f.progress, s)
+	f.mu.Unlock()
+	f.signal()
+}
+
+func (f *fakeJobs) addResult(id string, r protocol.JobResult) {
+	f.mu.Lock()
+	f.pending = append(f.pending, updater.Finished{ID: id, Result: r})
+	f.mu.Unlock()
+	f.signal()
+}
+
+func (f *fakeJobs) submitted(id string) (protocol.JobStart, bool) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	s, ok := f.submits[id]
+	return s, ok
+}
+
+func (f *fakeJobs) acked() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.acks...)
+}
+
 func startAgent(t *testing.T, fm *fakeMaster, fingerprint string, src Source) *Agent {
+	t.Helper()
+	return startAgentWith(t, fm, fingerprint, src, newFakeJobs(), 0)
+}
+
+func startAgentWith(t *testing.T, fm *fakeMaster, fingerprint string, src Source, jobs Jobs, progressInterval time.Duration) *Agent {
 	t.Helper()
 	a := New(Config{
 		URL: fm.url(), Token: "pua_test", Fingerprint: fingerprint, Version: "v-test",
 		MinBackoff: 10 * time.Millisecond, MaxBackoff: 50 * time.Millisecond,
-	}, src, slog.New(slog.NewTextHandler(io.Discard, nil)))
+		ProgressInterval: progressInterval,
+	}, src, jobs, slog.New(slog.NewTextHandler(io.Discard, nil)))
 	ctx, cancel := context.WithCancel(context.Background())
 	done := make(chan struct{})
 	go func() { _ = a.Run(ctx); close(done) }()
@@ -137,7 +237,7 @@ func TestAgent_SendsHelloThenSnapshotWithToken(t *testing.T) {
 	require.Equal(t, protocol.TypeHello, hello.Type)
 	h, err := protocol.Decode[protocol.Hello](hello)
 	require.NoError(t, err)
-	assert.Equal(t, protocol.Hello{Hostname: "node-2", AgentVersion: "v-test"}, h)
+	assert.Equal(t, protocol.Hello{Hostname: "node-2", AgentVersion: "v-test", ProtocolVersion: protocol.Version}, h)
 
 	snap := fm.next(t)
 	require.Equal(t, protocol.TypeSnapshot, snap.Type)
@@ -198,3 +298,212 @@ func TestAgent_RefusesMasterWithWrongKey(t *testing.T) {
 	time.Sleep(200 * time.Millisecond)
 	assert.Empty(t, fm.auth, "the token must never be sent to a server with the wrong key")
 }
+
+func sendEnv(t *testing.T, c *websocket.Conn, env protocol.Envelope) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	require.NoError(t, wsjson.Write(ctx, c, env))
+}
+
+func TestAgent_HelloCarriesVersionAndKnownJobs(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	jobs.known = []string{"3", "7"}
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, time.Millisecond)
+
+	h, err := protocol.Decode[protocol.Hello](fm.next(t))
+	require.NoError(t, err)
+	assert.Equal(t, protocol.Version, h.ProtocolVersion)
+	assert.Equal(t, []string{"3", "7"}, h.KnownJobs)
+}
+
+func TestAgent_JobStartSubmitsAndAckAcks(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, time.Millisecond)
+	fm.next(t)
+	fm.next(t)
+	c := <-fm.conns
+
+	start := protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"}
+	env, err := protocol.EncodeJob(protocol.TypeJobStart, "42", start)
+	require.NoError(t, err)
+	sendEnv(t, c, env)
+	require.Eventually(t, func() bool { _, ok := jobs.submitted("42"); return ok }, 2*time.Second, 5*time.Millisecond)
+	got, _ := jobs.submitted("42")
+	assert.Equal(t, start, got)
+
+	ack, err := protocol.EncodeJob(protocol.TypeJobAck, "42", protocol.JobAck{})
+	require.NoError(t, err)
+	sendEnv(t, c, ack)
+	require.Eventually(t, func() bool { return len(jobs.acked()) == 1 }, 2*time.Second, 5*time.Millisecond)
+	assert.Equal(t, []string{"42"}, jobs.acked())
+}
+
+func TestAgent_SendsPendingResultsAfterSnapshot(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	jobs.pending = []updater.Finished{
+		{ID: "5", Result: protocol.JobResult{Status: protocol.ResultSucceeded, Ref: "a:1@sha256:aa", Output: "ok\n"}},
+		{ID: "6", Result: protocol.JobResult{Status: protocol.ResultFailed, Error: "boom"}},
+	}
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, time.Millisecond)
+
+	assert.Equal(t, protocol.TypeHello, fm.next(t).Type)
+	assert.Equal(t, protocol.TypeSnapshot, fm.next(t).Type)
+	r5 := fm.next(t)
+	assert.Equal(t, protocol.TypeJobResult, r5.Type)
+	assert.Equal(t, "5", r5.ID)
+	res, err := protocol.Decode[protocol.JobResult](r5)
+	require.NoError(t, err)
+	assert.Equal(t, "a:1@sha256:aa", res.Ref)
+	r6 := fm.next(t)
+	assert.Equal(t, "6", r6.ID)
+}
+
+func TestAgent_ResendsUnackedResultAfterReconnect(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	jobs.addResult("5", protocol.JobResult{Status: protocol.ResultSucceeded})
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, time.Millisecond)
+
+	fm.next(t) // hello
+	fm.next(t) // snapshot
+	assert.Equal(t, "5", fm.next(t).ID)
+	(<-fm.conns).CloseNow() // dropped without ack
+
+	assert.Equal(t, protocol.TypeHello, fm.next(t).Type)
+	assert.Equal(t, protocol.TypeSnapshot, fm.next(t).Type)
+	again := fm.next(t)
+	assert.Equal(t, protocol.TypeJobResult, again.Type)
+	assert.Equal(t, "5", again.ID)
+
+	c := <-fm.conns
+	ack, err := protocol.EncodeJob(protocol.TypeJobAck, "5", protocol.JobAck{})
+	require.NoError(t, err)
+	sendEnv(t, c, ack)
+	require.Eventually(t, func() bool { return len(jobs.acked()) == 1 }, 2*time.Second, 5*time.Millisecond)
+	c.CloseNow()
+
+	assert.Equal(t, protocol.TypeHello, fm.next(t).Type)
+	assert.Equal(t, protocol.TypeSnapshot, fm.next(t).Type)
+	select {
+	case e := <-fm.envs:
+		t.Fatalf("acked result was sent again: %+v", e)
+	case <-time.After(100 * time.Millisecond):
+	}
+}
+
+func TestAgent_ProgressIsThrottled(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, 100*time.Millisecond)
+	fm.next(t)
+	fm.next(t)
+
+	for range 20 {
+		jobs.addProgress("x")
+		time.Sleep(2500 * time.Microsecond)
+	}
+	var batches int
+	var total strings.Builder
+	collect := func(d time.Duration) {
+		deadline := time.After(d)
+		for {
+			select {
+			case e := <-fm.envs:
+				if e.Type != protocol.TypeJobProgress {
+					continue
+				}
+				p, err := protocol.Decode[protocol.JobProgress](e)
+				require.NoError(t, err)
+				batches++
+				total.WriteString(p.Output)
+			case <-deadline:
+				return
+			}
+		}
+	}
+	collect(100 * time.Millisecond)
+	assert.LessOrEqual(t, batches, 2)
+	collect(400 * time.Millisecond)
+	assert.Equal(t, strings.Repeat("x", 20), total.String(), "the last chunk must be delivered")
+}
+
+func TestAgent_IgnoresUnknownType(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, time.Millisecond)
+	fm.next(t)
+	fm.next(t)
+	c := <-fm.conns
+
+	sendEnv(t, c, protocol.Envelope{Type: "future.thing", Payload: []byte(`{"a":1}`)})
+	env, err := protocol.EncodeJob(protocol.TypeJobStart, "9", protocol.JobStart{Kind: protocol.KindCompose, Ref: "a:1"})
+	require.NoError(t, err)
+	sendEnv(t, c, env)
+	require.Eventually(t, func() bool { _, ok := jobs.submitted("9"); return ok }, 2*time.Second, 5*time.Millisecond)
+	assert.Zero(t, len(fm.conns), "no reconnect expected")
+}
+
+func TestAgent_ClosesOnMalformedJobStart(t *testing.T) {
+	fm := newFakeMaster(t)
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, newFakeJobs(), time.Millisecond)
+	fm.next(t)
+	fm.next(t)
+	c := <-fm.conns
+
+	env, err := protocol.Encode(protocol.TypeJobStart, protocol.JobStart{Ref: "a:1"}) // no ID
+	require.NoError(t, err)
+	sendEnv(t, c, env)
+
+	select {
+	case <-fm.conns:
+	case <-time.After(2 * time.Second):
+		t.Fatal("agent did not reconnect after a malformed job.start")
+	}
+}
+
+func TestAgent_InventoryAndProgressConcurrently(t *testing.T) {
+	fm := newFakeMaster(t)
+	src := &fakeSource{}
+	src.set(svcA)
+	jobs := newFakeJobs()
+	a := startAgentWith(t, fm, fm.fingerprint, src, jobs, time.Millisecond)
+	fm.next(t)
+	fm.next(t)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := range 30 {
+			jobs.addProgress("y")
+			if i%2 == 0 {
+				src.set(svcB)
+			} else {
+				src.set(svcA)
+			}
+			a.Notify()
+			time.Sleep(time.Millisecond)
+		}
+	}()
+	<-done
+	var progress, inventory int
+	deadline := time.After(500 * time.Millisecond)
+	for {
+		select {
+		case e := <-fm.envs:
+			switch e.Type {
+			case protocol.TypeJobProgress:
+				progress++
+			case protocol.TypeUpsert, protocol.TypeRemove:
+				inventory++
+			}
+		case <-deadline:
+			assert.Positive(t, progress)
+			assert.Positive(t, inventory)
+			return
+		}
+	}
+}

internal/agent/inventory/collector.go

@@ -5,8 +5,12 @@ package inventory
 import (
 	"context"
 	"fmt"
+	"log/slog"
+	"os"
 	"sort"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/image"
@@ -21,6 +25,8 @@ const (
 	labelProject     = "com.docker.compose.project"
 	labelService     = "com.docker.compose.service"
 	labelWorkingDir  = "com.docker.compose.project.working_dir"
+	labelConfigFiles = "com.docker.compose.project.config_files"
+	labelEnvFile     = "com.docker.compose.project.environment_file"
 	labelSwarmTask   = "com.docker.swarm.service.id"
 )
 
@@ -34,16 +40,56 @@ type DockerClient interface {
 	Info(ctx context.Context) (system.Info, error)
 }
 
+// BuildCheck finds the services of a Compose project that are built from
+// source. The arguments are labels of one of the project's containers.
+// *updater.BuildChecker satisfies it.
+type BuildCheck interface {
+	BuiltServices(ctx context.Context, project, workingDir, configFilesLabel, envFileLabel string) (map[string]bool, error)
+}
+
+// buildRetry is how long a failed build check is not repeated while the
+// project's files stay the same.
+const buildRetry = 5 * time.Minute
+
 // Collector reads the local daemon's inventory.
 type Collector struct {
 	cli    DockerClient
 	selfID string
+
+	check BuildCheck
+	log   *slog.Logger
+	now   func() time.Time
+
+	mu     sync.Mutex
+	cache  map[string]buildEntry // by Compose project
+	warned map[string]bool       // logged warnings: "<project>" (check failed) and "<project>/<service>" (service not in config)
 }
 
+// buildEntry is the cached outcome of a build check. It is valid as long as
+// key (the project's labels plus its files' sizes and mtimes) is unchanged.
+type buildEntry struct {
+	key   string
+	built map[string]bool
+	err   error
+	at    time.Time
+}
+
+// stackLabels are the Compose labels a build check needs.
+type stackLabels struct{ workingDir, configFiles, envFile string }
+
 // New returns a Collector. selfID is a prefix of the agent's own container
 // ID (inside a container, the hostname); that container is marked Self.
 func New(cli DockerClient, selfID string) *Collector {
-	return &Collector{cli: cli, selfID: selfID}
+	return &Collector{cli: cli, selfID: selfID, now: time.Now, cache: map[string]buildEntry{}, warned: map[string]bool{}}
+}
+
+// WithBuildCheck makes the collector mark Compose services that have a
+// build: section as Built. The check re-runs only when a project's files
+// change. When it fails, the project's services are reported as not built and
+// the failure is logged once. It returns c.
+func (c *Collector) WithBuildCheck(check BuildCheck, log *slog.Logger) *Collector {
+	c.check, c.log = check, log
+	return c
 }
 
 // Hello returns the machine facts sent on connect. AgentVersion is left for
@@ -80,6 +126,7 @@ func (c *Collector) Snapshot(ctx context.Context) ([]protocol.Service, error) {
 	}
 
 	byKey := make(map[string]protocol.Service)
+	stacks := make(map[string]stackLabels)
 	for _, ct := range containers {
 		if _, isTask := ct.Labels[labelSwarmTask]; isTask {
 			continue
@@ -89,12 +136,19 @@ func (c *Collector) Snapshot(ctx context.Context) ([]protocol.Service, error) {
 			return nil, err
 		}
 		svc := c.fromContainer(ct, ref, repoDigests[ct.ImageID])
+		if svc.Kind == protocol.KindCompose {
+			if _, ok := stacks[svc.ComposeProject]; !ok {
+				stacks[svc.ComposeProject] = stackLabels{svc.WorkingDir, svc.ConfigFiles, ct.Labels[labelEnvFile]}
+			}
+		}
 		if prev, ok := byKey[svc.Key()]; ok {
 			svc = mergeReplicas(prev, svc)
 		}
 		byKey[svc.Key()] = svc
 	}
 
+	c.markBuilt(ctx, byKey, stacks)
+
 	if info.Swarm.ControlAvailable {
 		services, err := c.cli.ServiceList(ctx, swarm.ServiceListOptions{Status: true})
 		if err != nil {
@@ -148,6 +202,7 @@ func (c *Collector) fromContainer(ct container.Summary, ref string, repoDigests
 		svc.ComposeProject = project
 		svc.ComposeService = service
 		svc.WorkingDir = ct.Labels[labelWorkingDir]
+		svc.ConfigFiles = ct.Labels[labelConfigFiles]
 		return svc
 	}
 	svc.Kind = protocol.KindContainer
@@ -214,3 +269,93 @@ func swarmStatus(st *swarm.ServiceStatus) string {
 		return "degraded"
 	}
 }
+
+// markBuilt sets Built on Compose services that have a build: section.
+func (c *Collector) markBuilt(ctx context.Context, byKey map[string]protocol.Service, stacks map[string]stackLabels) {
+	if c.check == nil {
+		return
+	}
+	for key, svc := range byKey {
+		if svc.Kind != protocol.KindCompose {
+			continue
+		}
+		built := c.builtServices(ctx, svc.ComposeProject, stacks[svc.ComposeProject])
+		if built == nil {
+			continue // check failed; already logged
+		}
+		isBuilt, known := built[svc.ComposeService]
+		if !known {
+			// Fail closed: a service compose does not list cannot be shown to
+			// be free of build:, so it is not offered for update.
+			c.warnOnce(svc.ComposeProject+"/"+svc.ComposeService, "compose_service_not_in_config",
+				"project", svc.ComposeProject, "service", svc.ComposeService)
+		}
+		if isBuilt || !known {
+			svc.Built = true
+			byKey[key] = svc
+		}
+	}
+}
+
+// builtServices returns the project's built services, from the cache while
+// the project's files are unchanged.
+func (c *Collector) builtServices(ctx context.Context, project string, l stackLabels) map[string]bool {
+	// The key covers the files named in the labels. Changes in files they
+	// pull in (extends:, include:, the default .env) do not invalidate the
+	// cache, so Built can be stale until another change or an agent restart.
+	// The job-time check is always fresh, so jobs stay safe.
+	key := l.workingDir + "|" + l.configFiles + "|" + l.envFile + "|" + fileStamps(l.configFiles+","+l.envFile)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if e, ok := c.cache[project]; ok && e.key == key && (e.err == nil || c.now().Sub(e.at) < buildRetry) {
+		return e.built
+	}
+	built, err := c.check.BuiltServices(ctx, project, l.workingDir, l.configFiles, l.envFile)
+	c.cache[project] = buildEntry{key: key, built: built, err: err, at: c.now()}
+	if err == nil {
+		// Forget the project's warnings so they are logged again if the
+		// problem returns. Project names cannot contain "/".
+		for k := range c.warned {
+			if k == project || strings.HasPrefix(k, project+"/") {
+				delete(c.warned, k)
+			}
+		}
+	} else {
+		c.warnOnceLocked(project, "compose_build_check_failed", "project", project, "error", err)
+	}
+	return built
+}
+
+// warnOnce logs msg the first time it is called for key. key is a project
+// name (its build check failed) or "<project>/<service>" (the service is not
+// in the project's config); a successful check of the project forgets both.
+func (c *Collector) warnOnce(key, msg string, args ...any) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.warnOnceLocked(key, msg, args...)
+}
+
+func (c *Collector) warnOnceLocked(key, msg string, args ...any) {
+	if c.warned[key] || c.log == nil {
+		return
+	}
+	c.warned[key] = true
+	c.log.Warn(msg, args...)
+}
+
+// fileStamps summarises size and mtime of the comma-separated files, so a
+// changed file changes the string.
+func fileStamps(list string) string {
+	var b strings.Builder
+	for _, f := range strings.Split(list, ",") {
+		if f = strings.TrimSpace(f); f == "" {
+			continue
+		}
+		if fi, err := os.Stat(f); err == nil {
+			fmt.Fprintf(&b, "%s:%d:%d;", f, fi.Size(), fi.ModTime().UnixNano())
+		} else {
+			fmt.Fprintf(&b, "%s:missing;", f)
+		}
+	}
+	return b.String()
+}

internal/agent/inventory/collector_test.go

@@ -1,9 +1,16 @@
 package inventory
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/image"
@@ -200,3 +207,132 @@ func TestHello(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, protocol.Hello{Hostname: "swarm-mgr", SwarmManager: true, SwarmClusterID: "cluster-1"}, got)
 }
+
+func TestSnapshot_ComposeConfigFilesIsRawLabel(t *testing.T) {
+	ct := composeContainer("abc123", "slusk", "web", "slusk:1", "sha256:img1", "running")
+	ct.Labels[labelConfigFiles] = "/s/a/compose.yaml,/s/a/compose.override.yaml"
+	got, err := New(&fakeDocker{containers: []container.Summary{ct}}, "").Snapshot(context.Background())
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, "/s/a/compose.yaml,/s/a/compose.override.yaml", got[0].ConfigFiles)
+}
+
+type fakeBuildCheck struct {
+	built map[string]bool
+	err   error
+	calls int
+}
+
+func (f *fakeBuildCheck) BuiltServices(context.Context, string, string, string, string) (map[string]bool, error) {
+	f.calls++
+	return f.built, f.err
+}
+
+func buildCollector(t *testing.T, check BuildCheck, files string) (*Collector, *fakeDocker) {
+	t.Helper()
+	ct := composeContainer("abc", "app", "web", "app-web", "sha256:img1", "running")
+	ct.Labels[labelConfigFiles] = files
+	other := composeContainer("def", "app", "db", "pg:16", "sha256:img2", "running")
+	other.Labels[labelConfigFiles] = files
+	cli := &fakeDocker{containers: []container.Summary{ct, other}}
+	return New(cli, "").WithBuildCheck(check, slog.New(slog.NewTextHandler(io.Discard, nil))), cli
+}
+
+func TestSnapshot_BuiltServiceIsMarked(t *testing.T) {
+	check := &fakeBuildCheck{built: map[string]bool{"web": true, "db": false}}
+	c, _ := buildCollector(t, check, "/s/app/compose.yaml")
+	got, err := c.Snapshot(context.Background())
+	require.NoError(t, err)
+	byName := map[string]protocol.Service{}
+	for _, s := range got {
+		byName[s.ComposeService] = s
+	}
+	assert.True(t, byName["web"].Built)
+	assert.False(t, byName["db"].Built)
+}
+
+func TestSnapshot_BuildCheckIsCachedUntilFilesChange(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "compose.yaml")
+	require.NoError(t, os.WriteFile(file, []byte("services: {}\n"), 0o644))
+	check := &fakeBuildCheck{built: map[string]bool{"web": true}}
+	c, _ := buildCollector(t, check, file)
+
+	for range 3 {
+		_, err := c.Snapshot(context.Background())
+		require.NoError(t, err)
+	}
+	assert.Equal(t, 1, check.calls, "one check per project while files are unchanged")
+
+	require.NoError(t, os.WriteFile(file, []byte("services:\n  web: {}\n"), 0o644))
+	_, err := c.Snapshot(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 2, check.calls, "a changed file re-runs the check")
+}
+
+func TestSnapshot_BuildCheckFailureIsNotFatalAndRetriesLater(t *testing.T) {
+	check := &fakeBuildCheck{err: errors.New("no mounts")}
+	c, _ := buildCollector(t, check, "/s/app/compose.yaml")
+	now := time.Now()
+	c.now = func() time.Time { return now }
+	var logs bytes.Buffer
+	c.log = slog.New(slog.NewTextHandler(&logs, nil))
+
+	got, err := c.Snapshot(context.Background())
+	require.NoError(t, err)
+	for _, s := range got {
+		assert.False(t, s.Built)
+	}
+	_, _ = c.Snapshot(context.Background())
+	assert.Equal(t, 1, check.calls, "a failure is not retried on every snapshot")
+
+	now = now.Add(buildRetry + time.Second)
+	_, _ = c.Snapshot(context.Background())
+	assert.Equal(t, 2, check.calls)
+	assert.Equal(t, 1, strings.Count(logs.String(), "compose_build_check_failed"), "logged once per project")
+}
+
+func TestSnapshot_ServiceAbsentFromConfigFailsClosed(t *testing.T) {
+	check := &fakeBuildCheck{built: map[string]bool{"db": false}} // "web" is missing
+	c, _ := buildCollector(t, check, "/s/app/compose.yaml")
+	var logs bytes.Buffer
+	c.log = slog.New(slog.NewTextHandler(&logs, nil))
+
+	for range 2 {
+		got, err := c.Snapshot(context.Background())
+		require.NoError(t, err)
+		byName := map[string]protocol.Service{}
+		for _, s := range got {
+			byName[s.ComposeService] = s
+		}
+		assert.True(t, byName["web"].Built, "an unknown service is not offered for update")
+		assert.False(t, byName["db"].Built)
+	}
+	assert.Equal(t, 1, strings.Count(logs.String(), "compose_service_not_in_config"), "logged once per project")
+}
+
+func TestSnapshot_MissingServiceIsLoggedAgainAfterItCameBack(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "compose.yaml")
+	write := func(body string) {
+		require.NoError(t, os.WriteFile(file, []byte(body), 0o644))
+	}
+	write("v1")
+	check := &fakeBuildCheck{built: map[string]bool{"db": false}} // "web" missing
+	c, _ := buildCollector(t, check, file)
+	var logs bytes.Buffer
+	c.log = slog.New(slog.NewTextHandler(&logs, nil))
+	snap := func() {
+		_, err := c.Snapshot(context.Background())
+		require.NoError(t, err)
+	}
+
+	snap()
+	write("v2 changed")
+	check.built = map[string]bool{"db": false, "web": false} // back
+	snap()
+	assert.Empty(t, c.warned, "warnings are forgotten once the project checks out")
+	write("v3 changed again")
+	check.built = map[string]bool{"db": false} // missing again
+	snap()
+
+	assert.Equal(t, 2, strings.Count(logs.String(), "compose_service_not_in_config"))
+}

internal/agent/updater/build.go

@@ -0,0 +1,117 @@
+package updater
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// OutputRunner runs "docker <args>" in dir and returns its stdout. A failure
+// carries the tail of stderr in the error.
+type OutputRunner func(ctx context.Context, dir string, args ...string) ([]byte, error)
+
+// ExecDockerOutput is the real OutputRunner. It uses the same minimal
+// environment as ExecDocker.
+func ExecDockerOutput(ctx context.Context, dir string, args ...string) ([]byte, error) {
+	cmd := exec.CommandContext(ctx, "docker", args...)
+	cmd.Dir = dir
+	cmd.Env = dockerEnv()
+	cmd.WaitDelay = 10 * time.Second
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		tail := &tailBuffer{max: errorOutputTail}
+		tail.Write(stderr.Bytes())
+		return nil, fmt.Errorf("%w (output: %s)", err, strings.TrimSpace(tail.String()))
+	}
+	return stdout.Bytes(), nil
+}
+
+// BuildChecker finds out which services of a Compose project are built from
+// source, by asking `docker compose config`. Such a service has a local
+// image that an update would replace with the upstream one, so it is never
+// updated.
+type BuildChecker struct {
+	cli    SelfInspector
+	selfID string
+	run    OutputRunner
+}
+
+// NewBuildChecker returns a BuildChecker. cli and selfID name the agent's own
+// container, whose mounts decide which files may be read; run is
+// ExecDockerOutput outside tests.
+func NewBuildChecker(cli SelfInspector, selfID string, run OutputRunner) *BuildChecker {
+	return &BuildChecker{cli: cli, selfID: selfID, run: run}
+}
+
+// BuiltServices returns every service of project (all profiles) mapped to
+// whether it has a build: section. A service that is not a key is not in the
+// project's configuration.
+// The arguments are the labels of one of the project's containers. Files that
+// are not inside the agent's read-write same-path mounts are refused.
+func (b *BuildChecker) BuiltServices(ctx context.Context, project, workingDir, configFilesLabel, envFileLabel string) (map[string]bool, error) {
+	roots, err := MountRoots(ctx, b.cli, b.selfID)
+	if err != nil {
+		return nil, err
+	}
+	files, dir, err := stackFiles(roots, configFilesLabel)
+	if err != nil {
+		return nil, err
+	}
+	if workingDir == "" {
+		workingDir = dir
+	} else if !insideAny(roots, workingDir) {
+		return nil, notMounted(workingDir)
+	}
+	envs, err := envFiles(roots, envFileLabel)
+	if err != nil {
+		return nil, err
+	}
+	return b.builtServices(ctx, project, workingDir, files, envs)
+}
+
+// builtServices runs `docker compose config --format json` on files that are
+// already checked.
+func (b *BuildChecker) builtServices(ctx context.Context, project, workingDir string, files, envs []string) (map[string]bool, error) {
+	args := composeBase(project, workingDir, files, envs)
+	// Without --profile * compose omits services whose profile is not active,
+	// and a built service behind a profile would read as absent.
+	args = append(args, "--profile", "*", "config", "--format", "json")
+	raw, err := b.run(ctx, workingDir, args...)
+	if err != nil {
+		return nil, fmt.Errorf("docker compose config: %w", err)
+	}
+	var cfg struct {
+		Services map[string]struct {
+			Build json.RawMessage `json:"build"`
+		} `json:"services"`
+	}
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return nil, fmt.Errorf("parse docker compose config: %w", err)
+	}
+	built := make(map[string]bool, len(cfg.Services))
+	for name, svc := range cfg.Services {
+		built[name] = len(svc.Build) > 0 && string(svc.Build) != "null"
+	}
+	return built, nil
+}
+
+// composeBase returns the leading `docker compose` arguments every call
+// shares. --project-directory is explicit because, with -f, compose would
+// take it from the first file, which differs for stacks started with the
+// flag.
+func composeBase(project, workingDir string, files, envs []string) []string {
+	args := []string{"compose", "-p", project, "--project-directory", workingDir}
+	for _, e := range envs {
+		args = append(args, "--env-file", e)
+	}
+	for _, f := range files {
+		args = append(args, "-f", f)
+	}
+	return args
+}

internal/agent/updater/build_test.go

@@ -0,0 +1,44 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuildChecker_BuiltServices(t *testing.T) {
+	root, _ := filepath.EvalSymlinks(t.TempDir())
+	stack := filepath.Join(root, "app")
+	require.NoError(t, os.MkdirAll(stack, 0o755))
+	self := fakeSelf{mounts: []container.MountPoint{{Type: mount.TypeBind, Source: root, Destination: root, RW: true}}}
+	files := filepath.Join(stack, "compose.yaml") + "," + filepath.Join(stack, OverrideName)
+
+	t.Run("reports only services with build", func(t *testing.T) {
+		o := &outRunner{json: `{"services":{"web":{"image":"nginx","build":{"context":"."}},"db":{"image":"pg"},"x":{"build":null}}}`}
+		got, err := NewBuildChecker(self, "me", o.run).BuiltServices(context.Background(), "app", stack, files, "")
+		require.NoError(t, err)
+		assert.Equal(t, map[string]bool{"web": true, "db": false, "x": false}, got)
+		require.Len(t, o.cmds, 1)
+		assert.Equal(t, stack, o.cmds[0].dir)
+		assert.Equal(t, []string{"compose", "-p", "app", "--project-directory", stack, "-f", filepath.Join(stack, "compose.yaml"), "--profile", "*", "config", "--format", "json"}, o.cmds[0].args)
+	})
+	t.Run("files outside the roots are refused without running docker", func(t *testing.T) {
+		o := &outRunner{json: plainConfig}
+		_, err := NewBuildChecker(self, "me", o.run).BuiltServices(context.Background(), "app", "", "/elsewhere/compose.yaml", "")
+		assert.ErrorContains(t, err, "not mounted")
+		assert.Empty(t, o.cmds)
+	})
+	t.Run("runner and parse errors", func(t *testing.T) {
+		_, err := NewBuildChecker(self, "me", (&outRunner{err: errors.New("boom")}).run).BuiltServices(context.Background(), "app", stack, files, "")
+		assert.ErrorContains(t, err, "boom")
+		_, err = NewBuildChecker(self, "me", (&outRunner{json: "not json"}).run).BuiltServices(context.Background(), "app", stack, files, "")
+		assert.ErrorContains(t, err, "parse")
+	})
+}

internal/agent/updater/compose.go

@@ -0,0 +1,169 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+const (
+	labelComposeProject = "com.docker.compose.project"
+	labelComposeService = "com.docker.compose.service"
+	labelWorkingDir     = "com.docker.compose.project.working_dir"
+	labelConfigFiles    = "com.docker.compose.project.config_files"
+	labelEnvFile        = "com.docker.compose.project.environment_file"
+)
+
+// errorOutputTail is how much of a failed command's output goes into the
+// job's error message; the full output is in the job log.
+const errorOutputTail = 2 << 10
+
+// tailWriter adapts a tailBuffer to io.Writer.
+type tailWriter struct{ b *tailBuffer }
+
+func (w tailWriter) Write(p []byte) (int, error) {
+	w.b.Write(p)
+	return len(p), nil
+}
+
+// CommandRunner runs "docker <args>" in dir with stdout and stderr to out.
+type CommandRunner func(ctx context.Context, dir string, out io.Writer, args ...string) error
+
+// ExecDocker is the real CommandRunner. When ctx ends the process is killed
+// and, after a grace period, its output pipes are closed.
+func ExecDocker(ctx context.Context, dir string, out io.Writer, args ...string) error {
+	cmd := exec.CommandContext(ctx, "docker", args...)
+	cmd.Dir = dir
+	cmd.Stdout = out
+	cmd.Stderr = out
+	cmd.Env = dockerEnv()
+	cmd.WaitDelay = 10 * time.Second
+	return cmd.Run()
+}
+
+// dockerEnvVars are the only variables from the agent's environment that
+// reach docker compose. Compose prefers process variables over the stack's
+// .env file when it interpolates, so passing the agent's own (HOSTNAME,
+// AGENT_TOKEN, ...) would change stack configuration and expose the token.
+var dockerEnvVars = []string{"PATH", "HOME", "DOCKER_HOST", "DOCKER_CONFIG", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"}
+
+// dockerEnv returns the allowed variables that are set in the agent's
+// environment, as KEY=value pairs.
+func dockerEnv() []string {
+	var env []string
+	for _, k := range dockerEnvVars {
+		if v := os.Getenv(k); v != "" {
+			env = append(env, k+"="+v)
+		}
+	}
+	return env
+}
+
+// ComposeDocker is the subset of the Docker SDK ComposeExecutor needs.
+type ComposeDocker interface {
+	SelfInspector
+	ContainerList(ctx context.Context, opts container.ListOptions) ([]container.Summary, error)
+}
+
+// ComposeExecutor updates a Compose service by writing the agent-owned
+// override file and running `docker compose pull` and `up -d`.
+type ComposeExecutor struct {
+	cli     ComposeDocker
+	resolve *Resolver
+	selfID  string
+	run     CommandRunner
+	build   *BuildChecker
+}
+
+// NewComposeExecutor returns an executor for Compose jobs. selfID names the
+// agent's own container, whose mounts decide which directories may be
+// touched; run and runOut are ExecDocker and ExecDockerOutput outside tests.
+func NewComposeExecutor(cli ComposeDocker, resolve *Resolver, selfID string, run CommandRunner, runOut OutputRunner) *ComposeExecutor {
+	return &ComposeExecutor{cli: cli, resolve: resolve, selfID: selfID, run: run, build: NewBuildChecker(cli, selfID, runOut)}
+}
+
+// Execute pins job.Ref to its current digest, writes it to the stack's
+// override file and recreates the service. Working directory and config
+// files come from the service's container labels on this machine, never from
+// the job. It returns the pinned ref.
+func (e *ComposeExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	project, service := job.ComposeProject, job.ComposeService
+	cts, err := e.cli.ContainerList(ctx, container.ListOptions{
+		All: true,
+		Filters: filters.NewArgs(
+			filters.Arg("label", labelComposeProject+"="+project),
+			filters.Arg("label", labelComposeService+"="+service),
+		),
+	})
+	if err != nil {
+		return "", fmt.Errorf("list containers: %w", err)
+	}
+	if len(cts) == 0 {
+		return "", fmt.Errorf("no container for compose service %s/%s on this machine", project, service)
+	}
+	workingDir := cts[0].Labels[labelWorkingDir]
+	configFiles := cts[0].Labels[labelConfigFiles]
+	envFileLabel := cts[0].Labels[labelEnvFile]
+
+	roots, err := MountRoots(ctx, e.cli, e.selfID)
+	if err != nil {
+		return "", err
+	}
+	files, dir, err := composeFiles(roots, configFiles)
+	if err != nil {
+		return "", err
+	}
+	if workingDir == "" {
+		workingDir = dir
+	} else if !insideAny(roots, workingDir) {
+		return "", notMounted(workingDir)
+	}
+
+	envs, err := envFiles(roots, envFileLabel)
+	if err != nil {
+		return "", err
+	}
+	// Fresh check, not the inventory's cache: a build: section may have been
+	// added since. Nothing is written for such a service.
+	built, err := e.build.builtServices(ctx, project, workingDir, files[:len(files)-1], envs)
+	if err != nil {
+		return "", err
+	}
+	isBuilt, known := built[service]
+	if !known {
+		return "", fmt.Errorf("service %s not found in compose config", service)
+	}
+	if isBuilt {
+		return "", fmt.Errorf("service %s is built from source (build:), not updated", service)
+	}
+
+	pinned, err := e.resolve.Resolve(ctx, job.Ref)
+	if err != nil {
+		return "", err
+	}
+	fmt.Fprintf(out, "resolved %s\n", pinned)
+
+	if _, err := WriteOverride(dir, service, pinned); err != nil {
+		return "", err
+	}
+
+	base := composeBase(project, workingDir, files, envs)
+	for _, step := range [][]string{{"pull", service}, {"up", "-d", service}} {
+		args := append(append([]string(nil), base...), step...)
+		fmt.Fprintf(out, "$ docker %s\n", strings.Join(args, " "))
+		tail := &tailBuffer{max: errorOutputTail}
+		if err := e.run(ctx, workingDir, io.MultiWriter(out, tailWriter{tail}), args...); err != nil {
+			return "", fmt.Errorf("docker compose %s: %w (output: %s)", step[0], err, strings.TrimSpace(tail.String()))
+		}
+	}
+	return pinned, nil
+}

internal/agent/updater/compose_test.go

@@ -0,0 +1,289 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+type fakeCompose struct {
+	fakeSelf
+	containers []container.Summary
+}
+
+func (f fakeCompose) ContainerList(context.Context, container.ListOptions) ([]container.Summary, error) {
+	return f.containers, nil
+}
+
+// outRunner is a fake OutputRunner for `docker compose config`.
+type outRunner struct {
+	json string
+	err  error
+	cmds []ranCmd
+}
+
+func (o *outRunner) run(_ context.Context, dir string, args ...string) ([]byte, error) {
+	o.cmds = append(o.cmds, ranCmd{dir, args})
+	return []byte(o.json), o.err
+}
+
+const plainConfig = `{"services":{"web":{"image":"x"},"db":{"image":"y"}}}`
+
+type ranCmd struct {
+	dir  string
+	args []string
+}
+
+type recorder struct {
+	cmds       []ranCmd
+	failOn     string // second arg after -f list, e.g. "pull"
+	failOutput string
+	onFirst    func()
+}
+
+func (r *recorder) run(_ context.Context, dir string, out io.Writer, args ...string) error {
+	if len(r.cmds) == 0 && r.onFirst != nil {
+		r.onFirst()
+	}
+	r.cmds = append(r.cmds, ranCmd{dir, args})
+	if r.failOn != "" && strings.Contains(strings.Join(args, " "), " "+r.failOn+" ") {
+		io.WriteString(out, r.failOutput)
+		return errors.New("exit status 1")
+	}
+	return nil
+}
+
+func TestComposeExecutor(t *testing.T) {
+	const d = "sha256:d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1"
+	root, _ := filepath.EvalSymlinks(t.TempDir())
+	stack := filepath.Join(root, "slusk")
+	require.NoError(t, os.MkdirAll(stack, 0o755))
+	job := protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "gitea.shcizo.se/shcizo/slusk:1.4"}
+	pinned := job.Ref + "@" + d
+
+	ct := container.Summary{Labels: map[string]string{
+		labelWorkingDir:  stack,
+		labelConfigFiles: filepath.Join(stack, "compose.yaml"),
+	}}
+	rw := fakeSelf{mounts: []container.MountPoint{{Type: mount.TypeBind, Source: root, Destination: root, RW: true}}}
+	ro := fakeSelf{mounts: []container.MountPoint{{Type: mount.TypeBind, Source: root, Destination: root, RW: false}}}
+	wantFiles := []string{"-f", filepath.Join(stack, "compose.yaml"), "-f", filepath.Join(stack, OverrideName)}
+	cmd := func(rest ...string) []string {
+		return append(append([]string{"compose", "-p", "slusk", "--project-directory", stack}, wantFiles...), rest...)
+	}
+	overridePath := filepath.Join(stack, OverrideName)
+	cleanOverride := func() { os.Remove(overridePath) }
+
+	outr := &outRunner{json: plainConfig}
+	setup := func(cli ComposeDocker, dist *fakeDist, rec *recorder) *ComposeExecutor {
+		*outr = outRunner{json: plainConfig}
+		return NewComposeExecutor(cli, NewResolver(dist, nil), "me", rec.run, outr.run)
+	}
+
+	t.Run("happy path", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{onFirst: func() {
+			_, err := os.Stat(overridePath)
+			assert.NoError(t, err, "override must exist before the first command")
+		}}
+		var out strings.Builder
+		got, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, &out)
+		require.NoError(t, err)
+		assert.Equal(t, pinned, got)
+		require.Len(t, rec.cmds, 2)
+		assert.Equal(t, ranCmd{stack, cmd("pull", "web")}, rec.cmds[0])
+		assert.Equal(t, ranCmd{stack, cmd("up", "-d", "web")}, rec.cmds[1])
+		_, doc := readOverride(t, stack)
+		assert.Equal(t, pinned, serviceImage(doc, "web"))
+		assert.Contains(t, out.String(), "resolved "+pinned)
+	})
+	t.Run("built from source is refused before anything is written", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{}
+		e := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec)
+		outr.json = `{"services":{"web":{"image":"nginx","build":{"context":"."}}}}`
+		_, err := e.Execute(context.Background(), job, io.Discard)
+		assert.EqualError(t, err, "service web is built from source (build:), not updated")
+		assert.Empty(t, rec.cmds, "no pull/up")
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("build check uses the job's files without the override", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		require.NoError(t, err)
+		require.Len(t, outr.cmds, 1)
+		assert.Equal(t, ranCmd{stack, []string{"compose", "-p", "slusk", "--project-directory", stack, "-f", filepath.Join(stack, "compose.yaml"), "--profile", "*", "config", "--format", "json"}}, outr.cmds[0])
+	})
+	t.Run("service missing from the config is refused", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{}
+		e := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec)
+		outr.json = `{"services":{"db":{"image":"y"}}}`
+		dist := &fakeDist{digest: d}
+		e.resolve = NewResolver(dist, nil)
+		_, err := e.Execute(context.Background(), job, io.Discard)
+		assert.EqualError(t, err, "service web not found in compose config")
+		assert.Empty(t, rec.cmds)
+		assert.Empty(t, dist.refs, "nothing resolved")
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("build check failure stops the job", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{}
+		e := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec)
+		outr.err = errors.New("invalid compose file")
+		_, err := e.Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "invalid compose file")
+		assert.Empty(t, rec.cmds)
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("env files are passed to every compose call", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		envFile := filepath.Join(stack, ".env.prod")
+		require.NoError(t, os.WriteFile(envFile, []byte("A=1\n"), 0o644))
+		t.Cleanup(func() { os.Remove(envFile); os.Remove(filepath.Join(stack, ".env.local")) })
+		envFile2 := filepath.Join(stack, ".env.local")
+		require.NoError(t, os.WriteFile(envFile2, []byte("B=2\n"), 0o644))
+		withEnv := container.Summary{Labels: map[string]string{
+			labelWorkingDir: stack, labelConfigFiles: filepath.Join(stack, "compose.yaml"), labelEnvFile: envFile + "," + envFile2,
+		}}
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{withEnv}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		require.NoError(t, err)
+		prefix := []string{"compose", "-p", "slusk", "--project-directory", stack, "--env-file", envFile, "--env-file", envFile2}
+		require.Len(t, outr.cmds, 1)
+		assert.Equal(t, prefix, outr.cmds[0].args[:len(prefix)])
+		require.Len(t, rec.cmds, 2)
+		assert.Equal(t, prefix, rec.cmds[0].args[:len(prefix)])
+		assert.Equal(t, prefix, rec.cmds[1].args[:len(prefix)])
+	})
+	t.Run("env file outside the roots is refused", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		outside := filepath.Join(t.TempDir(), "secrets.env")
+		require.NoError(t, os.WriteFile(outside, []byte("A=1\n"), 0o644))
+		bad := container.Summary{Labels: map[string]string{
+			labelWorkingDir: stack, labelConfigFiles: filepath.Join(stack, "compose.yaml"), labelEnvFile: outside,
+		}}
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{bad}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "env file")
+		assert.Empty(t, rec.cmds)
+		assert.Empty(t, outr.cmds)
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("env file symlinked outside the roots is refused", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		outside := filepath.Join(t.TempDir(), "secrets.env")
+		require.NoError(t, os.WriteFile(outside, []byte("A=1\n"), 0o644))
+		link := filepath.Join(stack, ".env.link")
+		require.NoError(t, os.Symlink(outside, link))
+		t.Cleanup(func() { os.Remove(link) })
+		bad := container.Summary{Labels: map[string]string{
+			labelWorkingDir: stack, labelConfigFiles: filepath.Join(stack, "compose.yaml"), labelEnvFile: link,
+		}}
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{bad}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "env file")
+		assert.Empty(t, rec.cmds)
+	})
+	t.Run("no matching container", func(t *testing.T) {
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, nil}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "no container for compose service slusk/web on this machine")
+		assert.Empty(t, rec.cmds)
+	})
+	t.Run("resolve fails", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{err: errors.New("denied")}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "denied")
+		assert.Empty(t, rec.cmds)
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("pull fails", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{failOn: "pull", failOutput: "pull access denied\n"}
+		_, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "pull access denied")
+		require.Len(t, rec.cmds, 1, "up must not run")
+	})
+	t.Run("override refused", func(t *testing.T) {
+		other := filepath.Join(stack, "docker-compose.override.yml")
+		require.NoError(t, os.WriteFile(other, []byte("services: {}\n"), 0o644))
+		t.Cleanup(func() { os.Remove(other) })
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "refused")
+		assert.Empty(t, rec.cmds)
+	})
+	t.Run("working dir outside the roots", func(t *testing.T) {
+		outside := filepath.Join(t.TempDir(), "elsewhere")
+		bad := container.Summary{Labels: map[string]string{
+			labelWorkingDir:  outside,
+			labelConfigFiles: filepath.Join(stack, "compose.yaml"),
+		}}
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{bad}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "not mounted")
+		assert.Empty(t, rec.cmds)
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("read-only mount", func(t *testing.T) {
+		rec := &recorder{}
+		_, err := setup(fakeCompose{ro, []container.Summary{ct}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "no stack directory is mounted")
+		assert.Empty(t, rec.cmds)
+		assert.NoFileExists(t, overridePath)
+	})
+}
+
+func TestDockerEnv(t *testing.T) {
+	t.Setenv("AGENT_TOKEN", "secret")
+	t.Setenv("HOSTNAME", "abc")
+	t.Setenv("MASTER_URL", "wss://x")
+	t.Setenv("PATH", "/usr/bin")
+	t.Setenv("DOCKER_CONFIG", "/docker-config")
+	t.Setenv("DOCKER_HOST", "")
+	os.Unsetenv("DOCKER_CERT_PATH")
+
+	env := dockerEnv()
+	assert.Contains(t, env, "PATH=/usr/bin")
+	assert.Contains(t, env, "DOCKER_CONFIG=/docker-config")
+	assert.NotContains(t, env, "DOCKER_HOST=", "empty variables are not passed")
+	for _, e := range env {
+		assert.NotContains(t, e, "AGENT_TOKEN")
+		assert.NotContains(t, e, "HOSTNAME")
+		assert.NotContains(t, e, "MASTER_URL")
+		assert.False(t, strings.HasPrefix(e, "DOCKER_CERT_PATH="), "unset variables are not passed")
+	}
+}
+
+func TestExecDocker_UsesMinimalEnv(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "docker"), []byte("#!/bin/sh\nenv\n"), 0o755))
+	t.Setenv("PATH", dir+":/usr/bin:/bin")
+	t.Setenv("AGENT_TOKEN", "secret")
+	t.Setenv("DOCKER_CONFIG", "/docker-config")
+
+	var out strings.Builder
+	require.NoError(t, ExecDocker(context.Background(), dir, &out, "compose", "ps"))
+	assert.NotContains(t, out.String(), "AGENT_TOKEN")
+	assert.Contains(t, out.String(), "DOCKER_CONFIG=/docker-config")
+
+	raw, err := ExecDockerOutput(context.Background(), dir, "compose", "config")
+	require.NoError(t, err)
+	assert.NotContains(t, string(raw), "AGENT_TOKEN")
+	assert.Contains(t, string(raw), "DOCKER_CONFIG=/docker-config")
+}

internal/agent/updater/dispatch.go

@@ -0,0 +1,28 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+type dispatch struct{ compose, swarm Executor }
+
+// NewDispatch returns an Executor that picks compose or swarm by job.Kind.
+// KindContainer and unknown kinds fail.
+func NewDispatch(compose, swarm Executor) Executor {
+	return dispatch{compose: compose, swarm: swarm}
+}
+
+func (d dispatch) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	switch job.Kind {
+	case protocol.KindCompose:
+		return d.compose.Execute(ctx, job, out)
+	case protocol.KindSwarm:
+		return d.swarm.Execute(ctx, job, out)
+	default:
+		return "", fmt.Errorf("cannot update a service of kind %q", job.Kind)
+	}
+}

internal/agent/updater/dispatch_test.go

@@ -0,0 +1,31 @@
+package updater
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+func TestDispatch(t *testing.T) {
+	compose := execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "compose", nil })
+	swarm := execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "swarm", nil })
+	d := NewDispatch(compose, swarm)
+
+	got, err := d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindCompose}, io.Discard)
+	require.NoError(t, err)
+	assert.Equal(t, "compose", got)
+
+	got, err = d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindSwarm}, io.Discard)
+	require.NoError(t, err)
+	assert.Equal(t, "swarm", got)
+
+	_, err = d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindContainer}, io.Discard)
+	assert.ErrorContains(t, err, "cannot update")
+	_, err = d.Execute(context.Background(), protocol.JobStart{Kind: "weird"}, io.Discard)
+	assert.ErrorContains(t, err, "cannot update")
+}

internal/agent/updater/override.go

@@ -0,0 +1,235 @@
+package updater
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"gopkg.in/yaml.v3"
+)
+
+// OverrideName is the override file the agent owns. docker compose loads it
+// automatically next to compose.yaml or docker-compose.yml.
+const OverrideName = "compose.override.yaml"
+
+// overrideHeader marks a file the agent may rewrite. A file without it
+// belongs to the operator and is never touched.
+const overrideHeader = "# managed by package-updater — do not edit\n"
+
+// overrideNames are the override files docker compose auto-loads. It loads
+// only the first one present (in this order), so any other one next to ours
+// would either hide ours or be hidden by it.
+var overrideNames = []string{
+	"compose.override.yml", "compose.override.yaml",
+	"docker-compose.override.yml", "docker-compose.override.yaml",
+}
+
+// WriteOverride sets services.<service>.image to ref in dir/OverrideName,
+// keeping every other entry in the file, and returns the file's path. It
+// refuses when the file exists without the header, or when another
+// auto-loaded override file exists in dir. The write is atomic.
+func WriteOverride(dir, service, ref string) (string, error) {
+	for _, name := range overrideNames {
+		if name == OverrideName {
+			continue
+		}
+		if _, err := os.Lstat(filepath.Join(dir, name)); err == nil {
+			return "", fmt.Errorf("refused: %s exists; the agent only manages %s and never edits operator files", name, OverrideName)
+		} else if !errors.Is(err, os.ErrNotExist) {
+			return "", fmt.Errorf("check %s: %w", name, err)
+		}
+	}
+	path := filepath.Join(dir, OverrideName)
+	doc := map[string]any{}
+	raw, err := os.ReadFile(path)
+	switch {
+	case errors.Is(err, os.ErrNotExist):
+	case err != nil:
+		return "", fmt.Errorf("read override: %w", err)
+	case !bytes.HasPrefix(raw, []byte(overrideHeader)):
+		return "", fmt.Errorf("refused: %s exists and was not written by package-updater", path)
+	default:
+		if err := yaml.Unmarshal(raw, &doc); err != nil {
+			return "", fmt.Errorf("parse override: %w", err)
+		}
+		if doc == nil {
+			doc = map[string]any{}
+		}
+	}
+	services, _ := doc["services"].(map[string]any)
+	if services == nil {
+		services = map[string]any{}
+	}
+	entry, _ := services[service].(map[string]any)
+	if entry == nil {
+		entry = map[string]any{}
+	}
+	entry["image"] = ref
+	services[service] = entry
+	doc["services"] = services
+	body, err := yaml.Marshal(doc)
+	if err != nil {
+		return "", fmt.Errorf("encode override: %w", err)
+	}
+	// Temp file in the same directory + rename: a crash never leaves a
+	// half-written override that compose would then load.
+	tmp, err := os.CreateTemp(dir, "."+OverrideName+".*")
+	if err != nil {
+		return "", fmt.Errorf("write override: %w", err)
+	}
+	defer os.Remove(tmp.Name()) // no-op after a successful rename
+	if _, err := tmp.WriteString(overrideHeader + string(body)); err != nil {
+		tmp.Close()
+		return "", fmt.Errorf("write override: %w", err)
+	}
+	if err := tmp.Chmod(0o644); err != nil {
+		tmp.Close()
+		return "", fmt.Errorf("write override: %w", err)
+	}
+	if err := tmp.Close(); err != nil {
+		return "", fmt.Errorf("write override: %w", err)
+	}
+	if err := os.Rename(tmp.Name(), path); err != nil {
+		return "", fmt.Errorf("write override: %w", err)
+	}
+	return path, nil
+}
+
+// SelfInspector is the subset of the Docker SDK MountRoots needs.
+type SelfInspector interface {
+	ContainerInspect(ctx context.Context, containerID string) (container.InspectResponse, error)
+}
+
+// errNoRoots is returned when the agent has no directory it may write stacks in.
+var errNoRoots = errors.New("refused: no stack directory is mounted into the agent at the same path (read-write); " +
+	"mount each stacks directory as /path:/path and check that SELF_ID (default: the hostname) names the agent's container")
+
+// MountRoots returns the directories the agent may update Compose stacks in:
+// its own container's read-write bind mounts whose host path equals the
+// path inside the container. Compose labels hold host paths, so only those
+// mounts make a label path usable as is.
+func MountRoots(ctx context.Context, cli SelfInspector, selfID string) ([]string, error) {
+	if selfID == "" {
+		return nil, errNoRoots
+	}
+	insp, err := cli.ContainerInspect(ctx, selfID)
+	if err != nil {
+		return nil, fmt.Errorf("inspect own container %s: %w", selfID, err)
+	}
+	var roots []string
+	for _, m := range insp.Mounts {
+		if m.Type == mount.TypeBind && m.RW && filepath.Clean(m.Source) == filepath.Clean(m.Destination) {
+			roots = append(roots, filepath.Clean(m.Destination))
+		}
+	}
+	return roots, nil
+}
+
+// composeFiles returns the -f arguments for a job: the stack's config files
+// from the container label with any copy of the override removed, then the
+// override last so it wins. Every file, and the real location of its
+// directory after resolving symlinks, must be inside one of roots.
+func composeFiles(roots []string, configFilesLabel string) (files []string, dir string, err error) {
+	files, dir, err = stackFiles(roots, configFilesLabel)
+	if err != nil {
+		return nil, "", err
+	}
+	return append(files, filepath.Join(dir, OverrideName)), dir, nil
+}
+
+// stackFiles is composeFiles without the override: the operator's own config
+// files, checked against roots. dir is the first file's directory.
+func stackFiles(roots []string, configFilesLabel string) (files []string, dir string, err error) {
+	if len(roots) == 0 {
+		return nil, "", errNoRoots
+	}
+	for _, f := range strings.Split(configFilesLabel, ",") {
+		if f = strings.TrimSpace(f); f != "" {
+			files = append(files, f)
+		}
+	}
+	if len(files) == 0 {
+		return nil, "", errors.New("container has no com.docker.compose.project.config_files label")
+	}
+	dir = filepath.Dir(files[0])
+	override := filepath.Join(dir, OverrideName)
+
+	realRoots := realDirs(roots)
+	out := files[:0]
+	for _, f := range files {
+		if !filepath.IsAbs(f) || !insideAny(roots, f) {
+			return nil, "", notMounted(filepath.Dir(f))
+		}
+		real, err := filepath.EvalSymlinks(filepath.Dir(f))
+		if err != nil {
+			return nil, "", fmt.Errorf("resolve %s: %w", filepath.Dir(f), err)
+		}
+		if !insideAny(realRoots, real) {
+			return nil, "", notMounted(filepath.Dir(f))
+		}
+		if filepath.Clean(f) != override {
+			out = append(out, f)
+		}
+	}
+	return out, dir, nil
+}
+
+// envFiles returns the --env-file paths from the container's
+// com.docker.compose.project.environment_file label (empty label: none).
+// Like config files, each must be inside one of roots, also after resolving
+// symlinks of the file itself.
+func envFiles(roots []string, label string) ([]string, error) {
+	var files []string
+	realRoots := realDirs(roots)
+	for _, f := range strings.Split(label, ",") {
+		if f = strings.TrimSpace(f); f == "" {
+			continue
+		}
+		if !filepath.IsAbs(f) || !insideAny(roots, f) {
+			return nil, fmt.Errorf("env file %s is not inside a directory mounted into the agent at the same path (read-write)", f)
+		}
+		real, err := filepath.EvalSymlinks(f)
+		if err != nil {
+			return nil, fmt.Errorf("resolve env file %s: %w", f, err)
+		}
+		if !insideAny(realRoots, real) {
+			return nil, fmt.Errorf("env file %s is not inside a directory mounted into the agent at the same path (read-write)", f)
+		}
+		files = append(files, f)
+	}
+	return files, nil
+}
+
+// realDirs returns roots with symlinks resolved. Only directories count once
+// resolved: this drops a same-path mount of a single file such as
+// /var/run/docker.sock.
+func realDirs(roots []string) []string {
+	var out []string
+	for _, r := range roots {
+		if rr, err := filepath.EvalSymlinks(r); err == nil {
+			if fi, err := os.Stat(rr); err == nil && fi.IsDir() {
+				out = append(out, rr)
+			}
+		}
+	}
+	return out
+}
+
+func insideAny(roots []string, path string) bool {
+	for _, r := range roots {
+		if IsInsideRoot(r, path) {
+			return true
+		}
+	}
+	return false
+}
+
+func notMounted(dir string) error {
+	return fmt.Errorf("stack directory %s is not mounted into the agent at the same path (read-write)", dir)
+}

internal/agent/updater/override_test.go

@@ -0,0 +1,204 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func readOverride(t *testing.T, dir string) (string, map[string]any) {
+	t.Helper()
+	raw, err := os.ReadFile(filepath.Join(dir, OverrideName))
+	require.NoError(t, err)
+	var doc map[string]any
+	require.NoError(t, yaml.Unmarshal(raw, &doc))
+	return string(raw), doc
+}
+
+func serviceImage(doc map[string]any, name string) any {
+	svcs, _ := doc["services"].(map[string]any)
+	entry, _ := svcs[name].(map[string]any)
+	return entry["image"]
+}
+
+func TestWriteOverride(t *testing.T) {
+	t.Run("new file", func(t *testing.T) {
+		dir := t.TempDir()
+		path, err := WriteOverride(dir, "web", "a:1@sha256:aa")
+		require.NoError(t, err)
+		assert.Equal(t, filepath.Join(dir, OverrideName), path)
+		raw, doc := readOverride(t, dir)
+		assert.True(t, len(raw) > len(overrideHeader) && raw[:len(overrideHeader)] == overrideHeader)
+		assert.Equal(t, "a:1@sha256:aa", serviceImage(doc, "web"))
+	})
+	t.Run("keeps other services", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, OverrideName), []byte(overrideHeader+"services:\n  db:\n    image: db:1\n"), 0o644))
+		_, err := WriteOverride(dir, "web", "a:1@sha256:aa")
+		require.NoError(t, err)
+		_, doc := readOverride(t, dir)
+		assert.Equal(t, "db:1", serviceImage(doc, "db"))
+		assert.Equal(t, "a:1@sha256:aa", serviceImage(doc, "web"))
+	})
+	t.Run("replaces own entry", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, OverrideName), []byte(overrideHeader+"services:\n  web:\n    image: old\n"), 0o644))
+		_, err := WriteOverride(dir, "web", "a:2@sha256:bb")
+		require.NoError(t, err)
+		_, doc := readOverride(t, dir)
+		assert.Equal(t, "a:2@sha256:bb", serviceImage(doc, "web"))
+	})
+	t.Run("operator file without header is never touched", func(t *testing.T) {
+		dir := t.TempDir()
+		const body = "services:\n  web:\n    environment: [A=1]\n"
+		require.NoError(t, os.WriteFile(filepath.Join(dir, OverrideName), []byte(body), 0o644))
+		_, err := WriteOverride(dir, "web", "a:1@sha256:aa")
+		assert.ErrorContains(t, err, "refused")
+		got, _ := os.ReadFile(filepath.Join(dir, OverrideName))
+		assert.Equal(t, body, string(got))
+	})
+	for _, other := range []string{"compose.override.yml", "docker-compose.override.yml", "docker-compose.override.yaml"} {
+		t.Run("refuses next to "+other, func(t *testing.T) {
+			dir := t.TempDir()
+			require.NoError(t, os.WriteFile(filepath.Join(dir, other), []byte("services: {}\n"), 0o644))
+			_, err := WriteOverride(dir, "web", "a:1@sha256:aa")
+			assert.ErrorContains(t, err, other)
+			_, statErr := os.Stat(filepath.Join(dir, OverrideName))
+			assert.True(t, errors.Is(statErr, os.ErrNotExist))
+		})
+	}
+	t.Run("leaves no temp files", func(t *testing.T) {
+		dir := t.TempDir()
+		_, err := WriteOverride(dir, "web", "a:1@sha256:aa")
+		require.NoError(t, err)
+		entries, err := os.ReadDir(dir)
+		require.NoError(t, err)
+		require.Len(t, entries, 1)
+		assert.Equal(t, OverrideName, entries[0].Name())
+	})
+}
+
+type fakeSelf struct {
+	mounts []container.MountPoint
+	err    error
+}
+
+func (f fakeSelf) ContainerInspect(context.Context, string) (container.InspectResponse, error) {
+	return container.InspectResponse{Mounts: f.mounts}, f.err
+}
+
+func TestMountRoots(t *testing.T) {
+	bind := func(src, dst string, rw bool) container.MountPoint {
+		return container.MountPoint{Type: mount.TypeBind, Source: src, Destination: dst, RW: rw}
+	}
+	cases := []struct {
+		name    string
+		self    fakeSelf
+		selfID  string
+		want    []string
+		wantErr string
+	}{
+		{"rw same path", fakeSelf{mounts: []container.MountPoint{bind("/srv/stacks", "/srv/stacks", true)}}, "me", []string{"/srv/stacks"}, ""},
+		{"read-only", fakeSelf{mounts: []container.MountPoint{bind("/srv/stacks", "/srv/stacks", false)}}, "me", nil, ""},
+		{"path translated", fakeSelf{mounts: []container.MountPoint{bind("/home/x/stacks", "/stacks", true)}}, "me", nil, ""},
+		{"named volume", fakeSelf{mounts: []container.MountPoint{{Type: mount.TypeVolume, Source: "/v", Destination: "/v", RW: true}}}, "me", nil, ""},
+		{"inspect error", fakeSelf{err: errors.New("no such container")}, "me", nil, "inspect own container"},
+		{"no self id", fakeSelf{}, "", nil, "no stack directory is mounted"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got, err := MountRoots(context.Background(), c.self, c.selfID)
+			if c.wantErr != "" {
+				assert.ErrorContains(t, err, c.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, c.want, got)
+		})
+	}
+}
+
+func TestComposeFiles(t *testing.T) {
+	root := t.TempDir()
+	root, err := filepath.EvalSymlinks(root) // macOS temp dirs are symlinked
+	require.NoError(t, err)
+	mk := func(p ...string) string {
+		dir := filepath.Join(append([]string{root}, p...)...)
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		return dir
+	}
+	a := mk("a")
+	nested := mk("apps", "x", "y")
+	second := t.TempDir()
+	second, _ = filepath.EvalSymlinks(second)
+	secondDir := filepath.Join(second, "s")
+	require.NoError(t, os.MkdirAll(secondDir, 0o755))
+
+	outside, _ := filepath.EvalSymlinks(t.TempDir())
+	require.NoError(t, os.Symlink(outside, filepath.Join(root, "evil")))
+
+	sock := filepath.Join(root, "docker.sock")
+	require.NoError(t, os.WriteFile(sock, nil, 0o644))
+
+	t.Run("adds override last", func(t *testing.T) {
+		files, dir, err := composeFiles([]string{root}, filepath.Join(a, "compose.yaml"))
+		require.NoError(t, err)
+		assert.Equal(t, []string{filepath.Join(a, "compose.yaml"), filepath.Join(a, OverrideName)}, files)
+		assert.Equal(t, a, dir)
+	})
+	t.Run("override in label listed once, last", func(t *testing.T) {
+		label := filepath.Join(a, OverrideName) + "," + filepath.Join(a, "compose.yaml") + "," + filepath.Join(a, "extra.yaml")
+		files, _, err := composeFiles([]string{root}, label)
+		require.NoError(t, err)
+		assert.Equal(t, []string{filepath.Join(a, "compose.yaml"), filepath.Join(a, "extra.yaml"), filepath.Join(a, OverrideName)}, files)
+	})
+	t.Run("nested", func(t *testing.T) {
+		_, dir, err := composeFiles([]string{root}, filepath.Join(nested, "compose.yaml"))
+		require.NoError(t, err)
+		assert.Equal(t, nested, dir)
+	})
+	t.Run("second root", func(t *testing.T) {
+		_, _, err := composeFiles([]string{root, second}, filepath.Join(secondDir, "compose.yaml"))
+		require.NoError(t, err)
+	})
+	t.Run("empty label", func(t *testing.T) {
+		_, _, err := composeFiles([]string{root}, "")
+		assert.ErrorContains(t, err, "config_files")
+	})
+	t.Run("no roots", func(t *testing.T) {
+		_, _, err := composeFiles(nil, filepath.Join(a, "compose.yaml"))
+		assert.ErrorContains(t, err, "no stack directory is mounted")
+	})
+	t.Run("outside every root", func(t *testing.T) {
+		_, _, err := composeFiles([]string{root}, filepath.Join(second, "s", "compose.yaml"))
+		assert.EqualError(t, err, "stack directory "+filepath.Join(second, "s")+" is not mounted into the agent at the same path (read-write)")
+	})
+	t.Run("symlink escaping the root", func(t *testing.T) {
+		_, _, err := composeFiles([]string{root}, filepath.Join(root, "evil", "compose.yaml"))
+		assert.EqualError(t, err, "stack directory "+filepath.Join(root, "evil")+" is not mounted into the agent at the same path (read-write)")
+	})
+	t.Run("second file under a symlink escaping the root", func(t *testing.T) {
+		label := filepath.Join(a, "compose.yaml") + "," + filepath.Join(root, "evil", "extra.yaml")
+		_, _, err := composeFiles([]string{root}, label)
+		assert.ErrorContains(t, err, "not mounted")
+	})
+	t.Run("prefix trick", func(t *testing.T) {
+		evil := root + "-evil"
+		require.NoError(t, os.MkdirAll(evil, 0o755))
+		t.Cleanup(func() { os.RemoveAll(evil) })
+		_, _, err := composeFiles([]string{root}, filepath.Join(evil, "compose.yaml"))
+		assert.ErrorContains(t, err, "not mounted")
+	})
+	t.Run("a file root allows nothing", func(t *testing.T) {
+		_, _, err := composeFiles([]string{sock}, filepath.Join(sock, "compose.yaml"))
+		assert.Error(t, err)
+	})
+}

internal/agent/updater/pathcheck.go

@@ -1,4 +1,4 @@
-package discovery
+package updater
 
 import (
 	"path/filepath"

internal/agent/updater/pathcheck_test.go

@@ -1,9 +1,9 @@
-package discovery_test
+package updater_test
 
 import (
 	"testing"
 
-	"github.com/shcizo/package-updater/internal/discovery"
+	"github.com/shcizo/package-updater/internal/agent/updater"
 	"github.com/stretchr/testify/require"
 )
 
@@ -26,7 +26,7 @@ func TestIsInsideRoot(t *testing.T) {
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			require.Equal(t, c.want, discovery.IsInsideRoot(c.root, c.path))
+			require.Equal(t, c.want, updater.IsInsideRoot(c.root, c.path))
 		})
 	}
 }

internal/agent/updater/registryauth.go

@@ -0,0 +1,105 @@
+package updater
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/distribution/reference"
+	"github.com/docker/docker/api/types/registry"
+)
+
+// RegistryAuth holds inline credentials from a docker config.json, already
+// encoded as X-Registry-Auth values. A nil *RegistryAuth means anonymous
+// access.
+type RegistryAuth struct {
+	byHost map[string]string
+}
+
+type dockerConfigFile struct {
+	Auths map[string]struct {
+		Auth          string `json:"auth"`
+		IdentityToken string `json:"identitytoken"`
+	} `json:"auths"`
+	CredsStore  string            `json:"credsStore"`
+	CredHelpers map[string]string `json:"credHelpers"`
+}
+
+// LoadRegistryAuth reads dockerConfigDir/config.json. An empty dir returns
+// nil, nil. Only inline `auth` and `identitytoken` entries are supported;
+// credential helpers are not available inside the agent container, which is
+// logged once.
+func LoadRegistryAuth(dockerConfigDir string, log *slog.Logger) (*RegistryAuth, error) {
+	if dockerConfigDir == "" {
+		return nil, nil
+	}
+	path := filepath.Join(dockerConfigDir, "config.json")
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	var cfg dockerConfigFile
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return nil, fmt.Errorf("parse %s: %w", path, err)
+	}
+	if (cfg.CredsStore != "" || len(cfg.CredHelpers) > 0) && log != nil {
+		log.Warn("registry_cred_helpers_unsupported", "path", path)
+	}
+	a := &RegistryAuth{byHost: make(map[string]string)}
+	for key, entry := range cfg.Auths {
+		host := normaliseRegistryKey(key)
+		ac := registry.AuthConfig{ServerAddress: host}
+		switch {
+		case entry.IdentityToken != "":
+			ac.IdentityToken = entry.IdentityToken
+		case entry.Auth != "":
+			dec, err := base64.StdEncoding.DecodeString(entry.Auth)
+			if err != nil {
+				return nil, fmt.Errorf("%s: auth for %s is not base64", path, key)
+			}
+			user, pass, ok := strings.Cut(string(dec), ":")
+			if !ok {
+				return nil, fmt.Errorf("%s: auth for %s is not user:password", path, key)
+			}
+			ac.Username, ac.Password = user, pass
+		default:
+			continue
+		}
+		enc, err := registry.EncodeAuthConfig(ac)
+		if err != nil {
+			return nil, fmt.Errorf("encode auth for %s: %w", key, err)
+		}
+		a.byHost[host] = enc
+	}
+	return a, nil
+}
+
+// normaliseRegistryKey reduces a config.json key to a bare registry host,
+// mapping every Docker Hub spelling to "docker.io".
+func normaliseRegistryKey(key string) string {
+	k := strings.TrimPrefix(strings.TrimPrefix(key, "https://"), "http://")
+	if i := strings.Index(k, "/"); i >= 0 {
+		k = k[:i]
+	}
+	switch k {
+	case "index.docker.io", "docker.io", "registry-1.docker.io":
+		return "docker.io"
+	}
+	return k
+}
+
+// For returns the X-Registry-Auth value for repo's registry, or "".
+func (a *RegistryAuth) For(repo string) string {
+	if a == nil {
+		return ""
+	}
+	named, err := reference.ParseNormalizedNamed(repo)
+	if err != nil {
+		return ""
+	}
+	return a.byHost[reference.Domain(named)]
+}

internal/agent/updater/registryauth_test.go

@@ -0,0 +1,72 @@
+package updater
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/api/types/registry"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func writeDockerConfig(t *testing.T, body string) string {
+	t.Helper()
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(body), 0o600))
+	return dir
+}
+
+func decodeAuth(t *testing.T, enc string) registry.AuthConfig {
+	t.Helper()
+	raw, err := base64.URLEncoding.DecodeString(enc)
+	require.NoError(t, err)
+	var ac registry.AuthConfig
+	require.NoError(t, json.Unmarshal(raw, &ac))
+	return ac
+}
+
+func TestRegistryAuth(t *testing.T) {
+	user := base64.StdEncoding.EncodeToString([]byte("bob:s3cret"))
+	log := slog.New(slog.NewTextHandler(io.Discard, nil))
+
+	t.Run("docker hub spellings", func(t *testing.T) {
+		for _, key := range []string{"https://index.docker.io/v1/", "index.docker.io", "docker.io", "registry-1.docker.io"} {
+			dir := writeDockerConfig(t, `{"auths":{"`+key+`":{"auth":"`+user+`"}}}`)
+			a, err := LoadRegistryAuth(dir, log)
+			require.NoError(t, err)
+			for _, repo := range []string{"gitea/gitea", "docker.io/library/nginx", "nginx"} {
+				ac := decodeAuth(t, a.For(repo))
+				assert.Equal(t, "bob", ac.Username, "%s for %s", key, repo)
+				assert.Equal(t, "s3cret", ac.Password)
+			}
+			assert.Empty(t, a.For("gitea.shcizo.se/shcizo/slusk"))
+		}
+	})
+	t.Run("custom registry with scheme", func(t *testing.T) {
+		dir := writeDockerConfig(t, `{"auths":{"https://gitea.shcizo.se":{"auth":"`+user+`"},"ghcr.io":{"identitytoken":"tok"}}}`)
+		a, err := LoadRegistryAuth(dir, log)
+		require.NoError(t, err)
+		assert.Equal(t, "bob", decodeAuth(t, a.For("gitea.shcizo.se/shcizo/slusk")).Username)
+		assert.Equal(t, "tok", decodeAuth(t, a.For("ghcr.io/x/y")).IdentityToken)
+		assert.Empty(t, a.For("registry.example.com/x"))
+	})
+	t.Run("empty dir is anonymous", func(t *testing.T) {
+		a, err := LoadRegistryAuth("", log)
+		require.NoError(t, err)
+		assert.Nil(t, a)
+		assert.Empty(t, a.For("gitea/gitea"))
+	})
+	t.Run("malformed json", func(t *testing.T) {
+		_, err := LoadRegistryAuth(writeDockerConfig(t, "{"), log)
+		assert.Error(t, err)
+	})
+	t.Run("missing file", func(t *testing.T) {
+		_, err := LoadRegistryAuth(t.TempDir(), log)
+		assert.Error(t, err)
+	})
+}

internal/agent/updater/resolve.go

@@ -0,0 +1,48 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/docker/docker/api/types/registry"
+
+	"github.com/shcizo/package-updater/internal/imageref"
+)
+
+// DistributionClient is the subset of the Docker SDK the Resolver needs.
+// *client.Client satisfies it.
+type DistributionClient interface {
+	DistributionInspect(ctx context.Context, imageRef, encodedRegistryAuth string) (registry.DistributionInspect, error)
+}
+
+// Resolver turns "repo:tag" into "repo:tag@digest" by asking the local
+// daemon, which contacts the registry.
+type Resolver struct {
+	cli  DistributionClient
+	auth *RegistryAuth
+}
+
+// NewResolver returns a Resolver. auth may be nil (anonymous access).
+func NewResolver(cli DistributionClient, auth *RegistryAuth) *Resolver {
+	return &Resolver{cli: cli, auth: auth}
+}
+
+// Resolve returns ref pinned to the digest its tag currently points at. It
+// fails for a ref without a tag or with a digest already.
+func (r *Resolver) Resolve(ctx context.Context, ref string) (string, error) {
+	p := imageref.Parse(ref)
+	switch {
+	case p.Digest != "":
+		return "", fmt.Errorf("image %q already has a digest", ref)
+	case p.Tag == "":
+		return "", fmt.Errorf("image %q has no tag to resolve", ref)
+	}
+	info, err := r.cli.DistributionInspect(ctx, ref, r.auth.For(p.Repo))
+	if err != nil {
+		return "", fmt.Errorf("resolve %s: %w", ref, err)
+	}
+	if info.Descriptor.Digest == "" {
+		return "", fmt.Errorf("resolve %s: registry returned no digest", ref)
+	}
+	return ref + "@" + info.Descriptor.Digest.String(), nil
+}

internal/agent/updater/resolve_test.go

@@ -0,0 +1,38 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolver(t *testing.T) {
+	const d = "sha256:d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1"
+	auth := &RegistryAuth{byHost: map[string]string{"gitea.shcizo.se": "enc-gitea"}}
+
+	t.Run("resolves tag with registry auth", func(t *testing.T) {
+		dist := &fakeDist{digest: d}
+		got, err := NewResolver(dist, auth).Resolve(context.Background(), "gitea.shcizo.se/shcizo/slusk:1.4")
+		require.NoError(t, err)
+		assert.Equal(t, "gitea.shcizo.se/shcizo/slusk:1.4@"+d, got)
+		assert.Equal(t, []string{"gitea.shcizo.se/shcizo/slusk:1.4"}, dist.refs)
+		assert.Equal(t, []string{"enc-gitea"}, dist.auths)
+	})
+	t.Run("digest or missing tag is refused without asking the daemon", func(t *testing.T) {
+		for _, ref := range []string{"repo@sha256:x", "repo:1@sha256:x", "repo"} {
+			dist := &fakeDist{digest: d}
+			_, err := NewResolver(dist, nil).Resolve(context.Background(), ref)
+			assert.Error(t, err, ref)
+			assert.Empty(t, dist.refs)
+		}
+	})
+	t.Run("daemon error is wrapped", func(t *testing.T) {
+		cause := errors.New("unauthorized")
+		_, err := NewResolver(&fakeDist{err: cause}, nil).Resolve(context.Background(), "repo:1")
+		assert.ErrorIs(t, err, cause)
+		assert.ErrorContains(t, err, "repo:1")
+	})
+}

internal/agent/updater/runner.go

@@ -0,0 +1,272 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// maxProgress caps the output buffered between two TakeProgress calls, so a
+// chatty job cannot grow a single job.progress message without bound.
+const maxProgress = 32 << 10
+
+// Executor deploys one job and returns the "repo:tag@digest" it deployed.
+// Everything it writes to out is streamed to the master as job output.
+type Executor interface {
+	Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error)
+}
+
+// Progress is output a running job produced since the previous TakeProgress.
+type Progress struct{ ID, Output string }
+
+// Finished is a job's outcome, kept until the master acknowledges it.
+type Finished struct {
+	ID     string
+	Result protocol.JobResult
+}
+
+// tailBuffer keeps the last max bytes written to it.
+type tailBuffer struct {
+	max       int
+	buf       []byte
+	truncated bool
+}
+
+func (b *tailBuffer) Write(p []byte) {
+	b.buf = append(b.buf, p...)
+	if over := len(b.buf) - b.max; over > 0 {
+		b.buf = append(b.buf[:0], b.buf[over:]...)
+		b.truncated = true
+	}
+}
+
+func (b *tailBuffer) String() string  { return string(b.buf) }
+func (b *tailBuffer) Truncated() bool { return b.truncated }
+func (b *tailBuffer) Reset()          { b.buf, b.truncated = nil, false }
+
+type jobState struct {
+	start    protocol.JobStart
+	running  bool
+	result   *protocol.JobResult
+	output   tailBuffer // everything, capped at protocol.MaxJobOutput
+	unsent   tailBuffer // since the last TakeProgress, capped at maxProgress
+	reported bool       // a Progress has been taken since the job started
+	finished time.Time  // orders Pending
+}
+
+// Runner executes jobs one at a time in the order they arrive and keeps each
+// job's result until Ack. Two docker compose calls against the same stack
+// would race, so there is deliberately exactly one worker.
+//
+// Runner satisfies conn.Jobs.
+type Runner struct {
+	exec    Executor
+	timeout time.Duration
+	now     func() time.Time
+
+	mu    sync.Mutex // guards jobs and queue
+	jobs  map[string]*jobState
+	queue []string
+
+	wake    chan struct{} // wakes the worker
+	changed chan struct{} // tells the connection there is something to send
+}
+
+// NewRunner returns a Runner that gives every job at most timeout. Call Run
+// to start the worker.
+func NewRunner(exec Executor, timeout time.Duration) *Runner {
+	return &Runner{
+		exec:    exec,
+		timeout: timeout,
+		now:     time.Now,
+		jobs:    make(map[string]*jobState),
+		wake:    make(chan struct{}, 1),
+		changed: make(chan struct{}, 1),
+	}
+}
+
+// signal does a non-blocking send so neither side ever waits on the other.
+func signal(ch chan struct{}) {
+	select {
+	case ch <- struct{}{}:
+	default:
+	}
+}
+
+// Submit queues a job. A job whose ID the runner already holds (queued,
+// running, or finished and not yet acknowledged) is ignored.
+func (r *Runner) Submit(id string, start protocol.JobStart) {
+	r.mu.Lock()
+	if _, ok := r.jobs[id]; ok {
+		r.mu.Unlock()
+		return
+	}
+	r.jobs[id] = &jobState{
+		start:  start,
+		output: tailBuffer{max: protocol.MaxJobOutput},
+		unsent: tailBuffer{max: maxProgress},
+	}
+	r.queue = append(r.queue, id)
+	r.mu.Unlock()
+	signal(r.wake)
+}
+
+// Ack forgets a finished job. Acks for unknown, queued or running jobs are
+// ignored.
+func (r *Runner) Ack(id string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if j, ok := r.jobs[id]; ok && j.result != nil {
+		delete(r.jobs, id)
+	}
+}
+
+// Known returns the sorted IDs of every job the runner holds.
+func (r *Runner) Known() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	ids := make([]string, 0, len(r.jobs))
+	for id := range r.jobs {
+		ids = append(ids, id)
+	}
+	sort.Strings(ids)
+	return ids
+}
+
+// TakeProgress returns the output running jobs produced since the previous
+// call. A job's first entry (possibly with empty Output) tells the master it
+// is running.
+func (r *Runner) TakeProgress() []Progress {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	var out []Progress
+	for id, j := range r.jobs {
+		if !j.running {
+			continue
+		}
+		s := j.unsent.String()
+		if j.reported && s == "" {
+			continue
+		}
+		j.reported = true
+		j.unsent.Reset()
+		out = append(out, Progress{ID: id, Output: s})
+	}
+	sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID })
+	return out
+}
+
+// Pending returns the results not yet acknowledged, oldest first.
+func (r *Runner) Pending() []Finished {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	var out []Finished
+	var at []time.Time
+	for id, j := range r.jobs {
+		if j.result != nil {
+			out = append(out, Finished{ID: id, Result: *j.result})
+			at = append(at, j.finished)
+		}
+	}
+	idx := make([]int, len(out))
+	for i := range idx {
+		idx[i] = i
+	}
+	sort.Slice(idx, func(a, b int) bool {
+		if !at[idx[a]].Equal(at[idx[b]]) {
+			return at[idx[a]].Before(at[idx[b]])
+		}
+		return out[idx[a]].ID < out[idx[b]].ID
+	})
+	sorted := make([]Finished, len(out))
+	for i, k := range idx {
+		sorted[i] = out[k]
+	}
+	return sorted
+}
+
+// Changed returns a channel that receives when there is new progress or a
+// new result to send.
+func (r *Runner) Changed() <-chan struct{} { return r.changed }
+
+// Run works through the queue until ctx ends. Jobs still queued then are
+// dropped; the master marks them lost on the agent's next hello.
+func (r *Runner) Run(ctx context.Context) {
+	for {
+		id, start, ok := r.next()
+		if !ok {
+			select {
+			case <-ctx.Done():
+				return
+			case <-r.wake:
+				continue
+			}
+		}
+		if ctx.Err() != nil {
+			return
+		}
+		r.runOne(ctx, id, start)
+	}
+}
+
+// next pops the next queued job and marks it running.
+func (r *Runner) next() (string, protocol.JobStart, bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if len(r.queue) == 0 {
+		return "", protocol.JobStart{}, false
+	}
+	id := r.queue[0]
+	r.queue = r.queue[1:]
+	j := r.jobs[id]
+	j.running = true
+	signal(r.changed)
+	return id, j.start, true
+}
+
+func (r *Runner) runOne(ctx context.Context, id string, start protocol.JobStart) {
+	started := r.now()
+	jobCtx, cancel := context.WithTimeout(ctx, r.timeout)
+	ref, err := r.exec.Execute(jobCtx, start, &jobWriter{r: r, id: id})
+	timedOut := errors.Is(jobCtx.Err(), context.DeadlineExceeded)
+	cancel()
+
+	res := protocol.JobResult{Status: protocol.ResultSucceeded, Ref: ref, StartedAt: started, FinishedAt: r.now()}
+	switch {
+	case timedOut:
+		res = protocol.JobResult{Status: protocol.ResultFailed, Error: fmt.Sprintf("timed out after %s", r.timeout), StartedAt: started, FinishedAt: res.FinishedAt}
+	case err != nil:
+		res = protocol.JobResult{Status: protocol.ResultFailed, Error: err.Error(), StartedAt: started, FinishedAt: res.FinishedAt}
+	}
+
+	r.mu.Lock()
+	j := r.jobs[id]
+	res.Output, res.OutputTruncated = j.output.String(), j.output.Truncated()
+	j.running, j.result, j.finished = false, &res, res.FinishedAt
+	j.unsent.Reset()
+	r.mu.Unlock()
+	signal(r.changed)
+}
+
+// jobWriter feeds an executor's output into the job's buffers.
+type jobWriter struct {
+	r  *Runner
+	id string
+}
+
+func (w *jobWriter) Write(p []byte) (int, error) {
+	w.r.mu.Lock()
+	if j, ok := w.r.jobs[w.id]; ok {
+		j.output.Write(p)
+		j.unsent.Write(p)
+	}
+	w.r.mu.Unlock()
+	signal(w.r.changed)
+	return len(p), nil
+}

internal/agent/updater/runner_test.go

@@ -0,0 +1,200 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"io"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// fakeExec writes "pulling <ref>" and then waits on gate (if set). Refs
+// starting with "bad" fail. It records the highest concurrency it saw.
+type fakeExec struct {
+	gate    chan struct{}
+	cur     atomic.Int32
+	maxSeen atomic.Int32
+	started chan string
+}
+
+func newFakeExec() *fakeExec {
+	return &fakeExec{gate: make(chan struct{}), started: make(chan string, 10)}
+}
+
+func (f *fakeExec) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	n := f.cur.Add(1)
+	defer f.cur.Add(-1)
+	for {
+		m := f.maxSeen.Load()
+		if n <= m || f.maxSeen.CompareAndSwap(m, n) {
+			break
+		}
+	}
+	f.started <- job.Ref
+	io.WriteString(out, "pulling "+job.Ref+"\n")
+	select {
+	case <-f.gate:
+	case <-ctx.Done():
+		return "", ctx.Err()
+	}
+	if strings.HasPrefix(job.Ref, "bad") {
+		return "", errors.New("boom")
+	}
+	return job.Ref + "@sha256:aa", nil
+}
+
+func startRunner(t *testing.T, exec Executor, timeout time.Duration) *Runner {
+	t.Helper()
+	r := NewRunner(exec, timeout)
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() { defer wg.Done(); r.Run(ctx) }()
+	t.Cleanup(func() { cancel(); wg.Wait() })
+	return r
+}
+
+func waitPending(t *testing.T, r *Runner, n int) []Finished {
+	t.Helper()
+	var got []Finished
+	require.Eventually(t, func() bool { got = r.Pending(); return len(got) >= n }, 2*time.Second, 5*time.Millisecond)
+	return got
+}
+
+func TestRunner_FIFOSingleWorker(t *testing.T) {
+	fe := newFakeExec()
+	r := startRunner(t, fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	r.Submit("2", protocol.JobStart{Ref: "bad:2"})
+	assert.Equal(t, "a:1", <-fe.started)
+	fe.gate <- struct{}{}
+	assert.Equal(t, "bad:2", <-fe.started)
+	fe.gate <- struct{}{}
+
+	got := waitPending(t, r, 2)
+	assert.Equal(t, int32(1), fe.maxSeen.Load())
+	require.Len(t, got, 2)
+	assert.Equal(t, "1", got[0].ID)
+	assert.Equal(t, protocol.ResultSucceeded, got[0].Result.Status)
+	assert.Equal(t, "a:1@sha256:aa", got[0].Result.Ref)
+	assert.Equal(t, "pulling a:1\n", got[0].Result.Output)
+	assert.Equal(t, "2", got[1].ID)
+	assert.Equal(t, protocol.ResultFailed, got[1].Result.Status)
+	assert.Equal(t, "boom", got[1].Result.Error)
+	assert.False(t, got[0].Result.StartedAt.IsZero())
+}
+
+func TestRunner_DuplicateSubmitIgnored(t *testing.T) {
+	fe := newFakeExec()
+	r := startRunner(t, fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	assert.Equal(t, "a:1", <-fe.started)
+	r.Submit("1", protocol.JobStart{Ref: "other:9"})
+	fe.gate <- struct{}{}
+	got := waitPending(t, r, 1)
+	assert.Equal(t, "a:1@sha256:aa", got[0].Result.Ref)
+	select {
+	case ref := <-fe.started:
+		t.Fatalf("duplicate job ran: %s", ref)
+	case <-time.After(50 * time.Millisecond):
+	}
+}
+
+func TestRunner_KnownAndAck(t *testing.T) {
+	fe := newFakeExec()
+	r := startRunner(t, fe, time.Minute)
+	r.Submit("2", protocol.JobStart{Ref: "a:2"})
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	assert.Equal(t, "a:2", <-fe.started)
+	assert.Equal(t, []string{"1", "2"}, r.Known())
+
+	r.Ack("2") // running: ignored
+	assert.Equal(t, []string{"1", "2"}, r.Known())
+
+	fe.gate <- struct{}{}
+	waitPending(t, r, 1)
+	r.Ack("2")
+	assert.Equal(t, []string{"1"}, r.Known())
+	<-fe.started
+	fe.gate <- struct{}{}
+	waitPending(t, r, 1)
+}
+
+func TestRunner_TakeProgress(t *testing.T) {
+	fe := newFakeExec()
+	r := startRunner(t, fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	<-fe.started
+	require.Eventually(t, func() bool { return len(r.Pending()) == 0 && len(r.Known()) == 1 }, time.Second, 5*time.Millisecond)
+
+	var first []Progress
+	require.Eventually(t, func() bool { first = r.TakeProgress(); return len(first) > 0 }, time.Second, 5*time.Millisecond)
+	assert.Equal(t, []Progress{{ID: "1", Output: "pulling a:1\n"}}, first)
+	assert.Empty(t, r.TakeProgress(), "nothing new since the last call")
+
+	fe.gate <- struct{}{}
+	waitPending(t, r, 1)
+	assert.Empty(t, r.TakeProgress(), "a finished job no longer reports progress")
+}
+
+func TestRunner_TakeProgressReportsStartedJobOnce(t *testing.T) {
+	gate := make(chan struct{})
+	started := make(chan struct{})
+	r := startRunner(t, execFunc(func(ctx context.Context, _ protocol.JobStart, _ io.Writer) (string, error) {
+		close(started)
+		<-gate
+		return "x", nil
+	}), time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	<-started
+	assert.Equal(t, []Progress{{ID: "1", Output: ""}}, r.TakeProgress())
+	assert.Empty(t, r.TakeProgress())
+	close(gate)
+}
+
+type execFunc func(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error)
+
+func (f execFunc) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	return f(ctx, job, out)
+}
+
+func TestRunner_Timeout(t *testing.T) {
+	r := startRunner(t, execFunc(func(ctx context.Context, _ protocol.JobStart, _ io.Writer) (string, error) {
+		<-ctx.Done()
+		return "", ctx.Err()
+	}), 20*time.Millisecond)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	got := waitPending(t, r, 1)
+	assert.Equal(t, protocol.ResultFailed, got[0].Result.Status)
+	assert.Equal(t, "timed out after 20ms", got[0].Result.Error)
+}
+
+func TestRunner_OutputCapped(t *testing.T) {
+	r := startRunner(t, execFunc(func(_ context.Context, _ protocol.JobStart, out io.Writer) (string, error) {
+		io.WriteString(out, strings.Repeat("a", 10))
+		io.WriteString(out, strings.Repeat("b", protocol.MaxJobOutput))
+		return "x", nil
+	}), time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	got := waitPending(t, r, 1)
+	assert.Equal(t, strings.Repeat("b", protocol.MaxJobOutput), got[0].Result.Output)
+	assert.True(t, got[0].Result.OutputTruncated)
+}
+
+func TestRunner_ChangedSignals(t *testing.T) {
+	r := startRunner(t, execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "x", nil }), time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	select {
+	case <-r.Changed():
+	case <-time.After(time.Second):
+		t.Fatal("no change signal")
+	}
+}

internal/agent/updater/swarm.go

@@ -0,0 +1,93 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/system"
+
+	"github.com/shcizo/package-updater/internal/imageref"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// SwarmClient is the subset of the Docker SDK SwarmExecutor depends on.
+// Defined as an interface so tests can supply a fake.
+type SwarmClient interface {
+	Info(ctx context.Context) (system.Info, error)
+	ServiceInspectWithRaw(ctx context.Context, serviceID string, opts swarm.ServiceInspectOptions) (swarm.Service, []byte, error)
+	ServiceUpdate(ctx context.Context, serviceID string, version swarm.Version, service swarm.ServiceSpec, opts swarm.ServiceUpdateOptions) (swarm.ServiceUpdateResponse, error)
+}
+
+// SwarmExecutor updates a Swarm service's image via the Docker API,
+// equivalent to `docker service update --image`.
+type SwarmExecutor struct {
+	cli     SwarmClient
+	resolve *Resolver
+	auth    *RegistryAuth
+}
+
+// NewSwarmExecutor returns an executor that drives Swarm service updates
+// through the Docker API. auth may be nil (anonymous registry access).
+func NewSwarmExecutor(cli SwarmClient, resolve *Resolver, auth *RegistryAuth) *SwarmExecutor {
+	return &SwarmExecutor{cli: cli, resolve: resolve, auth: auth}
+}
+
+// Execute checks that the local daemon is a manager of the job's cluster,
+// pins job.Ref to its current digest and updates the service to it. The
+// update is addressed by the ID and version of the service that was
+// inspected, so it changes exactly that object. Success means the daemon
+// accepted the new spec, not that the tasks have converged. It returns the
+// pinned ref.
+func (e *SwarmExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	info, err := e.cli.Info(ctx)
+	if err != nil {
+		return "", fmt.Errorf("docker info: %w", err)
+	}
+	if !info.Swarm.ControlAvailable {
+		return "", fmt.Errorf("refused: this agent's daemon is not a swarm manager")
+	}
+	actual := ""
+	if info.Swarm.Cluster != nil {
+		actual = info.Swarm.Cluster.ID
+	}
+	if actual != job.SwarmClusterID {
+		return "", fmt.Errorf("refused: job is for swarm cluster %s, this daemon is in %s", job.SwarmClusterID, actual)
+	}
+
+	svc, _, err := e.cli.ServiceInspectWithRaw(ctx, job.SwarmService, swarm.ServiceInspectOptions{})
+	if err != nil {
+		return "", fmt.Errorf("inspect service %s: %w", job.SwarmService, err)
+	}
+	if svc.Spec.Name != job.SwarmService {
+		return "", fmt.Errorf("inspect service %s: daemon returned service %q", job.SwarmService, svc.Spec.Name)
+	}
+	if svc.Spec.TaskTemplate.ContainerSpec == nil {
+		return "", fmt.Errorf("service %s has no container spec", job.SwarmService)
+	}
+
+	pinned, err := e.resolve.Resolve(ctx, job.Ref)
+	if err != nil {
+		return "", err
+	}
+	fmt.Fprintf(out, "resolved %s\n", pinned)
+
+	spec := svc.Spec
+	spec.TaskTemplate.ContainerSpec.Image = pinned
+	opts := swarm.ServiceUpdateOptions{QueryRegistry: true}
+	if enc := e.auth.For(imageref.Parse(job.Ref).Repo); enc != "" {
+		opts.EncodedRegistryAuth = enc
+	} else {
+		opts.RegistryAuthFrom = swarm.RegistryAuthFromSpec
+	}
+	resp, err := e.cli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, opts)
+	if err != nil {
+		return "", fmt.Errorf("update service %s: %w", job.SwarmService, err)
+	}
+	for _, w := range resp.Warnings {
+		fmt.Fprintf(out, "warning: %s\n", w)
+	}
+	fmt.Fprintf(out, "service %s updated\n", job.SwarmService)
+	return pinned, nil
+}

internal/agent/updater/swarm_test.go

@@ -0,0 +1,142 @@
+package updater
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/docker/docker/api/types/registry"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/system"
+	"github.com/opencontainers/go-digest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+type fakeSwarm struct {
+	info       system.Info
+	svc        swarm.Service
+	inspectErr error
+	updateErr  error
+	warnings   []string
+
+	inspectCalls int
+	updateCalls  int
+	updID        string
+	updVersion   swarm.Version
+	updSpec      swarm.ServiceSpec
+	updOpts      swarm.ServiceUpdateOptions
+}
+
+func (f *fakeSwarm) Info(context.Context) (system.Info, error) { return f.info, nil }
+
+func (f *fakeSwarm) ServiceInspectWithRaw(context.Context, string, swarm.ServiceInspectOptions) (swarm.Service, []byte, error) {
+	f.inspectCalls++
+	return f.svc, nil, f.inspectErr
+}
+
+func (f *fakeSwarm) ServiceUpdate(_ context.Context, id string, v swarm.Version, spec swarm.ServiceSpec, opts swarm.ServiceUpdateOptions) (swarm.ServiceUpdateResponse, error) {
+	f.updateCalls++
+	f.updID, f.updVersion, f.updSpec, f.updOpts = id, v, spec, opts
+	return swarm.ServiceUpdateResponse{Warnings: f.warnings}, f.updateErr
+}
+
+type fakeDist struct {
+	digest string
+	err    error
+	refs   []string
+	auths  []string
+}
+
+func (f *fakeDist) DistributionInspect(_ context.Context, ref, auth string) (registry.DistributionInspect, error) {
+	f.refs = append(f.refs, ref)
+	f.auths = append(f.auths, auth)
+	var d registry.DistributionInspect
+	d.Descriptor.Digest = digest.Digest(f.digest)
+	return d, f.err
+}
+
+func managerOf(cluster string) system.Info {
+	var info system.Info
+	info.Swarm.ControlAvailable = true
+	info.Swarm.Cluster = &swarm.ClusterInfo{ID: cluster}
+	return info
+}
+
+func swarmSvc(name string) swarm.Service {
+	return swarm.Service{
+		ID:   "svc-id",
+		Meta: swarm.Meta{Version: swarm.Version{Index: 7}},
+		Spec: swarm.ServiceSpec{
+			Annotations:  swarm.Annotations{Name: name},
+			TaskTemplate: swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{Image: "gitea/gitea:1.26"}},
+		},
+	}
+}
+
+func TestSwarmExecutor(t *testing.T) {
+	job := protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"}
+	workerInfo := managerOf("c1")
+	workerInfo.Swarm.ControlAvailable = false
+	noContainer := swarmSvc("gitea_gitea")
+	noContainer.Spec.TaskTemplate.ContainerSpec = nil
+
+	cases := []struct {
+		name        string
+		fake        *fakeSwarm
+		auth        *RegistryAuth
+		wantErr     string
+		wantUpdate  bool
+		wantEncAuth bool
+	}{
+		{name: "happy path, no auth", fake: &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_gitea")}, wantUpdate: true},
+		{name: "happy path, auth known", fake: &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_gitea"), warnings: []string{"w1"}},
+			auth: &RegistryAuth{byHost: map[string]string{"docker.io": "enc"}}, wantUpdate: true, wantEncAuth: true},
+		{name: "cluster mismatch", fake: &fakeSwarm{info: managerOf("other"), svc: swarmSvc("gitea_gitea")},
+			wantErr: "refused: job is for swarm cluster c1, this daemon is in other"},
+		{name: "worker node", fake: &fakeSwarm{info: workerInfo, svc: swarmSvc("gitea_gitea")}, wantErr: "not a swarm manager"},
+		{name: "other service returned", fake: &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_other")}, wantErr: "daemon returned service"},
+		{name: "no container spec", fake: &fakeSwarm{info: managerOf("c1"), svc: noContainer}, wantErr: "no container spec"},
+		{name: "inspect fails", fake: &fakeSwarm{info: managerOf("c1"), inspectErr: errors.New("gone")}, wantErr: "gone"},
+		{name: "update fails", fake: &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_gitea"), updateErr: errors.New("rejected")},
+			wantErr: "rejected", wantUpdate: true},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			dist := &fakeDist{digest: "sha256:" + string(bytes.Repeat([]byte("a"), 64))}
+			e := NewSwarmExecutor(c.fake, NewResolver(dist, c.auth), c.auth)
+			var out bytes.Buffer
+			ref, err := e.Execute(context.Background(), job, &out)
+			if c.wantErr != "" {
+				assert.ErrorContains(t, err, c.wantErr)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, "gitea/gitea:1.27@"+dist.digest, ref)
+				assert.Contains(t, out.String(), "resolved "+ref)
+			}
+			if c.wantErr == "refused: job is for swarm cluster c1, this daemon is in other" || c.wantErr == "not a swarm manager" {
+				assert.Zero(t, c.fake.inspectCalls)
+			}
+			if !c.wantUpdate {
+				assert.Zero(t, c.fake.updateCalls)
+				return
+			}
+			assert.Equal(t, 1, c.fake.updateCalls)
+			assert.Equal(t, "svc-id", c.fake.updID)
+			assert.Equal(t, uint64(7), c.fake.updVersion.Index)
+			assert.Equal(t, "gitea/gitea:1.27@"+dist.digest, c.fake.updSpec.TaskTemplate.ContainerSpec.Image)
+			assert.True(t, c.fake.updOpts.QueryRegistry)
+			if c.wantEncAuth {
+				assert.Equal(t, "enc", c.fake.updOpts.EncodedRegistryAuth)
+				assert.Empty(t, c.fake.updOpts.RegistryAuthFrom)
+				assert.Contains(t, out.String(), "warning: w1")
+			} else {
+				assert.Empty(t, c.fake.updOpts.EncodedRegistryAuth)
+				assert.Equal(t, swarm.RegistryAuthFromSpec, c.fake.updOpts.RegistryAuthFrom)
+			}
+		})
+	}
+}

internal/api/handlers.go

@@ -1,148 +0,0 @@
-package api
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-
-	"github.com/docker/docker/api/types"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/shcizo/package-updater/internal/metrics"
-	"github.com/shcizo/package-updater/internal/updater"
-)
-
-// Finder is the discovery interface used by the update handler.
-type Finder interface {
-	FindJobs(ctx context.Context, image string) ([]discovery.Job, error)
-}
-
-// Submitter is the queue interface used by the update handler.
-type Submitter interface {
-	Submit(ctx context.Context, jobs []discovery.Job) []updater.Result
-}
-
-// Pinger pings the Docker daemon for the healthcheck.
-type Pinger interface {
-	Ping(ctx context.Context) (types.Ping, error)
-}
-
-// Handlers wires the HTTP endpoints to the rest of the service.
-type Handlers struct {
-	finder    Finder
-	submitter Submitter
-	pinger    Pinger
-	version   string
-	commit    string
-	buildTime string
-	metrics   *metrics.Metrics
-}
-
-// NewHandlers constructs a Handlers value with all dependencies injected.
-// m may be nil, in which case metrics recording is silently skipped.
-func NewHandlers(f Finder, s Submitter, p Pinger, version, commit, buildTime string, m *metrics.Metrics) *Handlers {
-	return &Handlers{finder: f, submitter: s, pinger: p, version: version, commit: commit, buildTime: buildTime, metrics: m}
-}
-
-// Update implements POST /update.
-func (h *Handlers) Update(w http.ResponseWriter, r *http.Request) {
-	var req UpdateRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeJSONError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-	if req.Image == "" {
-		writeJSONError(w, http.StatusBadRequest, "image is required")
-		return
-	}
-
-	requestedImage := req.Image
-	if req.Tag != "" {
-		requestedImage = req.Image + ":" + req.Tag
-	}
-
-	jobs, err := h.finder.FindJobs(r.Context(), requestedImage)
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "discovery failed: "+err.Error())
-		return
-	}
-
-	resp := UpdateResponse{
-		RequestID: logging.RequestIDFrom(r.Context()),
-		Image:     req.Image,
-		Tag:       req.Tag,
-		Matched:   len(jobs),
-		Results:   []ResultDTO{},
-	}
-
-	if len(jobs) == 0 {
-		writeJSON(w, http.StatusOK, resp)
-		return
-	}
-
-	results := h.submitter.Submit(r.Context(), jobs)
-	updated, failed := 0, 0
-	for _, res := range results {
-		composeFile := ""
-		if len(res.Job.ConfigFiles) > 0 {
-			composeFile = res.Job.ConfigFiles[0]
-		}
-		resp.Results = append(resp.Results, ResultDTO{
-			Project:     res.Job.Project,
-			Service:     res.Job.Service,
-			ComposeFile: composeFile,
-			ServiceID:   res.Job.ServiceID,
-			Status:      string(res.Status),
-			Error:       res.Error,
-			DurationMs:  res.DurationMs,
-		})
-		if res.Status == updater.StatusUpdated {
-			updated++
-		} else {
-			failed++
-		}
-	}
-
-	switch {
-	case failed == 0:
-		writeJSON(w, http.StatusOK, resp)
-	case updated == 0:
-		writeJSON(w, http.StatusInternalServerError, resp)
-	default:
-		writeJSON(w, http.StatusMultiStatus, resp)
-	}
-}
-
-// Healthz implements GET /healthz.
-func (h *Handlers) Healthz(w http.ResponseWriter, r *http.Request) {
-	if _, err := h.pinger.Ping(r.Context()); err != nil {
-		if h.metrics != nil {
-			h.metrics.DockerPingUp.Set(0)
-		}
-		writeJSON(w, http.StatusServiceUnavailable, HealthResponse{Status: "unhealthy", Docker: "unreachable"})
-		return
-	}
-	if h.metrics != nil {
-		h.metrics.DockerPingUp.Set(1)
-	}
-	writeJSON(w, http.StatusOK, HealthResponse{Status: "ok", Docker: "ok"})
-}
-
-// Version implements GET /version.
-func (h *Handlers) Version(w http.ResponseWriter, _ *http.Request) {
-	writeJSON(w, http.StatusOK, VersionResponse{
-		Version:   h.version,
-		Commit:    h.commit,
-		BuildTime: h.buildTime,
-	})
-}
-
-func writeJSON(w http.ResponseWriter, code int, body any) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(code)
-	_ = json.NewEncoder(w).Encode(body)
-}
-
-func writeJSONError(w http.ResponseWriter, code int, msg string) {
-	writeJSON(w, code, map[string]string{"error": msg})
-}

internal/api/handlers_test.go

@@ -1,216 +0,0 @@
-package api_test
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/docker/docker/api/types"
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/updater"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeFinder struct {
-	jobs []discovery.Job
-	err  error
-
-	// gotImage records the image argument passed to FindJobs, for
-	// assertions on the exact reference the handler built.
-	gotImage string
-	// swarmShaped, when true, makes FindJobs return a Job whose Image
-	// field carries the received image argument, simulating
-	// discovery.SwarmDiscovery.FindJobs.
-	swarmShaped bool
-}
-
-func (f *fakeFinder) FindJobs(_ context.Context, image string) ([]discovery.Job, error) {
-	f.gotImage = image
-	if f.swarmShaped {
-		return []discovery.Job{{Service: "myapp_web", ServiceID: "svc1", Image: image}}, f.err
-	}
-	return f.jobs, f.err
-}
-
-type fakeSubmitter struct {
-	results []updater.Result
-}
-
-func (f *fakeSubmitter) Submit(_ context.Context, jobs []discovery.Job) []updater.Result {
-	if f.results != nil {
-		return f.results
-	}
-	out := make([]updater.Result, len(jobs))
-	for i, j := range jobs {
-		out[i] = updater.Result{Job: j, Status: updater.StatusUpdated}
-	}
-	return out
-}
-
-type fakePinger struct{ err error }
-
-func (f *fakePinger) Ping(_ context.Context) (types.Ping, error) {
-	return types.Ping{}, f.err
-}
-
-func decode[T any](t *testing.T, body io.Reader) T {
-	t.Helper()
-	var v T
-	require.NoError(t, json.NewDecoder(body).Decode(&v))
-	return v
-}
-
-func TestUpdate_ValidationError(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodPost, "/update",
-		strings.NewReader(`{"tag":"v1.2.3"}`))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusBadRequest, w.Code)
-}
-
-func TestUpdate_BadJSON(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`not json`))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusBadRequest, w.Code)
-}
-
-func TestUpdate_DiscoveryFailureReturns500(t *testing.T) {
-	finder := &fakeFinder{err: errors.New("daemon unreachable")}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusInternalServerError, w.Code)
-}
-
-func TestUpdate_ZeroMatchesReturns200(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	resp := decode[api.UpdateResponse](t, w.Body)
-	require.Equal(t, 0, resp.Matched)
-}
-
-func TestUpdate_AllSucceeded200(t *testing.T) {
-	finder := &fakeFinder{jobs: []discovery.Job{
-		{Project: "p", Service: "s", WorkingDir: "/x", ConfigFiles: []string{"/x/c.yml"}},
-	}}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x", Tag: "v1"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	resp := decode[api.UpdateResponse](t, w.Body)
-	require.Equal(t, 1, resp.Matched)
-	require.Equal(t, "updated", resp.Results[0].Status)
-	require.Equal(t, "/x/c.yml", resp.Results[0].ComposeFile)
-}
-
-func TestUpdate_MixedReturns207(t *testing.T) {
-	jobs := []discovery.Job{
-		{Project: "p1", Service: "s", ConfigFiles: []string{"/x/c.yml"}},
-		{Project: "p2", Service: "s", ConfigFiles: []string{"/y/c.yml"}},
-	}
-	finder := &fakeFinder{jobs: jobs}
-	submitter := &fakeSubmitter{results: []updater.Result{
-		{Job: jobs[0], Status: updater.StatusUpdated},
-		{Job: jobs[1], Status: updater.StatusFailed, Error: "boom"},
-	}}
-	h := api.NewHandlers(finder, submitter, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusMultiStatus, w.Code)
-}
-
-func TestUpdate_AllFailedReturns500(t *testing.T) {
-	jobs := []discovery.Job{{Project: "p", Service: "s", ConfigFiles: []string{"/x/c.yml"}}}
-	finder := &fakeFinder{jobs: jobs}
-	submitter := &fakeSubmitter{results: []updater.Result{
-		{Job: jobs[0], Status: updater.StatusFailed, Error: "boom"},
-	}}
-	h := api.NewHandlers(finder, submitter, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusInternalServerError, w.Code)
-}
-
-func TestHealthz_OKWhenDockerUp(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
-	w := httptest.NewRecorder()
-	h.Healthz(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-}
-
-func TestHealthz_503WhenDockerDown(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{err: errors.New("ping fail")}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
-	w := httptest.NewRecorder()
-	h.Healthz(w, req)
-	require.Equal(t, http.StatusServiceUnavailable, w.Code)
-}
-
-func TestUpdate_FindJobsCalledWithTaggedImage(t *testing.T) {
-	finder := &fakeFinder{}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "registry.example.com/myapp", Tag: "v2"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, "registry.example.com/myapp:v2", finder.gotImage)
-}
-
-func TestUpdate_FindJobsCalledWithBareImageWhenNoTag(t *testing.T) {
-	finder := &fakeFinder{}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "registry.example.com/myapp"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, "registry.example.com/myapp", finder.gotImage)
-}
-
-func TestUpdate_SwarmModeThreadsTagThroughDiscovery(t *testing.T) {
-	finder := &fakeFinder{swarmShaped: true}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "registry.example.com/myapp", Tag: "v2"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	// The fake Finder's returned Job.Image (as SwarmDiscovery would build
-	// it) reflects the image argument it received from the handler.
-	// Asserting on the captured argument proves the full request ->
-	// discovery flow carries the tag through to what would reach the
-	// SwarmExecutor.
-	require.Equal(t, "registry.example.com/myapp:v2", finder.gotImage)
-}
-
-func TestVersion(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v1.2.3", "abcdef", "2026-05-22T00:00:00Z", nil)
-	req := httptest.NewRequest(http.MethodGet, "/version", nil)
-	w := httptest.NewRecorder()
-	h.Version(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	resp := decode[api.VersionResponse](t, w.Body)
-	require.Equal(t, "v1.2.3", resp.Version)
-}

internal/api/middleware.go

@@ -1,112 +0,0 @@
-// Package api contains HTTP handlers, middleware, and request/response DTOs.
-package api
-
-import (
-	"crypto/rand"
-	"crypto/subtle"
-	"fmt"
-	"log/slog"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/shcizo/package-updater/internal/metrics"
-)
-
-// Auth returns middleware that requires a matching bearer token.
-// Compares with constant-time to defeat timing attacks.
-func Auth(token string) func(http.Handler) http.Handler {
-	tokenBytes := []byte(token)
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			h := r.Header.Get("Authorization")
-			const prefix = "Bearer "
-			if !strings.HasPrefix(h, prefix) {
-				writeAuthError(w)
-				return
-			}
-			provided := []byte(strings.TrimPrefix(h, prefix))
-			if len(provided) == 0 ||
-				subtle.ConstantTimeCompare(provided, tokenBytes) != 1 {
-				writeAuthError(w)
-				return
-			}
-			next.ServeHTTP(w, r)
-		})
-	}
-}
-
-func writeAuthError(w http.ResponseWriter) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(http.StatusUnauthorized)
-	_, _ = w.Write([]byte(`{"error":"unauthorized"}`))
-}
-
-// newUUID generates a random UUID v4 string using crypto/rand.
-func newUUID() string {
-	var b [16]byte
-	_, _ = rand.Read(b[:])
-	b[6] = (b[6] & 0x0f) | 0x40 // version 4
-	b[8] = (b[8] & 0x3f) | 0x80 // variant bits
-	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
-		b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
-}
-
-// RequestID middleware ensures every request has an X-Request-ID
-// header (generated if absent) and stores it in the request context.
-func RequestID(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		id := r.Header.Get("X-Request-ID")
-		if id == "" {
-			id = newUUID()
-		}
-		w.Header().Set("X-Request-ID", id)
-		ctx := logging.WithRequestID(r.Context(), id)
-		next.ServeHTTP(w, r.WithContext(ctx))
-	})
-}
-
-// RequestLogger emits a structured access-log line per request. m may be nil,
-// in which case metrics recording is silently skipped.
-func RequestLogger(base *slog.Logger, m *metrics.Metrics) func(http.Handler) http.Handler {
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			start := time.Now()
-			sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
-			next.ServeHTTP(sw, r)
-			logger := logging.FromContext(r.Context(), base)
-			logger.Info("http_request",
-				"method", r.Method,
-				"path", r.URL.Path,
-				"status", sw.status,
-				"duration_ms", time.Since(start).Milliseconds(),
-				"client_ip", clientIP(r),
-			)
-			if m != nil {
-				m.HTTPRequests.WithLabelValues(r.URL.Path, strconv.Itoa(sw.status)).Inc()
-			}
-		})
-	}
-}
-
-type statusWriter struct {
-	http.ResponseWriter
-	status int
-}
-
-func (s *statusWriter) WriteHeader(code int) {
-	s.status = code
-	s.ResponseWriter.WriteHeader(code)
-}
-
-func clientIP(r *http.Request) string {
-	if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
-		if comma := strings.Index(xff, ","); comma >= 0 {
-			return strings.TrimSpace(xff[:comma])
-		}
-		return xff
-	}
-	return r.RemoteAddr
-}

internal/api/middleware_test.go

@@ -1,92 +0,0 @@
-package api_test
-
-import (
-	"io"
-	"log/slog"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/stretchr/testify/require"
-)
-
-func newTestLogger() *slog.Logger {
-	return slog.New(slog.NewJSONHandler(io.Discard, nil))
-}
-
-func TestAuth_AllowsMatchingToken(t *testing.T) {
-	called := false
-	h := api.Auth("secret")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		called = true
-		w.WriteHeader(http.StatusOK)
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	req.Header.Set("Authorization", "Bearer secret")
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.True(t, called)
-	require.Equal(t, http.StatusOK, w.Code)
-}
-
-func TestAuth_Rejects(t *testing.T) {
-	cases := []struct{ name, header string }{
-		{"missing", ""},
-		{"wrong scheme", "Token secret"},
-		{"wrong value", "Bearer nope"},
-		{"empty bearer", "Bearer "},
-	}
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			h := api.Auth("secret")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-				t.Fatal("handler must not be called")
-			}))
-			req := httptest.NewRequest(http.MethodPost, "/update", nil)
-			if c.header != "" {
-				req.Header.Set("Authorization", c.header)
-			}
-			w := httptest.NewRecorder()
-			h.ServeHTTP(w, req)
-			require.Equal(t, http.StatusUnauthorized, w.Code)
-		})
-	}
-}
-
-func TestRequestID_GeneratesIfMissing(t *testing.T) {
-	var seenID string
-	h := api.RequestID(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
-		seenID = logging.RequestIDFrom(r.Context())
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.NotEmpty(t, seenID)
-	require.Equal(t, seenID, w.Header().Get("X-Request-ID"))
-}
-
-func TestRequestID_UsesIncoming(t *testing.T) {
-	var seenID string
-	h := api.RequestID(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
-		seenID = logging.RequestIDFrom(r.Context())
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	req.Header.Set("X-Request-ID", "given-id")
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.Equal(t, "given-id", seenID)
-}
-
-func TestRequestLogger_LogsAndDelegates(t *testing.T) {
-	logger := newTestLogger()
-	called := false
-	h := api.RequestLogger(logger, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		called = true
-		w.WriteHeader(http.StatusTeapot)
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.True(t, called)
-	require.Equal(t, http.StatusTeapot, w.Code)
-}

internal/api/swarm_result_test.go

@@ -1,48 +0,0 @@
-package api_test
-
-import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/docker/docker/api/types"
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/updater"
-	"github.com/stretchr/testify/require"
-)
-
-type swarmResultFinder struct{ jobs []discovery.Job }
-
-func (f *swarmResultFinder) FindJobs(_ context.Context, _ string) ([]discovery.Job, error) {
-	return f.jobs, nil
-}
-
-type swarmResultSubmitter struct{ results []updater.Result }
-
-func (s *swarmResultSubmitter) Submit(_ context.Context, _ []discovery.Job) []updater.Result {
-	return s.results
-}
-
-type swarmResultPinger struct{}
-
-func (swarmResultPinger) Ping(_ context.Context) (types.Ping, error) { return types.Ping{}, nil }
-
-func TestUpdate_SwarmJobPopulatesServiceID(t *testing.T) {
-	job := discovery.Job{Service: "myapp_web", ServiceID: "svc123", Image: "registry.example.com/myapp:v2"}
-	finder := &swarmResultFinder{jobs: []discovery.Job{job}}
-	submitter := &swarmResultSubmitter{results: []updater.Result{
-		{Job: job, Status: updater.StatusUpdated, DurationMs: 42},
-	}}
-	h := api.NewHandlers(finder, submitter, swarmResultPinger{}, "v", "c", "b", nil)
-
-	req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"image":"registry.example.com/myapp"}`))
-	rec := httptest.NewRecorder()
-	h.Update(rec, req)
-
-	require.Equal(t, http.StatusOK, rec.Code)
-	require.Contains(t, rec.Body.String(), `"service_id":"svc123"`)
-	require.Contains(t, rec.Body.String(), `"service":"myapp_web"`)
-}

internal/api/types.go

@@ -1,40 +0,0 @@
-package api
-
-// UpdateRequest is the body of POST /update.
-type UpdateRequest struct {
-	Image string `json:"image"`
-	Tag   string `json:"tag,omitempty"`
-}
-
-// UpdateResponse is the body of POST /update.
-type UpdateResponse struct {
-	RequestID string      `json:"request_id"`
-	Image     string      `json:"image"`
-	Tag       string      `json:"tag,omitempty"`
-	Matched   int         `json:"matched"`
-	Results   []ResultDTO `json:"results"`
-}
-
-// ResultDTO is one row in UpdateResponse.Results.
-type ResultDTO struct {
-	Project     string `json:"project"`
-	Service     string `json:"service"`
-	ComposeFile string `json:"compose_file"`
-	ServiceID   string `json:"service_id,omitempty"`
-	Status      string `json:"status"`
-	Error       string `json:"error,omitempty"`
-	DurationMs  int64  `json:"duration_ms"`
-}
-
-// HealthResponse is the body of GET /healthz.
-type HealthResponse struct {
-	Status string `json:"status"`
-	Docker string `json:"docker"`
-}
-
-// VersionResponse is the body of GET /version.
-type VersionResponse struct {
-	Version   string `json:"version"`
-	Commit    string `json:"commit"`
-	BuildTime string `json:"build_time"`
-}

internal/config/config.go

@@ -1,58 +0,0 @@
-// Package config loads service configuration from environment variables.
-package config
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"time"
-)
-
-// Config holds all runtime configuration for the service.
-type Config struct {
-	APIKey        string
-	StacksRoot    string
-	Port          string
-	LogLevel      string
-	UpdateTimeout time.Duration
-	OptInLabel    string
-	Mode          string
-}
-
-// Load reads configuration from environment variables, applies defaults,
-// and validates required fields. Returns an error if validation fails so
-// the service can fail-fast at startup.
-func Load() (*Config, error) {
-	cfg := &Config{
-		APIKey:     os.Getenv("UPDATER_API_KEY"),
-		StacksRoot: getenvDefault("STACKS_ROOT", "/home/shcizo/self-hosted"),
-		Port:       getenvDefault("PORT", "8080"),
-		LogLevel:   getenvDefault("LOG_LEVEL", "info"),
-		OptInLabel: getenvDefault("OPT_IN_LABEL", "se.shcizo.auto-update"),
-		Mode:       getenvDefault("MODE", "compose"),
-	}
-
-	if cfg.APIKey == "" {
-		return nil, errors.New("UPDATER_API_KEY is required")
-	}
-
-	if cfg.Mode != "compose" && cfg.Mode != "swarm" {
-		return nil, fmt.Errorf("MODE %q is invalid: must be %q or %q", cfg.Mode, "compose", "swarm")
-	}
-
-	timeoutStr := getenvDefault("UPDATE_TIMEOUT", "5m")
-	d, err := time.ParseDuration(timeoutStr)
-	if err != nil {
-		return nil, fmt.Errorf("UPDATE_TIMEOUT %q is not a valid duration: %w", timeoutStr, err)
-	}
-	cfg.UpdateTimeout = d
-
-	return cfg, nil
-}
-
-func getenvDefault(key, fallback string) string {
-	if v := os.Getenv(key); v != "" {
-		return v
-	}
-	return fallback
-}

internal/config/config_test.go

@@ -1,82 +0,0 @@
-package config_test
-
-import (
-	"testing"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/config"
-	"github.com/stretchr/testify/require"
-)
-
-func TestLoad_RequiresAPIKey(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "")
-	_, err := config.Load()
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "UPDATER_API_KEY")
-}
-
-func TestLoad_AppliesDefaults(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("STACKS_ROOT", "")
-	t.Setenv("PORT", "")
-	t.Setenv("LOG_LEVEL", "")
-	t.Setenv("UPDATE_TIMEOUT", "")
-	t.Setenv("OPT_IN_LABEL", "")
-
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "secret", cfg.APIKey)
-	require.Equal(t, "/home/shcizo/self-hosted", cfg.StacksRoot)
-	require.Equal(t, "8080", cfg.Port)
-	require.Equal(t, "info", cfg.LogLevel)
-	require.Equal(t, 5*time.Minute, cfg.UpdateTimeout)
-	require.Equal(t, "se.shcizo.auto-update", cfg.OptInLabel)
-}
-
-func TestLoad_OverridesViaEnv(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("STACKS_ROOT", "/srv/stacks")
-	t.Setenv("PORT", "9090")
-	t.Setenv("LOG_LEVEL", "debug")
-	t.Setenv("UPDATE_TIMEOUT", "30s")
-	t.Setenv("OPT_IN_LABEL", "io.example.update")
-
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "/srv/stacks", cfg.StacksRoot)
-	require.Equal(t, "9090", cfg.Port)
-	require.Equal(t, "debug", cfg.LogLevel)
-	require.Equal(t, 30*time.Second, cfg.UpdateTimeout)
-	require.Equal(t, "io.example.update", cfg.OptInLabel)
-}
-
-func TestLoad_InvalidTimeoutErrors(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("UPDATE_TIMEOUT", "not-a-duration")
-	_, err := config.Load()
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "UPDATE_TIMEOUT")
-}
-
-func TestLoad_DefaultsToComposeMode(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "compose", cfg.Mode)
-}
-
-func TestLoad_AcceptsSwarmMode(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("MODE", "swarm")
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "swarm", cfg.Mode)
-}
-
-func TestLoad_RejectsInvalidMode(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("MODE", "kubernetes")
-	_, err := config.Load()
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "MODE")
-}

internal/discovery/discovery.go

@@ -1,96 +0,0 @@
-package discovery
-
-import (
-	"context"
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/docker/docker/api/types/container"
-)
-
-// Job describes a single (project, service, config_files) update to execute.
-type Job struct {
-	Project     string
-	Service     string
-	WorkingDir  string
-	ConfigFiles []string
-	// Image is the full image reference the update request asked for.
-	// Compose jobs ignore it (compose.yml already pins the reference to
-	// pull); Swarm jobs need it to know what to set on the service spec.
-	Image string
-	// ServiceID is the Swarm service ID. Empty for Compose jobs.
-	ServiceID string
-	// Refused is true when the WorkingDir falls outside STACKS_ROOT.
-	// The job is returned so the caller can surface a per-job "refused"
-	// result, but it MUST NOT be executed.
-	Refused       bool
-	RefusedReason string
-}
-
-// Discovery orchestrates "given an image, what jobs should we enqueue?".
-type Discovery struct {
-	cli        DockerClient
-	stacksRoot string
-	optInLabel string
-}
-
-// New returns a Discovery bound to the given Docker client and config.
-func New(cli DockerClient, stacksRoot, optInLabel string) *Discovery {
-	return &Discovery{cli: cli, stacksRoot: stacksRoot, optInLabel: optInLabel}
-}
-
-// FindJobs lists running containers, filters by image match + opt-in label,
-// extracts Compose info, applies the path safety check, and deduplicates.
-func (d *Discovery) FindJobs(ctx context.Context, image string) ([]Job, error) {
-	all, err := d.cli.ContainerList(ctx, container.ListOptions{All: true})
-	if err != nil {
-		return nil, fmt.Errorf("docker container list: %w", err)
-	}
-
-	seen := make(map[string]struct{})
-	var jobs []Job
-
-	for _, c := range all {
-		if !ImagesMatch(image, c.Image) {
-			continue
-		}
-		if !HasOptIn(c.Labels, d.optInLabel) {
-			continue
-		}
-		cl, err := ParseComposeLabels(c.Labels)
-		if err != nil {
-			continue
-		}
-
-		refused := !IsInsideRoot(d.stacksRoot, cl.WorkingDir)
-		reason := ""
-		if refused {
-			reason = fmt.Sprintf("working_dir %q outside STACKS_ROOT %q", cl.WorkingDir, d.stacksRoot)
-		}
-
-		key := dedupKey(cl)
-		if _, dup := seen[key]; dup {
-			continue
-		}
-		seen[key] = struct{}{}
-
-		jobs = append(jobs, Job{
-			Project:       cl.Project,
-			Service:       cl.Service,
-			WorkingDir:    cl.WorkingDir,
-			ConfigFiles:   cl.ConfigFiles,
-			Image:         image,
-			Refused:       refused,
-			RefusedReason: reason,
-		})
-	}
-
-	return jobs, nil
-}
-
-func dedupKey(cl ComposeLabels) string {
-	files := append([]string(nil), cl.ConfigFiles...)
-	sort.Strings(files)
-	return cl.Project + "|" + cl.Service + "|" + strings.Join(files, ",")
-}

internal/discovery/discovery_test.go

@@ -1,172 +0,0 @@
-package discovery_test
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeDockerClient struct {
-	containers []types.Container
-	err        error
-}
-
-func (f *fakeDockerClient) ContainerList(_ context.Context, _ container.ListOptions) ([]types.Container, error) {
-	return f.containers, f.err
-}
-
-func (f *fakeDockerClient) Ping(_ context.Context) (types.Ping, error) {
-	return types.Ping{}, nil
-}
-
-func mkContainer(image string, labels map[string]string) types.Container {
-	return types.Container{Image: image, Labels: labels}
-}
-
-func mkComposeLabels(project, service, workingDir, configFile string, optIn bool) map[string]string {
-	m := map[string]string{
-		"com.docker.compose.project":              project,
-		"com.docker.compose.service":              service,
-		"com.docker.compose.project.working_dir":  workingDir,
-		"com.docker.compose.project.config_files": configFile,
-	}
-	if optIn {
-		m["se.shcizo.auto-update"] = "true"
-	}
-	return m
-}
-
-func TestFindJobs_MatchAndOptIn(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			true,
-		)),
-		mkContainer("registry.example.com/other:v1", mkComposeLabels(
-			"other", "web",
-			"/home/shcizo/self-hosted/other",
-			"/home/shcizo/self-hosted/other/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.Equal(t, "myapp-prod", jobs[0].Project)
-	require.Equal(t, "web", jobs[0].Service)
-	require.Equal(t, "/home/shcizo/self-hosted/myapp-prod", jobs[0].WorkingDir)
-}
-
-func TestFindJobs_SkipsWithoutOptIn(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			false,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}
-
-func TestFindJobs_DedupReplicas(t *testing.T) {
-	labels := mkComposeLabels(
-		"myapp", "web",
-		"/home/shcizo/self-hosted/myapp",
-		"/home/shcizo/self-hosted/myapp/docker-compose.yml",
-		true,
-	)
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", labels),
-		mkContainer("registry.example.com/myapp:v1", labels),
-		mkContainer("registry.example.com/myapp:v1", labels),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-}
-
-func TestFindJobs_OutsideRootProducesRefusedJob(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp", "web",
-			"/opt/elsewhere/myapp",
-			"/opt/elsewhere/myapp/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.True(t, jobs[0].Refused)
-}
-
-func TestFindJobs_MultipleStacksSameImage(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			true,
-		)),
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-staging", "web",
-			"/home/shcizo/self-hosted/myapp-staging",
-			"/home/shcizo/self-hosted/myapp-staging/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 2)
-}
-
-func TestFindJobs_DockerError(t *testing.T) {
-	fake := &fakeDockerClient{err: errors.New("connection refused")}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	_, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.Error(t, err)
-}
-
-func TestFindJobs_NonComposeContainerIsSkipped(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", map[string]string{
-			"se.shcizo.auto-update": "true",
-		}),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}
-
-func TestFindJobs_PopulatesImageField(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.Equal(t, "registry.example.com/myapp", jobs[0].Image)
-	require.Empty(t, jobs[0].ServiceID)
-}

internal/discovery/docker_client.go

@@ -1,15 +0,0 @@
-package discovery
-
-import (
-	"context"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-)
-
-// DockerClient is the subset of the Docker SDK we depend on.
-// Defined as an interface so tests can supply a fake.
-type DockerClient interface {
-	ContainerList(ctx context.Context, opts container.ListOptions) ([]types.Container, error)
-	Ping(ctx context.Context) (types.Ping, error)
-}

internal/discovery/labels.go

@@ -1,71 +0,0 @@
-package discovery
-
-import (
-	"fmt"
-	"strings"
-)
-
-// ComposeLabels captures the four Compose-managed labels we need
-// to drive a `docker compose pull`/`up -d` against the right stack.
-type ComposeLabels struct {
-	Project     string
-	Service     string
-	WorkingDir  string
-	ConfigFiles []string
-}
-
-const (
-	labelProject     = "com.docker.compose.project"
-	labelService     = "com.docker.compose.service"
-	labelWorkingDir  = "com.docker.compose.project.working_dir"
-	labelConfigFiles = "com.docker.compose.project.config_files"
-)
-
-// ParseComposeLabels extracts the Compose labels we need. Returns an
-// error naming the missing label if any required field is absent —
-// in practice this should only happen if the container was not
-// started by Compose.
-func ParseComposeLabels(labels map[string]string) (ComposeLabels, error) {
-	get := func(key string) (string, error) {
-		v, ok := labels[key]
-		if !ok || v == "" {
-			return "", fmt.Errorf("missing required label: %s", key)
-		}
-		return v, nil
-	}
-
-	project, err := get(labelProject)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-	service, err := get(labelService)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-	workingDir, err := get(labelWorkingDir)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-	configFilesRaw, err := get(labelConfigFiles)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-
-	files := strings.Split(configFilesRaw, ",")
-	for i, f := range files {
-		files[i] = strings.TrimSpace(f)
-	}
-
-	return ComposeLabels{
-		Project:     project,
-		Service:     service,
-		WorkingDir:  workingDir,
-		ConfigFiles: files,
-	}, nil
-}
-
-// HasOptIn reports whether the labels include the opt-in marker with
-// value "true" (exact, case-sensitive — anything else is excluded).
-func HasOptIn(labels map[string]string, key string) bool {
-	return labels[key] == "true"
-}

internal/discovery/labels_test.go

@@ -1,88 +0,0 @@
-package discovery_test
-
-import (
-	"testing"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-func TestParseComposeLabels_Success(t *testing.T) {
-	labels := map[string]string{
-		"com.docker.compose.project":                  "myapp-prod",
-		"com.docker.compose.service":                  "web",
-		"com.docker.compose.project.working_dir":      "/home/shcizo/self-hosted/myapp-prod",
-		"com.docker.compose.project.config_files":     "/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-		"se.shcizo.auto-update":                       "true",
-	}
-	got, err := discovery.ParseComposeLabels(labels)
-	require.NoError(t, err)
-	require.Equal(t, "myapp-prod", got.Project)
-	require.Equal(t, "web", got.Service)
-	require.Equal(t, "/home/shcizo/self-hosted/myapp-prod", got.WorkingDir)
-	require.Equal(t, []string{"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml"}, got.ConfigFiles)
-}
-
-func TestParseComposeLabels_MultipleConfigFiles(t *testing.T) {
-	labels := map[string]string{
-		"com.docker.compose.project":                  "myapp",
-		"com.docker.compose.service":                  "web",
-		"com.docker.compose.project.working_dir":      "/srv/myapp",
-		"com.docker.compose.project.config_files":     "/srv/myapp/docker-compose.yml,/srv/myapp/docker-compose.prod.yml",
-	}
-	got, err := discovery.ParseComposeLabels(labels)
-	require.NoError(t, err)
-	require.Equal(t, []string{
-		"/srv/myapp/docker-compose.yml",
-		"/srv/myapp/docker-compose.prod.yml",
-	}, got.ConfigFiles)
-}
-
-func TestParseComposeLabels_MissingFieldsError(t *testing.T) {
-	cases := []struct {
-		name    string
-		missing string
-		labels  map[string]string
-	}{
-		{"project", "com.docker.compose.project", map[string]string{
-			"com.docker.compose.service":              "web",
-			"com.docker.compose.project.working_dir":  "/x",
-			"com.docker.compose.project.config_files": "/x/y.yml",
-		}},
-		{"service", "com.docker.compose.service", map[string]string{
-			"com.docker.compose.project":              "p",
-			"com.docker.compose.project.working_dir":  "/x",
-			"com.docker.compose.project.config_files": "/x/y.yml",
-		}},
-		{"working_dir", "com.docker.compose.project.working_dir", map[string]string{
-			"com.docker.compose.project":              "p",
-			"com.docker.compose.service":              "web",
-			"com.docker.compose.project.config_files": "/x/y.yml",
-		}},
-		{"config_files", "com.docker.compose.project.config_files", map[string]string{
-			"com.docker.compose.project":             "p",
-			"com.docker.compose.service":             "web",
-			"com.docker.compose.project.working_dir": "/x",
-		}},
-	}
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			_, err := discovery.ParseComposeLabels(c.labels)
-			require.Error(t, err)
-			require.Contains(t, err.Error(), c.missing)
-		})
-	}
-}
-
-func TestHasOptIn(t *testing.T) {
-	require.True(t, discovery.HasOptIn(map[string]string{
-		"se.shcizo.auto-update": "true",
-	}, "se.shcizo.auto-update"))
-	require.False(t, discovery.HasOptIn(map[string]string{
-		"se.shcizo.auto-update": "false",
-	}, "se.shcizo.auto-update"))
-	require.False(t, discovery.HasOptIn(map[string]string{
-		"se.shcizo.auto-update": "TRUE",
-	}, "se.shcizo.auto-update"))
-	require.False(t, discovery.HasOptIn(map[string]string{}, "se.shcizo.auto-update"))
-}

internal/discovery/matching.go

@@ -1,33 +0,0 @@
-package discovery
-
-import "strings"
-
-// NormaliseImage strips the tag and digest from an image reference,
-// returning the bare repository name.
-//
-// Tricky case: "localhost:5000/foo:v1" — the first colon is a port,
-// not a tag. We disambiguate by splitting on "/" first and only
-// treating colons in the last segment as tag separators.
-func NormaliseImage(ref string) string {
-	if at := strings.Index(ref, "@"); at >= 0 {
-		ref = ref[:at]
-	}
-	slash := strings.LastIndex(ref, "/")
-	if slash < 0 {
-		if colon := strings.Index(ref, ":"); colon >= 0 {
-			return ref[:colon]
-		}
-		return ref
-	}
-	prefix, last := ref[:slash], ref[slash+1:]
-	if colon := strings.Index(last, ":"); colon >= 0 {
-		last = last[:colon]
-	}
-	return prefix + "/" + last
-}
-
-// ImagesMatch reports whether two image references resolve to the same
-// repository, ignoring tag and digest. Case-sensitive per spec section 5.2.
-func ImagesMatch(a, b string) bool {
-	return NormaliseImage(a) == NormaliseImage(b)
-}

internal/discovery/matching_test.go

@@ -1,51 +0,0 @@
-package discovery_test
-
-import (
-	"testing"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNormaliseImage(t *testing.T) {
-	cases := []struct {
-		in   string
-		want string
-	}{
-		{"registry.example.com/myapp", "registry.example.com/myapp"},
-		{"registry.example.com/myapp:v1.2.3", "registry.example.com/myapp"},
-		{"registry.example.com/myapp:latest", "registry.example.com/myapp"},
-		{"registry.example.com/myapp@sha256:abc123", "registry.example.com/myapp"},
-		{"registry.example.com/myapp:v1.2.3@sha256:abc123", "registry.example.com/myapp"},
-		{"nginx", "nginx"},
-		{"nginx:1.25-alpine", "nginx"},
-		{"library/nginx:latest", "library/nginx"},
-		{"gcr.io/proj/svc:tag", "gcr.io/proj/svc"},
-		{"localhost:5000/myimg:v1", "localhost:5000/myimg"},
-	}
-	for _, c := range cases {
-		t.Run(c.in, func(t *testing.T) {
-			require.Equal(t, c.want, discovery.NormaliseImage(c.in))
-		})
-	}
-}
-
-func TestImagesMatch(t *testing.T) {
-	require.True(t, discovery.ImagesMatch(
-		"registry.example.com/myapp",
-		"registry.example.com/myapp:v1.2.3",
-	))
-	require.True(t, discovery.ImagesMatch(
-		"registry.example.com/myapp:v1.0.0",
-		"registry.example.com/myapp:v9.9.9",
-	))
-	require.False(t, discovery.ImagesMatch(
-		"registry.example.com/myapp",
-		"registry.example.com/otherapp",
-	))
-	// Case-sensitive per spec section 5.2
-	require.False(t, discovery.ImagesMatch(
-		"registry.example.com/MyApp",
-		"registry.example.com/myapp",
-	))
-}

internal/discovery/swarm.go

@@ -1,60 +0,0 @@
-package discovery
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/docker/docker/api/types/swarm"
-)
-
-// SwarmDockerClient is the subset of the Docker SDK SwarmDiscovery depends
-// on. Defined as an interface so tests can supply a fake.
-type SwarmDockerClient interface {
-	ServiceList(ctx context.Context, opts swarm.ServiceListOptions) ([]swarm.Service, error)
-}
-
-// SwarmDiscovery orchestrates "given an image, which Swarm services should
-// we update?". Unlike Discovery, it has no filesystem path to check — the
-// opt-in label on the service itself is the only gate, since Swarm services
-// have no local compose file to anchor a STACKS_ROOT check against.
-type SwarmDiscovery struct {
-	cli        SwarmDockerClient
-	optInLabel string
-}
-
-// NewSwarm returns a SwarmDiscovery bound to the given Docker client and
-// opt-in label.
-func NewSwarm(cli SwarmDockerClient, optInLabel string) *SwarmDiscovery {
-	return &SwarmDiscovery{cli: cli, optInLabel: optInLabel}
-}
-
-// FindJobs lists Swarm services, filters by image match + opt-in label, and
-// returns one Job per matching service. Signature matches Discovery.FindJobs
-// so both satisfy api.Finder.
-func (d *SwarmDiscovery) FindJobs(ctx context.Context, image string) ([]Job, error) {
-	services, err := d.cli.ServiceList(ctx, swarm.ServiceListOptions{})
-	if err != nil {
-		return nil, fmt.Errorf("docker service list: %w", err)
-	}
-
-	var jobs []Job
-	for _, svc := range services {
-		if svc.Spec.TaskTemplate.ContainerSpec == nil {
-			continue
-		}
-		if !ImagesMatch(image, svc.Spec.TaskTemplate.ContainerSpec.Image) {
-			continue
-		}
-		if !HasOptIn(svc.Spec.Labels, d.optInLabel) {
-			continue
-		}
-
-		jobs = append(jobs, Job{
-			Service:   svc.Spec.Name,
-			ServiceID: svc.ID,
-			Image:     image,
-		})
-	}
-
-	return jobs, nil
-}

internal/discovery/swarm_test.go

@@ -1,86 +0,0 @@
-package discovery_test
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/docker/docker/api/types/swarm"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeSwarmDockerClient struct {
-	services []swarm.Service
-	err      error
-}
-
-func (f *fakeSwarmDockerClient) ServiceList(_ context.Context, _ swarm.ServiceListOptions) ([]swarm.Service, error) {
-	return f.services, f.err
-}
-
-func mkService(id, name, image string, labels map[string]string) swarm.Service {
-	return swarm.Service{
-		ID: id,
-		Spec: swarm.ServiceSpec{
-			Annotations: swarm.Annotations{Name: name, Labels: labels},
-			TaskTemplate: swarm.TaskSpec{
-				ContainerSpec: &swarm.ContainerSpec{Image: image},
-			},
-		},
-	}
-}
-
-func TestSwarmFindJobs_MatchAndOptIn(t *testing.T) {
-	fake := &fakeSwarmDockerClient{services: []swarm.Service{
-		mkService("svc-myapp", "myapp_web", "registry.example.com/myapp:v1", map[string]string{
-			"se.shcizo.auto-update": "true",
-		}),
-		mkService("svc-other", "other_web", "registry.example.com/other:v1", map[string]string{
-			"se.shcizo.auto-update": "true",
-		}),
-	}}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.Equal(t, "myapp_web", jobs[0].Service)
-	require.Equal(t, "svc-myapp", jobs[0].ServiceID)
-	require.Equal(t, "registry.example.com/myapp", jobs[0].Image)
-	require.False(t, jobs[0].Refused)
-	require.Empty(t, jobs[0].WorkingDir)
-	require.Empty(t, jobs[0].ConfigFiles)
-}
-
-func TestSwarmFindJobs_SkipsWithoutOptIn(t *testing.T) {
-	fake := &fakeSwarmDockerClient{services: []swarm.Service{
-		mkService("svc-myapp", "myapp_web", "registry.example.com/myapp:v1", nil),
-	}}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}
-
-func TestSwarmFindJobs_DockerError(t *testing.T) {
-	fake := &fakeSwarmDockerClient{err: errors.New("connection refused")}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	_, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.Error(t, err)
-}
-
-func TestSwarmFindJobs_NoContainerSpecIsSkipped(t *testing.T) {
-	fake := &fakeSwarmDockerClient{services: []swarm.Service{
-		{
-			ID: "svc-weird",
-			Spec: swarm.ServiceSpec{
-				Annotations:  swarm.Annotations{Name: "weird", Labels: map[string]string{"se.shcizo.auto-update": "true"}},
-				TaskTemplate: swarm.TaskSpec{ContainerSpec: nil},
-			},
-		},
-	}}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}

internal/e2e/agent_master_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/shcizo/package-updater/internal/agent/conn"
+	"github.com/shcizo/package-updater/internal/agent/updater"
 	"github.com/shcizo/package-updater/internal/agenttoken"
 	"github.com/shcizo/package-updater/internal/master/events"
 	"github.com/shcizo/package-updater/internal/master/hub"
@@ -35,6 +36,129 @@ type memStore struct {
 	lookups   int
 	snapshots int
 	services  map[string]protocol.Service
+
+	// Jobs: in-memory twins of the store methods in C5, with the same
+	// ownership and "only active jobs change" rules.
+	svcRows       map[int64]model.Service
+	jobs          map[int64]*model.Job
+	nextJob       int64
+	finishApplied int // FinishJob calls that returned true
+}
+
+// seedService adds a service row that jobs can be triggered for.
+func (m *memStore) seedService(svc model.Service) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.svcRows == nil {
+		m.svcRows = map[int64]model.Service{}
+	}
+	m.svcRows[svc.ID] = svc
+}
+
+func (m *memStore) GetService(_ context.Context, id int64) (model.Service, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	svc, ok := m.svcRows[id]
+	if !ok {
+		return model.Service{}, model.ErrNotFound
+	}
+	return svc, nil
+}
+
+func (m *memStore) CreateJob(_ context.Context, nj model.NewJob) (model.Job, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.jobs == nil {
+		m.jobs = map[int64]*model.Job{}
+	}
+	for _, j := range m.jobs {
+		if j.ServiceID == nj.ServiceID && j.Active() {
+			return *j, model.ErrJobActive
+		}
+	}
+	m.nextJob++
+	j := &model.Job{ID: m.nextJob, ServiceID: nj.ServiceID, AgentID: nj.AgentID, FromRef: nj.FromRef, ToRef: nj.ToRef,
+		Trigger: nj.Trigger, RequestedBy: nj.RequestedBy, Status: model.JobQueued, CreatedAt: nj.CreatedAt}
+	m.jobs[j.ID] = j
+	return *j, nil
+}
+
+func (m *memStore) FailJob(_ context.Context, id int64, msg string, at time.Time) (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	j, ok := m.jobs[id]
+	if !ok || !j.Active() {
+		return false, nil
+	}
+	j.Status, j.Error, j.FinishedAt = model.JobFailed, msg, &at
+	return true, nil
+}
+
+func (m *memStore) GetJob(_ context.Context, id int64) (model.Job, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	j, ok := m.jobs[id]
+	if !ok {
+		return model.Job{}, model.ErrNotFound
+	}
+	return *j, nil
+}
+
+func (m *memStore) FailLostJobs(_ context.Context, agentID int64, known []int64, at time.Time) ([]int64, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	var lost []int64
+	for _, j := range m.jobs {
+		if j.AgentID == nil || *j.AgentID != agentID || !j.Active() {
+			continue
+		}
+		isKnown := false
+		for _, k := range known {
+			isKnown = isKnown || k == j.ID
+		}
+		if !isKnown {
+			j.Status, j.Error, j.FinishedAt = model.JobFailed, "lost: the agent did not know this job when it reconnected", &at
+			lost = append(lost, j.ID)
+		}
+	}
+	return lost, nil
+}
+
+func (m *memStore) JobProgress(_ context.Context, agentID, jobID int64, chunk string, at time.Time) (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	j, ok := m.jobs[jobID]
+	if !ok || j.AgentID == nil || *j.AgentID != agentID || !j.Active() {
+		return false, nil
+	}
+	j.Status, j.Output = model.JobRunning, j.Output+chunk
+	if j.StartedAt == nil {
+		j.StartedAt = &at
+	}
+	return true, nil
+}
+
+func (m *memStore) FinishJob(_ context.Context, agentID, jobID int64, f model.JobFinish) (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	j, ok := m.jobs[jobID]
+	if !ok || j.AgentID == nil || *j.AgentID != agentID || !j.Active() {
+		return false, nil
+	}
+	j.Status, j.Error, j.Output, j.OutputTruncated, j.FinishedAt = f.Status, f.Error, f.Output, f.OutputTruncated, &f.FinishedAt
+	if f.Ref != "" {
+		j.ToRef = f.Ref
+	}
+	if j.StartedAt == nil {
+		j.StartedAt = &f.StartedAt
+	}
+	if f.Status == model.JobSucceeded {
+		svc := m.svcRows[j.ServiceID]
+		svc.DesiredRef = f.Ref
+		m.svcRows[j.ServiceID] = svc
+	}
+	m.finishApplied++
+	return true, nil
 }
 
 func (m *memStore) AgentByTokenHash(_ context.Context, hash string) (model.Agent, error) {
@@ -112,8 +236,21 @@ func (s *staticSource) set(v ...protocol.Service) {
 type testMaster struct {
 	url         string
 	fingerprint string
-	hub         *hub.Hub
+	hub         *currentHub   // the hub behind /agent; follows restart()
 	attempts    *atomic.Int64 // TCP connections accepted by the server
+	broker      *events.Broker
+	restart     func() // replaces the hub and drops every connection
+}
+
+// currentHub forwards to whichever hub is live, so a master restart can swap
+// the hub without the mux or the jobs service noticing.
+type currentHub struct{ p atomic.Pointer[hub.Hub] }
+
+func (c *currentHub) ServeHTTP(w http.ResponseWriter, r *http.Request) { c.p.Load().ServeHTTP(w, r) }
+func (c *currentHub) Disconnect(id int64)                              { c.p.Load().Disconnect(id) }
+func (c *currentHub) Route(svc model.Service) (model.Route, error)     { return c.p.Load().Route(svc) }
+func (c *currentHub) StartJob(r model.Route, id int64, s protocol.JobStart) error {
+	return c.p.Load().StartJob(r, id, s)
 }
 
 func startMaster(t *testing.T, store *memStore) testMaster {
@@ -125,7 +262,10 @@ func startMaster(t *testing.T, store *memStore) testMaster {
 	fingerprint, err := pin.Fingerprint(key.Public())
 	require.NoError(t, err)
 
-	h := hub.New(store, events.NewBroker(), slog.New(slog.NewTextHandler(io.Discard, nil)))
+	quiet := slog.New(slog.NewTextHandler(io.Discard, nil))
+	broker := events.NewBroker()
+	h := &currentHub{}
+	h.p.Store(hub.New(store, broker, quiet))
 	mux := http.NewServeMux()
 	mux.Handle("GET /agent", h)
 	attempts := &atomic.Int64{}
@@ -139,14 +279,27 @@ func startMaster(t *testing.T, store *memStore) testMaster {
 	srv.TLS = &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13}
 	srv.StartTLS()
 	t.Cleanup(srv.Close)
-	return testMaster{url: "wss" + strings.TrimPrefix(srv.URL, "https") + "/agent", fingerprint: fingerprint, hub: h, attempts: attempts}
+	restart := func() {
+		// The hub holds no job state, so a new hub over the same store is a
+		// faithful master restart.
+		old := h.p.Swap(hub.New(store, broker, quiet))
+		// CloseClientConnections skips hijacked (WebSocket) connections, so
+		// cut each agent through the old hub.
+		store.mu.Lock()
+		defer store.mu.Unlock()
+		for _, a := range store.agents {
+			old.Disconnect(a.ID)
+		}
+	}
+	return testMaster{url: "wss" + strings.TrimPrefix(srv.URL, "https") + "/agent", fingerprint: fingerprint,
+		hub: h, attempts: attempts, broker: broker, restart: restart}
 }
 
 func runAgent(t *testing.T, url, fingerprint, token string, src conn.Source) *conn.Agent {
 	t.Helper()
 	a := conn.New(conn.Config{URL: url, Token: token, Fingerprint: fingerprint, Version: "e2e",
 		MinBackoff: 10 * time.Millisecond, MaxBackoff: 50 * time.Millisecond,
-	}, src, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	}, src, updater.NewRunner(nil, time.Minute), slog.New(slog.NewTextHandler(io.Discard, nil)))
 	ctx, cancel := context.WithCancel(context.Background())
 	done := make(chan struct{})
 	go func() { _ = a.Run(ctx); close(done) }()

internal/e2e/jobs_test.go

@@ -0,0 +1,257 @@
+package e2e
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coder/websocket"
+	"github.com/coder/websocket/wsjson"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/agent/conn"
+	"github.com/shcizo/package-updater/internal/agent/updater"
+	"github.com/shcizo/package-updater/internal/agenttoken"
+	"github.com/shcizo/package-updater/internal/master/jobs"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+const (
+	slusk       = "gitea.shcizo.se/shcizo/slusk"
+	agentID     = int64(2)
+	waitFor     = 3 * time.Second
+	pollEvery   = 10 * time.Millisecond
+	lostMessage = "lost: the agent did not know this job when it reconnected"
+)
+
+// gateExecutor writes a line of output, then blocks until release is closed
+// or the job is cancelled.
+type gateExecutor struct {
+	started chan string // receives the job's Ref when it begins
+	release chan struct{}
+}
+
+func newGateExecutor() *gateExecutor {
+	return &gateExecutor{started: make(chan string, 8), release: make(chan struct{})}
+}
+
+func (g *gateExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	_, _ = io.WriteString(out, "pulling "+job.Ref+"\n")
+	g.started <- job.Ref
+	select {
+	case <-g.release:
+	case <-ctx.Done():
+		return "", ctx.Err()
+	}
+	_, _ = io.WriteString(out, "recreated\n")
+	return job.Ref + "@sha256:e2e", nil
+}
+
+// jobAgent is a connected agent with a real runner and its two lifetimes: the
+// connection and the runner can be stopped separately.
+type jobAgent struct {
+	runner   *updater.Runner
+	stopConn func()
+	stopRun  func()
+}
+
+func startJobAgent(t *testing.T, m testMaster, token string, exec updater.Executor) *jobAgent {
+	t.Helper()
+	src := &staticSource{}
+	src.set(web)
+	runner := updater.NewRunner(exec, time.Minute)
+	runCtx, stopRun := context.WithCancel(context.Background())
+	runDone := make(chan struct{})
+	go func() { runner.Run(runCtx); close(runDone) }()
+
+	a := conn.New(conn.Config{URL: m.url, Token: token, Fingerprint: m.fingerprint, Version: "e2e",
+		MinBackoff: 10 * time.Millisecond, MaxBackoff: 50 * time.Millisecond, ProgressInterval: 10 * time.Millisecond,
+	}, src, runner, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	connCtx, cancelConn := context.WithCancel(context.Background())
+	connDone := make(chan struct{})
+	go func() { _ = a.Run(connCtx); close(connDone) }()
+
+	ja := &jobAgent{
+		runner:   runner,
+		stopConn: func() { cancelConn(); <-connDone },
+		stopRun:  func() { stopRun(); <-runDone },
+	}
+	t.Cleanup(func() { ja.stopConn(); ja.stopRun() })
+	return ja
+}
+
+// jobEnv is a master with one agent (ID 2, token "pua_ok") that owns a Compose
+// service, and a jobs service wired to the master's current hub.
+type jobEnv struct {
+	store *memStore
+	m     testMaster
+	jobs  *jobs.Service
+	exec  *gateExecutor
+	agent *jobAgent
+}
+
+func newJobEnv(t *testing.T) *jobEnv {
+	t.Helper()
+	store := &memStore{agents: map[string]model.Agent{agenttoken.Hash("pua_ok"): {ID: agentID, Name: "node-2"}}}
+	store.seedService(model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
+		AgentID: ptr(agentID), ActualRef: slusk + ":1.4.2@sha256:old"})
+	m := startMaster(t, store)
+	exec := newGateExecutor()
+	env := &jobEnv{store: store, m: m, exec: exec,
+		jobs:  jobs.New(store, m.hub, m.broker, "gitea.shcizo.se/shcizo/package-updater:latest", slog.New(slog.NewTextHandler(io.Discard, nil))),
+		agent: startJobAgent(t, m, "pua_ok", exec)}
+	env.waitRoutable(t)
+	return env
+}
+
+func ptr[T any](v T) *T { return &v }
+
+// waitRoutable waits until the master can send jobs to the agent.
+func (e *jobEnv) waitRoutable(t *testing.T) {
+	t.Helper()
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { _, err := e.m.hub.Route(svc); return err == nil }, waitFor, pollEvery)
+}
+
+func (e *jobEnv) job(t *testing.T, id int64) model.Job {
+	t.Helper()
+	j, err := e.store.GetJob(context.Background(), id)
+	require.NoError(t, err)
+	return j
+}
+
+func (e *jobEnv) waitStatus(t *testing.T, id int64, want model.JobStatus) model.Job {
+	t.Helper()
+	require.Eventually(t, func() bool { return e.job(t, id).Status == want }, waitFor, pollEvery,
+		"job %d should become %s, is %s (%s)", id, want, e.job(t, id).Status, e.job(t, id).Error)
+	return e.job(t, id)
+}
+
+func (e *jobEnv) waitStarted(t *testing.T) {
+	t.Helper()
+	select {
+	case <-e.exec.started:
+	case <-time.After(waitFor):
+		t.Fatal("the agent never started the job")
+	}
+}
+
+func (e *jobEnv) trigger(t *testing.T) model.Job {
+	t.Helper()
+	j, err := e.jobs.Trigger(context.Background(), 1, "sub-1")
+	require.NoError(t, err)
+	return j
+}
+
+func TestManualJobFlowsToAgentAndBack(t *testing.T) {
+	e := newJobEnv(t)
+	close(e.exec.release)
+
+	j := e.trigger(t)
+	done := e.waitStatus(t, j.ID, model.JobSucceeded)
+
+	assert.Equal(t, slusk+":1.4.2@sha256:e2e", done.ToRef)
+	assert.Contains(t, done.Output, "pulling "+slusk+":1.4.2")
+	assert.Contains(t, done.Output, "recreated")
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, slusk+":1.4.2@sha256:e2e", svc.DesiredRef)
+	require.Eventually(t, func() bool { return len(e.agent.runner.Known()) == 0 }, waitFor, pollEvery, "the result must be acked")
+}
+
+func TestJobFinishedWhileDisconnectedIsRecordedOnce(t *testing.T) {
+	e := newJobEnv(t)
+	j := e.trigger(t)
+	e.waitStarted(t)
+
+	e.m.restart() // drops the agent's connection while the executor blocks
+	close(e.exec.release)
+
+	done := e.waitStatus(t, j.ID, model.JobSucceeded)
+	assert.NotEqual(t, lostMessage, done.Error)
+	assert.GreaterOrEqual(t, e.m.attempts.Load(), int64(2), "the agent must have reconnected")
+	require.Eventually(t, func() bool { return len(e.agent.runner.Known()) == 0 }, waitFor, pollEvery)
+	e.store.mu.Lock()
+	defer e.store.mu.Unlock()
+	assert.Equal(t, 1, e.store.finishApplied)
+}
+
+func TestMasterRestartKeepsJobTheAgentKnows(t *testing.T) {
+	e := newJobEnv(t)
+	j := e.trigger(t)
+	e.waitStarted(t)
+
+	e.m.restart()
+	e.waitRoutable(t) // the agent reconnected and said hello with its known jobs
+	assert.True(t, e.job(t, j.ID).Active(), "the agent still has the job, so it is not lost")
+	close(e.exec.release)
+
+	done := e.waitStatus(t, j.ID, model.JobSucceeded)
+	assert.Empty(t, done.Error)
+}
+
+func TestAgentRestartMarksJobLost(t *testing.T) {
+	e := newJobEnv(t)
+	j := e.trigger(t)
+	e.waitStarted(t)
+
+	// The agent process dies: connection first, so the cancelled job cannot
+	// report a result, then the runner.
+	e.agent.stopConn()
+	e.agent.stopRun()
+	fresh := startJobAgent(t, e.m, "pua_ok", newGateExecutor())
+	_ = fresh
+
+	lost := e.waitStatus(t, j.ID, model.JobFailed)
+	assert.Equal(t, lostMessage, lost.Error)
+}
+
+func TestPhaseOneAgentGetsNoJobs(t *testing.T) {
+	store := &memStore{agents: map[string]model.Agent{agenttoken.Hash("pua_old"): {ID: agentID, Name: "node-2"}}}
+	svcRow := model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
+		AgentID: ptr(agentID), ActualRef: slusk + ":1.4.2@sha256:old"}
+	store.seedService(svcRow)
+	m := startMaster(t, store)
+	js := jobs.New(store, m.hub, m.broker, "gitea.shcizo.se/shcizo/package-updater:latest", slog.New(slog.NewTextHandler(io.Discard, nil)))
+
+	// A raw client speaking the phase-1 protocol: its hello has no
+	// protocol_version.
+	ctx := context.Background()
+	c, _, err := websocket.Dial(ctx, m.url, &websocket.DialOptions{
+		HTTPHeader: http.Header{"Authorization": {"Bearer pua_old"}},
+		HTTPClient: &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}},
+	})
+	require.NoError(t, err)
+	defer c.CloseNow()
+	hello, err := protocol.Encode(protocol.TypeHello, map[string]any{"agent_version": "1", "hostname": "node-2", "swarm_manager": false})
+	require.NoError(t, err)
+	require.NoError(t, wsjson.Write(ctx, c, hello))
+	snap, err := protocol.Encode(protocol.TypeSnapshot, protocol.Snapshot{Services: []protocol.Service{web}})
+	require.NoError(t, err)
+	require.NoError(t, wsjson.Write(ctx, c, snap))
+
+	require.Eventually(t, func() bool {
+		_, err := m.hub.Route(svcRow)
+		return errors.Is(err, model.ErrAgentTooOld)
+	}, waitFor, pollEvery)
+
+	j, err := js.Trigger(ctx, 1, "sub-1")
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, j.Status)
+	assert.Equal(t, "agent too old for updates", j.Error)
+
+	rctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
+	defer cancel()
+	var env protocol.Envelope
+	err = wsjson.Read(rctx, c, &env)
+	assert.True(t, err != nil && strings.Contains(err.Error(), "deadline"), "no job.start expected, got %q (err %v)", env.Type, err)
+}

internal/master/events/broker.go

@@ -2,7 +2,10 @@
 // open browser streams.
 package events
 
-import "sync"
+import (
+	"strconv"
+	"sync"
+)
 
 // Topic names what changed. Subscribers refetch that part of the UI.
 type Topic string
@@ -11,8 +14,13 @@ type Topic string
 const (
 	Services Topic = "services"
 	Agents   Topic = "agents"
+	Jobs     Topic = "jobs" // a job was created or changed status
 )
 
+// JobTopic is signalled when job id gets new output or changes status. The
+// name avoids ":" so it is safe inside hx-trigger="sse:job-42".
+func JobTopic(id int64) Topic { return Topic("job-" + strconv.FormatInt(id, 10)) }
+
 // Broker is an in-process publish/subscribe hub.
 type Broker struct {
 	mu   sync.Mutex

internal/master/events/broker_test.go

@@ -46,3 +46,7 @@ func TestBroker_SlowSubscriberDoesNotBlockPublish(t *testing.T) {
 		t.Fatal("Publish blocked on a slow subscriber")
 	}
 }
+
+func TestJobTopic(t *testing.T) {
+	assert.Equal(t, Topic("job-42"), JobTopic(42))
+}

internal/master/hub/hub.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/coder/websocket"
@@ -25,6 +26,13 @@ import (
 // small for a full snapshot from a busy machine.
 const readLimit = 1 << 20
 
+// writeTimeout bounds one websocket write. A peer that stops reading is cut
+// off after this long.
+const writeTimeout = 10 * time.Second
+
+// outQueue is how many messages may wait for a connection's writer.
+const outQueue = 64
+
 // Store is the storage the hub writes to. *store.Store satisfies it.
 type Store interface {
 	AgentByTokenHash(ctx context.Context, hash string) (model.Agent, error)
@@ -33,6 +41,9 @@ type Store interface {
 	UpsertService(ctx context.Context, agentID int64, clusterID string, svc protocol.Service, at time.Time) error
 	RemoveService(ctx context.Context, agentID int64, clusterID, agentKey string, at time.Time) error
 	MarkSeen(ctx context.Context, agentID int64, at time.Time) error
+	FailLostJobs(ctx context.Context, agentID int64, known []int64, at time.Time) ([]int64, error)
+	JobProgress(ctx context.Context, agentID, jobID int64, chunk string, at time.Time) (bool, error)
+	FinishJob(ctx context.Context, agentID, jobID int64, f model.JobFinish) (bool, error)
 }
 
 // Publisher receives "something changed" signals. *events.Broker satisfies it.
@@ -47,13 +58,57 @@ type Hub struct {
 	log   *slog.Logger
 	now   func() time.Time
 
+	connSeq atomic.Uint64
+
 	mu    sync.Mutex
-	conns map[int64]*websocket.Conn
+	conns map[int64]*agentConn
+}
+
+// agentConn is one live agent connection. Its writer goroutine is the only
+// code that writes to ws; everything else queues envelopes on out.
+type agentConn struct {
+	ws     *websocket.Conn
+	id     uint64
+	out    chan protocol.Envelope
+	ctx    context.Context
+	cancel context.CancelFunc
+	hello  *protocol.Hello // guarded by Hub.mu; nil until the hello is reconciled
+}
+
+// send queues env for the writer. It returns false if the connection ended
+// before the message could be queued.
+func (c *agentConn) send(env protocol.Envelope) bool {
+	select {
+	case c.out <- env:
+		return true
+	case <-c.ctx.Done():
+		return false
+	}
+}
+
+// writeLoop writes queued envelopes until the connection's context ends. A
+// failed or slow write closes the connection.
+func (c *agentConn) writeLoop() {
+	for {
+		select {
+		case <-c.ctx.Done():
+			return
+		case env := <-c.out:
+			wctx, cancel := context.WithTimeout(c.ctx, writeTimeout)
+			err := wsjson.Write(wctx, c.ws, env)
+			cancel()
+			if err != nil {
+				c.ws.CloseNow()
+				c.cancel()
+				return
+			}
+		}
+	}
 }
 
 // New returns a Hub. Mount it at GET /agent on the TLS listener.
 func New(store Store, pub Publisher, log *slog.Logger) *Hub {
-	return &Hub{store: store, pub: pub, log: log, now: time.Now, conns: make(map[int64]*websocket.Conn)}
+	return &Hub{store: store, pub: pub, log: log, now: time.Now, conns: make(map[int64]*agentConn)}
 }
 
 // ServeHTTP authenticates the agent's token and runs its connection. The token
@@ -78,8 +133,12 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	c.SetReadLimit(readLimit)
-	h.register(agent, r.RemoteAddr, c)
-	defer h.unregister(agent.ID, c)
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
+	ac := &agentConn{ws: c, id: h.connSeq.Add(1), out: make(chan protocol.Envelope, outQueue), ctx: ctx, cancel: cancel}
+	go ac.writeLoop()
+	h.register(agent, r.RemoteAddr, ac)
+	defer h.unregister(agent.ID, ac)
 
 	// Revoking commits revoked_at before calling Disconnect. Re-checking after
 	// register closes the window where a revoke lands between the first lookup
@@ -90,8 +149,8 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	s := &session{hub: h, agent: agent}
-	err = s.run(r.Context(), c)
+	s := &session{hub: h, agent: agent, conn: ac}
+	err = s.run(ctx, c)
 	h.log.Info("agent_disconnected", "agent", agent.Name, "reason", err)
 }
 
@@ -110,25 +169,25 @@ func (h *Hub) Disconnect(agentID int64) {
 	c := h.conns[agentID]
 	h.mu.Unlock()
 	if c != nil {
-		c.CloseNow()
+		c.ws.CloseNow()
 	}
 }
 
 // register makes c the agent's live connection, closing any older one. A
 // replacement is logged: it is normal after a network blip, but repeated
 // replacements mean two agents share a token.
-func (h *Hub) register(agent model.Agent, remoteAddr string, c *websocket.Conn) {
+func (h *Hub) register(agent model.Agent, remoteAddr string, c *agentConn) {
 	h.mu.Lock()
 	old := h.conns[agent.ID]
 	h.conns[agent.ID] = c
 	h.mu.Unlock()
 	if old != nil {
 		h.log.Warn("agent_replaced", "agent_id", agent.ID, "agent", agent.Name, "remote_addr", remoteAddr)
-		old.CloseNow()
+		old.ws.CloseNow()
 	}
 }
 
-func (h *Hub) unregister(id int64, c *websocket.Conn) {
+func (h *Hub) unregister(id int64, c *agentConn) {
 	h.mu.Lock()
 	current := h.conns[id] == c
 	if current {
@@ -148,6 +207,7 @@ func (h *Hub) unregister(id int64, c *websocket.Conn) {
 type session struct {
 	hub   *Hub
 	agent model.Agent
+	conn  *agentConn
 	hello *protocol.Hello
 }
 
@@ -188,7 +248,25 @@ func (s *session) handle(ctx context.Context, env protocol.Envelope) error {
 		if err := h.store.RecordHello(ctx, s.agent.ID, v, h.now()); err != nil {
 			return err
 		}
+		// Reconcile jobs before the connection becomes routable, so a job is
+		// never sent to an agent whose old jobs have not been settled yet.
+		known := parseJobIDs(v.KnownJobs, func(id string) {
+			h.log.Warn("known_job_not_numeric", "agent", s.agent.Name, "job_id", id)
+		})
+		lost, err := h.store.FailLostJobs(ctx, s.agent.ID, known, h.now())
+		if err != nil {
+			return err
+		}
+		if len(lost) > 0 {
+			h.pub.Publish(events.Jobs)
+			for _, id := range lost {
+				h.pub.Publish(events.JobTopic(id))
+			}
+		}
 		s.hello = &v
+		h.mu.Lock()
+		s.conn.hello = &v
+		h.mu.Unlock()
 		h.log.Info("agent_connected", "agent", s.agent.Name, "hostname", v.Hostname, "swarm_manager", v.SwarmManager)
 		h.pub.Publish(events.Agents)
 	case protocol.TypeSnapshot:
@@ -224,8 +302,12 @@ func (s *session) handle(ctx context.Context, env protocol.Envelope) error {
 			return err
 		}
 		h.pub.Publish(events.Services)
+	case protocol.TypeJobProgress:
+		return s.handleJobProgress(ctx, env)
+	case protocol.TypeJobResult:
+		return s.handleJobResult(ctx, env)
 	default:
-		return fmt.Errorf("unknown message type %q", env.Type)
+		h.log.Warn("unknown_message_type", "agent", s.agent.Name, "type", env.Type)
 	}
 	return nil
 }

internal/master/hub/hub_test.go

@@ -3,6 +3,7 @@ package hub
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
@@ -35,6 +36,18 @@ type fakeStore struct {
 	// revokeFromLookup, when > 0, makes AgentByTokenHash return the agent as
 	// revoked from that (1-based) lookup onwards.
 	revokeFromLookup int
+
+	// Job calls, recorded; the *Result fields script the answers.
+	lostCalls    [][]int64
+	lostResult   []int64
+	lostErr      error
+	lostGate     chan struct{} // when set, FailLostJobs blocks until it is closed
+	progress     []string      // "agent|job|chunk"
+	progressOK   bool
+	finishes     []model.JobFinish
+	finishAgents []int64
+	finishOK     bool
+	finishErr    error
 }
 
 func newFakeStore(agents ...model.Agent) *fakeStore {
@@ -94,6 +107,31 @@ func (f *fakeStore) MarkSeen(_ context.Context, id int64, _ time.Time) error {
 	return nil
 }
 
+func (f *fakeStore) FailLostJobs(_ context.Context, _ int64, known []int64, _ time.Time) ([]int64, error) {
+	if f.lostGate != nil {
+		<-f.lostGate
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.lostCalls = append(f.lostCalls, known)
+	return f.lostResult, f.lostErr
+}
+
+func (f *fakeStore) JobProgress(_ context.Context, agentID, jobID int64, chunk string, _ time.Time) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.progress = append(f.progress, fmt.Sprintf("%d|%d|%s", agentID, jobID, chunk))
+	return f.progressOK, nil
+}
+
+func (f *fakeStore) FinishJob(_ context.Context, agentID, _ int64, fin model.JobFinish) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.finishAgents = append(f.finishAgents, agentID)
+	f.finishes = append(f.finishes, fin)
+	return f.finishOK, f.finishErr
+}
+
 func (f *fakeStore) count(get func(*fakeStore) int) func() int {
 	return func() int {
 		f.mu.Lock()

internal/master/hub/jobs.go

@@ -0,0 +1,133 @@
+package hub
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	"github.com/shcizo/package-updater/internal/master/events"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// StartJob sends job.start over exactly the connection r names. It returns
+// model.ErrAgentOffline if that connection is gone, was replaced by a newer
+// one, or has not finished its hello.
+func (h *Hub) StartJob(r model.Route, jobID int64, start protocol.JobStart) error {
+	h.mu.Lock()
+	c := h.conns[r.AgentID]
+	usable := c != nil && c.id == r.ConnID && c.hello != nil
+	h.mu.Unlock()
+	if !usable {
+		return model.ErrAgentOffline
+	}
+	env, err := protocol.EncodeJob(protocol.TypeJobStart, strconv.FormatInt(jobID, 10), start)
+	if err != nil {
+		return err
+	}
+	// A job whose write later fails stays queued; the agent's next hello
+	// settles it as lost.
+	if !c.send(env) {
+		return model.ErrAgentOffline
+	}
+	return nil
+}
+
+// parseJobIDs converts the agent's job IDs to integers, calling skip for each
+// one that is not a number.
+func parseJobIDs(ids []string, skip func(id string)) []int64 {
+	out := make([]int64, 0, len(ids))
+	for _, s := range ids {
+		id, err := strconv.ParseInt(s, 10, 64)
+		if err != nil {
+			skip(s)
+			continue
+		}
+		out = append(out, id)
+	}
+	return out
+}
+
+// cleanOutput drops NUL bytes, which Postgres text columns reject. Without
+// this a result containing one could never be stored and would be resent
+// forever.
+func cleanOutput(s string) string { return strings.ReplaceAll(s, "\x00", "") }
+
+func (s *session) handleJobProgress(ctx context.Context, env protocol.Envelope) error {
+	h := s.hub
+	v, err := protocol.Decode[protocol.JobProgress](env)
+	if err != nil {
+		return err
+	}
+	id, err := strconv.ParseInt(env.ID, 10, 64)
+	if err != nil {
+		h.log.Warn("job_progress_ignored", "agent", s.agent.Name, "job_id", env.ID, "reason", "not a number")
+		return nil
+	}
+	applied, err := h.store.JobProgress(ctx, s.agent.ID, id, cleanOutput(v.Output), h.now())
+	if err != nil {
+		return err
+	}
+	if applied {
+		h.pub.Publish(events.Jobs)
+		h.pub.Publish(events.JobTopic(id))
+	}
+	return nil
+}
+
+func (s *session) handleJobResult(ctx context.Context, env protocol.Envelope) error {
+	h := s.hub
+	v, err := protocol.Decode[protocol.JobResult](env)
+	if err != nil {
+		return err
+	}
+	ack, err := protocol.EncodeJob(protocol.TypeJobAck, env.ID, protocol.JobAck{})
+	if err != nil {
+		return err
+	}
+	id, err := strconv.ParseInt(env.ID, 10, 64)
+	if err != nil {
+		h.log.Warn("job_result_ignored", "agent", s.agent.Name, "job_id", env.ID, "reason", "not a number")
+		s.conn.send(ack)
+		return nil
+	}
+
+	fin := model.JobFinish{
+		Status:          model.JobStatus(v.Status),
+		Ref:             v.Ref,
+		Error:           v.Error,
+		Output:          cleanOutput(v.Output),
+		OutputTruncated: v.OutputTruncated,
+		StartedAt:       v.StartedAt,
+		FinishedAt:      v.FinishedAt,
+	}
+	if v.Status != protocol.ResultSucceeded && v.Status != protocol.ResultFailed {
+		fin.Status = model.JobFailed
+		fin.Ref = ""
+		fin.Error = "agent sent invalid status " + strconv.Quote(v.Status)
+	}
+	if fin.StartedAt.IsZero() {
+		fin.StartedAt = h.now()
+	}
+	if fin.FinishedAt.IsZero() {
+		fin.FinishedAt = h.now()
+	}
+
+	// A store error closes the connection without an ack, so the agent
+	// resends the result after it reconnects.
+	applied, err := h.store.FinishJob(ctx, s.agent.ID, id, fin)
+	if err != nil {
+		return err
+	}
+	if applied {
+		h.pub.Publish(events.Jobs)
+		h.pub.Publish(events.JobTopic(id))
+		if fin.Status == model.JobSucceeded {
+			h.pub.Publish(events.Services)
+		}
+	} else {
+		h.log.Warn("job_result_ignored", "agent", s.agent.Name, "job_id", env.ID, "reason", "no such active job for this agent")
+	}
+	s.conn.send(ack)
+	return nil
+}

internal/master/hub/jobs_test.go

@@ -0,0 +1,274 @@
+package hub
+
+import (
+	"context"
+	"errors"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/coder/websocket"
+	"github.com/coder/websocket/wsjson"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/events"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+var t0 = time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
+
+func sendJob(t *testing.T, c *websocket.Conn, typ, id string, payload any) {
+	t.Helper()
+	env, err := protocol.EncodeJob(typ, id, payload)
+	require.NoError(t, err)
+	require.NoError(t, wsjson.Write(context.Background(), c, env))
+}
+
+func readEnv(t *testing.T, c *websocket.Conn) protocol.Envelope {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	var env protocol.Envelope
+	require.NoError(t, wsjson.Read(ctx, c, &env))
+	return env
+}
+
+func waitRoute(t *testing.T, h *Hub) model.Route {
+	t.Helper()
+	svc := model.Service{Kind: protocol.KindCompose, AgentID: ptr(node2.ID)}
+	var r model.Route
+	require.Eventually(t, func() bool {
+		var err error
+		r, err = h.Route(svc)
+		return err == nil
+	}, 2*time.Second, 5*time.Millisecond)
+	return r
+}
+
+func TestHub_ReconcilesBeforeRoutable(t *testing.T) {
+	store := newFakeStore(node2)
+	store.lostGate = make(chan struct{})
+	store.lostResult = []int64{5}
+	h, pub, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+
+	send(t, c, protocol.TypeHello, protocol.Hello{Hostname: "node-2", ProtocolVersion: 2, KnownJobs: []string{"7", "9", "x"}})
+	svc := model.Service{Kind: protocol.KindCompose, AgentID: ptr(node2.ID)}
+	time.Sleep(50 * time.Millisecond) // hello is now blocked inside FailLostJobs
+	_, err = h.Route(svc)
+	assert.ErrorIs(t, err, model.ErrAgentOffline)
+
+	close(store.lostGate)
+	waitRoute(t, h)
+
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	assert.Equal(t, [][]int64{{7, 9}}, store.lostCalls)
+	assert.True(t, pub.has(events.Jobs))
+	assert.True(t, pub.has(events.JobTopic(5)))
+}
+
+func TestHub_StartJobSendsEnvelope(t *testing.T) {
+	store := newFakeStore(node2)
+	h, _, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	r := waitRoute(t, h)
+
+	start := protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "r:1"}
+	require.NoError(t, h.StartJob(r, 42, start))
+
+	env := readEnv(t, c)
+	assert.Equal(t, protocol.TypeJobStart, env.Type)
+	assert.Equal(t, "42", env.ID)
+	got, err := protocol.Decode[protocol.JobStart](env)
+	require.NoError(t, err)
+	assert.Equal(t, start, got)
+}
+
+func TestHub_StartJobOnReplacedConnectionFails(t *testing.T) {
+	store := newFakeStore(node2)
+	h, _, srv := newHub(t, store)
+	first, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer first.CloseNow()
+	send(t, first, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	old := waitRoute(t, h)
+
+	second, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer second.CloseNow()
+
+	// dial returns before the hub has registered the connection, so wait for
+	// the replacement to take effect: the old route is then refused. Until the
+	// new connection says hello, nothing is routable.
+	require.Eventually(t, func() bool {
+		return errors.Is(h.StartJob(old, 1, protocol.JobStart{}), model.ErrAgentOffline)
+	}, 2*time.Second, 5*time.Millisecond)
+	send(t, second, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	fresh := waitRoute(t, h)
+	require.NotEqual(t, old.ConnID, fresh.ConnID)
+	assert.ErrorIs(t, h.StartJob(old, 1, protocol.JobStart{}), model.ErrAgentOffline)
+
+	require.NoError(t, h.StartJob(fresh, 2, protocol.JobStart{Ref: "r:2"}))
+	env := readEnv(t, second)
+	assert.Equal(t, "2", env.ID, "the new connection got only the job sent to it")
+}
+
+func TestHub_ProgressAndResult(t *testing.T) {
+	store := newFakeStore(node2)
+	store.progressOK, store.finishOK = true, true
+	_, pub, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+
+	sendJob(t, c, protocol.TypeJobProgress, "42", protocol.JobProgress{Output: "out"})
+	require.Eventually(t, func() bool { return pub.has(events.JobTopic(42)) }, time.Second, 5*time.Millisecond)
+	store.mu.Lock()
+	assert.Equal(t, []string{"2|42|out"}, store.progress)
+	store.mu.Unlock()
+
+	pub.reset()
+	res := protocol.JobResult{Status: protocol.ResultSucceeded, Ref: "r:1@sha256:d", Output: "done", StartedAt: t0, FinishedAt: t0}
+	sendJob(t, c, protocol.TypeJobResult, "42", res)
+	ack := readEnv(t, c)
+	assert.Equal(t, protocol.TypeJobAck, ack.Type)
+	assert.Equal(t, "42", ack.ID)
+	assert.True(t, pub.has(events.Services))
+	assert.True(t, pub.has(events.JobTopic(42)))
+	store.mu.Lock()
+	require.Len(t, store.finishes, 1)
+	assert.Equal(t, model.JobSucceeded, store.finishes[0].Status)
+	assert.Equal(t, "r:1@sha256:d", store.finishes[0].Ref)
+	store.mu.Unlock()
+
+	// Not applied (already recorded, or not this agent's): still acked.
+	store.mu.Lock()
+	store.finishOK = false
+	store.mu.Unlock()
+	pub.reset()
+	sendJob(t, c, protocol.TypeJobResult, "43", res)
+	assert.Equal(t, "43", readEnv(t, c).ID)
+	assert.False(t, pub.has(events.JobTopic(43)))
+
+	// Non-numeric ID: acked, no store call.
+	store.mu.Lock()
+	before := len(store.finishes)
+	store.mu.Unlock()
+	sendJob(t, c, protocol.TypeJobResult, "abc", res)
+	assert.Equal(t, "abc", readEnv(t, c).ID)
+	store.mu.Lock()
+	assert.Len(t, store.finishes, before)
+	store.mu.Unlock()
+
+	// Unknown status is stored as a failure.
+	store.mu.Lock()
+	store.finishOK = true
+	store.mu.Unlock()
+	sendJob(t, c, protocol.TypeJobResult, "44", protocol.JobResult{Status: "weird", Ref: "r:9"})
+	assert.Equal(t, "44", readEnv(t, c).ID)
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	last := store.finishes[len(store.finishes)-1]
+	assert.Equal(t, model.JobFailed, last.Status)
+	assert.Equal(t, `agent sent invalid status "weird"`, last.Error)
+	assert.Empty(t, last.Ref)
+}
+
+func TestHub_ResultStoreErrorClosesConnection(t *testing.T) {
+	store := newFakeStore(node2)
+	store.finishErr = assert.AnError
+	_, _, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	sendJob(t, c, protocol.TypeJobResult, "42", protocol.JobResult{Status: protocol.ResultFailed})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	_, _, err = c.Read(ctx)
+	assert.Equal(t, websocket.StatusPolicyViolation, websocket.CloseStatus(err), "no ack, connection closed")
+}
+
+func TestHub_IgnoresUnknownType(t *testing.T) {
+	store := newFakeStore(node2)
+	_, _, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	require.NoError(t, wsjson.Write(context.Background(), c, protocol.Envelope{Type: "future.thing"}))
+	send(t, c, protocol.TypeUpsert, protocol.Upsert{Service: protocol.Service{Kind: protocol.KindContainer, ContainerName: "y"}})
+
+	require.Eventually(t, func() bool { return store.count(func(f *fakeStore) int { return len(f.upserts) })() == 1 }, time.Second, 5*time.Millisecond)
+}
+
+func TestHub_LargeResultArrives(t *testing.T) {
+	store := newFakeStore(node2)
+	store.finishOK = true
+	_, _, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+
+	out := strings.Repeat("\x1b", protocol.MaxJobOutput)
+	sendJob(t, c, protocol.TypeJobResult, "42", protocol.JobResult{Status: protocol.ResultFailed, Output: out})
+	assert.Equal(t, "42", readEnv(t, c).ID)
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	require.Len(t, store.finishes, 1)
+	assert.Equal(t, out, store.finishes[0].Output)
+}
+
+func TestHub_ConcurrentWritesAreSerialised(t *testing.T) {
+	const n = 50
+	store := newFakeStore(node2)
+	store.finishOK = true
+	h, _, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	r := waitRoute(t, h)
+
+	var wg sync.WaitGroup
+	for i := 0; i < n; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			assert.NoError(t, h.StartJob(r, int64(1000+i), protocol.JobStart{Ref: "r:1"}))
+		}()
+	}
+	go func() {
+		for i := 0; i < n; i++ {
+			sendJob(t, c, protocol.TypeJobResult, strconv.Itoa(i), protocol.JobResult{Status: protocol.ResultFailed})
+		}
+	}()
+
+	starts, acks := 0, 0
+	for starts+acks < 2*n {
+		switch env := readEnv(t, c); env.Type {
+		case protocol.TypeJobStart:
+			starts++
+		case protocol.TypeJobAck:
+			acks++
+		default:
+			t.Fatalf("unexpected %q", env.Type)
+		}
+	}
+	wg.Wait()
+	assert.Equal(t, n, starts)
+	assert.Equal(t, n, acks)
+}

internal/master/hub/route.go

@@ -0,0 +1,69 @@
+package hub
+
+import (
+	"sort"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// candidate is a connection that has finished its hello and may get jobs.
+type candidate struct {
+	agentID int64
+	connID  uint64
+	hello   protocol.Hello
+}
+
+// Route picks the connection a job for svc goes to. It returns
+// model.ErrAgentOffline or model.ErrAgentTooOld when there is none.
+func (h *Hub) Route(svc model.Service) (model.Route, error) {
+	h.mu.Lock()
+	conns := make([]candidate, 0, len(h.conns))
+	for id, c := range h.conns {
+		if c.hello != nil {
+			conns = append(conns, candidate{agentID: id, connID: c.id, hello: *c.hello})
+		}
+	}
+	h.mu.Unlock()
+	return route(svc, conns)
+}
+
+// route picks the connection a job for svc goes to. Compose services and
+// containers can only run on the agent that reported them. Swarm services
+// run on the reporting agent if it qualifies, else on any other connected
+// manager of the same cluster (lowest agent ID first). Too-old wins over
+// offline only when a connection would otherwise have qualified.
+func route(svc model.Service, conns []candidate) (model.Route, error) {
+	sort.Slice(conns, func(a, b int) bool { return conns[a].agentID < conns[b].agentID })
+	tooOld := false
+	try := func(c candidate) bool {
+		if svc.Kind == protocol.KindSwarm {
+			if !c.hello.SwarmManager || c.hello.SwarmClusterID == "" || c.hello.SwarmClusterID != svc.SwarmClusterID {
+				return false
+			}
+		}
+		if c.hello.ProtocolVersion < protocol.MinJobVersion {
+			tooOld = true
+			return false
+		}
+		return true
+	}
+	if svc.AgentID != nil {
+		for _, c := range conns {
+			if c.agentID == *svc.AgentID && try(c) {
+				return model.Route{AgentID: c.agentID, ConnID: c.connID}, nil
+			}
+		}
+	}
+	if svc.Kind == protocol.KindSwarm {
+		for _, c := range conns {
+			if try(c) {
+				return model.Route{AgentID: c.agentID, ConnID: c.connID}, nil
+			}
+		}
+	}
+	if tooOld {
+		return model.Route{}, model.ErrAgentTooOld
+	}
+	return model.Route{}, model.ErrAgentOffline
+}

internal/master/hub/route_test.go

@@ -0,0 +1,50 @@
+package hub
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+func TestRoute(t *testing.T) {
+	composeSvc := model.Service{Kind: protocol.KindCompose, AgentID: ptr(int64(2))}
+	swarmSvc := model.Service{Kind: protocol.KindSwarm, AgentID: ptr(int64(1)), SwarmClusterID: "c1"}
+	v2 := protocol.Hello{ProtocolVersion: protocol.Version}
+	mgrV2 := protocol.Hello{ProtocolVersion: protocol.Version, SwarmManager: true, SwarmClusterID: "c1"}
+
+	tests := []struct {
+		name  string
+		svc   model.Service
+		conns []candidate
+		want  model.Route
+		err   error
+	}{
+		{"compose on own agent", composeSvc, []candidate{{2, 20, v2}}, model.Route{AgentID: 2, ConnID: 20}, nil},
+		{"compose, own agent too old", composeSvc, []candidate{{2, 20, protocol.Hello{}}}, model.Route{}, model.ErrAgentTooOld},
+		{"compose, own agent missing", composeSvc, nil, model.Route{}, model.ErrAgentOffline},
+		{"compose, only another agent online", composeSvc, []candidate{{5, 50, v2}}, model.Route{}, model.ErrAgentOffline},
+		{"swarm on reporting agent", swarmSvc, []candidate{{1, 10, mgrV2}, {3, 30, mgrV2}}, model.Route{AgentID: 1, ConnID: 10}, nil},
+		{"swarm, reporting agent offline, other manager", swarmSvc, []candidate{{3, 30, mgrV2}}, model.Route{AgentID: 3, ConnID: 30}, nil},
+		{"swarm, lowest agent id first", swarmSvc, []candidate{{4, 40, mgrV2}, {3, 30, mgrV2}}, model.Route{AgentID: 3, ConnID: 30}, nil},
+		{"swarm, other manager in another cluster", swarmSvc,
+			[]candidate{{3, 30, protocol.Hello{ProtocolVersion: 2, SwarmManager: true, SwarmClusterID: "c2"}}}, model.Route{}, model.ErrAgentOffline},
+		{"swarm, only a v0 manager of the cluster", swarmSvc,
+			[]candidate{{1, 10, protocol.Hello{SwarmManager: true, SwarmClusterID: "c1"}}}, model.Route{}, model.ErrAgentTooOld},
+		{"swarm, v0 manager but a v2 manager exists", swarmSvc,
+			[]candidate{{1, 10, protocol.Hello{SwarmManager: true, SwarmClusterID: "c1"}}, {3, 30, mgrV2}}, model.Route{AgentID: 3, ConnID: 30}, nil},
+		{"swarm, worker of the cluster", swarmSvc,
+			[]candidate{{1, 10, protocol.Hello{ProtocolVersion: 2, SwarmClusterID: "c1"}}}, model.Route{}, model.ErrAgentOffline},
+		{"swarm, manager with empty cluster id", model.Service{Kind: protocol.KindSwarm},
+			[]candidate{{1, 10, protocol.Hello{ProtocolVersion: 2, SwarmManager: true}}}, model.Route{}, model.ErrAgentOffline},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := route(tt.svc, tt.conns)
+			assert.ErrorIs(t, err, tt.err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

internal/master/jobs/jobs.go

@@ -0,0 +1,146 @@
+// Package jobs decides whether a service may be updated and starts update
+// jobs on the right agent.
+package jobs
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"time"
+
+	"github.com/shcizo/package-updater/internal/imageref"
+	"github.com/shcizo/package-updater/internal/master/events"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// Store is the storage jobs needs. *store.Store satisfies it.
+type Store interface {
+	GetService(ctx context.Context, id int64) (model.Service, error)
+	CreateJob(ctx context.Context, j model.NewJob) (model.Job, error)
+	FailJob(ctx context.Context, id int64, msg string, at time.Time) (bool, error)
+	GetJob(ctx context.Context, id int64) (model.Job, error)
+}
+
+// Hub routes jobs to agent connections. *hub.Hub satisfies it.
+type Hub interface {
+	// Route picks the connection for svc: model.ErrAgentOffline or
+	// model.ErrAgentTooOld when there is none.
+	Route(svc model.Service) (model.Route, error)
+	// StartJob sends job.start over exactly the routed connection.
+	// model.ErrAgentOffline if that connection is gone or was replaced.
+	StartJob(r model.Route, jobID int64, start protocol.JobStart) error
+}
+
+// Publisher receives change signals. *events.Broker satisfies it.
+type Publisher interface{ Publish(t events.Topic) }
+
+// Service starts update jobs.
+type Service struct {
+	store    Store
+	hub      Hub
+	pub      Publisher
+	selfRepo string
+	log      *slog.Logger
+	now      func() time.Time
+}
+
+// New returns a Service. agentImage is the master's AGENT_IMAGE; services
+// whose image repo equals its repo are the updater itself and are refused.
+func New(store Store, hub Hub, pub Publisher, agentImage string, log *slog.Logger) *Service {
+	return &Service{
+		store: store, hub: hub, pub: pub, log: log, now: time.Now,
+		selfRepo: imageref.Parse(agentImage).Repo,
+	}
+}
+
+// target splits a service's actual ref into the repo and tag to deploy. A bare
+// image ("postgres", no tag and no digest) means "latest", as it does for
+// Docker. A ref with a digest but no tag follows no tag (ok is false).
+func target(actualRef string) (repo, tag string, ok bool) {
+	ref := imageref.Parse(actualRef)
+	switch {
+	case ref.Tag != "":
+		return ref.Repo, ref.Tag, true
+	case ref.Digest == "":
+		return ref.Repo, "latest", true
+	}
+	return ref.Repo, "", false
+}
+
+// Updatable returns nil or a *model.NotUpdatableError. It looks only at the
+// service row, so the UI can call it for every page render.
+func (s *Service) Updatable(svc model.Service) error {
+	switch {
+	case svc.RemovedAt != nil:
+		return &model.NotUpdatableError{Reason: model.ReasonRemoved}
+	case svc.Self || imageref.Parse(svc.ActualRef).Repo == s.selfRepo:
+		return &model.NotUpdatableError{Reason: model.ReasonSelf}
+	case svc.Built:
+		return &model.NotUpdatableError{Reason: model.ReasonBuilt}
+	case svc.Kind == protocol.KindContainer:
+		return &model.NotUpdatableError{Reason: model.ReasonContainer}
+	}
+	if _, _, ok := target(svc.ActualRef); !ok {
+		return &model.NotUpdatableError{Reason: model.ReasonNoTag}
+	}
+	return nil
+}
+
+// Trigger starts a manual update of the service to its current tag. It
+// returns the job, which may already be failed ("agent offline", "agent too
+// old for updates"); a *model.NotUpdatableError when the service cannot be
+// updated (no job created); the active job and model.ErrJobActive when one
+// is queued or running; model.ErrNotFound for an unknown service.
+func (s *Service) Trigger(ctx context.Context, serviceID int64, requestedBy string) (model.Job, error) {
+	svc, err := s.store.GetService(ctx, serviceID)
+	if err != nil {
+		return model.Job{}, err
+	}
+	if err := s.Updatable(svc); err != nil {
+		return model.Job{}, err
+	}
+	repo, tag, _ := target(svc.ActualRef) // Updatable checked the tag
+	ref := repo + ":" + tag
+
+	r, routeErr := s.hub.Route(svc)
+	nj := model.NewJob{
+		ServiceID: svc.ID, FromRef: svc.ActualRef, ToRef: ref,
+		Trigger: model.TriggerManual, RequestedBy: requestedBy, CreatedAt: s.now(),
+	}
+	if routeErr == nil {
+		nj.AgentID = &r.AgentID
+	}
+	job, err := s.store.CreateJob(ctx, nj)
+	if errors.Is(err, model.ErrAgentOffline) {
+		// The routed agent was revoked after Route. Record the attempt as a
+		// failed job that was never sent anywhere.
+		routeErr, nj.AgentID = err, nil
+		job, err = s.store.CreateJob(ctx, nj)
+	}
+	if err != nil {
+		return job, err
+	}
+
+	failure := routeErr
+	if failure == nil {
+		start := protocol.JobStart{Kind: svc.Kind, Ref: ref}
+		if svc.Kind == protocol.KindSwarm {
+			start.SwarmClusterID, start.SwarmService = svc.SwarmClusterID, svc.SwarmService
+		} else {
+			start.ComposeProject, start.ComposeService = svc.ComposeProject, svc.ComposeService
+		}
+		if err := s.hub.StartJob(r, job.ID, start); err != nil {
+			s.log.Warn("job_start_failed", "job_id", job.ID, "error", err)
+			failure = model.ErrAgentOffline
+		}
+	}
+	if failure != nil {
+		if _, err := s.store.FailJob(ctx, job.ID, failure.Error(), s.now()); err != nil {
+			return model.Job{}, err
+		}
+	}
+	s.pub.Publish(events.Jobs)
+	s.pub.Publish(events.JobTopic(job.ID))
+	return s.store.GetJob(ctx, job.ID)
+}

internal/master/jobs/jobs_test.go

@@ -0,0 +1,244 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/events"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+type fakeStore struct {
+	services map[int64]model.Service
+	jobs     map[int64]model.Job
+	created  []model.NewJob
+	active   *model.Job // returned with ErrJobActive when set
+
+	revokedAgents map[int64]bool // CreateJob returns ErrAgentOffline for these
+}
+
+func (f *fakeStore) GetService(_ context.Context, id int64) (model.Service, error) {
+	svc, ok := f.services[id]
+	if !ok {
+		return model.Service{}, model.ErrNotFound
+	}
+	return svc, nil
+}
+
+func (f *fakeStore) CreateJob(_ context.Context, nj model.NewJob) (model.Job, error) {
+	if f.active != nil {
+		return *f.active, model.ErrJobActive
+	}
+	if nj.AgentID != nil && f.revokedAgents[*nj.AgentID] {
+		return model.Job{}, model.ErrAgentOffline
+	}
+	f.created = append(f.created, nj)
+	j := model.Job{ID: int64(len(f.created)), ServiceID: nj.ServiceID, AgentID: nj.AgentID, FromRef: nj.FromRef,
+		ToRef: nj.ToRef, Trigger: nj.Trigger, RequestedBy: nj.RequestedBy, Status: model.JobQueued, CreatedAt: nj.CreatedAt}
+	f.jobs[j.ID] = j
+	return j, nil
+}
+
+func (f *fakeStore) FailJob(_ context.Context, id int64, msg string, at time.Time) (bool, error) {
+	j := f.jobs[id]
+	j.Status, j.Error, j.FinishedAt = model.JobFailed, msg, &at
+	f.jobs[id] = j
+	return true, nil
+}
+
+func (f *fakeStore) GetJob(_ context.Context, id int64) (model.Job, error) { return f.jobs[id], nil }
+
+type startCall struct {
+	route model.Route
+	id    int64
+	start protocol.JobStart
+}
+
+type fakeHub struct {
+	route    model.Route
+	routeErr error
+	startErr error
+	starts   []startCall
+}
+
+func (h *fakeHub) Route(model.Service) (model.Route, error) { return h.route, h.routeErr }
+
+func (h *fakeHub) StartJob(r model.Route, id int64, s protocol.JobStart) error {
+	h.starts = append(h.starts, startCall{r, id, s})
+	return h.startErr
+}
+
+type fakePub struct{ topics []events.Topic }
+
+func (p *fakePub) Publish(t events.Topic) { p.topics = append(p.topics, t) }
+
+const selfImage = "gitea.shcizo.se/shcizo/package-updater:latest"
+
+var (
+	composeSvc = model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
+		AgentID: ptr(int64(7)), ActualRef: "gitea.shcizo.se/shcizo/slusk:1.4.2@sha256:d1"}
+	bareSvc = model.Service{ID: 3, Kind: protocol.KindCompose, ComposeProject: "db", ComposeService: "pg",
+		AgentID: ptr(int64(7)), ActualRef: "postgres"}
+	swarmSvc = model.Service{ID: 2, Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea",
+		ActualRef: "gitea/gitea:1.27.3@sha256:87a6"}
+)
+
+func ptr[T any](v T) *T { return &v }
+
+func newService(st *fakeStore, h *fakeHub) (*Service, *fakePub) {
+	pub := &fakePub{}
+	return New(st, h, pub, selfImage, slog.New(slog.NewTextHandler(io.Discard, nil))), pub
+}
+
+func TestUpdatable(t *testing.T) {
+	tests := []struct {
+		name   string
+		svc    model.Service
+		reason string
+	}{
+		{"removed", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", RemovedAt: ptr(time.Now())}, model.ReasonRemoved},
+		{"flagged self", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, model.ReasonSelf},
+		{"same repo as the agent image", model.Service{Kind: protocol.KindCompose,
+			ActualRef: "gitea.shcizo.se/shcizo/package-updater:1.2.0@sha256:abc"}, model.ReasonSelf},
+		{"built from source", model.Service{Kind: protocol.KindCompose, ActualRef: "slusk:1", Built: true}, model.ReasonBuilt},
+		{"container", model.Service{Kind: protocol.KindContainer, ActualRef: "redis:7"}, model.ReasonContainer},
+		{"digest without tag", model.Service{Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}, model.ReasonNoTag},
+		{"bare image means latest", model.Service{Kind: protocol.KindCompose, ActualRef: "postgres"}, ""},
+		{"bare image on a registry with port", model.Service{Kind: protocol.KindCompose, ActualRef: "localhost:5000/foo"}, ""},
+		{"tag", model.Service{Kind: protocol.KindCompose, ActualRef: "postgres:16"}, ""},
+		{"tag and digest", model.Service{Kind: protocol.KindCompose, ActualRef: "postgres:17.10@sha256:abc"}, ""},
+		{"registry with port and tag", model.Service{Kind: protocol.KindSwarm, ActualRef: "localhost:5000/foo:v1"}, ""},
+	}
+	s, _ := newService(&fakeStore{}, &fakeHub{})
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := s.Updatable(tt.svc)
+			if tt.reason == "" {
+				assert.NoError(t, err)
+				return
+			}
+			var nu *model.NotUpdatableError
+			require.True(t, errors.As(err, &nu))
+			assert.Equal(t, tt.reason, nu.Reason)
+		})
+	}
+}
+
+func TestTrigger(t *testing.T) {
+	ok := model.Route{AgentID: 7, ConnID: 70}
+	activeJob := model.Job{ID: 99, Status: model.JobRunning}
+
+	tests := []struct {
+		name       string
+		serviceID  int64
+		hub        *fakeHub
+		active     *model.Job
+		wantErr    error
+		wantStatus model.JobStatus
+		wantError  string
+		wantStart  *protocol.JobStart
+		wantCreate bool
+	}{
+		{name: "compose happy path", serviceID: 1, hub: &fakeHub{route: ok}, wantStatus: model.JobQueued, wantCreate: true,
+			wantStart: &protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "gitea.shcizo.se/shcizo/slusk:1.4.2"}},
+		{name: "swarm", serviceID: 2, hub: &fakeHub{route: ok}, wantStatus: model.JobQueued, wantCreate: true,
+			wantStart: &protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27.3"}},
+		{name: "bare image deploys latest", serviceID: 3, hub: &fakeHub{route: ok}, wantStatus: model.JobQueued, wantCreate: true,
+			wantStart: &protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "db", ComposeService: "pg", Ref: "postgres:latest"}},
+		{name: "active job", serviceID: 1, hub: &fakeHub{route: ok}, active: &activeJob, wantErr: model.ErrJobActive, wantStatus: model.JobRunning},
+		{name: "agent offline", serviceID: 1, hub: &fakeHub{routeErr: model.ErrAgentOffline}, wantStatus: model.JobFailed,
+			wantError: "agent offline", wantCreate: true},
+		{name: "agent too old", serviceID: 1, hub: &fakeHub{routeErr: model.ErrAgentTooOld}, wantStatus: model.JobFailed,
+			wantError: "agent too old for updates", wantCreate: true},
+		{name: "start fails", serviceID: 1, hub: &fakeHub{route: ok, startErr: model.ErrAgentOffline}, wantStatus: model.JobFailed,
+			wantError: "agent offline", wantCreate: true,
+			wantStart: &protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "gitea.shcizo.se/shcizo/slusk:1.4.2"}},
+		{name: "unknown service", serviceID: 5, hub: &fakeHub{route: ok}, wantErr: model.ErrNotFound},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			st := &fakeStore{
+				services: map[int64]model.Service{1: composeSvc, 2: swarmSvc, 3: bareSvc},
+				jobs:     map[int64]model.Job{99: activeJob},
+				active:   tt.active,
+			}
+			s, pub := newService(st, tt.hub)
+
+			job, err := s.Trigger(context.Background(), tt.serviceID, "sub-1")
+
+			if tt.wantErr != nil {
+				assert.ErrorIs(t, err, tt.wantErr)
+			} else {
+				require.NoError(t, err)
+			}
+			if tt.wantErr == nil || tt.active != nil {
+				assert.Equal(t, tt.wantStatus, job.Status)
+				assert.Equal(t, tt.wantError, job.Error)
+			}
+			if tt.wantStart == nil {
+				assert.Empty(t, tt.hub.starts)
+			} else {
+				require.Len(t, tt.hub.starts, 1)
+				assert.Equal(t, *tt.wantStart, tt.hub.starts[0].start)
+				assert.Equal(t, ok, tt.hub.starts[0].route)
+				assert.Equal(t, job.ID, tt.hub.starts[0].id)
+			}
+			if tt.wantCreate {
+				require.Len(t, st.created, 1)
+				assert.Equal(t, "sub-1", st.created[0].RequestedBy)
+				assert.Equal(t, model.TriggerManual, st.created[0].Trigger)
+				assert.Equal(t, st.services[tt.serviceID].ActualRef, st.created[0].FromRef)
+				assert.Contains(t, pub.topics, events.Jobs)
+				assert.Contains(t, pub.topics, events.JobTopic(job.ID))
+			} else {
+				assert.Empty(t, st.created)
+				assert.Empty(t, pub.topics)
+			}
+		})
+	}
+}
+
+// The routed agent can be revoked between Route and CreateJob; the store then
+// refuses the job. Trigger records a failed job that was never sent.
+func TestTrigger_AgentRevokedAfterRoute(t *testing.T) {
+	st := &fakeStore{
+		services:      map[int64]model.Service{1: composeSvc},
+		jobs:          map[int64]model.Job{},
+		revokedAgents: map[int64]bool{7: true},
+	}
+	h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
+	s, _ := newService(st, h)
+
+	job, err := s.Trigger(context.Background(), 1, "sub-1")
+
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, job.Status)
+	assert.Equal(t, "agent offline", job.Error)
+	assert.Nil(t, job.AgentID)
+	assert.Empty(t, h.starts)
+}
+
+func TestTrigger_NotUpdatableCreatesNoJob(t *testing.T) {
+	st := &fakeStore{
+		services: map[int64]model.Service{1: {ID: 1, Kind: protocol.KindContainer, ActualRef: "redis:7"}},
+		jobs:     map[int64]model.Job{},
+	}
+	h := &fakeHub{}
+	s, pub := newService(st, h)
+
+	_, err := s.Trigger(context.Background(), 1, "sub-1")
+
+	var nu *model.NotUpdatableError
+	require.True(t, errors.As(err, &nu))
+	assert.Empty(t, st.created)
+	assert.Empty(t, h.starts)
+	assert.Empty(t, pub.topics)
+}

internal/master/model/job.go

@@ -0,0 +1,117 @@
+package model
+
+import (
+	"errors"
+	"time"
+)
+
+// Sentinel errors for jobs.
+var (
+	// ErrJobActive means the service already has a queued or running job.
+	// CreateJob and jobs.Service.Trigger return it together with that job.
+	ErrJobActive = errors.New("an update is already in progress for this service")
+	// ErrAgentOffline means no connected agent can run the job.
+	ErrAgentOffline = errors.New("agent offline")
+	// ErrAgentTooOld means the only agent that could run the job speaks a
+	// protocol version without jobs.
+	ErrAgentTooOld = errors.New("agent too old for updates")
+)
+
+// NotUpdatableError says why a service cannot be updated. Reason is shown
+// in the UI as is.
+type NotUpdatableError struct {
+	Reason string
+}
+
+func (e *NotUpdatableError) Error() string { return "not updatable: " + e.Reason }
+
+// Reasons a service cannot be updated.
+const (
+	ReasonSelf      = "updated manually outside the system"
+	ReasonContainer = "plain containers cannot be updated"
+	ReasonNoTag     = "follows no tag"
+	ReasonRemoved   = "service no longer exists"
+	ReasonBuilt     = "built from source (build:)"
+)
+
+// JobStatus is the state of an update job.
+type JobStatus string
+
+// Job statuses. A job moves queued -> running -> succeeded|failed, or
+// straight from queued to failed.
+const (
+	JobQueued    JobStatus = "queued"
+	JobRunning   JobStatus = "running"
+	JobSucceeded JobStatus = "succeeded"
+	JobFailed    JobStatus = "failed"
+)
+
+// JobTrigger says what created a job. Only TriggerManual is used in phase 2.
+type JobTrigger string
+
+// Job triggers.
+const (
+	TriggerManual  JobTrigger = "manual"
+	TriggerWebhook JobTrigger = "webhook"
+	TriggerPoll    JobTrigger = "poll"
+)
+
+// Job is one update of one service.
+type Job struct {
+	ID              int64
+	ServiceID       int64
+	ServiceName     string // Service.DisplayName() of the job's service
+	AgentID         *int64 // agent the job was sent to; nil if it never was
+	AgentName       string
+	FromRef         string
+	ToRef           string // "repo:tag" when created; "repo:tag@digest" after success
+	Trigger         JobTrigger
+	RequestedBy     string // OIDC subject for manual jobs
+	Status          JobStatus
+	Output          string // empty in ListJobs results
+	OutputTruncated bool
+	Error           string
+	CreatedAt       time.Time
+	StartedAt       *time.Time
+	FinishedAt      *time.Time
+}
+
+// Active reports whether the job is queued or running.
+func (j Job) Active() bool { return j.Status == JobQueued || j.Status == JobRunning }
+
+// NewJob is what CreateJob needs to insert a queued job.
+type NewJob struct {
+	ServiceID   int64
+	AgentID     *int64
+	FromRef     string
+	ToRef       string
+	Trigger     JobTrigger
+	RequestedBy string
+	CreatedAt   time.Time
+}
+
+// JobFinish is a job's outcome as reported by the agent.
+type JobFinish struct {
+	Status          JobStatus // JobSucceeded or JobFailed
+	Ref             string    // deployed "repo:tag@digest"; set on success
+	Error           string
+	Output          string
+	OutputTruncated bool
+	StartedAt       time.Time // agent clock
+	FinishedAt      time.Time // agent clock
+}
+
+// JobFilter selects jobs for ListJobs. ServiceID 0 means all services.
+// Limit 0 means 100.
+type JobFilter struct {
+	ServiceID int64
+	Limit     int
+}
+
+// Route is the hub's choice of agent connection for a job. ConnID identifies
+// the connection, so a job is never sent over a connection that replaced the
+// one chosen.
+type Route struct {
+	AgentID int64
+	ConnID  uint64
+}

internal/master/model/model.go

@@ -41,17 +41,21 @@ type Service struct {
 	ComposeProject string
 	ComposeService string
 	WorkingDir     string
+	ConfigFiles    string // raw config_files label, paths joined by ","
 	SwarmClusterID string
 	SwarmService   string
 	ContainerName  string
 	ImageRepo      string
+	DesiredRef     string // set by the last successful job; "" if never updated
 	ActualRef      string
 	ActualDigest   string
 	Status         string
 	Self           bool
+	Built          bool // Compose service with a build: section; never updated
 	UpdateMode     string
 	FirstSeenAt    time.Time
 	UpdatedAt      time.Time
+	RemovedAt      *time.Time
 }
 
 // DisplayName is the human-facing name of the service.

internal/master/model/model_test.go

@@ -39,3 +39,20 @@ func TestDisplayName(t *testing.T) {
 		})
 	}
 }
+
+func TestJob_Active(t *testing.T) {
+	cases := []struct {
+		status JobStatus
+		want   bool
+	}{
+		{JobQueued, true},
+		{JobRunning, true},
+		{JobSucceeded, false},
+		{JobFailed, false},
+	}
+	for _, c := range cases {
+		t.Run(string(c.status), func(t *testing.T) {
+			assert.Equal(t, c.want, Job{Status: c.status}.Active())
+		})
+	}
+}

internal/master/store/agents.go

@@ -63,30 +63,39 @@ func (s *Store) AgentByTokenHash(ctx context.Context, hash string) (model.Agent,
 	return scanAgent(s.pool.QueryRow(ctx, `SELECT `+agentColumns+` FROM agents WHERE token_hash = $1`, hash))
 }
 
-// RevokeAgent marks the agent revoked and its Compose and plain-container
-// services removed, in one transaction. Swarm rows are shared with other
-// managers and left alone. Its token stops working immediately; closing a
-// live connection is the hub's job.
-func (s *Store) RevokeAgent(ctx context.Context, id int64, at time.Time) error {
+// RevokeAgent marks the agent revoked, its Compose and plain-container
+// services removed and its queued and running jobs failed, in one
+// transaction, and returns the IDs of the jobs it failed. Swarm rows are
+// shared with other managers and left alone. Its token stops working
+// immediately; closing a live connection is the hub's job.
+func (s *Store) RevokeAgent(ctx context.Context, id int64, at time.Time) ([]int64, error) {
 	tx, err := s.pool.Begin(ctx)
 	if err != nil {
-		return fmt.Errorf("revoke agent: begin: %w", err)
+		return nil, fmt.Errorf("revoke agent: begin: %w", err)
 	}
 	defer func() { _ = tx.Rollback(ctx) }()
 
 	if _, err := tx.Exec(ctx,
 		`UPDATE agents SET revoked_at = $2 WHERE id = $1 AND revoked_at IS NULL`, id, at); err != nil {
-		return fmt.Errorf("revoke agent: %w", err)
+		return nil, fmt.Errorf("revoke agent: %w", err)
 	}
 	if _, err := tx.Exec(ctx,
 		`UPDATE services SET removed_at = $2
 		 WHERE agent_id = $1 AND kind IN ('compose', 'container') AND removed_at IS NULL`, id, at); err != nil {
-		return fmt.Errorf("revoke agent: remove services: %w", err)
+		return nil, fmt.Errorf("revoke agent: remove services: %w", err)
+	}
+	rows, err := tx.Query(ctx, failActiveJobsSQL, id, "agent revoked", at)
+	if err != nil {
+		return nil, fmt.Errorf("revoke agent: fail jobs: %w", err)
+	}
+	failed, err := pgx.CollectRows(rows, pgx.RowTo[int64])
+	if err != nil {
+		return nil, fmt.Errorf("revoke agent: fail jobs: %w", err)
 	}
 	if err := tx.Commit(ctx); err != nil {
-		return fmt.Errorf("revoke agent: commit: %w", err)
+		return nil, fmt.Errorf("revoke agent: commit: %w", err)
 	}
-	return nil
+	return failed, nil
 }
 
 // RecordHello stores what the agent reported about itself on connect.

internal/master/store/agents_integration_test.go

@@ -46,7 +46,8 @@ func TestAgents_CreateLookupRevoke(t *testing.T) {
 	_, err = s.AgentByTokenHash(ctx, "nope")
 	assert.ErrorIs(t, err, model.ErrNotFound)
 
-	require.NoError(t, s.RevokeAgent(ctx, created.ID, time.Now()))
+	_, err = s.RevokeAgent(ctx, created.ID, time.Now())
+	require.NoError(t, err)
 	found, err = s.AgentByTokenHash(ctx, "hash-2")
 	require.NoError(t, err)
 	assert.NotNil(t, found.RevokedAt)
@@ -88,7 +89,8 @@ func TestAgents_RevokedNameCanBeReused(t *testing.T) {
 	ctx := context.Background()
 	old, err := s.CreateAgent(ctx, "node-2", "h1")
 	require.NoError(t, err)
-	require.NoError(t, s.RevokeAgent(ctx, old.ID, time.Now()))
+	_, err = s.RevokeAgent(ctx, old.ID, time.Now())
+	require.NoError(t, err)
 
 	fresh, err := s.CreateAgent(ctx, "node-2", "h2")
 	require.NoError(t, err)
@@ -104,7 +106,8 @@ func TestAgents_RevokeRemovesOwnComposeAndContainerServices(t *testing.T) {
 	id := newAgent(t, s, "swarm-mgr")
 	require.NoError(t, s.ApplySnapshot(ctx, id, manager, []protocol.Service{slusk, adhoc, gitea}, t0))
 
-	require.NoError(t, s.RevokeAgent(ctx, id, t0.Add(time.Hour)))
+	_, err := s.RevokeAgent(ctx, id, t0.Add(time.Hour))
+	require.NoError(t, err)
 
 	got, err := s.ListServices(ctx)
 	require.NoError(t, err)
@@ -115,7 +118,8 @@ func TestRevokedAgent_InventoryWritesRefused(t *testing.T) {
 	s := openTestStore(t)
 	ctx := context.Background()
 	id := newAgent(t, s, "node-2")
-	require.NoError(t, s.RevokeAgent(ctx, id, t0))
+	_, err := s.RevokeAgent(ctx, id, t0)
+	require.NoError(t, err)
 
 	assert.ErrorIs(t, s.ApplySnapshot(ctx, id, worker, []protocol.Service{slusk}, t0.Add(time.Minute)), model.ErrRevoked)
 	assert.ErrorIs(t, s.UpsertService(ctx, id, "", adhoc, t0.Add(time.Minute)), model.ErrRevoked)
@@ -130,7 +134,8 @@ func TestRevokedAgent_UpsertDoesNotResurrectRemovedRow(t *testing.T) {
 	ctx := context.Background()
 	id := newAgent(t, s, "node-2")
 	require.NoError(t, s.ApplySnapshot(ctx, id, worker, []protocol.Service{slusk}, t0))
-	require.NoError(t, s.RevokeAgent(ctx, id, t0.Add(time.Hour)))
+	_, err := s.RevokeAgent(ctx, id, t0.Add(time.Hour))
+	require.NoError(t, err)
 
 	assert.ErrorIs(t, s.UpsertService(ctx, id, "", slusk, t0.Add(2*time.Hour)), model.ErrRevoked)
 
@@ -138,3 +143,26 @@ func TestRevokedAgent_UpsertDoesNotResurrectRemovedRow(t *testing.T) {
 	require.NoError(t, err)
 	assert.Empty(t, got, "the revoked agent's removed row must stay removed")
 }
+
+func TestRevokeAgent_FailsActiveJobs(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svcA := jobFixture(t, s, "a")
+	b, svcB := jobFixture(t, s, "b")
+	ja, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	jb, err := s.CreateJob(ctx, newJob(svcB, b))
+	require.NoError(t, err)
+
+	failed, err := s.RevokeAgent(ctx, a, t0)
+	require.NoError(t, err)
+	assert.Equal(t, []int64{ja.ID}, failed, "the failed job IDs are returned so the UI can signal them")
+
+	got, err := s.GetJob(ctx, ja.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, got.Status)
+	assert.Equal(t, "agent revoked", got.Error)
+	got, err = s.GetJob(ctx, jb.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobQueued, got.Status)
+}

internal/master/store/jobs.go

@@ -0,0 +1,229 @@
+package store
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+const lostJobMessage = "lost: the agent did not know this job when it reconnected"
+
+// failActiveJobsSQL fails every queued or running job of agent $1 with
+// message $2 at time $3.
+const failActiveJobsSQL = `
+UPDATE update_jobs SET status = 'failed', error = $2, finished_at = $3
+WHERE agent_id = $1 AND status IN ('queued', 'running')
+RETURNING id`
+
+const jobSelectSQL = `
+SELECT j.id, j.service_id, s.kind, s.compose_project, s.compose_service, s.swarm_service, s.container_name,
+       j.agent_id, COALESCE(a.name, ''), j.from_ref, j.to_ref, j.trigger, j.requested_by, j.status,
+       %s, j.output_truncated, j.error, j.created_at, j.started_at, j.finished_at
+FROM update_jobs j
+JOIN services s ON s.id = j.service_id
+LEFT JOIN agents a ON a.id = j.agent_id`
+
+func scanJob(row pgx.Row) (model.Job, error) {
+	var j model.Job
+	var svc model.Service
+	var kind, trigger, status string
+	err := row.Scan(&j.ID, &j.ServiceID, &kind, &svc.ComposeProject, &svc.ComposeService, &svc.SwarmService, &svc.ContainerName,
+		&j.AgentID, &j.AgentName, &j.FromRef, &j.ToRef, &trigger, &j.RequestedBy, &status,
+		&j.Output, &j.OutputTruncated, &j.Error, &j.CreatedAt, &j.StartedAt, &j.FinishedAt)
+	svc.Kind = protocol.Kind(kind)
+	j.ServiceName = svc.DisplayName()
+	j.Trigger = model.JobTrigger(trigger)
+	j.Status = model.JobStatus(status)
+	return j, err
+}
+
+func (s *Store) getJob(ctx context.Context, id int64) (model.Job, error) {
+	j, err := scanJob(s.pool.QueryRow(ctx, fmt.Sprintf(jobSelectSQL, "j.output")+` WHERE j.id = $1`, id))
+	if errors.Is(err, pgx.ErrNoRows) {
+		return model.Job{}, model.ErrNotFound
+	}
+	if err != nil {
+		return model.Job{}, fmt.Errorf("get job: %w", err)
+	}
+	return j, nil
+}
+
+// CreateJob inserts a queued job. If the service already has a queued or
+// running job, it returns that job and model.ErrJobActive (detected via the
+// unique index, so it holds under concurrent calls). A job for a revoked
+// agent is refused with model.ErrAgentOffline: the agent's revoke fails its
+// jobs once, so a job created afterwards would never be settled and would
+// block the service. The share lock on the agent row makes a concurrent
+// revoke wait for this insert, so its job cleanup sees the new row.
+func (s *Store) CreateJob(ctx context.Context, j model.NewJob) (model.Job, error) {
+	// The active job can finish between the unique violation and the lookup
+	// below; then the insert is simply tried once more.
+	for attempt := 0; attempt < 2; attempt++ {
+		var id int64
+		err := s.pool.QueryRow(ctx, `
+			INSERT INTO update_jobs (service_id, agent_id, from_ref, to_ref, trigger, requested_by, status, created_at)
+			SELECT $1::bigint, $2::bigint, $3::text, $4::text, $5::text, $6::text, 'queued', $7::timestamptz
+			WHERE $2::bigint IS NULL
+			   OR EXISTS (SELECT 1 FROM agents WHERE id = $2 AND revoked_at IS NULL FOR SHARE)
+			RETURNING id`,
+			j.ServiceID, j.AgentID, j.FromRef, j.ToRef, string(j.Trigger), j.RequestedBy, j.CreatedAt).Scan(&id)
+		if errors.Is(err, pgx.ErrNoRows) {
+			return model.Job{}, model.ErrAgentOffline
+		}
+		var pgErr *pgconn.PgError
+		if errors.As(err, &pgErr) && pgErr.Code == "23505" && pgErr.ConstraintName == "update_jobs_one_active" {
+			active, err := scanJob(s.pool.QueryRow(ctx, fmt.Sprintf(jobSelectSQL, "j.output")+
+				` WHERE j.service_id = $1 AND j.status IN ('queued', 'running')`, j.ServiceID))
+			if errors.Is(err, pgx.ErrNoRows) {
+				continue
+			}
+			if err != nil {
+				return model.Job{}, fmt.Errorf("find active job: %w", err)
+			}
+			return active, model.ErrJobActive
+		}
+		if err != nil {
+			return model.Job{}, fmt.Errorf("create job: %w", err)
+		}
+		return s.getJob(ctx, id)
+	}
+	return model.Job{}, fmt.Errorf("create job: active job vanished twice")
+}
+
+// FailJob sets a queued or running job to failed with msg and finished_at =
+// at. Returns false (no error) if the job is no longer active.
+func (s *Store) FailJob(ctx context.Context, id int64, msg string, at time.Time) (bool, error) {
+	tag, err := s.pool.Exec(ctx, `
+		UPDATE update_jobs SET status = 'failed', error = $2, finished_at = $3
+		WHERE id = $1 AND status IN ('queued', 'running')`, id, msg, at)
+	if err != nil {
+		return false, fmt.Errorf("fail job: %w", err)
+	}
+	return tag.RowsAffected() == 1, nil
+}
+
+// GetJob returns one job with its output. Returns model.ErrNotFound if missing.
+func (s *Store) GetJob(ctx context.Context, id int64) (model.Job, error) {
+	return s.getJob(ctx, id)
+}
+
+// ListJobs returns jobs newest first, without Output.
+func (s *Store) ListJobs(ctx context.Context, f model.JobFilter) ([]model.Job, error) {
+	limit := f.Limit
+	if limit <= 0 {
+		limit = 100
+	}
+	rows, err := s.pool.Query(ctx, fmt.Sprintf(jobSelectSQL, "''")+`
+		WHERE ($1 = 0 OR j.service_id = $1)
+		ORDER BY j.id DESC
+		LIMIT $2`, f.ServiceID, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list jobs: %w", err)
+	}
+	defer rows.Close()
+	var out []model.Job
+	for rows.Next() {
+		j, err := scanJob(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan job: %w", err)
+		}
+		out = append(out, j)
+	}
+	return out, rows.Err()
+}
+
+// FailLostJobs fails every queued or running job sent to agentID whose ID is
+// not in known, with a "lost" error. Returns the IDs it failed.
+func (s *Store) FailLostJobs(ctx context.Context, agentID int64, known []int64, at time.Time) ([]int64, error) {
+	if known == nil {
+		known = []int64{}
+	}
+	rows, err := s.pool.Query(ctx, `
+		UPDATE update_jobs SET status = 'failed', error = $3, finished_at = $4
+		WHERE agent_id = $1 AND status IN ('queued', 'running') AND NOT (id = ANY($2))
+		RETURNING id`, agentID, known, lostJobMessage, at)
+	if err != nil {
+		return nil, fmt.Errorf("fail lost jobs: %w", err)
+	}
+	defer rows.Close()
+	var ids []int64
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			return nil, fmt.Errorf("scan lost job: %w", err)
+		}
+		ids = append(ids, id)
+	}
+	return ids, rows.Err()
+}
+
+// JobProgress appends chunk to the output of a queued or running job sent to
+// agentID, keeping the last protocol.MaxJobOutput characters, and moves a
+// queued job to running (started_at = at if unset). Returns false if no such
+// active job belongs to agentID.
+func (s *Store) JobProgress(ctx context.Context, agentID, jobID int64, chunk string, at time.Time) (bool, error) {
+	tag, err := s.pool.Exec(ctx, `
+		UPDATE update_jobs SET
+			status           = 'running',
+			started_at       = COALESCE(started_at, $4),
+			output_truncated = output_truncated OR length(output) + length($3::text) > $5,
+			output           = right(output || $3::text, $5)
+		WHERE id = $1 AND agent_id = $2 AND status IN ('queued', 'running')`,
+		jobID, agentID, chunk, at, protocol.MaxJobOutput)
+	if err != nil {
+		return false, fmt.Errorf("job progress: %w", err)
+	}
+	return tag.RowsAffected() == 1, nil
+}
+
+// FinishJob records the agent's result for a queued or running job sent to
+// agentID: status, error, output (replacing what progress assembled),
+// started_at (the agent's, or the existing one if the agent sent none),
+// finished_at; to_ref = f.Ref when set. On success it also sets the
+// service's desired_ref = f.Ref, in the same transaction. Returns false if no
+// such active job belongs to agentID.
+func (s *Store) FinishJob(ctx context.Context, agentID, jobID int64, f model.JobFinish) (bool, error) {
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return false, fmt.Errorf("finish job: begin: %w", err)
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+
+	// The agent's clock wins when it sent a start time; otherwise the time the
+	// first progress arrived stays.
+	var startedAt *time.Time
+	if !f.StartedAt.IsZero() {
+		startedAt = &f.StartedAt
+	}
+	var serviceID int64
+	err = tx.QueryRow(ctx, `
+		UPDATE update_jobs SET
+			status = $3, error = $4, output = $5, output_truncated = $6,
+			to_ref = CASE WHEN $7::text <> '' THEN $7::text ELSE to_ref END,
+			started_at = COALESCE($8, started_at), finished_at = $9
+		WHERE id = $1 AND agent_id = $2 AND status IN ('queued', 'running')
+		RETURNING service_id`,
+		jobID, agentID, string(f.Status), f.Error, f.Output, f.OutputTruncated, f.Ref, startedAt, f.FinishedAt).Scan(&serviceID)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("finish job: %w", err)
+	}
+	if f.Status == model.JobSucceeded {
+		if _, err := tx.Exec(ctx, `UPDATE services SET desired_ref = $2 WHERE id = $1`, serviceID, f.Ref); err != nil {
+			return false, fmt.Errorf("finish job: desired ref: %w", err)
+		}
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return false, fmt.Errorf("finish job: commit: %w", err)
+	}
+	return true, nil
+}

internal/master/store/jobs_integration_test.go

@@ -0,0 +1,379 @@
+//go:build integration
+
+package store
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// jobFixture seeds one agent with one Compose service and returns their IDs.
+func jobFixture(t *testing.T, s *Store, agentName string) (agentID, serviceID int64) {
+	t.Helper()
+	ctx := context.Background()
+	agentID = newAgent(t, s, agentName)
+	require.NoError(t, s.ApplySnapshot(ctx, agentID, worker, []protocol.Service{slusk}, t0))
+	svcs, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	for _, svc := range svcs {
+		if svc.AgentID != nil && *svc.AgentID == agentID {
+			return agentID, svc.ID
+		}
+	}
+	t.Fatal("service not found")
+	return
+}
+
+func newJob(serviceID, agentID int64) model.NewJob {
+	return model.NewJob{
+		ServiceID: serviceID, AgentID: &agentID,
+		FromRef: "r:1@sha256:a", ToRef: "r:1", Trigger: model.TriggerManual,
+		RequestedBy: "sub", CreatedAt: t0,
+	}
+}
+
+func TestCreateJob_OneActivePerService(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	agentID, svcID := jobFixture(t, s, "a")
+
+	var wg sync.WaitGroup
+	jobs := make([]model.Job, 2)
+	errs := make([]error, 2)
+	for i := range jobs {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			jobs[i], errs[i] = s.CreateJob(ctx, newJob(svcID, agentID))
+		}()
+	}
+	wg.Wait()
+
+	assert.Equal(t, jobs[0].ID, jobs[1].ID)
+	var failures int
+	for _, err := range errs {
+		if err != nil {
+			assert.ErrorIs(t, err, model.ErrJobActive)
+			failures++
+		}
+	}
+	assert.Equal(t, 1, failures)
+
+	ok, err := s.FailJob(ctx, jobs[0].ID, "x", t0)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	ok, err = s.FailJob(ctx, jobs[0].ID, "x", t0)
+	require.NoError(t, err)
+	assert.False(t, ok)
+
+	next, err := s.CreateJob(ctx, newJob(svcID, agentID))
+	require.NoError(t, err)
+	assert.NotEqual(t, jobs[0].ID, next.ID)
+}
+
+func TestFailLostJobs(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svcA := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	require.NoError(t, s.ApplySnapshot(ctx, b, worker, []protocol.Service{adhoc}, t0))
+	all, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	var svcB int64
+	for _, svc := range all {
+		if svc.ID != svcA {
+			svcB = svc.ID
+		}
+	}
+
+	// Only one active job per service, so A's jobs go through the lifecycle.
+	j3, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	ok, err := s.FinishJob(ctx, a, j3.ID, model.JobFinish{Status: model.JobSucceeded, Ref: "r:1@sha256:d", StartedAt: t0, FinishedAt: t0})
+	require.NoError(t, err)
+	require.True(t, ok)
+	j1, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	j4, err := s.CreateJob(ctx, newJob(svcB, b))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, b, j4.ID, "x", t0)
+	require.NoError(t, err)
+
+	ids, err := s.FailLostJobs(ctx, a, []int64{j3.ID}, t0)
+	require.NoError(t, err)
+	assert.Equal(t, []int64{j1.ID}, ids)
+
+	got, err := s.GetJob(ctx, j1.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, got.Status)
+	assert.Contains(t, got.Error, "lost")
+	got, err = s.GetJob(ctx, j3.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobSucceeded, got.Status)
+	got, err = s.GetJob(ctx, j4.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobRunning, got.Status)
+
+	// A running job the agent still knows is kept.
+	j5, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j5.ID, "", t0)
+	require.NoError(t, err)
+	ids, err = s.FailLostJobs(ctx, a, []int64{j5.ID}, t0)
+	require.NoError(t, err)
+	assert.Empty(t, ids)
+}
+
+func TestJobProgress_OwnerOnly(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+
+	started := t0.Add(time.Second)
+	ok, err := s.JobProgress(ctx, a, j.ID, "hello ", started)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobRunning, got.Status)
+	require.NotNil(t, got.StartedAt)
+	assert.True(t, started.Equal(*got.StartedAt))
+
+	ok, err = s.JobProgress(ctx, b, j.ID, "evil", t0.Add(time.Minute))
+	require.NoError(t, err)
+	assert.False(t, ok)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "hello ", got.Output)
+
+	ok, err = s.JobProgress(ctx, a, j.ID, "world", t0.Add(time.Minute))
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "hello world", got.Output)
+	assert.True(t, started.Equal(*got.StartedAt))
+
+	_, err = s.FailJob(ctx, j.ID, "x", t0)
+	require.NoError(t, err)
+	ok, err = s.JobProgress(ctx, a, j.ID, "late", t0)
+	require.NoError(t, err)
+	assert.False(t, ok)
+}
+
+func TestJobProgress_KeepsTail(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+
+	_, err = s.JobProgress(ctx, a, j.ID, strings.Repeat("a", protocol.MaxJobOutput), t0)
+	require.NoError(t, err)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.False(t, got.OutputTruncated)
+
+	_, err = s.JobProgress(ctx, a, j.ID, "TAIL", t0)
+	require.NoError(t, err)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.True(t, got.OutputTruncated)
+	assert.Len(t, got.Output, protocol.MaxJobOutput)
+	assert.True(t, strings.HasSuffix(got.Output, "aaTAIL"))
+}
+
+func TestFinishJob_OnlyOnceAndOnlyByOwner(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j.ID, "partial", t0)
+	require.NoError(t, err)
+
+	fin := model.JobFinish{Status: model.JobSucceeded, Ref: "r:1@sha256:d", Output: "full", StartedAt: t0.Add(time.Hour), FinishedAt: t0.Add(2 * time.Hour)}
+	ok, err := s.FinishJob(ctx, b, j.ID, fin)
+	require.NoError(t, err)
+	assert.False(t, ok)
+
+	ok, err = s.FinishJob(ctx, a, j.ID, fin)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobSucceeded, got.Status)
+	assert.Equal(t, "r:1@sha256:d", got.ToRef)
+	assert.Equal(t, "full", got.Output)
+	assert.True(t, fin.StartedAt.Equal(*got.StartedAt), "the agent's started_at wins over the master's progress clock")
+	assert.True(t, fin.FinishedAt.Equal(*got.FinishedAt))
+	service, err := s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, "r:1@sha256:d", service.DesiredRef)
+
+	again := fin
+	again.Ref = "r:2@sha256:e"
+	again.Output = "other"
+	ok, err = s.FinishJob(ctx, a, j.ID, again)
+	require.NoError(t, err)
+	assert.False(t, ok)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "full", got.Output)
+	service, err = s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, "r:1@sha256:d", service.DesiredRef)
+
+	j2, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+	ok, err = s.FinishJob(ctx, a, j2.ID, model.JobFinish{Status: model.JobFailed, Error: "boom", Output: "o", StartedAt: t0, FinishedAt: t0})
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err = s.GetJob(ctx, j2.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, got.Status)
+	assert.Equal(t, "boom", got.Error)
+	assert.Equal(t, "r:1", got.ToRef)
+	service, err = s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, "r:1@sha256:d", service.DesiredRef)
+}
+
+func TestGetJobAndListJobs(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svcA := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	require.NoError(t, s.ApplySnapshot(ctx, b, worker, []protocol.Service{adhoc}, t0))
+	all, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	var svcB int64
+	for _, svc := range all {
+		if svc.ID != svcA {
+			svcB = svc.ID
+		}
+	}
+
+	j1, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j1.ID, "out", t0)
+	require.NoError(t, err)
+	_, err = s.FailJob(ctx, j1.ID, "x", t0)
+	require.NoError(t, err)
+	j2, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	j3, err := s.CreateJob(ctx, newJob(svcB, b))
+	require.NoError(t, err)
+
+	got, err := s.GetJob(ctx, j1.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "slusk/web", got.ServiceName)
+	assert.Equal(t, "a", got.AgentName)
+	assert.Equal(t, "out", got.Output)
+
+	_, err = s.GetJob(ctx, 99999)
+	assert.ErrorIs(t, err, model.ErrNotFound)
+
+	list, err := s.ListJobs(ctx, model.JobFilter{})
+	require.NoError(t, err)
+	require.Len(t, list, 3)
+	assert.Equal(t, []int64{j3.ID, j2.ID, j1.ID}, []int64{list[0].ID, list[1].ID, list[2].ID})
+	for _, j := range list {
+		assert.Empty(t, j.Output)
+	}
+
+	list, err = s.ListJobs(ctx, model.JobFilter{ServiceID: svcA})
+	require.NoError(t, err)
+	assert.Len(t, list, 2)
+
+	list, err = s.ListJobs(ctx, model.JobFilter{Limit: 1})
+	require.NoError(t, err)
+	assert.Len(t, list, 1)
+}
+
+func TestFinishJob_ZeroStartedAtKeepsProgressClock(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j.ID, "x", t0)
+	require.NoError(t, err)
+
+	ok, err := s.FinishJob(ctx, a, j.ID, model.JobFinish{Status: model.JobFailed, FinishedAt: t0.Add(time.Hour)})
+	require.NoError(t, err)
+	require.True(t, ok)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.True(t, t0.Equal(*got.StartedAt))
+}
+
+func TestCreateJob_RefusesRevokedAgent(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	_, err := s.RevokeAgent(ctx, a, t0)
+	require.NoError(t, err)
+
+	_, err = s.CreateJob(ctx, newJob(svc, a))
+	assert.ErrorIs(t, err, model.ErrAgentOffline)
+	list, err := s.ListJobs(ctx, model.JobFilter{})
+	require.NoError(t, err)
+	assert.Empty(t, list, "no job is left behind to block the service")
+
+	nj := newJob(svc, a)
+	nj.AgentID = nil
+	_, err = s.CreateJob(ctx, nj)
+	assert.NoError(t, err, "a job that was never routed has no agent to check")
+}
+
+// CreateJob must wait for a revoke that is still in flight. Without the share
+// lock it would read the agent as active, insert the job, and the revoke's
+// job cleanup (which ran before the insert committed) would never fail it.
+func TestCreateJob_WaitsForConcurrentRevoke(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+
+	tx, err := s.pool.Begin(ctx)
+	require.NoError(t, err)
+	defer func() { _ = tx.Rollback(ctx) }()
+	_, err = tx.Exec(ctx, `UPDATE agents SET revoked_at = now() WHERE id = $1`, a)
+	require.NoError(t, err)
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := s.CreateJob(ctx, newJob(svc, a))
+		done <- err
+	}()
+
+	select {
+	case err := <-done:
+		t.Fatalf("CreateJob returned (%v) while the revoke was uncommitted", err)
+	case <-time.After(500 * time.Millisecond):
+	}
+
+	require.NoError(t, tx.Commit(ctx))
+	select {
+	case err := <-done:
+		assert.ErrorIs(t, err, model.ErrAgentOffline)
+	case <-time.After(5 * time.Second):
+		t.Fatal("CreateJob did not return after the revoke committed")
+	}
+	list, err := s.ListJobs(ctx, model.JobFilter{})
+	require.NoError(t, err)
+	assert.Empty(t, list)
+}

internal/master/store/migrations/0002_jobs.sql

@@ -0,0 +1,28 @@
+ALTER TABLE services ADD COLUMN config_files text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN desired_ref  text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN built        boolean NOT NULL DEFAULT false;
+
+CREATE TABLE update_jobs (
+    id               bigserial PRIMARY KEY,
+    service_id       bigint NOT NULL REFERENCES services (id),
+    agent_id         bigint REFERENCES agents (id),
+    from_ref         text NOT NULL DEFAULT '',
+    to_ref           text NOT NULL,
+    trigger          text NOT NULL CHECK (trigger IN ('manual', 'webhook', 'poll')),
+    requested_by     text NOT NULL DEFAULT '',
+    status           text NOT NULL CHECK (status IN ('queued', 'running', 'succeeded', 'failed')),
+    output           text NOT NULL DEFAULT '',
+    output_truncated boolean NOT NULL DEFAULT false,
+    error            text NOT NULL DEFAULT '',
+    created_at       timestamptz NOT NULL,
+    started_at       timestamptz,
+    finished_at      timestamptz
+);
+
+-- One queued or running job per service. CreateJob relies on this index to
+-- turn a second concurrent click into ErrJobActive.
+CREATE UNIQUE INDEX update_jobs_one_active ON update_jobs (service_id)
+    WHERE status IN ('queued', 'running');
+CREATE INDEX update_jobs_by_service ON update_jobs (service_id, id DESC);
+CREATE INDEX update_jobs_active_by_agent ON update_jobs (agent_id)
+    WHERE status IN ('queued', 'running');

internal/master/store/services.go

@@ -39,13 +39,15 @@ func lockActiveAgent(ctx context.Context, tx pgx.Tx, agentID int64) error {
 
 const upsertServiceSQL = `
 INSERT INTO services (
-	kind, location_key, agent_id, compose_project, compose_service, working_dir,
-	swarm_cluster_id, swarm_service, container_name, image_repo, actual_ref, actual_digest,
+	kind, location_key, agent_id, compose_project, compose_service, working_dir, config_files,
+	built, swarm_cluster_id, swarm_service, container_name, image_repo, actual_ref, actual_digest,
 	status, self, first_seen_at, updated_at)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $15)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $17)
 ON CONFLICT (location_key) DO UPDATE SET
 	agent_id      = EXCLUDED.agent_id,
 	working_dir   = EXCLUDED.working_dir,
+	config_files  = EXCLUDED.config_files,
+	built         = EXCLUDED.built,
 	image_repo    = EXCLUDED.image_repo,
 	actual_ref    = EXCLUDED.actual_ref,
 	actual_digest = EXCLUDED.actual_digest,
@@ -61,7 +63,7 @@ func upsertService(ctx context.Context, db execer, agentID int64, clusterID stri
 	}
 	_, err := db.Exec(ctx, upsertServiceSQL,
 		string(svc.Kind), model.LocationKey(agentID, clusterID, svc.Key()), agentID,
-		svc.ComposeProject, svc.ComposeService, svc.WorkingDir,
+		svc.ComposeProject, svc.ComposeService, svc.WorkingDir, svc.ConfigFiles, svc.Built,
 		clusterCol, svc.SwarmService, svc.ContainerName,
 		imageref.Parse(svc.Image).Repo, svc.Image, svc.Digest,
 		svc.Status, svc.Self, at)
@@ -144,14 +146,27 @@ func (s *Store) RemoveService(ctx context.Context, agentID int64, clusterID, age
 	return nil
 }
 
+const serviceColumns = `s.id, s.kind, s.location_key, s.agent_id, COALESCE(a.name, ''),
+	s.compose_project, s.compose_service, s.working_dir, s.config_files, s.built, s.swarm_cluster_id,
+	s.swarm_service, s.container_name, s.image_repo, s.desired_ref, s.actual_ref, s.actual_digest,
+	s.status, s.self, s.update_mode, s.first_seen_at, s.updated_at, s.removed_at`
+
+func scanService(row pgx.Row) (model.Service, error) {
+	var m model.Service
+	var kind string
+	err := row.Scan(&m.ID, &kind, &m.LocationKey, &m.AgentID, &m.AgentName,
+		&m.ComposeProject, &m.ComposeService, &m.WorkingDir, &m.ConfigFiles, &m.Built, &m.SwarmClusterID,
+		&m.SwarmService, &m.ContainerName, &m.ImageRepo, &m.DesiredRef, &m.ActualRef, &m.ActualDigest,
+		&m.Status, &m.Self, &m.UpdateMode, &m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt)
+	m.Kind = protocol.Kind(kind)
+	return m, err
+}
+
 // ListServices returns all services not marked removed: machine-bound
 // services grouped by agent name first, Swarm services last.
 func (s *Store) ListServices(ctx context.Context) ([]model.Service, error) {
 	rows, err := s.pool.Query(ctx, `
-		SELECT s.id, s.kind, s.location_key, s.agent_id, COALESCE(a.name, ''),
-		       s.compose_project, s.compose_service, s.working_dir, s.swarm_cluster_id,
-		       s.swarm_service, s.container_name, s.image_repo, s.actual_ref, s.actual_digest,
-		       s.status, s.self, s.update_mode, s.first_seen_at, s.updated_at
+		SELECT `+serviceColumns+`
 		FROM services s LEFT JOIN agents a ON a.id = s.agent_id
 		WHERE s.removed_at IS NULL
 		ORDER BY s.kind = 'swarm', COALESCE(a.name, ''), s.location_key`)
@@ -161,16 +176,27 @@ func (s *Store) ListServices(ctx context.Context) ([]model.Service, error) {
 	defer rows.Close()
 	var out []model.Service
 	for rows.Next() {
-		var m model.Service
-		var kind string
-		if err := rows.Scan(&m.ID, &kind, &m.LocationKey, &m.AgentID, &m.AgentName,
-			&m.ComposeProject, &m.ComposeService, &m.WorkingDir, &m.SwarmClusterID,
-			&m.SwarmService, &m.ContainerName, &m.ImageRepo, &m.ActualRef, &m.ActualDigest,
-			&m.Status, &m.Self, &m.UpdateMode, &m.FirstSeenAt, &m.UpdatedAt); err != nil {
+		m, err := scanService(rows)
+		if err != nil {
 			return nil, fmt.Errorf("scan service: %w", err)
 		}
-		m.Kind = protocol.Kind(kind)
 		out = append(out, m)
 	}
 	return out, rows.Err()
 }
+
+// GetService returns one service, including a removed one (RemovedAt set).
+// Returns model.ErrNotFound if the ID does not exist.
+func (s *Store) GetService(ctx context.Context, id int64) (model.Service, error) {
+	m, err := scanService(s.pool.QueryRow(ctx, `
+		SELECT `+serviceColumns+`
+		FROM services s LEFT JOIN agents a ON a.id = s.agent_id
+		WHERE s.id = $1`, id))
+	if errors.Is(err, pgx.ErrNoRows) {
+		return model.Service{}, model.ErrNotFound
+	}
+	if err != nil {
+		return model.Service{}, fmt.Errorf("get service: %w", err)
+	}
+	return m, nil
+}

internal/master/store/services_integration_test.go

@@ -160,3 +160,56 @@ func TestListServices_SwarmLast(t *testing.T) {
 }
 
 func itoa(n int64) string { return strconv.FormatInt(n, 10) }
+
+func TestGetService_IncludesRemoved(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	id := newAgent(t, s, "node-2")
+	require.NoError(t, s.ApplySnapshot(ctx, id, worker, []protocol.Service{slusk}, t0))
+	svcs, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	require.Len(t, svcs, 1)
+
+	require.NoError(t, s.RemoveService(ctx, id, "", slusk.Key(), t0))
+	got, err := s.GetService(ctx, svcs[0].ID)
+	require.NoError(t, err)
+	assert.NotNil(t, got.RemovedAt)
+	assert.Equal(t, "node-2", got.AgentName)
+
+	_, err = s.GetService(ctx, 99999)
+	assert.ErrorIs(t, err, model.ErrNotFound)
+}
+
+func TestApplySnapshot_StoresConfigFiles(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	id := newAgent(t, s, "node-2")
+	withFiles := slusk
+	withFiles.ConfigFiles = "/srv/slusk/compose.yaml,/srv/slusk/compose.override.yaml"
+	require.NoError(t, s.ApplySnapshot(ctx, id, worker, []protocol.Service{withFiles}, t0))
+
+	got, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, withFiles.ConfigFiles, got[0].ConfigFiles)
+}
+
+func TestApplySnapshot_StoresBuilt(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	id := newAgent(t, s, "node-2")
+	built := slusk
+	built.Built = true
+	require.NoError(t, s.ApplySnapshot(ctx, id, worker, []protocol.Service{built}, t0))
+
+	got, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.True(t, got[0].Built)
+
+	// A later report without the flag clears it.
+	require.NoError(t, s.UpsertService(ctx, id, "", slusk, t0.Add(time.Minute)))
+	got, err = s.ListServices(ctx)
+	require.NoError(t, err)
+	assert.False(t, got[0].Built)
+}

internal/master/web/server.go

@@ -23,14 +23,21 @@ import (
 //go:embed templates/*.html static/*
 var assets embed.FS
 
+// serviceJobsLimit is how many jobs the service page lists.
+const serviceJobsLimit = 20
+
 var agentNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
 
 // Store is the storage the UI reads and writes. *store.Store satisfies it.
 type Store interface {
 	ListServices(ctx context.Context) ([]model.Service, error)
+	GetService(ctx context.Context, id int64) (model.Service, error)
+	GetJob(ctx context.Context, id int64) (model.Job, error)
+	ListJobs(ctx context.Context, f model.JobFilter) ([]model.Job, error)
 	ListAgents(ctx context.Context) ([]model.Agent, error)
 	CreateAgent(ctx context.Context, name, tokenHash string) (model.Agent, error)
-	RevokeAgent(ctx context.Context, id int64, at time.Time) error
+	// RevokeAgent returns the IDs of the jobs it failed.
+	RevokeAgent(ctx context.Context, id int64, at time.Time) ([]int64, error)
 }
 
 // Hub reports and controls live agent connections. *hub.Hub satisfies it.
@@ -39,6 +46,13 @@ type Hub interface {
 	Disconnect(agentID int64)
 }
 
+// Jobs starts updates and explains why a service cannot be updated.
+// *jobs.Service satisfies it.
+type Jobs interface {
+	Updatable(svc model.Service) error
+	Trigger(ctx context.Context, serviceID int64, requestedBy string) (model.Job, error)
+}
+
 // Broker delivers change signals to open streams and accepts signals from
 // UI actions. *events.Broker satisfies it.
 type Broker interface {
@@ -57,6 +71,7 @@ type Config struct {
 type Server struct {
 	store  Store
 	hub    Hub
+	jobs   Jobs
 	broker Broker
 	cfg    Config
 	log    *slog.Logger
@@ -66,17 +81,17 @@ type Server struct {
 
 // New parses the templates. Each page is its own template set so every page
 // can define its own "content" block.
-func New(store Store, hub Hub, broker Broker, cfg Config, log *slog.Logger) (*Server, error) {
-	funcs := template.FuncMap{"shortRef": shortRef}
+func New(store Store, hub Hub, jobs Jobs, broker Broker, cfg Config, log *slog.Logger) (*Server, error) {
+	funcs := template.FuncMap{"shortRef": shortRef, "duration": jobDuration}
 	pages := map[string]*template.Template{}
-	for _, page := range []string{"overview.html", "agents.html", "agent_created.html"} {
+	for _, page := range []string{"overview.html", "agents.html", "agent_created.html", "service.html", "jobs.html", "job.html"} {
 		t, err := template.New(page).Funcs(funcs).ParseFS(assets, "templates/layout.html", "templates/"+page)
 		if err != nil {
 			return nil, fmt.Errorf("parse %s: %w", page, err)
 		}
 		pages[page] = t
 	}
-	return &Server{store: store, hub: hub, broker: broker, cfg: cfg, log: log, now: time.Now, pages: pages}, nil
+	return &Server{store: store, hub: hub, jobs: jobs, broker: broker, cfg: cfg, log: log, now: time.Now, pages: pages}, nil
 }
 
 // Register adds the UI routes to mux. protect wraps every route except
@@ -86,6 +101,13 @@ func (s *Server) Register(mux *http.ServeMux, protect func(http.Handler) http.Ha
 	route := func(pattern string, h http.HandlerFunc) { mux.Handle(pattern, protect(h)) }
 	route("GET /{$}", s.overview)
 	route("GET /fragments/services", s.servicesFragment)
+	route("GET /services/{id}", s.servicePage)
+	route("GET /fragments/services/{id}/jobs", s.serviceJobsFragment)
+	route("POST /services/{id}/update", s.updateService)
+	route("GET /jobs", s.jobsPage)
+	route("GET /fragments/jobs", s.jobsFragment)
+	route("GET /jobs/{id}", s.jobPage)
+	route("GET /fragments/jobs/{id}", s.jobFragment)
 	route("GET /agents", s.agents)
 	route("GET /fragments/agents", s.agentsFragment)
 	route("POST /agents", s.createAgent)
@@ -101,6 +123,12 @@ type pageData struct {
 	Error   string
 	Created model.Agent
 	Snippet string
+
+	Service     model.Service
+	ConfigFiles []string
+	Reason      string // why Service cannot be updated; "" when it can
+	Jobs        []model.Job
+	Job         model.Job
 }
 
 func (s *Server) base(r *http.Request, title string) pageData {
@@ -144,6 +172,151 @@ func (s *Server) servicesFragment(w http.ResponseWriter, r *http.Request) {
 	s.render(w, http.StatusOK, "overview.html", "services", data)
 }
 
+// pathID parses the {id} path value. On failure it has already answered 400.
+func pathID(w http.ResponseWriter, r *http.Request) (int64, bool) {
+	id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
+	if err != nil {
+		http.Error(w, "invalid id", http.StatusBadRequest)
+		return 0, false
+	}
+	return id, true
+}
+
+// failLookup answers 404 for model.ErrNotFound and 500 otherwise.
+func (s *Server) failLookup(w http.ResponseWriter, err error) {
+	if errors.Is(err, model.ErrNotFound) {
+		http.Error(w, "not found", http.StatusNotFound)
+		return
+	}
+	s.fail(w, err)
+}
+
+func (s *Server) serviceDetail(r *http.Request, id int64) (pageData, error) {
+	svc, err := s.store.GetService(r.Context(), id)
+	if err != nil {
+		return pageData{}, err
+	}
+	data := s.base(r, svc.DisplayName())
+	data.Service = svc
+	if svc.ConfigFiles != "" {
+		data.ConfigFiles = strings.Split(svc.ConfigFiles, ",")
+	}
+	var nu *model.NotUpdatableError
+	if err := s.jobs.Updatable(svc); errors.As(err, &nu) {
+		data.Reason = nu.Reason
+	} else if err != nil {
+		return data, err
+	}
+	data.Jobs, err = s.store.ListJobs(r.Context(), model.JobFilter{ServiceID: id, Limit: serviceJobsLimit})
+	return data, err
+}
+
+func (s *Server) servicePage(w http.ResponseWriter, r *http.Request) {
+	id, ok := pathID(w, r)
+	if !ok {
+		return
+	}
+	data, err := s.serviceDetail(r, id)
+	if err != nil {
+		s.failLookup(w, err)
+		return
+	}
+	s.render(w, http.StatusOK, "service.html", "layout", data)
+}
+
+func (s *Server) serviceJobsFragment(w http.ResponseWriter, r *http.Request) {
+	id, ok := pathID(w, r)
+	if !ok {
+		return
+	}
+	jobs, err := s.store.ListJobs(r.Context(), model.JobFilter{ServiceID: id, Limit: serviceJobsLimit})
+	if err != nil {
+		s.fail(w, err)
+		return
+	}
+	data := s.base(r, "Service")
+	data.Jobs = jobs
+	s.render(w, http.StatusOK, "service.html", "history", data)
+}
+
+func (s *Server) updateService(w http.ResponseWriter, r *http.Request) {
+	id, ok := pathID(w, r)
+	if !ok {
+		return
+	}
+	sess, _ := webauth.SessionFrom(r.Context())
+	job, err := s.jobs.Trigger(r.Context(), id, sess.Subject)
+	var nu *model.NotUpdatableError
+	switch {
+	case err == nil || errors.Is(err, model.ErrJobActive):
+		http.Redirect(w, r, "/jobs/"+strconv.FormatInt(job.ID, 10), http.StatusSeeOther)
+	case errors.As(err, &nu):
+		data, derr := s.serviceDetail(r, id)
+		if derr != nil {
+			s.failLookup(w, derr)
+			return
+		}
+		data.Error = "Cannot update: " + nu.Reason + "."
+		s.render(w, http.StatusConflict, "service.html", "layout", data)
+	default:
+		s.failLookup(w, err)
+	}
+}
+
+func (s *Server) jobsData(r *http.Request) (pageData, error) {
+	data := s.base(r, "Jobs")
+	var err error
+	data.Jobs, err = s.store.ListJobs(r.Context(), model.JobFilter{Limit: 100})
+	return data, err
+}
+
+func (s *Server) jobsPage(w http.ResponseWriter, r *http.Request) {
+	data, err := s.jobsData(r)
+	if err != nil {
+		s.fail(w, err)
+		return
+	}
+	s.render(w, http.StatusOK, "jobs.html", "layout", data)
+}
+
+func (s *Server) jobsFragment(w http.ResponseWriter, r *http.Request) {
+	data, err := s.jobsData(r)
+	if err != nil {
+		s.fail(w, err)
+		return
+	}
+	s.render(w, http.StatusOK, "jobs.html", "jobs", data)
+}
+
+// jobData always reads the job from the store, so a fragment fetched after
+// a dropped signal still shows the full current output.
+func (s *Server) jobData(w http.ResponseWriter, r *http.Request) (pageData, bool) {
+	id, ok := pathID(w, r)
+	if !ok {
+		return pageData{}, false
+	}
+	job, err := s.store.GetJob(r.Context(), id)
+	if err != nil {
+		s.failLookup(w, err)
+		return pageData{}, false
+	}
+	data := s.base(r, fmt.Sprintf("Job #%d", job.ID))
+	data.Job = job
+	return data, true
+}
+
+func (s *Server) jobPage(w http.ResponseWriter, r *http.Request) {
+	if data, ok := s.jobData(w, r); ok {
+		s.render(w, http.StatusOK, "job.html", "layout", data)
+	}
+}
+
+func (s *Server) jobFragment(w http.ResponseWriter, r *http.Request) {
+	if data, ok := s.jobData(w, r); ok {
+		s.render(w, http.StatusOK, "job.html", "job", data)
+	}
+}
+
 func (s *Server) agentData(r *http.Request) (pageData, error) {
 	data := s.base(r, "Agents")
 	agents, err := s.store.ListAgents(r.Context())
@@ -217,12 +390,17 @@ func (s *Server) revokeAgent(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "invalid agent id", http.StatusBadRequest)
 		return
 	}
-	if err := s.store.RevokeAgent(r.Context(), id, s.now()); err != nil {
+	failedJobs, err := s.store.RevokeAgent(r.Context(), id, s.now())
+	if err != nil {
 		s.fail(w, err)
 		return
 	}
 	s.hub.Disconnect(id)
 	s.broker.Publish(events.Services) // the agent's services were marked removed
+	s.broker.Publish(events.Jobs)     // the agent's active jobs were failed
+	for _, jobID := range failedJobs {
+		s.broker.Publish(events.JobTopic(jobID)) // open job pages show the failure
+	}
 	http.Redirect(w, r, "/agents", http.StatusSeeOther)
 }
 
@@ -282,7 +460,12 @@ func agentSnippet(cfg Config, token string) string {
       MASTER_URL: %s
       MASTER_KEY: %s
       AGENT_TOKEN: %s
+      UPDATE_TIMEOUT: 5m
+      # Optional registry credentials (a config.json written by docker login):
+      # DOCKER_CONFIG: /docker-config
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      # - /path/to/stacks:/path/to/stacks  # mount each stacks directory at the same path, read-write
+      # - /root/.docker/config.json:/docker-config/config.json:ro
 `, cfg.AgentImage, cfg.AgentURL, cfg.MasterKey, token)
 }

internal/master/web/server_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/shcizo/package-updater/internal/agenttoken"
 	"github.com/shcizo/package-updater/internal/master/events"
 	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/master/webauth"
 	"github.com/shcizo/package-updater/internal/protocol"
 )
 
@@ -28,12 +29,42 @@ import (
 type fakeStore struct {
 	mu           sync.Mutex
 	services     []model.Service
+	jobs         []model.Job
 	agents       []model.Agent
 	created      map[string]string // name → token hash
 	revokedID    int64
+	revokeFailed []int64 // job IDs RevokeAgent reports as failed
 }
 
 func (f *fakeStore) ListServices(context.Context) ([]model.Service, error) { return f.services, nil }
+func (f *fakeStore) GetService(_ context.Context, id int64) (model.Service, error) {
+	for _, s := range f.services {
+		if s.ID == id {
+			return s, nil
+		}
+	}
+	return model.Service{}, model.ErrNotFound
+}
+
+func (f *fakeStore) GetJob(_ context.Context, id int64) (model.Job, error) {
+	for _, j := range f.jobs {
+		if j.ID == id {
+			return j, nil
+		}
+	}
+	return model.Job{}, model.ErrNotFound
+}
+
+func (f *fakeStore) ListJobs(_ context.Context, flt model.JobFilter) ([]model.Job, error) {
+	var out []model.Job
+	for _, j := range f.jobs {
+		if flt.ServiceID == 0 || j.ServiceID == flt.ServiceID {
+			out = append(out, j)
+		}
+	}
+	return out, nil
+}
+
 func (f *fakeStore) ListAgents(context.Context) ([]model.Agent, error) { return f.agents, nil }
 
 func (f *fakeStore) CreateAgent(_ context.Context, name, hash string) (model.Agent, error) {
@@ -46,11 +77,11 @@ func (f *fakeStore) CreateAgent(_ context.Context, name, hash string) (model.Age
 	return model.Agent{ID: 9, Name: name}, nil
 }
 
-func (f *fakeStore) RevokeAgent(_ context.Context, id int64, _ time.Time) error {
+func (f *fakeStore) RevokeAgent(_ context.Context, id int64, _ time.Time) ([]int64, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	f.revokedID = id
-	return nil
+	return f.revokeFailed, nil
 }
 
 func (f *fakeStore) createdHash(name string) string {
@@ -65,6 +96,31 @@ func (f *fakeStore) revoked() int64 {
 	return f.revokedID
 }
 
+// fakeJobs returns scripted results and records who asked for the update.
+type fakeJobs struct {
+	updatable error
+	job       model.Job
+	err       error
+
+	mu          sync.Mutex
+	requestedBy string
+}
+
+func (f *fakeJobs) Updatable(model.Service) error { return f.updatable }
+
+func (f *fakeJobs) Trigger(_ context.Context, _ int64, by string) (model.Job, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.requestedBy = by
+	return f.job, f.err
+}
+
+func (f *fakeJobs) triggeredBy() string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.requestedBy
+}
+
 type fakeHub struct {
 	mu           sync.Mutex
 	online       map[int64]bool
@@ -90,16 +146,22 @@ func (h *fakeHub) lastDisconnected() int64 {
 }
 
 func newTestServer(t *testing.T) (*httptest.Server, *fakeStore, *fakeHub, *events.Broker) {
+	t.Helper()
+	srv, store, hub, broker, _ := newTestEnv(t)
+	return srv, store, hub, broker
+}
+
+func newTestEnv(t *testing.T) (*httptest.Server, *fakeStore, *fakeHub, *events.Broker, *fakeJobs) {
 	t.Helper()
 	agentID := int64(2)
 	store := &fakeStore{
 		created: map[string]string{"node-2": "x"},
 		services: []model.Service{
-			{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", AgentID: &agentID, AgentName: "node-2",
+			{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", AgentID: &agentID, AgentName: "node-2",
 				ActualRef: "gitea.shcizo.se/shcizo/slusk:1.4.2", Status: "running", UpdateMode: "manual"},
-			{Kind: protocol.KindSwarm, SwarmService: "jellyfin_jellyfin",
+			{ID: 2, Kind: protocol.KindSwarm, SwarmService: "jellyfin_jellyfin",
 				ActualRef: "lscr.io/linuxserver/jellyfin@sha256:0f42497a69fa0441bfd5f9d6bba8694f2a656ec984e571d0ac04c6dd91250039", Status: "degraded", UpdateMode: "manual"},
-			{Kind: protocol.KindContainer, ContainerName: "updater-agent", AgentID: &agentID, AgentName: "node-2",
+			{ID: 3, Kind: protocol.KindContainer, ContainerName: "updater-agent", AgentID: &agentID, AgentName: "node-2",
 				ActualRef: "pu:1", Status: "running", UpdateMode: "manual", Self: true},
 		},
 		agents: []model.Agent{
@@ -110,15 +172,21 @@ func newTestServer(t *testing.T) (*httptest.Server, *fakeStore, *fakeHub, *event
 	}
 	hub := &fakeHub{online: map[int64]bool{2: true}}
 	broker := events.NewBroker()
-	s, err := New(store, hub, broker, Config{
+	jobs := &fakeJobs{}
+	s, err := New(store, hub, jobs, broker, Config{
 		AgentURL: "wss://10.0.0.5:8090/agent", MasterKey: "sha256:abc", AgentImage: "gitea.shcizo.se/shcizo/package-updater:latest",
 	}, slog.New(slog.NewTextHandler(io.Discard, nil)))
 	require.NoError(t, err)
 	mux := http.NewServeMux()
-	s.Register(mux, func(h http.Handler) http.Handler { return h })
+	s.Register(mux, func(h http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx := webauth.WithSession(r.Context(), model.Session{Subject: "alice", CSRFToken: "csrf"})
+			h.ServeHTTP(w, r.WithContext(ctx))
+		})
+	})
 	srv := httptest.NewServer(mux)
 	t.Cleanup(srv.Close)
-	return srv, store, hub, broker
+	return srv, store, hub, broker, jobs
 }
 
 func ptr[T any](v T) *T { return &v }
@@ -209,6 +277,7 @@ func TestCreateAgent_Validation(t *testing.T) {
 
 func TestRevokeAgent_RevokesAndDisconnects(t *testing.T) {
 	srv, store, hub, broker := newTestServer(t)
+	store.revokeFailed = []int64{5, 6}
 	ch, cancel := broker.Subscribe()
 	defer cancel()
 	resp, err := noRedirectClient().Post(srv.URL+"/agents/2/revoke", "application/x-www-form-urlencoded", nil)
@@ -219,13 +288,18 @@ func TestRevokeAgent_RevokesAndDisconnects(t *testing.T) {
 	assert.Equal(t, "/agents", resp.Header.Get("Location"))
 	assert.Equal(t, int64(2), store.revoked())
 	assert.Equal(t, int64(2), hub.lastDisconnected(), "a revoked agent must be cut off immediately")
+	var got []events.Topic
+	for len(got) < 4 {
 		select {
 		case topic := <-ch:
-		assert.Equal(t, events.Services, topic, "the overview must drop the revoked agent's services")
+			got = append(got, topic)
 		default:
-		t.Fatal("revoke published no services event")
+			t.Fatalf("revoke published %v, want services, jobs and one topic per failed job", got)
 		}
 	}
+	assert.ElementsMatch(t, []events.Topic{events.Services, events.Jobs, events.JobTopic(5), events.JobTopic(6)}, got,
+		"the overview drops the agent's services; job lists and open job pages show the failures")
+}
 
 func TestRevokeAgent_BadID(t *testing.T) {
 	srv, _, _, _ := newTestServer(t)
@@ -263,3 +337,170 @@ func TestEvents_StreamsTopics(t *testing.T) {
 	}
 	assert.Equal(t, "event: services\n", line)
 }
+
+func TestAgentSnippet(t *testing.T) {
+	snippet := agentSnippet(Config{AgentURL: "wss://m/agent", MasterKey: "sha256:abc", AgentImage: "img:1"}, "tok")
+	assert.NotContains(t, snippet, "STACKS_ROOT")
+	assert.Contains(t, snippet, "UPDATE_TIMEOUT: 5m")
+	assert.Contains(t, snippet, "      # DOCKER_CONFIG: /docker-config\n")
+	assert.Contains(t, snippet, "      # - /path/to/stacks:/path/to/stacks  # mount each stacks directory at the same path, read-write\n")
+	assert.Contains(t, snippet, "AGENT_TOKEN: tok")
+}
+
+func TestOverview_LinksToServicePage(t *testing.T) {
+	srv, _, _, _ := newTestServer(t)
+	_, body := get(t, srv, "/")
+	assert.Contains(t, body, `href="/services/1"`)
+	assert.Contains(t, body, `href="/jobs"`)
+}
+
+func TestServicePage_ShowsUpdateButton(t *testing.T) {
+	srv, store, _, _, _ := newTestEnv(t)
+	store.services[0].ConfigFiles = "/srv/slusk/compose.yaml,/srv/slusk/compose.override.yaml"
+	store.jobs = []model.Job{{ID: 7, ServiceID: 1, Status: model.JobSucceeded, FromRef: "a:1", ToRef: "a:1", RequestedBy: "alice", CreatedAt: time.Now()}}
+	code, body := get(t, srv, "/services/1")
+	assert.Equal(t, http.StatusOK, code)
+	assert.Contains(t, body, `action="/services/1/update"`)
+	assert.Contains(t, body, `name="csrf_token" value="csrf"`)
+	assert.Contains(t, body, "/srv/slusk/compose.override.yaml")
+	assert.Contains(t, body, `href="/jobs/7"`)
+	assert.Contains(t, body, "sse:jobs")
+}
+
+func TestServicePage_DisabledReason(t *testing.T) {
+	for _, reason := range []string{model.ReasonSelf, model.ReasonContainer, model.ReasonNoTag, model.ReasonRemoved, model.ReasonBuilt} {
+		t.Run(reason, func(t *testing.T) {
+			srv, _, _, _, jobs := newTestEnv(t)
+			jobs.updatable = &model.NotUpdatableError{Reason: reason}
+			code, body := get(t, srv, "/services/1")
+			assert.Equal(t, http.StatusOK, code)
+			assert.Contains(t, body, reason)
+			assert.NotContains(t, body, `action="/services/1/update"`)
+		})
+	}
+}
+
+func TestServicePage_NotFoundAndBadID(t *testing.T) {
+	srv, _, _, _ := newTestServer(t)
+	code, _ := get(t, srv, "/services/99")
+	assert.Equal(t, http.StatusNotFound, code)
+	code, _ = get(t, srv, "/services/abc")
+	assert.Equal(t, http.StatusBadRequest, code)
+}
+
+func postUpdate(t *testing.T, srv *httptest.Server) (*http.Response, string) {
+	t.Helper()
+	resp, err := noRedirectClient().PostForm(srv.URL+"/services/1/update", url.Values{"csrf_token": {"csrf"}})
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(resp.Body)
+	return resp, string(body)
+}
+
+func TestUpdate_RedirectsToJob(t *testing.T) {
+	srv, _, _, _, jobs := newTestEnv(t)
+	jobs.job = model.Job{ID: 42}
+	resp, _ := postUpdate(t, srv)
+	assert.Equal(t, http.StatusSeeOther, resp.StatusCode)
+	assert.Equal(t, "/jobs/42", resp.Header.Get("Location"))
+	assert.Equal(t, "alice", jobs.triggeredBy())
+}
+
+func TestUpdate_ActiveJobRedirectsToExisting(t *testing.T) {
+	srv, _, _, _, jobs := newTestEnv(t)
+	jobs.job, jobs.err = model.Job{ID: 41}, model.ErrJobActive
+	resp, _ := postUpdate(t, srv)
+	assert.Equal(t, http.StatusSeeOther, resp.StatusCode)
+	assert.Equal(t, "/jobs/41", resp.Header.Get("Location"))
+}
+
+func TestUpdate_NotUpdatableShows409(t *testing.T) {
+	srv, _, _, _, jobs := newTestEnv(t)
+	jobs.err = &model.NotUpdatableError{Reason: model.ReasonNoTag}
+	jobs.updatable = jobs.err
+	resp, body := postUpdate(t, srv)
+	assert.Equal(t, http.StatusConflict, resp.StatusCode)
+	assert.Contains(t, body, model.ReasonNoTag)
+}
+
+func TestUpdate_UnknownService404(t *testing.T) {
+	srv, _, _, _, jobs := newTestEnv(t)
+	jobs.err = model.ErrNotFound
+	resp, _ := postUpdate(t, srv)
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+}
+
+func TestServiceJobsFragment_IsNotAFullPage(t *testing.T) {
+	srv, store, _, _, _ := newTestEnv(t)
+	store.jobs = []model.Job{{ID: 7, ServiceID: 1, Status: model.JobFailed, CreatedAt: time.Now()}}
+	code, body := get(t, srv, "/fragments/services/1/jobs")
+	assert.Equal(t, http.StatusOK, code)
+	assert.NotContains(t, body, "<html")
+	assert.Contains(t, body, "/jobs/7")
+}
+
+func TestJobsPage_ListsJobs(t *testing.T) {
+	srv, store, _, _, _ := newTestEnv(t)
+	started, finished := time.Now(), time.Now().Add(42*time.Second)
+	store.jobs = []model.Job{
+		{ID: 7, ServiceID: 1, ServiceName: "slusk/web", Status: model.JobSucceeded, FromRef: "a:1", ToRef: "a:1@sha256:abc", RequestedBy: "alice", CreatedAt: started, StartedAt: &started, FinishedAt: &finished},
+		{ID: 8, ServiceID: 2, ServiceName: "jellyfin_jellyfin", Status: model.JobFailed, CreatedAt: started},
+	}
+	for _, path := range []string{"/jobs", "/fragments/jobs"} {
+		code, body := get(t, srv, path)
+		assert.Equal(t, http.StatusOK, code)
+		assert.Contains(t, body, `href="/jobs/7"`)
+		assert.Contains(t, body, `href="/services/2"`)
+		assert.Contains(t, body, "succeeded")
+		assert.Contains(t, body, "failed")
+		assert.Contains(t, body, "42s")
+		assert.Equal(t, path == "/jobs", strings.Contains(body, "<html"))
+	}
+}
+
+func TestJobPage_ShowsOutputAndError(t *testing.T) {
+	srv, store, _, _, _ := newTestEnv(t)
+	store.jobs = []model.Job{{ID: 42, ServiceID: 1, ServiceName: "slusk/web", Status: model.JobFailed,
+		Output: "Pulling web\n<b>boom</b>", Error: "pull failed", OutputTruncated: true, CreatedAt: time.Now()}}
+	code, body := get(t, srv, "/jobs/42")
+	assert.Equal(t, http.StatusOK, code)
+	assert.Contains(t, body, "Pulling web")
+	assert.Contains(t, body, "&lt;b&gt;boom", "output is escaped")
+	assert.Contains(t, body, "pull failed")
+	assert.Contains(t, body, "Earlier output truncated")
+	assert.Contains(t, body, `hx-trigger="sse:job-42"`)
+}
+
+func TestJobFragment_RereadsFromStore(t *testing.T) {
+	srv, store, _, _, _ := newTestEnv(t)
+	store.jobs = []model.Job{{ID: 42, ServiceID: 1, Status: model.JobRunning, Output: "step 1", CreatedAt: time.Now()}}
+	code, body := get(t, srv, "/fragments/jobs/42")
+	assert.Equal(t, http.StatusOK, code)
+	assert.NotContains(t, body, "<html")
+	assert.Contains(t, body, "step 1")
+
+	store.jobs[0].Output = "step 1\nstep 2"
+	_, body = get(t, srv, "/fragments/jobs/42")
+	assert.Contains(t, body, "step 2")
+}
+
+func TestJobFragment_PollsOnlyWhileActive(t *testing.T) {
+	srv, store, _, _, _ := newTestEnv(t)
+	store.jobs = []model.Job{
+		{ID: 42, ServiceID: 1, Status: model.JobRunning, CreatedAt: time.Now()},
+		{ID: 43, ServiceID: 1, Status: model.JobSucceeded, CreatedAt: time.Now()},
+	}
+	_, body := get(t, srv, "/fragments/jobs/42")
+	assert.Contains(t, body, `hx-trigger="sse:job-42, every 2s"`)
+	_, body = get(t, srv, "/fragments/jobs/43")
+	assert.Contains(t, body, `hx-trigger="sse:job-43"`)
+	assert.NotContains(t, body, "every 2s")
+}
+
+func TestJob_NotFound(t *testing.T) {
+	srv, _, _, _ := newTestServer(t)
+	for _, path := range []string{"/jobs/99", "/fragments/jobs/99"} {
+		code, _ := get(t, srv, path)
+		assert.Equal(t, http.StatusNotFound, code)
+	}
+}

internal/master/web/static/app.css

@@ -21,3 +21,10 @@ pre { padding: 12px; border: 1px solid var(--line); overflow-x: auto; }
 .warning { color: var(--warn); }
 form.inline { display: flex; gap: 8px; margin-bottom: 16px; }
 @media (max-width: 600px) { table { display: block; overflow-x: auto; } }
+.muted { color: var(--muted); }
+.status-job-succeeded { color: var(--ok); }
+.status-job-queued, .status-job-running { color: var(--warn); }
+.status-job-failed { color: var(--bad); }
+table.details { width: auto; }
+table.details th { padding-right: 24px; }
+pre.log { white-space: pre-wrap; }

internal/master/web/templates/job.html

@@ -0,0 +1,22 @@
+{{define "content"}}
+<h1>Job #{{.Job.ID}}</h1>
+<p><a href="/services/{{.Job.ServiceID}}">{{.Job.ServiceName}}</a></p>
+{{template "job" .}}
+{{end}}
+
+{{/* The wrapper is part of the fragment and swapped as outerHTML, so a finished job drops the polling timer. The timer recovers from a dropped SSE signal. */}}
+{{define "job"}}
+<div id="job" hx-get="/fragments/jobs/{{.Job.ID}}" hx-trigger="sse:job-{{.Job.ID}}{{if .Job.Active}}, every 2s{{end}}" hx-swap="outerHTML">
+<table class="details">
+  <tr><th>Status</th><td class="status status-job-{{.Job.Status}}">{{.Job.Status}}</td></tr>
+  <tr><th>From</th><td><code>{{shortRef .Job.FromRef}}</code></td></tr>
+  <tr><th>To</th><td><code>{{shortRef .Job.ToRef}}</code></td></tr>
+  <tr><th>Requested by</th><td>{{.Job.RequestedBy}}</td></tr>
+  <tr><th>Created</th><td>{{.Job.CreatedAt.Format "2006-01-02 15:04:05"}}</td></tr>
+  <tr><th>Duration</th><td>{{duration .Job}}</td></tr>
+</table>
+{{if .Job.Error}}<p class="error">{{.Job.Error}}</p>{{end}}
+{{if .Job.OutputTruncated}}<p class="muted">Earlier output truncated.</p>{{end}}
+<pre class="log">{{.Job.Output}}</pre>
+</div>
+{{end}}

internal/master/web/templates/jobs.html

@@ -0,0 +1,27 @@
+{{define "content"}}
+<h1>Jobs</h1>
+<div id="jobs" hx-get="/fragments/jobs" hx-trigger="sse:jobs" hx-swap="innerHTML">
+  {{template "jobs" .}}
+</div>
+{{end}}
+
+{{define "jobs"}}
+<table>
+  <thead><tr><th>Job</th><th>Service</th><th>Status</th><th>From → to</th><th>Requested by</th><th>Created</th><th>Duration</th></tr></thead>
+  <tbody>
+  {{range .Jobs}}
+    <tr>
+      <td><a href="/jobs/{{.ID}}">#{{.ID}}</a></td>
+      <td><a href="/services/{{.ServiceID}}">{{.ServiceName}}</a></td>
+      <td class="status status-job-{{.Status}}">{{.Status}}</td>
+      <td><code>{{shortRef .FromRef}}</code> → <code>{{shortRef .ToRef}}</code></td>
+      <td>{{.RequestedBy}}</td>
+      <td>{{.CreatedAt.Format "2006-01-02 15:04"}}</td>
+      <td>{{duration .}}</td>
+    </tr>
+  {{else}}
+    <tr><td colspan="7">No jobs yet.</td></tr>
+  {{end}}
+  </tbody>
+</table>
+{{end}}

internal/master/web/templates/layout.html

@@ -11,6 +11,7 @@
 <body hx-headers='{"X-CSRF-Token": "{{.CSRF}}"}' hx-ext="sse" sse-connect="/events">
   <nav>
     <a href="/">Services</a>
+    <a href="/jobs">Jobs</a>
     <a href="/agents">Agents</a>
     <form method="post" action="/auth/logout">
       <input type="hidden" name="csrf_token" value="{{.CSRF}}">

internal/master/web/templates/overview.html

@@ -14,7 +14,7 @@
     <tbody>
     {{range .Services}}
       <tr>
-        <td>{{.DisplayName}}{{if .Self}} <span class="badge">managed outside</span>{{end}}</td>
+        <td><a href="/services/{{.ID}}">{{.DisplayName}}</a>{{if .Self}} <span class="badge">managed outside</span>{{end}}</td>
         <td><code>{{shortRef .ActualRef}}</code></td>
         <td class="status status-{{.Status}}">{{.Status}}</td>
         <td>{{.UpdateMode}}</td>

internal/master/web/templates/service.html

@@ -0,0 +1,46 @@
+{{define "content"}}
+<h1>{{.Service.DisplayName}}</h1>
+{{if .Error}}<p class="error">{{.Error}}</p>{{end}}
+<table class="details">
+  <tr><th>Kind</th><td>{{.Service.Kind}}</td></tr>
+  <tr><th>Agent</th><td>{{if .Service.AgentName}}{{.Service.AgentName}}{{else}}Swarm{{end}}</td></tr>
+  <tr><th>Image</th><td><code>{{shortRef .Service.ActualRef}}</code></td></tr>
+  {{if .Service.ActualDigest}}<tr><th>Digest</th><td><code>{{.Service.ActualDigest}}</code></td></tr>{{end}}
+  <tr><th>Status</th><td class="status status-{{.Service.Status}}">{{.Service.Status}}</td></tr>
+  {{if .Service.DesiredRef}}<tr><th>Desired image</th><td><code>{{shortRef .Service.DesiredRef}}</code></td></tr>{{end}}
+  {{if .ConfigFiles}}<tr><th>Config files</th><td>{{range .ConfigFiles}}<code>{{.}}</code><br>{{end}}</td></tr>{{end}}
+</table>
+{{if .Reason}}
+<p>Update unavailable: {{.Reason}}.</p>
+{{else}}
+<form method="post" action="/services/{{.Service.ID}}/update">
+  <input type="hidden" name="csrf_token" value="{{.CSRF}}">
+  <button type="submit">Update</button>
+</form>
+<p class="muted">Pulls the current tag again and redeploys the service.</p>
+{{end}}
+<h2>Update history</h2>
+<div id="history" hx-get="/fragments/services/{{.Service.ID}}/jobs" hx-trigger="sse:jobs" hx-swap="innerHTML">
+  {{template "history" .}}
+</div>
+{{end}}
+
+{{define "history"}}
+<table>
+  <thead><tr><th>Job</th><th>Status</th><th>From → to</th><th>Requested by</th><th>Created</th><th>Duration</th></tr></thead>
+  <tbody>
+  {{range .Jobs}}
+    <tr>
+      <td><a href="/jobs/{{.ID}}">#{{.ID}}</a></td>
+      <td class="status status-job-{{.Status}}">{{.Status}}</td>
+      <td><code>{{shortRef .FromRef}}</code> → <code>{{shortRef .ToRef}}</code></td>
+      <td>{{.RequestedBy}}</td>
+      <td>{{.CreatedAt.Format "2006-01-02 15:04"}}</td>
+      <td>{{duration .}}</td>
+    </tr>
+  {{else}}
+    <tr><td colspan="6">No updates yet.</td></tr>
+  {{end}}
+  </tbody>
+</table>
+{{end}}

internal/master/web/views.go

@@ -2,6 +2,7 @@ package web
 
 import (
 	"strings"
+	"time"
 
 	"github.com/shcizo/package-updater/internal/imageref"
 	"github.com/shcizo/package-updater/internal/master/model"
@@ -49,3 +50,15 @@ type agentRow struct {
 	model.Agent
 	Online bool
 }
+
+// jobDuration is how long a job ran, or has been running; "" before it starts.
+func jobDuration(j model.Job) string {
+	if j.StartedAt == nil {
+		return ""
+	}
+	end := time.Now()
+	if j.FinishedAt != nil {
+		end = *j.FinishedAt
+	}
+	return end.Sub(*j.StartedAt).Round(time.Second).String()
+}

internal/master/webauth/auth.go

@@ -157,7 +157,7 @@ func (a *Auth) Require(next http.Handler) http.Handler {
 				return
 			}
 		}
-		next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), sessionKey{}, sess)))
+		next.ServeHTTP(w, r.WithContext(WithSession(r.Context(), sess)))
 	})
 }
 
@@ -171,6 +171,11 @@ func (a *Auth) session(r *http.Request) (model.Session, error) {
 
 type sessionKey struct{}
 
+// WithSession returns ctx carrying sess, as Require does.
+func WithSession(ctx context.Context, sess model.Session) context.Context {
+	return context.WithValue(ctx, sessionKey{}, sess)
+}
+
 // SessionFrom returns the session Require stored in ctx.
 func SessionFrom(ctx context.Context) (model.Session, bool) {
 	s, ok := ctx.Value(sessionKey{}).(model.Session)

internal/master/webauth/auth_test.go

@@ -247,3 +247,10 @@ func TestLogout_DeletesSession(t *testing.T) {
 	assert.Equal(t, http.StatusOK, rec.Code)
 	assert.Empty(t, store.sessions)
 }
+
+func TestWithSession_RoundTrips(t *testing.T) {
+	ctx := WithSession(context.Background(), model.Session{Subject: "alice"})
+	sess, ok := SessionFrom(ctx)
+	assert.True(t, ok)
+	assert.Equal(t, "alice", sess.Subject)
+}

internal/metrics/metrics.go

@@ -1,69 +0,0 @@
-// Package metrics defines and registers the Prometheus collectors
-// exported on /metrics.
-package metrics
-
-import (
-	"net/http"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-)
-
-// Metrics bundles all collectors. Each field is the public handle the
-// rest of the service uses to record observations.
-type Metrics struct {
-	BuildInfo      *prometheus.GaugeVec
-	HTTPRequests   *prometheus.CounterVec
-	UpdateJobs     *prometheus.CounterVec
-	UpdateDuration *prometheus.HistogramVec
-	QueueDepth     prometheus.Gauge
-	LastUpdateTime *prometheus.GaugeVec
-	DockerPingUp   prometheus.Gauge
-}
-
-// New constructs Metrics and registers them with the given registry.
-// Use prometheus.NewRegistry() in tests so collectors don't leak between
-// runs; production code uses prometheus.DefaultRegisterer.
-func New(reg prometheus.Registerer) *Metrics {
-	m := &Metrics{
-		BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{
-			Name: "package_updater_build_info",
-			Help: "Always 1. Labels carry version/commit for dashboards.",
-		}, []string{"version", "commit"}),
-		HTTPRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
-			Name: "package_updater_http_requests_total",
-			Help: "HTTP requests handled, labelled by endpoint and status.",
-		}, []string{"endpoint", "status_code"}),
-		UpdateJobs: prometheus.NewCounterVec(prometheus.CounterOpts{
-			Name: "package_updater_update_jobs_total",
-			Help: "Update jobs executed, labelled by project/service/status.",
-		}, []string{"project", "service", "status"}),
-		UpdateDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
-			Name:    "package_updater_update_duration_seconds",
-			Help:    "Time taken to pull + up a single service.",
-			Buckets: []float64{0.5, 1, 2, 5, 10, 30, 60, 120, 300},
-		}, []string{"project", "service"}),
-		QueueDepth: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "package_updater_queue_depth",
-			Help: "Current number of submissions waiting in the queue.",
-		}),
-		LastUpdateTime: prometheus.NewGaugeVec(prometheus.GaugeOpts{
-			Name: "package_updater_last_update_timestamp",
-			Help: "Unix timestamp of the most recent successful update per service.",
-		}, []string{"project", "service"}),
-		DockerPingUp: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "package_updater_docker_ping_up",
-			Help: "1 if the Docker socket responded to ping, 0 otherwise.",
-		}),
-	}
-	reg.MustRegister(
-		m.BuildInfo, m.HTTPRequests, m.UpdateJobs, m.UpdateDuration,
-		m.QueueDepth, m.LastUpdateTime, m.DockerPingUp,
-	)
-	return m
-}
-
-// Handler returns the /metrics HTTP handler bound to the given registry.
-func Handler(gatherer prometheus.Gatherer) http.Handler {
-	return promhttp.HandlerFor(gatherer, promhttp.HandlerOpts{})
-}

internal/protocol/protocol.go

@@ -1,33 +1,66 @@
 // Package protocol defines the messages agents and the master exchange over
 // their WebSocket connection. Every message is an Envelope whose Payload is
 // one of the payload types below, selected by Type.
+//
+// Both sides close the connection on a message they cannot parse (bad JSON,
+// or a known Type whose payload does not decode). A message with an unknown
+// Type is logged and ignored, so a newer peer can add message types without
+// breaking an older one.
 package protocol
 
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 )
 
+// Version is the protocol version this build speaks. Agents send it in
+// Hello. Phase-1 agents do not send the field, which reads as 0.
+const Version = 2
+
+// MinJobVersion is the lowest agent protocol version the master sends jobs to.
+const MinJobVersion = 2
+
+// MaxJobOutput caps the job output an agent keeps and the master stores, in
+// bytes. Only the tail is kept. With JSON escaping (worst case 6 bytes per
+// byte) a JobResult carrying this much output still fits the 1 MiB read limit.
+const MaxJobOutput = 128 << 10
+
 // Message types sent from agent to master.
 const (
 	TypeHello       = "hello"
 	TypeSnapshot    = "snapshot"
 	TypeUpsert      = "upsert"
 	TypeRemove      = "remove"
+	TypeJobProgress = "job.progress"
+	TypeJobResult   = "job.result"
 )
 
-// Envelope wraps every message on the wire.
+// Message types sent from master to agent.
+const (
+	TypeJobStart = "job.start"
+	TypeJobAck   = "job.ack"
+)
+
+// Envelope wraps every message on the wire. ID is the job ID (decimal) on
+// job.* messages and empty on inventory messages.
 type Envelope struct {
 	Type    string          `json:"type"`
+	ID      string          `json:"id,omitempty"`
 	Payload json.RawMessage `json:"payload,omitempty"`
 }
 
 // Hello is the first message an agent sends on every connection.
 type Hello struct {
 	AgentVersion    string `json:"agent_version"`
+	ProtocolVersion int    `json:"protocol_version,omitempty"`
 	Hostname        string `json:"hostname"`
 	SwarmManager    bool   `json:"swarm_manager"`
 	SwarmClusterID  string `json:"swarm_cluster_id,omitempty"`
+	// KnownJobs lists every job ID the agent holds: queued, running, or
+	// finished but not yet acknowledged. The master fails its queued and
+	// running jobs for this agent that are not listed ("lost").
+	KnownJobs []string `json:"known_jobs,omitempty"`
 }
 
 // Kind says how a service is deployed.
@@ -47,12 +80,19 @@ type Service struct {
 	ComposeProject string `json:"compose_project,omitempty"`
 	ComposeService string `json:"compose_service,omitempty"`
 	WorkingDir     string `json:"working_dir,omitempty"`
+	// ConfigFiles is the raw com.docker.compose.project.config_files label:
+	// absolute host paths joined by ",". Kept as a string so Service stays
+	// comparable with ==.
+	ConfigFiles   string `json:"config_files,omitempty"`
 	SwarmService  string `json:"swarm_service,omitempty"`
 	ContainerName string `json:"container_name,omitempty"`
 	Image         string `json:"image"`
 	Digest        string `json:"digest,omitempty"`
 	Status        string `json:"status"`
 	Self          bool   `json:"self,omitempty"`
+	// Built says the Compose service has a build: section; it is never
+	// updated by jobs.
+	Built bool `json:"built,omitempty"`
 }
 
 // Key identifies the service within one agent's report. It is stable across
@@ -83,6 +123,50 @@ type Remove struct {
 	Key string `json:"key"`
 }
 
+// JobStart asks the agent to deploy Ref to one service. Ref is "repo:tag";
+// the agent resolves the tag's current digest and deploys "repo:tag@digest".
+// The agent reads the Compose working directory and config files from its
+// own Docker daemon, not from this message.
+type JobStart struct {
+	Kind           Kind   `json:"kind"`
+	ComposeProject string `json:"compose_project,omitempty"`
+	ComposeService string `json:"compose_service,omitempty"`
+	SwarmClusterID string `json:"swarm_cluster_id,omitempty"`
+	SwarmService   string `json:"swarm_service,omitempty"`
+	Ref            string `json:"ref"`
+}
+
+// JobProgress carries output produced since the previous JobProgress for the
+// same job. The first JobProgress of a job (Output may be empty) tells the
+// master the job is running.
+type JobProgress struct {
+	Output string `json:"output"`
+}
+
+// Job result statuses.
+const (
+	ResultSucceeded = "succeeded"
+	ResultFailed    = "failed"
+)
+
+// JobResult is a job's outcome. The agent keeps it until the master sends
+// JobAck for the same ID, and resends it after every reconnect until then.
+type JobResult struct {
+	Status string `json:"status"`        // ResultSucceeded or ResultFailed
+	Ref    string `json:"ref,omitempty"` // deployed "repo:tag@digest", set on success
+	Error  string `json:"error,omitempty"`
+	// Output is the job's complete output, tail-capped at MaxJobOutput. It
+	// replaces whatever the master assembled from JobProgress messages.
+	Output          string    `json:"output"`
+	OutputTruncated bool      `json:"output_truncated,omitempty"`
+	StartedAt       time.Time `json:"started_at"`
+	FinishedAt      time.Time `json:"finished_at"`
+}
+
+// JobAck tells the agent the master has stored the JobResult with the same
+// ID. It has no fields.
+type JobAck struct{}
+
 // Encode wraps payload in an Envelope of the given type.
 func Encode(typ string, payload any) (Envelope, error) {
 	raw, err := json.Marshal(payload)
@@ -92,6 +176,16 @@ func Encode(typ string, payload any) (Envelope, error) {
 	return Envelope{Type: typ, Payload: raw}, nil
 }
 
+// EncodeJob wraps a job message's payload in an Envelope carrying the job ID.
+func EncodeJob(typ, jobID string, payload any) (Envelope, error) {
+	env, err := Encode(typ, payload)
+	if err != nil {
+		return Envelope{}, err
+	}
+	env.ID = jobID
+	return env, nil
+}
+
 // Decode unpacks env's payload into T.
 func Decode[T any](env Envelope) (T, error) {
 	var v T

internal/protocol/protocol_test.go

@@ -47,3 +47,35 @@ func TestDecodeRejectsMalformedPayload(t *testing.T) {
 	_, err := Decode[Hello](Envelope{Type: TypeHello, Payload: json.RawMessage(`{"hostname": 5}`)})
 	assert.ErrorContains(t, err, "decode hello")
 }
+
+func TestEncodeJob_CarriesID(t *testing.T) {
+	in := JobStart{Kind: KindSwarm, SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"}
+	env, err := EncodeJob(TypeJobStart, "42", in)
+	require.NoError(t, err)
+
+	raw, err := json.Marshal(env)
+	require.NoError(t, err)
+	var back Envelope
+	require.NoError(t, json.Unmarshal(raw, &back))
+
+	assert.Equal(t, "42", back.ID)
+	assert.Equal(t, TypeJobStart, back.Type)
+	out, err := Decode[JobStart](back)
+	require.NoError(t, err)
+	assert.Equal(t, in, out)
+}
+
+func TestHello_PhaseOneDecodesAsVersionZero(t *testing.T) {
+	var h Hello
+	require.NoError(t, json.Unmarshal([]byte(`{"agent_version":"x","hostname":"h","swarm_manager":false}`), &h))
+	assert.Equal(t, 0, h.ProtocolVersion)
+	assert.Nil(t, h.KnownJobs)
+}
+
+func TestEnvelope_InventoryHasNoID(t *testing.T) {
+	env, err := Encode(TypeSnapshot, Snapshot{})
+	require.NoError(t, err)
+	raw, err := json.Marshal(env)
+	require.NoError(t, err)
+	assert.NotContains(t, string(raw), `"id"`)
+}

internal/selfupdate/selfupdate.go

@@ -1,56 +0,0 @@
-// Package selfupdate handles the special case where the update target
-// is the running service's own container. We must finish writing the
-// HTTP response (and flush + close the connection) before exec'ing
-// `docker compose up -d` against ourselves, otherwise the response is
-// lost when the container is replaced.
-package selfupdate
-
-import (
-	"context"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-)
-
-// IsSelf reports whether job targets the running service.
-// Matches on Compose project name (which is also typically the
-// service name for single-service stacks).
-func IsSelf(job discovery.Job, selfProject string) bool {
-	return job.Project == selfProject
-}
-
-// innerExec is the executor abstraction we wrap.
-type innerExec interface {
-	Execute(ctx context.Context, job discovery.Job) error
-}
-
-// Wrapped wraps an Executor with self-update-aware deferred execution.
-type Wrapped struct {
-	inner       innerExec
-	selfProject string
-	delay       time.Duration
-}
-
-// Wrap returns a Wrapped that defers exec until after flush() for
-// self-updates. The delay is added after flush before exec, so the
-// kernel TCP buffer has time to drain.
-func Wrap(inner innerExec, selfProject string, delay time.Duration) *Wrapped {
-	return &Wrapped{inner: inner, selfProject: selfProject, delay: delay}
-}
-
-// ExecuteWithFlush runs job, invoking flush() before exec for self-updates
-// and waiting `delay` after flush. For non-self jobs, exec happens
-// normally and flush is not invoked at all (the HTTP layer decides
-// when to flush in that case).
-func (w *Wrapped) ExecuteWithFlush(ctx context.Context, job discovery.Job, flush func()) error {
-	if !IsSelf(job, w.selfProject) {
-		return w.inner.Execute(ctx, job)
-	}
-	flush()
-	select {
-	case <-time.After(w.delay):
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-	return w.inner.Execute(ctx, job)
-}

internal/selfupdate/selfupdate_test.go

@@ -1,87 +0,0 @@
-package selfupdate_test
-
-import (
-	"context"
-	"errors"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/selfupdate"
-	"github.com/stretchr/testify/require"
-)
-
-type recExec struct {
-	mu     sync.Mutex
-	called bool
-	at     time.Time
-}
-
-func (r *recExec) Execute(_ context.Context, _ discovery.Job) error {
-	r.mu.Lock()
-	r.called = true
-	r.at = time.Now()
-	r.mu.Unlock()
-	return nil
-}
-
-func TestIsSelf(t *testing.T) {
-	require.True(t, selfupdate.IsSelf(
-		discovery.Job{Project: "package-updater", Service: "package-updater"},
-		"package-updater",
-	))
-	require.False(t, selfupdate.IsSelf(
-		discovery.Job{Project: "other", Service: "web"},
-		"package-updater",
-	))
-}
-
-func TestWrap_DefersSelf(t *testing.T) {
-	inner := &recExec{}
-	wrapped := selfupdate.Wrap(inner, "package-updater", 30*time.Millisecond)
-
-	flushed := make(chan time.Time, 1)
-	flush := func() { flushed <- time.Now() }
-
-	job := discovery.Job{Project: "package-updater", Service: "package-updater"}
-	require.NoError(t, wrapped.ExecuteWithFlush(context.Background(), job, flush))
-
-	flushAt := <-flushed
-	inner.mu.Lock()
-	require.True(t, inner.called)
-	require.True(t, inner.at.After(flushAt))
-	inner.mu.Unlock()
-}
-
-func TestWrap_NormalJobIsImmediate(t *testing.T) {
-	inner := &recExec{}
-	wrapped := selfupdate.Wrap(inner, "package-updater", 30*time.Millisecond)
-
-	flushed := make(chan time.Time, 1)
-	flush := func() { flushed <- time.Now() }
-
-	job := discovery.Job{Project: "other", Service: "web"}
-	require.NoError(t, wrapped.ExecuteWithFlush(context.Background(), job, flush))
-
-	inner.mu.Lock()
-	require.True(t, inner.called)
-	inner.mu.Unlock()
-	select {
-	case <-flushed:
-		t.Fatal("flush should not be called for non-self jobs")
-	default:
-	}
-}
-
-func TestWrap_PropagatesError(t *testing.T) {
-	wrapped := selfupdate.Wrap(failExec{}, "x", 1*time.Millisecond)
-	err := wrapped.ExecuteWithFlush(context.Background(), discovery.Job{Project: "other"}, func() {})
-	require.Error(t, err)
-}
-
-type failExec struct{}
-
-func (failExec) Execute(_ context.Context, _ discovery.Job) error {
-	return errors.New("boom")
-}

internal/updater/compose_executor.go

@@ -1,54 +0,0 @@
-package updater
-
-import (
-	"context"
-	"fmt"
-	"os/exec"
-	"strings"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-)
-
-// ComposeExecutor invokes `docker compose` as a subprocess.
-type ComposeExecutor struct{}
-
-// NewComposeExecutor returns an executor that shells out to docker compose.
-func NewComposeExecutor() *ComposeExecutor {
-	return &ComposeExecutor{}
-}
-
-// Execute runs `docker compose -f <file>... -p <project> pull <service>`
-// followed by `... up -d <service>`. Working directory is set to
-// job.WorkingDir so any relative paths in the compose file resolve correctly.
-func (e *ComposeExecutor) Execute(ctx context.Context, job discovery.Job) error {
-	if job.Refused {
-		return fmt.Errorf("refused: %s", job.RefusedReason)
-	}
-	if err := e.run(ctx, job, "pull"); err != nil {
-		return fmt.Errorf("pull: %w", err)
-	}
-	if err := e.run(ctx, job, "up", "-d"); err != nil {
-		return fmt.Errorf("up: %w", err)
-	}
-	return nil
-}
-
-func (e *ComposeExecutor) run(ctx context.Context, job discovery.Job, args ...string) error {
-	cliArgs := []string{"compose"}
-	for _, f := range job.ConfigFiles {
-		cliArgs = append(cliArgs, "-f", f)
-	}
-	cliArgs = append(cliArgs, "-p", job.Project)
-	cliArgs = append(cliArgs, args...)
-	cliArgs = append(cliArgs, job.Service)
-
-	cmd := exec.CommandContext(ctx, "docker", cliArgs...)
-	cmd.Dir = job.WorkingDir
-
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("docker %s: %w (output: %s)",
-			strings.Join(args, " "), err, strings.TrimSpace(string(out)))
-	}
-	return nil
-}

internal/updater/executor.go

@@ -1,15 +0,0 @@
-// Package updater contains the FIFO job queue, the single worker that
-// drains it, and the abstraction over `docker compose` invocation.
-package updater
-
-import (
-	"context"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-)
-
-// Executor runs `docker compose pull` then `up -d` for a single job.
-// Defined as an interface so the worker can be tested with a fake.
-type Executor interface {
-	Execute(ctx context.Context, job discovery.Job) error
-}

internal/updater/queue.go

@@ -1,144 +0,0 @@
-package updater
-
-import (
-	"context"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/metrics"
-)
-
-// Status describes the outcome of executing a single Job.
-type Status string
-
-const (
-	StatusUpdated Status = "updated"
-	StatusFailed  Status = "failed"
-	StatusRefused Status = "refused"
-	StatusTimeout Status = "timeout"
-)
-
-// Result captures a per-Job outcome to be surfaced in the HTTP response.
-type Result struct {
-	Job        discovery.Job
-	Status     Status
-	Error      string
-	DurationMs int64
-}
-
-// Queue serialises Job execution through a single worker so we never
-// run two `docker compose` commands against the same stack concurrently.
-// It also satisfies the "single global FIFO worker" design choice in
-// spec section 5.7.
-type Queue struct {
-	exec    Executor
-	metrics *metrics.Metrics
-	ch      chan submission
-	stop    chan struct{}
-	done    chan struct{}
-}
-
-type submission struct {
-	ctx     context.Context
-	jobs    []discovery.Job
-	results chan []Result
-}
-
-// NewQueue constructs a queue bound to the given executor. m may be nil,
-// in which case metrics recording is silently skipped.
-func NewQueue(exec Executor, m *metrics.Metrics) *Queue {
-	return &Queue{
-		exec:    exec,
-		metrics: m,
-		ch:      make(chan submission, 16),
-		stop:    make(chan struct{}),
-		done:    make(chan struct{}),
-	}
-}
-
-// Start launches the single background worker. Call Stop to terminate.
-func (q *Queue) Start(ctx context.Context) {
-	go q.run(ctx)
-}
-
-// Stop signals the worker to exit and waits for it to finish.
-func (q *Queue) Stop() {
-	close(q.stop)
-	<-q.done
-}
-
-// Submit blocks until all jobs in the batch have been processed by the
-// worker, then returns per-job results in the same order. The ctx
-// timeout (if any) applies to each individual Job's execution and is
-// surfaced as StatusTimeout.
-func (q *Queue) Submit(ctx context.Context, jobs []discovery.Job) []Result {
-	resCh := make(chan []Result, 1)
-	q.ch <- submission{ctx: ctx, jobs: jobs, results: resCh}
-	if q.metrics != nil {
-		q.metrics.QueueDepth.Set(float64(len(q.ch)))
-	}
-	return <-resCh
-}
-
-func (q *Queue) run(_ context.Context) {
-	defer close(q.done)
-	for {
-		select {
-		case <-q.stop:
-			return
-		case s := <-q.ch:
-			results := make([]Result, len(s.jobs))
-			for i, j := range s.jobs {
-				results[i] = q.runOne(s.ctx, j)
-			}
-			s.results <- results
-		}
-	}
-}
-
-func (q *Queue) runOne(ctx context.Context, job discovery.Job) Result {
-	start := time.Now()
-	r := Result{Job: job}
-
-	if job.Refused {
-		r.Status = StatusRefused
-		r.Error = job.RefusedReason
-		r.DurationMs = time.Since(start).Milliseconds()
-		if q.metrics != nil {
-			q.metrics.UpdateJobs.WithLabelValues(job.Project, job.Service, string(StatusRefused)).Inc()
-			q.metrics.UpdateDuration.WithLabelValues(job.Project, job.Service).Observe(time.Since(start).Seconds())
-		}
-		return r
-	}
-
-	err := q.exec.Execute(ctx, job)
-	r.DurationMs = time.Since(start).Milliseconds()
-	switch {
-	case err == nil:
-		r.Status = StatusUpdated
-	case errorsIsContextDeadline(err) || ctxDeadlineExceeded(ctx):
-		r.Status = StatusTimeout
-		r.Error = err.Error()
-	default:
-		r.Status = StatusFailed
-		r.Error = err.Error()
-	}
-
-	if q.metrics != nil {
-		elapsed := time.Since(start).Seconds()
-		q.metrics.UpdateJobs.WithLabelValues(job.Project, job.Service, string(r.Status)).Inc()
-		q.metrics.UpdateDuration.WithLabelValues(job.Project, job.Service).Observe(elapsed)
-		if r.Status == StatusUpdated {
-			q.metrics.LastUpdateTime.WithLabelValues(job.Project, job.Service).SetToCurrentTime()
-		}
-	}
-	return r
-}
-
-func errorsIsContextDeadline(err error) bool {
-	return err == context.DeadlineExceeded
-}
-
-func ctxDeadlineExceeded(ctx context.Context) bool {
-	return ctx.Err() == context.DeadlineExceeded
-}

internal/updater/queue_test.go

@@ -1,129 +0,0 @@
-package updater_test
-
-import (
-	"context"
-	"errors"
-	"sync"
-	"testing"
-	"time"
-
-	dto "github.com/prometheus/client_model/go"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/metrics"
-	"github.com/shcizo/package-updater/internal/updater"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeExec struct {
-	mu        sync.Mutex
-	calls     []discovery.Job
-	delay     time.Duration
-	errForSvc map[string]error
-}
-
-func (f *fakeExec) Execute(ctx context.Context, job discovery.Job) error {
-	f.mu.Lock()
-	f.calls = append(f.calls, job)
-	f.mu.Unlock()
-	if f.delay > 0 {
-		select {
-		case <-time.After(f.delay):
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-	if err, ok := f.errForSvc[job.Service]; ok {
-		return err
-	}
-	return nil
-}
-
-func (f *fakeExec) callsCopy() []discovery.Job {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return append([]discovery.Job(nil), f.calls...)
-}
-
-func TestQueue_ProcessesFIFO(t *testing.T) {
-	exec := &fakeExec{delay: 20 * time.Millisecond}
-	q := updater.NewQueue(exec, nil)
-	q.Start(context.Background())
-	defer q.Stop()
-
-	job1 := discovery.Job{Project: "a", Service: "svc"}
-	job2 := discovery.Job{Project: "b", Service: "svc"}
-	job3 := discovery.Job{Project: "c", Service: "svc"}
-
-	results := make(chan []updater.Result, 3)
-	go func() { results <- q.Submit(context.Background(), []discovery.Job{job1}) }()
-	time.Sleep(5 * time.Millisecond)
-	go func() { results <- q.Submit(context.Background(), []discovery.Job{job2}) }()
-	time.Sleep(5 * time.Millisecond)
-	go func() { results <- q.Submit(context.Background(), []discovery.Job{job3}) }()
-
-	for i := 0; i < 3; i++ {
-		<-results
-	}
-
-	calls := exec.callsCopy()
-	require.Equal(t, []string{"a", "b", "c"}, []string{calls[0].Project, calls[1].Project, calls[2].Project})
-}
-
-func TestQueue_ReturnsPerJobResults(t *testing.T) {
-	exec := &fakeExec{errForSvc: map[string]error{"failing": errors.New("boom")}}
-	q := updater.NewQueue(exec, nil)
-	q.Start(context.Background())
-	defer q.Stop()
-
-	jobs := []discovery.Job{
-		{Project: "p1", Service: "ok"},
-		{Project: "p2", Service: "failing"},
-		{Project: "p3", Service: "refused-svc", Refused: true, RefusedReason: "outside root"},
-	}
-	results := q.Submit(context.Background(), jobs)
-
-	require.Len(t, results, 3)
-	require.Equal(t, updater.StatusUpdated, results[0].Status)
-	require.Equal(t, updater.StatusFailed, results[1].Status)
-	require.Contains(t, results[1].Error, "boom")
-	require.Equal(t, updater.StatusRefused, results[2].Status)
-	require.Contains(t, results[2].Error, "outside root")
-}
-
-func TestQueue_Timeout(t *testing.T) {
-	exec := &fakeExec{delay: 200 * time.Millisecond}
-	q := updater.NewQueue(exec, nil)
-	q.Start(context.Background())
-	defer q.Stop()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
-	defer cancel()
-
-	jobs := []discovery.Job{{Project: "slow", Service: "svc"}}
-	results := q.Submit(ctx, jobs)
-
-	require.Len(t, results, 1)
-	require.Equal(t, updater.StatusTimeout, results[0].Status)
-}
-
-func TestQueue_RecordsMetrics(t *testing.T) {
-	reg := prometheus.NewRegistry()
-	m := metrics.New(reg)
-	exec := &fakeExec{}
-	q := updater.NewQueue(exec, m)
-	q.Start(context.Background())
-	defer q.Stop()
-
-	q.Submit(context.Background(), []discovery.Job{{Project: "p", Service: "s"}})
-
-	// Verify UpdateJobs counter was incremented for the successful job.
-	metric := &dto.Metric{}
-	require.NoError(t, m.UpdateJobs.WithLabelValues("p", "s", "updated").Write(metric))
-	require.Equal(t, 1.0, metric.GetCounter().GetValue())
-
-	// Verify LastUpdateTime was set (non-zero).
-	tsMetric := &dto.Metric{}
-	require.NoError(t, m.LastUpdateTime.WithLabelValues("p", "s").Write(tsMetric))
-	require.Greater(t, tsMetric.GetGauge().GetValue(), 0.0)
-}

internal/updater/swarm_executor.go

@@ -1,52 +0,0 @@
-package updater
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/docker/docker/api/types/swarm"
-	"github.com/shcizo/package-updater/internal/discovery"
-)
-
-// SwarmDockerClient is the subset of the Docker SDK SwarmExecutor depends
-// on. Defined as an interface so tests can supply a fake.
-type SwarmDockerClient interface {
-	ServiceInspectWithRaw(ctx context.Context, serviceID string, opts swarm.ServiceInspectOptions) (swarm.Service, []byte, error)
-	ServiceUpdate(ctx context.Context, serviceID string, version swarm.Version, service swarm.ServiceSpec, opts swarm.ServiceUpdateOptions) (swarm.ServiceUpdateResponse, error)
-}
-
-// SwarmExecutor updates a Swarm service's image via the Docker API,
-// equivalent to `docker service update --image`.
-type SwarmExecutor struct {
-	cli SwarmDockerClient
-}
-
-// NewSwarmExecutor returns an executor that drives Swarm service updates
-// through the Docker API.
-func NewSwarmExecutor(cli SwarmDockerClient) *SwarmExecutor {
-	return &SwarmExecutor{cli: cli}
-}
-
-// Execute inspects the service to get its current spec and version (required
-// by the Docker API as an optimistic-concurrency token), sets the new image
-// on the container spec, and calls ServiceUpdate. QueryRegistry is set so a
-// floating tag (e.g. ":latest") resolves to a fresh digest and actually
-// triggers a rolling update instead of being treated as unchanged.
-func (e *SwarmExecutor) Execute(ctx context.Context, job discovery.Job) error {
-	if job.Refused {
-		return fmt.Errorf("refused: %s", job.RefusedReason)
-	}
-
-	svc, _, err := e.cli.ServiceInspectWithRaw(ctx, job.ServiceID, swarm.ServiceInspectOptions{})
-	if err != nil {
-		return fmt.Errorf("inspect service %s: %w", job.ServiceID, err)
-	}
-
-	spec := svc.Spec
-	spec.TaskTemplate.ContainerSpec.Image = job.Image
-
-	if _, err := e.cli.ServiceUpdate(ctx, job.ServiceID, svc.Version, spec, swarm.ServiceUpdateOptions{QueryRegistry: true}); err != nil {
-		return fmt.Errorf("update service %s: %w", job.ServiceID, err)
-	}
-	return nil
-}

internal/updater/swarm_executor_test.go

@@ -1,99 +0,0 @@
-package updater_test
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/docker/docker/api/types/swarm"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/updater"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeSwarmDockerClient struct {
-	inspectService swarm.Service
-	inspectErr     error
-	updateErr      error
-
-	inspectCalled         bool
-	updateCalled          bool
-	updateCalledServiceID string
-	updateCalledVersion   swarm.Version
-	updateCalledSpec      swarm.ServiceSpec
-	updateCalledOpts      swarm.ServiceUpdateOptions
-}
-
-func (f *fakeSwarmDockerClient) ServiceInspectWithRaw(_ context.Context, _ string, _ swarm.ServiceInspectOptions) (swarm.Service, []byte, error) {
-	f.inspectCalled = true
-	return f.inspectService, nil, f.inspectErr
-}
-
-func (f *fakeSwarmDockerClient) ServiceUpdate(_ context.Context, serviceID string, version swarm.Version, spec swarm.ServiceSpec, opts swarm.ServiceUpdateOptions) (swarm.ServiceUpdateResponse, error) {
-	f.updateCalled = true
-	f.updateCalledServiceID = serviceID
-	f.updateCalledVersion = version
-	f.updateCalledSpec = spec
-	f.updateCalledOpts = opts
-	return swarm.ServiceUpdateResponse{}, f.updateErr
-}
-
-func mkInspectService(version uint64, image string) swarm.Service {
-	return swarm.Service{
-		ID:   "svc-myapp",
-		Meta: swarm.Meta{Version: swarm.Version{Index: version}},
-		Spec: swarm.ServiceSpec{
-			Annotations:  swarm.Annotations{Name: "myapp_web"},
-			TaskTemplate: swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{Image: image}},
-		},
-	}
-}
-
-func TestSwarmExecutor_UpdatesImageAndVersion(t *testing.T) {
-	fake := &fakeSwarmDockerClient{inspectService: mkInspectService(7, "registry.example.com/myapp:v1")}
-	e := updater.NewSwarmExecutor(fake)
-
-	job := discovery.Job{Service: "myapp_web", ServiceID: "svc-myapp", Image: "registry.example.com/myapp:v2"}
-	err := e.Execute(context.Background(), job)
-
-	require.NoError(t, err)
-	require.True(t, fake.updateCalled)
-	require.Equal(t, "svc-myapp", fake.updateCalledServiceID)
-	require.Equal(t, uint64(7), fake.updateCalledVersion.Index)
-	require.Equal(t, "registry.example.com/myapp:v2", fake.updateCalledSpec.TaskTemplate.ContainerSpec.Image)
-	require.True(t, fake.updateCalledOpts.QueryRegistry)
-}
-
-func TestSwarmExecutor_InspectError(t *testing.T) {
-	fake := &fakeSwarmDockerClient{inspectErr: errors.New("service not found")}
-	e := updater.NewSwarmExecutor(fake)
-
-	err := e.Execute(context.Background(), discovery.Job{ServiceID: "svc-missing", Image: "x:v2"})
-
-	require.Error(t, err)
-	require.False(t, fake.updateCalled)
-}
-
-func TestSwarmExecutor_UpdateError(t *testing.T) {
-	fake := &fakeSwarmDockerClient{
-		inspectService: mkInspectService(1, "registry.example.com/myapp:v1"),
-		updateErr:      errors.New("update rejected"),
-	}
-	e := updater.NewSwarmExecutor(fake)
-
-	err := e.Execute(context.Background(), discovery.Job{ServiceID: "svc-myapp", Image: "registry.example.com/myapp:v2"})
-
-	require.Error(t, err)
-}
-
-func TestSwarmExecutor_RefusedJobIsNotExecuted(t *testing.T) {
-	fake := &fakeSwarmDockerClient{}
-	e := updater.NewSwarmExecutor(fake)
-	job := discovery.Job{ServiceID: "svc-myapp", Image: "x:v2", Refused: true, RefusedReason: "not opted in"}
-
-	err := e.Execute(context.Background(), job)
-
-	require.Error(t, err)
-	require.False(t, fake.inspectCalled)
-	require.False(t, fake.updateCalled)
-}