feat: master/agent phase 5 — drift and merge, backend (#10) (#30)

## Summary
- Drift detection: `model.Service.Drifted()` compares desired and running version by digest (tag when the digest is unknown). `services.awaiting_inventory` hides drift between a job result and the next inventory write.
- Poll and webhook jobs are refused for a drifted service; `POST /services/{id}/restore` (manual job to `desired_ref`) and `POST /services/{id}/accept` (running version becomes desired, guarded by `actual` = `Service.RunningRef`).
- Move suggestions (`store.MoveCandidates`, Compose only) and `POST /services/{id}/merge`: the old row keeps its id, mode, `desired_ref` and history and takes the new location; the duplicate is deleted.
- Backend only. The buttons and badges come with #28, on top of the Ledger UI (#18).

Closes #10

## Test plan
- [x] `gofmt -l .`, `go vet ./...`, `go test ./...`
- [x] `go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/`
- [x] `go test -tags integration ./internal/master/store/` against postgres:17-alpine
- [ ] On the server: migration 0005 applies; no drift shown after a normal update
- [ ] On the server: manual `docker compose up` with another tag gives drift; webhook for an auto service is refused
- [ ] On the server: restore and accept via POST; merge after moving a stack

Reviewed-on: #30

.dockerignore

@@ -7,4 +7,3 @@ README.md
 .env
 .env.*
 gitea-action/
-docker-compose.example.yml

CLAUDE.md

@@ -1,104 +1,183 @@
 # package-updater — Claude context
 
-Webhook-driven Docker Compose updater. Go service, single binary, deployed as
-a container with `/var/run/docker.sock` mounted.
+Master/agent fleet updater. A master (UI behind OIDC, Postgres) shows every Compose
+stack, Swarm service and container on every machine and starts updates; one agent per
+machine (`/var/run/docker.sock` mounted) reports its inventory and runs the jobs.
+Both are in one image: `updater-master` and `updater-agent`.
 
 See `README.md` for user-facing docs. This file is for working ON the code.
 
 ## Commands
 
 ```bash
-go test ./...                          # all tests, fast (no docker required)
-go test -run TestFoo ./internal/api    # single test
-go build ./cmd/server                  # produces ./server
-go build ./cmd/master ./cmd/agent      # new master/agent binaries (phase 1: inventory)
-DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/   # Postgres tests
+go test ./...                          # all tests, fast (no docker, no postgres)
+go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/
+go test -run TestFoo ./internal/master/hub    # single test
+go build ./cmd/master ./cmd/agent      # produce ./master and ./agent
 docker build -t package-updater:dev .  # multi-stage, builds golang:1.26-alpine
 
-# Local smoke test (builds from Dockerfile, runs against /var/run/docker.sock):
-cp env.sample .env && $EDITOR .env     # fill UPDATER_API_KEY
-docker compose -f docker-compose.local.yml up --build
-curl -sH "Authorization: Bearer $UPDATER_API_KEY" \
-  -d '{"image":"foo"}' http://localhost:8080/update | jq
+# Postgres tests (store), against any throwaway database:
+DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/
 ```
 
 ## Architecture (one-line per package)
 
-- `cmd/server`        — wiring only: config → docker client → discovery → queue → http
-- `internal/api`      — HTTP handlers, bearer-token auth, request-id + access-log middleware
-- `internal/config`   — env-var loading; fails fast if `UPDATER_API_KEY` missing
-- `internal/discovery`— given an image, return Compose `Job`s to run (label parsing, path check, dedup)
-- `internal/discovery/swarm.go`  — SwarmDiscovery: same FindJobs signature, lists `docker service ls`, gates on service-level opt-in label instead of STACKS_ROOT path-check
-- `internal/updater`  — FIFO queue + single worker + `docker compose` subprocess executor
-- `internal/updater/swarm_executor.go` — SwarmExecutor: `docker service update --image` via the Docker API instead of a `docker compose` subprocess
-- `internal/selfupdate` — flush-then-exec wrapper for updating ourselves (NOT wired in live, see gotcha)
-- `internal/metrics`  — Prometheus collectors
+- `cmd/master`        — wiring only: config → store → hub → jobs → web + webhook, two listeners (UI, agents over pinned TLS)
+- `cmd/agent`         — wiring only: config → docker client → inventory → job runner → master connection
+- `internal/protocol` — the WebSocket messages both sides speak (`Version`, `JobStart`, `JobResult`, ...)
+- `internal/pin`      — master key pair, certificate and fingerprint pinning
+- `internal/agenttoken` — agent token generation and hashing
+- `internal/imageref` — image reference parsing (repo, tag, digest)
 - `internal/logging`  — slog JSON, request-id context propagation
-- `cmd/master`, `cmd/agent` — master/agent architecture (spec `2026-10-12-master-agent-design.md`), phase 1 = inventory only. `cmd/server` keeps doing updates until phase 3.
-- `internal/protocol`  — agent↔master WebSocket messages
-- `internal/pin`       — master key pair, self-signed cert, fingerprint-pinned TLS client
-- `internal/agenttoken`— per-agent tokens (`pua_` prefix, only the SHA-256 hash is stored)
-- `internal/imageref`  — split `repo:tag@digest`
-- `internal/agent/...` — inventory collection, Docker event debounce, connection loop
-- `internal/master/...`— Postgres store, agent hub, change broker, OIDC auth, htmx UI
+- `internal/agent/config`    — env-var loading; fails fast on missing `MASTER_URL`/`MASTER_KEY`/`AGENT_TOKEN`
+- `internal/agent/inventory` — reads Compose, Swarm and plain containers from the Docker daemon
+- `internal/agent/conn`      — dial, hello, snapshot, job messages, reconnect with backoff; one writer goroutine per session
+- `internal/agent/updater`   — job `Runner` (single FIFO worker, outbox until acked), Compose and Swarm executors, override file, mount-root check, registry auth
+- `internal/master/config`   — env-var loading
+- `internal/master/store`    — Postgres access and migrations
+- `internal/master/hub`      — agent connections: auth, per-connection writer, routing, job reconcile on hello
+- `internal/master/registry` — HEAD-only digest lookup against a registry (go-containerregistry, `DOCKER_CONFIG` credentials)
+- `internal/master/poller`   — every `POLL_INTERVAL`: one lookup per followed `repo:tag`, sets the available version, starts auto jobs
+- `internal/master/jobs`     — decides whether a service may be updated and starts a job on the right agent
+- `internal/master/webhook`  — `POST /update`: records the available version, starts jobs for auto services, waits for them
+- `internal/master/web`, `webauth` — HTMX UI behind OIDC login, SSE signals
+- `internal/master/events`   — in-process publish/subscribe for UI refresh signals
+- `internal/master/model`    — domain types shared by the above
 
 ## Design intent (do not break without discussion)
 
-- **Single FIFO worker by design** (`internal/updater/queue.go`). Spec §5.7. Never
-  parallelise the queue — two `docker compose` calls against the same stack race.
-- **Defense in depth: token AND opt-in label AND STACKS_ROOT prefix** must all
-  hold before a container is touched. Weakening any of these breaks the security
-  model — discuss before changing.
-- **Auth on `/update` only**. `/healthz`, `/metrics`, `/version` are intentionally
-  unauthenticated (internal network, scraper/healthcheck need them). See
-  `cmd/server/main.go` `routeAuth`.
-- **`metrics *Metrics` parameters may be nil**; constructors and call sites check.
-  Tests rely on this — don't drop the nil-check.
-- **Stateless**: no DB, no config file, no on-disk audit log. Docker daemon is
-  the source of truth.
-- **Compose mode and Swarm mode are selected once per deployment via `MODE`**, never
-  mixed at request time. Swarm mode's security gate is opt-in label only — there is
-  no STACKS_ROOT-equivalent path check, since Swarm services have no local compose
-  file. Don't add one; don't weaken Compose mode's three-factor gate to match.
+- **Single FIFO worker per agent by design** (`internal/agent/updater/runner.go`).
+  Two `docker compose` calls against the same stack race.
+- **One queued or running job per service**, enforced by the unique index
+  `update_jobs_one_active` (not by application code).
+- **The agent writes exactly one file**, `<stack dir>/compose.override.yaml`, and only
+  if it is absent or starts with the header line. It refuses the job when any other
+  auto-loaded override name exists.
+- **Every config file and the override path must be inside a read-write bind mount of
+  the agent's own container whose source equals its destination.** Mounts are read per
+  job (`ContainerInspect(SELF_ID)`) and checked after symlink resolution. This replaces
+  the old `STACKS_ROOT`.
+- **Self-protection on the master**: a service is refused when its image repo equals the
+  repo of `AGENT_IMAGE` or the row has the `self` flag. The agent does not re-check.
+- **Unknown message types are logged and ignored; malformed messages close the connection.**
+- **Reconcile before routable**: the hub fails a reconnecting agent's jobs it does not
+  list in `known_jobs`, and only then accepts jobs for that connection.
+- **The webhook never creates services; new services always start `manual`.** The upsert
+  does not touch `update_mode`.
+- **Drift is never corrected automatically; poll/webhook jobs are refused for a drifted
+  service** (`jobs.Start`, `ReasonDrifted`). Only a manual job, `RestoreDesired` or an
+  accept gets it out.
+- **Merging is never automatic; merge is the only path that deletes a `services` row**
+  (`store.MergeService`). The old row keeps its id and takes the duplicate's location.
+- **Auth**: agents by token over pinned TLS; people by OIDC; CI by the `UPDATER_API_KEY`
+  bearer token on `POST /update` only. Every state-changing UI request is a POST with a
+  CSRF token.
 
 ## Gotchas
 
-- **`internal/selfupdate.Wrapped` is implemented and unit-tested but NOT wired
-  into the live queue.** Spec §15 calls this out as a v1 gap. If you "fix" this,
-  read the package doc — flush-before-exec ordering is subtle.
-- **Image matching is case-sensitive and tag/digest-agnostic.** The tricky case
-  is `localhost:5000/foo:v1` where the first colon is a port, not a tag. See
-  `internal/discovery/matching.go` `NormaliseImage`.
+- **Registry access is HEAD only** (`internal/master/registry`). Docker Hub counts a manifest
+  GET as a pull; the client never falls back to GET, and a test asserts this.
+- **`image_versions` uses `''` instead of NULL** for a missing tag or digest, so
+  `UNIQUE (repo, tag, digest)` deduplicates. The webhook records `(repo, tag, '')`; that
+  row is hidden from `RecentVersions` when a row with the same tag and a digest was first
+  seen at or after it.
+- **The UI picks a version by `image_versions.id`, never by a ref from the form**, and the row
+  must belong to the service's `image_repo`. A digest-pinned or tagless pick needs agent
+  protocol 3 (`protocol.MinPinnedRefVersion`); `Hub.Route` takes the minimum version.
+- **Picking a tagless version (`repo@digest`) leaves the service following no tag:** it is no
+  longer polled and the plain Update button is refused.
+- **`services.last_tag_digest`** is set by the poller when it starts an auto job and when it
+  sees the service already running the tag's digest, and by `FinishJob` after any successful
+  plain tag job (not pinned/tagless picks; the digest is the one the agent pinned). The poller
+  starts no job for that digest, nor for one `desired_ref` (`ref@digest`) already names, and
+  re-reads the service before `Start`; so a rollback of an auto service holds until the
+  registry shows a different digest. `ErrJobActive` and `NotUpdatableError` leave it unset so
+  the next cycle retries.
+- **A bare image's running version is recorded with tag `latest`** (like `jobs.FollowedRef`);
+  `repo@digest` is recorded with tag `''`. Only `sha256:<64 hex>` digests are recorded.
+- **Image matching is case-sensitive.** The tricky case is `localhost:5000/foo:v1` where
+  the first colon is a port, not a tag. See `internal/imageref`.
 - **Path safety uses `filepath.Rel` + ".." prefix check**, NOT `strings.HasPrefix`.
-  Prevents the `/foo-evil` vs `/foo` confusion. See `internal/discovery/pathcheck.go`.
-- **`STACKS_ROOT` defaults to `/home/shcizo/self-hosted`** in prod but `env.sample`
-  suggests `/tmp` for smoke tests. The default in `config.go` is the prod value
-  — set explicitly in test/dev envs.
+  Prevents the `/foo-evil` vs `/foo` confusion. See `internal/agent/updater/pathcheck.go`.
 - **Go 1.26.3** (`go.mod`). Dockerfile pins `golang:1.26-alpine`. Bumping one
   without the other has caused a fix commit already.
 - **`<summary>` C# convention from global CLAUDE.md does not apply here** — this
   is Go. Use idiomatic GoDoc (`// FuncName does X.`).
-- **Swarm mode requires manager-node API access.** `docker service update` fails
-  with a permission error against a worker-only node. This is an operator/deployment
-  concern (point `DOCKER_HOST` at a manager, or schedule the updater on a manager),
-  not something the code can detect or work around.
-- **WebSocket read limit is raised to 1 MiB on both sides.** coder/websocket defaults to 32 KiB, which a full snapshot from a busy machine exceeds — symptom is an agent stuck in a reconnect loop.
-- **The master's key lives in Postgres (`master_identity`).** Wiping the database means every agent needs a new `MASTER_KEY`.
-- **SSE responses must flush through every wrapper.** `web.events` uses `http.ResponseController`; a middleware that wraps `ResponseWriter` without `Unwrap()` silently breaks live updates. Nginx needs `X-Accel-Buffering: no` (set by the handler).
-- **The "stateless" and "label gate" design intents above describe `cmd/server` only.** The master is stateful by design and gates updates per service in the UI (spec 2026-10-12).
-- **The master does OIDC discovery once at startup and exits if the issuer is unreachable.** It does not retry; Swarm restarts the container, so a short Authentik outage during deploy resolves itself.
+- **Compose loads only one override file.** Of `compose.override.yml`,
+  `compose.override.yaml`, `docker-compose.override.yml`, `docker-compose.override.yaml`
+  only the first that exists is read. Hence the refusal described above.
+- **Stacks started with explicit `-f` do not load the override on a manual `up`.** The
+  agent's own jobs always pass it explicitly.
+- **Override entries are never pruned.** A service deleted from the stack file keeps its
+  `image:` line in `compose.override.yaml`.
+- **The override file gets the owner of the existing override, else of the stack's first
+  compose file.** The exception is a root-owned existing override (written before this
+  fix): then the compose file's owner is used, so it stops being root-owned. The agent runs
+  as root, so `WriteOverride` chowns the open temp file before the rename (never the path
+  afterwards: a symlink could be swapped in). A failure is a `warning:` line in the job
+  output, never a failed job.
+- **There is no `STACKS_ROOT`.** Stack directories must be mounted read-write at the host
+  path (`/path:/path`), and `SELF_ID` must resolve to the agent's own container. On Docker
+  Desktop/OrbStack bind sources are translated, so Compose jobs fail there.
+- **`job.start` carries no paths.** The agent reads working dir and config files from its
+  own Docker daemon at execution time.
+- **`protocol.Service.ConfigFiles` is the raw comma-joined label string**, so `Service`
+  stays comparable with `==` (`inventory.Diff` depends on it).
+- **Swarm "succeeded" means the daemon accepted the new spec**, not that tasks converged.
+- **Swarm updates need manager-node API access** for the agent doing the update.
+- **The image's default entrypoint is `updater-agent`.** The master overrides it with
+  `updater-master`.
+- **The example stack runs the master as nobody (`user: "65534:65534"`).** Never add `USER`
+  to the Dockerfile: the agent uses the same image and needs root for the Docker socket.
+- **The webhook waits via a broker signal plus a store re-read, with a fallback poll**
+  (the broker drops signals for slow subscribers). Marking and starting jobs runs on
+  `context.WithoutCancel`, so a client that disconnects cannot leave a queued job no agent
+  knows about; only the waiting stops. The master never fails or cancels a timed-out job.
+- **A webhook job's target is `<image_repo>:<request tag>`**, built from the service's own
+  repo, never from the request's image string. With an explicit target the "follows no tag"
+  refusal does not apply; removed, self, built and container still do.
+- **`available_*` is cleared by `FinishJob`** on success only when the job's `to_ref`
+  (as created, without `@digest`) equals `available_ref`, `available_at <= job.created_at`
+  (that version is now running) and, when the job's `to_ref` carries a digest, that digest
+  equals `available_digest`. A rollback to an older digest of the tag therefore keeps the
+  newer suggestion, and a digest-pinned pick does not clear a digest-less (webhook) one.
+- **Job output is capped at `protocol.MaxJobOutput` (128 KiB, tail).** NUL bytes are
+  stripped by the hub because Postgres text rejects them.
+- **Drift compares by digest, not by string** (`model.RunsRef`): `repo:tag@D` is run by
+  `repo:tag` reporting digest D. Same digest under another tag is not drift. Different
+  repo strings (`nginx` vs `docker.io/library/nginx`) are.
+- **`services.awaiting_inventory`** is true from a successful `FinishJob` until the next
+  inventory write for the row; `Service.Drifted()` is false meanwhile, since `desired_ref`
+  is ahead of `actual_ref` for a debounce interval after every job.
+- **Accept never touches `last_tag_digest`, `available_*` or `update_mode`**, and the
+  `actual` form field (`Service.RunningRef()`, ref plus digest) is only a guard against a
+  changed service; the value written comes from the row. `RestoreDesired` sets
+  `last_tag_digest` to the drifted digest when it is another digest of the desired tag and
+  `last_tag_digest` is empty or equals the desired digest (otherwise it holds back a newer
+  digest and stays).
+- **`awaiting_inventory` can hide real drift** until the next inventory change when a job
+  result arrives after the snapshot that shows the drift (an agent reconnect resends its
+  outbox after the snapshot), or when Swarm coalesces an update and a rollback into one
+  debounce interval. This is accepted.
+- **Merge lock order is agent, then service rows by id** (as in inventory writes), and the
+  duplicate row is deleted before the old row is updated, because `location_key` is unique
+  and not deferrable. A removal racing a merge can be lost; the next snapshot fixes it.
 
 ## References
 
-- Design spec: `docs/superpowers/specs/2026-05-22-package-updater-design.md` (489 lines, authoritative)
-- Implementation plan: `docs/superpowers/plans/2026-05-22-package-updater-implementation.md`
-- Consumer-side CI integration: `gitea-action/`
-- Master/agent spec: `docs/superpowers/specs/2026-10-12-master-agent-design.md`
+- Master/agent spec: `docs/superpowers/specs/2026-10-12-master-agent-design.md` (authoritative)
 - Phase 1 plan: `docs/superpowers/plans/2026-10-12-master-agent-phase-1-inventory.md`
+- Phase 2 plan: `docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md`
+- Phase 3 plan: `docs/superpowers/plans/2026-10-12-master-agent-phase-3-webhook.md`
+- Phase 4 plan: `docs/superpowers/plans/2026-10-13-master-agent-phase-4-polling.md`
+- Phase 5 plan: `docs/superpowers/plans/2026-10-13-master-agent-phase-5-drift-merge.md`
+- Original single-instance design (historical):
+  `docs/superpowers/specs/2026-05-22-package-updater-design.md`
+- Consumer-side CI integration: `gitea-action/` (calls the master's `POST /update`)
 
 ## Conventions
 
 - Conventional Commits (`feat:`, `fix:`, `docs:`, `ci:`, `build:`, `chore:`).
 - One package = one responsibility; interfaces defined at consumer site
-  (`api.Finder`, `api.Submitter`, `api.Pinger`) for testability.
+  (`jobs.Store`, `jobs.Hub`, `conn.Jobs`, `hub.Store`) for testability.
 - Table-driven tests with `stretchr/testify`. No mocks beyond hand-written fakes.

Dockerfile

@@ -8,7 +8,7 @@ COPY . .
 ARG VERSION=dev
 ARG COMMIT=unknown
 ARG BUILD_TIME=unknown
-RUN for cmd in server:package-updater master:updater-master agent:updater-agent; do \
+RUN for cmd in master:updater-master agent:updater-agent; do \
       CGO_ENABLED=0 go build \
         -ldflags="-s -w \
           -X main.version=${VERSION} \
@@ -20,5 +20,5 @@ RUN for cmd in server:package-updater master:updater-master agent:updater-agent;
 FROM alpine:3.20
 RUN apk add --no-cache docker-cli docker-cli-compose ca-certificates wget
 COPY --from=build /out/ /usr/local/bin/
-EXPOSE 8080
-ENTRYPOINT ["/usr/local/bin/package-updater"]
+EXPOSE 8080 8090
+ENTRYPOINT ["/usr/local/bin/updater-agent"]

README.md

@@ -1,121 +1,333 @@
 # package-updater
 
-Webhook-driven Docker Compose service updater. Fills the gap between Watchtower (polling, no CI integration) and full GitOps (Argo CD, Flux) for a self-hosted, single-host environment.
+Updates self-hosted Docker Compose and Docker Swarm services from one place. A
+**master** (web UI behind OIDC login, state in Postgres) shows every Compose stack,
+Swarm service and container on every machine, and starts updates when someone clicks
+*Update*. One **agent** per machine reports its inventory and runs the jobs. Both are in
+the same image: `updater-master` and `updater-agent` (the default entrypoint).
 
-**Trigger flow:**
+Design: [master/agent spec](docs/superpowers/specs/2026-10-12-master-agent-design.md).
+Phase 2 plan: [jobs and manual update](docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md).
+Phase 3 plan: [webhook and auto mode](docs/superpowers/plans/2026-10-12-master-agent-phase-3-webhook.md).
+Phase 4 plan: [registry polling and version history](docs/superpowers/plans/2026-10-13-master-agent-phase-4-polling.md).
+Phase 5 plan: [drift and merge](docs/superpowers/plans/2026-10-13-master-agent-phase-5-drift-merge.md).
 
-1. Gitea workflow builds and pushes a new image to your registry.
-2. Workflow calls `POST /update` on this service with the image name.
-3. Service finds the matching Compose-managed container(s) on the host via Docker labels.
-4. Runs `docker compose pull` + `up -d` for the relevant service(s).
+## What an update does
 
-See [design spec](docs/superpowers/specs/2026-05-22-package-updater-design.md) and [implementation plan](docs/superpowers/plans/2026-05-22-package-updater-implementation.md) for full design and rationale.
+Clicking *Update* on a service re-pulls the service's **current tag** (`repo:tag`). The
+agent asks its Docker daemon for the tag's digest and deploys `repo:tag@digest`, so what
+runs is exactly what was pulled. The log streams to the browser while it runs; job
+history is kept in Postgres. A service without a tag (`repo@digest`), a plain container,
+and the updater's own services cannot be updated, and the UI says why. A bare image
+(`image: postgres`, no tag and no digest) counts as `postgres:latest`; only a reference
+with a digest and no tag (`repo@sha256:...`) "follows no tag". Compose services with a
+`build:` section are shown but never updated ("built from source (build:)"). One update at
+a time per service, and one job at a time per agent.
 
-## How it finds the right stack
+## Deploying the master
 
-The service queries the Docker socket and reads the labels Compose itself attaches to every container:
-
-- `com.docker.compose.project`
-- `com.docker.compose.service`
-- `com.docker.compose.project.working_dir`
-- `com.docker.compose.project.config_files`
-
-A container is eligible for update only if it has **both**:
-
-- An image name matching the request (tag-agnostic), AND
-- The opt-in label `se.shcizo.auto-update=true`.
-
-Defense in depth: a valid bearer token AND the opt-in label must both be present before any container is touched.
-
-## Swarm mode
-
-Set `MODE=swarm` to update Docker Swarm services instead of Compose stacks. The
-service runs `docker service update --image` via the Docker API instead of
-shelling out to `docker compose`.
-
-Two things to get right when running in Swarm mode:
-
-- **The opt-in label goes on the service, not the container/task.** Swarm mode
-  reads `Service.Spec.Labels`, so add it with
-  `docker service update --label-add se.shcizo.auto-update=true <service>` or
-  set it under `deploy.labels` in the stack file — a plain `labels:` entry on
-  the service (container-level) is not visible to Swarm mode's discovery.
-- **The updater must talk to a Swarm manager.** `docker service update`
-  requires manager API access, so either point `DOCKER_HOST` at a manager node
-  or schedule the updater container itself on a manager with the socket
-  mounted. This is a deployment concern the service cannot detect or work
-  around.
-
-## Master/agent (in progress)
-
-A master with one agent per machine is replacing the single-instance webhook
-model; see the [design spec](docs/superpowers/specs/2026-10-12-master-agent-design.md).
-Phase 1 delivers a live inventory: every Compose stack, Swarm service and
-container on every machine, behind Authentik login. Updates still go through
-the existing `/update` webhook until phase 3.
-
-- Master: `updater-master` in the same image, example stack in `deploy/master.stack.yml`.
-  At startup it does OIDC discovery against `OIDC_ISSUER` and exits if the issuer
-  is unreachable; Swarm then restarts it.
-- Agents: add one per machine on the master's **Agents** page; it shows a
-  ready-to-run compose file with the agent's token and the master's key
-  fingerprint.
-
-## Quick start
-
-1. Build and push the image (e.g. via your own CI).
-2. Copy `docker-compose.example.yml` to `/home/shcizo/self-hosted/package-updater/docker-compose.yml`.
-3. Create `.env` next to it: `UPDATER_API_KEY=$(openssl rand -hex 32)`.
-4. Point your reverse proxy (NPM/Traefik/Caddy) at `package-updater:8080`. NPM should handle TLS.
-5. `docker compose up -d`.
-6. Add the opt-in label `se.shcizo.auto-update: "true"` to each service you want auto-updated.
-7. Use the [Gitea composite action](gitea-action/README.md) in your repos to call `/update` after a build.
-
-## Configuration
-
-All via environment variables.
+Use [`deploy/master.stack.yml`](deploy/master.stack.yml) as a starting point (Swarm stack;
+needs Postgres and an OIDC provider). The master listens for the UI on `PORT` (default
+`8080`, put it behind your reverse proxy) and for agents on `AGENT_PORT` (default `8090`,
+pinned TLS, reached directly on the internal network). The example runs the master as
+`nobody` (`65534`); it needs no root. The agent runs as root because access to the Docker
+socket is root on the host anyway.
 
 | Variable | Required | Default | Purpose |
 |---|---|---|---|
-| `UPDATER_API_KEY` | **yes** | — | Bearer token. Service refuses to start without it. |
-| `STACKS_ROOT` | no | `/home/shcizo/self-hosted` | Required parent for any stack eligible to update. |
-| `PORT` | no | `8080` | HTTP listen port. |
-| `LOG_LEVEL` | no | `info` | `debug` / `info` / `warn` / `error`. |
-| `UPDATE_TIMEOUT` | no | `5m` | Per-job timeout (Go duration). |
-| `OPT_IN_LABEL` | no | `se.shcizo.auto-update` | Label name to check; value must equal `"true"`. |
-| `MODE` | no | `compose` | `compose` or `swarm`. Selects the update mechanism for the whole deployment; not mixed per-request. |
+| `UPDATER_API_KEY` | no | — | Bearer token for `POST /update`. Without it the webhook answers `503` and the master logs `webhook_disabled`. |
+| `UPDATE_TIMEOUT` | no | `5m` | How long one `POST /update` waits for its auto jobs (Go duration). |
+| `POLL_INTERVAL` | no | `6h` | Time between registry polling cycles (Go duration, must be positive). The first cycle runs one minute after start. |
+| `DOCKER_CONFIG` | no | `~/.docker` | Directory with a `config.json` holding registry credentials for polling, readable by the master's user. Only inline `auths` entries work; see [Registry polling](#registry-polling). |
 
-## Endpoints
+The master's other settings (`DATABASE_URL`, `PUBLIC_URL`, `AGENT_URL`, `OIDC_*`,
+`ALLOWED_GROUP`, `AGENT_IMAGE`, `PORT`, `AGENT_PORT`, `LOG_LEVEL`) are shown in
+[`deploy/master.stack.yml`](deploy/master.stack.yml).
 
-| Endpoint | Auth | Purpose |
+## Webhook (CI)
+
+CI calls the master after pushing an image. The Gitea action in
+[`gitea-action/`](gitea-action/README.md) does this; any HTTP client works.
+
+```
+POST /update
+Authorization: Bearer <UPDATER_API_KEY>
+Content-Type: application/json
+
+{"image": "registry.example.com/myapp", "tag": "1.5.0"}
+```
+
+`image` is matched against the image repo of every service on every agent (tag and digest
+in the request are ignored for matching; matching is case-sensitive). `tag` is optional.
+For every match the master records the version as *available* (the overview shows an
+"update available" badge), including services that cannot be updated automatically
+(updater's own, `build:` and plain containers), so the badge shows for them too. Services
+in **manual** mode stop there. Services in **auto**
+mode also get an update job, and the request waits for those jobs to finish, at most
+`UPDATE_TIMEOUT`.
+
+**`tag` semantics.** The target of an auto job is `<service's image repo>:<tag>`, built
+from the service's own repo and never from the request's `image` string. The tag must be
+one that has been pushed. Compose services get pinned to it through
+`compose.override.yaml`, as in any update. Omit `tag` to re-pull the tag the service
+already follows (a bare image counts as `:latest`); a service that follows no tag
+(`repo@digest`) is then not touched and is reported as `refused` ("no tag given and the
+service follows no tag"), manual or auto.
+With a `tag`, the "follows no tag" refusal does not apply; the other refusals (the
+updater's own services, `build:` services, plain containers) still do.
+
+Response (`200` or `500`):
+
+```json
+{"request_id": "…", "image": "registry.example.com/myapp", "tag": "1.5.0", "matched": 2,
+ "results": [
+  {"service_id": 12, "service": "myapp/web", "kind": "compose", "mode": "auto",
+   "status": "succeeded", "job_id": 42, "job_url": "https://updater.example.com/jobs/42",
+   "to_ref": "registry.example.com/myapp:1.5.0", "duration_ms": 8210},
+  {"service_id": 15, "service": "myapp/worker", "kind": "compose", "mode": "manual",
+   "status": "marked", "to_ref": "registry.example.com/myapp:1.5.0", "duration_ms": 3}]}
+```
+
+| `status` | Meaning | Counts as failure |
 |---|---|---|
-| `POST /update` | Bearer token | Trigger pull + restart for matching services |
-| `GET /healthz` | none | Liveness + Docker socket reachability |
-| `GET /version` | none | Build info |
-| `GET /metrics` | none | Prometheus exposition |
+| `succeeded` | Auto job finished successfully. | no |
+| `failed` | Auto job failed (for example agent offline), or the version could not be recorded. `error` says why. | yes |
+| `timeout` | Job still running when `UPDATE_TIMEOUT` passed. The master leaves it running. | yes |
+| `marked` | Manual service; the version was recorded only. | no |
+| `busy` | The service already has a queued or running job; `job_id` is that job, which is not waited for. | no |
+| `refused` | The service may not be updated, or there is no target (no `tag` given and the service follows no tag). `error` has the reason. | no |
 
-`/healthz`, `/version`, and `/metrics` are intentionally unauthenticated — they're internal-network only behind the reverse proxy.
+| Code | When |
+|---|---|
+| `200` | No result is `failed` or `timeout`. |
+| `400` | Body is not JSON, `image` is missing, or `tag` is not a valid Docker tag. |
+| `401` | Missing or wrong bearer token. |
+| `404` | No service uses the image (catches typos in CI configuration). |
+| `500` | At least one result is `failed` or `timeout`. |
+| `503` | `UPDATER_API_KEY` is not set on the master. |
 
-## Observability
+A service that has **drifted** (see [Drift and moved services](#drift-and-moved-services))
+is refused with its reason, in auto mode too, until someone restores or accepts it.
 
-- **Logs**: JSON to stdout, picked up by Promtail/Alloy → Loki.
-- **Metrics**: Prometheus exposition on `/metrics`. Notable: `package_updater_update_jobs_total{project,service,status}`, `package_updater_last_update_timestamp{project,service}`, `package_updater_docker_ping_up`.
+The webhook never creates services: the first deployment of a stack is done by hand. A
+new service always starts in **manual** mode; switch it with the *Make auto* button on the
+overview or the service page. Auto is refused for the updater's own services, `build:`
+services and plain containers.
+
+> **Warning:** auto mode deploys whatever CI pushes, without anyone looking at it. Using
+> it on critical services is your own risk.
+
+The request is open for as long as the jobs run, so the reverse proxy in front of the
+master must allow a read timeout longer than `UPDATE_TIMEOUT`. If the client disconnects,
+marking and starting jobs still complete; only the waiting stops.
+
+## Registry polling
+
+Besides the webhook, the master asks each registry whether the tag a service follows has
+a new digest. Every `POLL_INTERVAL` it sends one `HEAD /v2/<repo>/manifests/<tag>` per
+distinct followed `repo:tag` (a bare image counts as `:latest`; `repo:tag@digest` is polled
+as `repo:tag`). It never downloads a manifest: Docker Hub counts a `GET` as a pull against
+the rate limit and a `HEAD` as nothing
+([source](https://docs.docker.com/docker-hub/usage/pulls/)), and Docker Hub also has a
+separate abuse limit for many pulls from one IP address, so the poller stays HEAD-only. For a
+multi-arch image the digest is the index digest, the same one `docker inspect` shows in
+`RepoDigests`.
+
+- **What counts as new.** A digest different from the one the service runs becomes the
+  service's *available* version (the overview badge, source `poll`). A digest the master
+  already suggested is not suggested again. A service in **auto** mode also gets an update
+  job to `repo:tag` (the agent resolves the digest itself, so agents on protocol version 2
+  keep working). The master remembers the newest digest it started or deployed for the tag
+  in `services.last_tag_digest` (set when polling starts a job, when polling sees the
+  service already running that digest, and when any plain tag job succeeds, whoever started
+  it) and starts no job for that digest again, whether the job
+  failed or the user rolled back afterwards. If the service already had a job when the
+  digest was found, the next cycle tries again.
+- **What is not polled.** Services that follow no tag (`repo@digest`), `build:` services, and
+  services without a registry digest (locally built images). The service page says which.
+  The updater's own services and plain containers are polled, so their badge works, but
+  jobs for them are still refused.
+- **Errors.** A failed lookup is stored for the services that follow that tag and shown as a
+  "poll failed" badge on the overview and in the "Polling" row of the service page; other
+  tags continue. A later success clears it.
+- **Credentials.** The master reads `$DOCKER_CONFIG/config.json` (a file written by
+  `docker login`), mounted read-only into the container; without it registries are accessed
+  anonymously. The directory and file must be readable by the master's user (`65534` in the
+  example stack) without being world-readable: `chown -R 65534:65534` the directory and keep
+  `0700`/`0600`. Only inline `auths` entries are supported. A `credsStore` or `credHelpers`
+  entry makes every lookup fail with `error getting credentials ... executable file not found
+  in $PATH`, because the container has no credential helper binaries. The master logs
+  `registry_credentials` at startup with the directory, `config_json_found` and
+  `config_json_readable`, never the contents. If the file exists but cannot be opened it also
+  logs a `registry_credentials_unreadable` warning with the error.
+- **Requests go wherever the image name points.** The master sends the lookup to the
+  registry host in the image name, including a `GET /v2/` ping that follows redirects.
+  Anyone who can start containers on a managed host can therefore make the master send
+  blind requests to internal hosts. The registry's answer is not shown: an error keeps only
+  the HTTP status and the registry's error codes; any other failure shows only "timeout",
+  "cannot reach registry" or "lookup failed". The full error goes to the master's log
+  (`registry_lookup_failed`).
+
+### Version history and rollback
+
+The master records every version it sees (webhook, polling, and what agents report as
+running) in `image_versions`. The service page lists the 10 most recently first seen
+versions of the service's image repo, marks the one that is running, and has a *Deploy*
+button on each row. A webhook announces a tag but not its digest, so such a row has no
+digest until polling or an agent reports one; it is hidden when a row with the same tag
+and a digest was first seen at or after it. A bare image (`postgres`) is recorded as tag
+`latest`.
+
+Picking a version starts a manual job to exactly that reference. Picking an older one is a
+rollback, including to a digest-pinned reference. A version without a tag deploys
+`repo@digest` and the service then follows no tag, so it is no longer polled or updatable
+with the plain *Update* button until it is moved to a tagged version again. Picks of
+digest-pinned or tagless versions need an agent speaking protocol version 3 (this
+release); on an older agent the job fails with "agent too old to deploy a pinned version
+(needs protocol 3)".
+
+A rollback made before the first poll cycle after the upgrade has seen the service is not
+protected, because the master has not yet recorded the digest it was running.
+
+A service in **auto** mode stays auto after a rollback. Polling does not undo it: the
+service stays on the older version until the registry shows a digest of the followed tag
+that differs from the newest one the master deployed. The newer version it was rolled back
+from stays marked as available. A webhook that names a tag still starts a job for an auto
+service every time, so a CI push moves it forward again.
+
+## Drift and moved services
+
+A service has **drifted** when it runs something other than the version the last update
+job deployed. A service the updater never touched has nothing to drift from. Drift is
+decided by what runs, not by how the reference is written: the digest decides when both
+sides know one, so `repo:tag@sha256:…` is run by a service reporting `repo:tag` with that
+digest, and a different tag with the same digest is not drift. (The design spec compares
+the two reference strings; this is deliberately looser.) Between a successful job and the
+agent's next inventory report the service is not counted as drifted, because the report
+has not arrived yet.
+
+The master never corrects drift on its own, because a manual `docker stack deploy` of a
+newer tag may be intentional. Polling and the webhook start no job for a drifted
+service, in auto mode too; a person can still start one by hand. There are two ways out:
+
+- **Restore desired** deploys the recorded version again (a manual job to the exact
+  `repo:tag@digest`; an agent speaking protocol 3 is needed). If the service had drifted
+  to another digest of the same tag, that digest is remembered, so polling does not
+  deploy it again until the registry shows a newer one (unless `last_tag_digest` already
+  holds back a newer digest, which then stays).
+- **Accept running** records what runs now as the desired version. It starts no job and
+  leaves the available suggestion and the update mode alone.
+
+When a Compose service disappears from one agent and exactly one service with the same
+project and service name appears on another, and the removed one has history (a desired
+version, auto mode or jobs) while the new one has none, the service page suggests that it
+moved. **Merge** moves the old row, with its history, mode and desired version, to the new
+location and deletes the duplicate. Merging is never automatic. It can also be done by
+hand for any removed service of the same kind (Compose or Swarm, names may differ), but
+not for plain containers, the updater itself, or a service with a queued or running job.
+A merge keeps the old row's mode and desired version, so a manually merged target with its
+own history may show drift right after the merge and can be accepted.
+
+The restore, accept and merge buttons arrive with the UI follow-up (issue #28); the endpoints
+(`POST /services/{id}/restore`, `/accept`, `/merge`) and the data for them are in place.
+
+## Adding agents
+
+Open the master's **Agents** page and add one. It shows a ready-to-run compose file with
+the agent's token (shown once) and the master's key fingerprint; the same layout is in
+[`deploy/agent.compose.yml`](deploy/agent.compose.yml).
+
+| Variable | Required | Default | Purpose |
+|---|---|---|---|
+| `MASTER_URL` | yes | — | `wss://host:port/agent` |
+| `MASTER_KEY` | yes | — | Master key fingerprint (`sha256:...`) shown in the UI. |
+| `AGENT_TOKEN` | yes | — | Token shown once when the agent was added. |
+| `SELF_ID` | no | hostname | Container ID or name of the agent itself; used to read its mounts. |
+| `UPDATE_TIMEOUT` | no | `5m` | Per-job timeout (Go duration). |
+| `DOCKER_CONFIG` | no | — | Directory with a `config.json` holding registry credentials. Only inline `auths` entries work; `credsStore`/`credHelpers` are not available in the container. |
+| `LOG_LEVEL` | no | `info` | `debug` / `info` / `warn` / `error`. |
+
+`docker compose` runs inside the agent with only `PATH`, `HOME` and the `DOCKER_*`
+variables. Variables a stack uses for interpolation must come from the stack's own `.env`
+file or an `--env-file` (which the agent honours), never from the agent container's
+environment.
+
+### Registry credentials
+
+With `DOCKER_CONFIG` set, Compose jobs pull with the same credentials as `docker compose`.
+For Swarm jobs the agent passes the matching registry's credentials with the service
+update (same as `docker service update --with-registry-auth`), so Swarm stores them in the
+cluster and hands them to the nodes that pull the image. Only the entry for the image's
+registry host is ever sent.
+
+### Mounting stack directories
+
+There is no `STACKS_ROOT`. The agent may only touch Compose stacks in directories that
+are bind-mounted into its own container **read-write at the same path as on the host**
+(Compose labels hold host paths). Add one line per directory:
+
+```yaml
+volumes:
+  - /var/run/docker.sock:/var/run/docker.sock
+  - /home/shcizo/self-hosted:/home/shcizo/self-hosted
+```
+
+Read-only mounts and mounts whose source differs from the destination are refused with a
+clear message. On Docker Desktop and OrbStack bind sources are translated host paths, so
+Compose jobs fail there; that is a development-only limitation.
+
+## How Compose updates work, and their limits
+
+The agent never edits your files. It writes one file next to the stack's base file,
+`compose.override.yaml`, with a header line and an `image:` entry per updated service
+(`repo:tag@digest`), then runs `docker compose pull` and `up -d` for that service with
+the override passed explicitly.
+
+- The agent only writes `compose.override.yaml` if it is absent or starts with its header
+  line, and **refuses the job** if any other auto-loaded override exists
+  (`compose.override.yml`, `docker-compose.override.yml`, `docker-compose.override.yaml`):
+  Compose loads only one of them, so ours would silently disable yours.
+- Stacks you start with explicit `-f` files do not load the override on a manual
+  `docker compose up`; the agent's own jobs always pass it.
+- **The override is never pruned.** If you delete a service from the stack file, its
+  `image:` entry stays in `compose.override.yaml`, and a manual `docker compose up -d`
+  would recreate it. Remove the entry by hand.
+
+**Check before enabling.** List the config files of every running stack to spot existing
+override files and stacks started with explicit `-f`:
+
+```bash
+docker ps --format '{{.Label "com.docker.compose.project.config_files"}}' | sort -u
+```
+
+## Swarm updates
+
+The agent updates the service with `ServiceUpdate` and the pinned image. "Succeeded"
+means the daemon accepted the new spec, not that the tasks converged; the inventory shows
+`degraded` until they do. The agent doing the update must talk to a manager node (run it
+on a manager, or point `DOCKER_HOST` at one).
+
+## Upgrading from the single-instance updater
+
+This release removes `cmd/server` and changes the image's default entrypoint to
+`updater-agent`. A running old-style updater (default entrypoint, `UPDATER_API_KEY`) will
+crash-loop the next time its image is pulled. Anything that pulls `:latest` for the old
+instance triggers this: its own `/update` webhook, Watchtower or Shepherd.
+
+Pinning means two things: pin the old instance to the last pre-phase-2 tag **and** remove
+its `se.shcizo.auto-update` label, so nothing updates it to the new image. The master
+serves `POST /update` again (see [Webhook](#webhook-ci)); point the Gitea action's
+`endpoint` there and set `UPDATER_API_KEY` on the master.
+
+Master and agents must be on this release together: a phase-1 agent connects but every
+job sent to it fails "agent too old for updates".
 
 ## Development
 
 ```bash
 go test ./...
-go build ./cmd/server
+go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/
+go build ./cmd/master ./cmd/agent
 docker build -t package-updater:dev .
+DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/
 ```
-
-## Known v1 gaps
-
-These are tracked in the spec's section 2 and section 15 as deliberate out-of-scope:
-
-- **Self-update wiring**: `internal/selfupdate.Wrapped` exists and is unit-tested but is not wired into the live queue. The HTTP response flush ordering for self-replacement is a future enhancement; for now, expect to manually rerun `docker compose up -d` on the host if pushing a new image of `package-updater` itself causes a mid-response interruption.
-- **No rollback**: Compose's "keep old container if new fails to start" is the only safety net.
-- **Single host only in Compose mode**. Swarm mode (`MODE=swarm`) is the
-  multi-node path, but only from a manager node's point of view — the updater
-  itself still needs manager API access (see "Swarm mode" above).
-- **No per-repo API keys**: a single shared bearer token is used.

cmd/agent/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/shcizo/package-updater/internal/agent/config"
 	"github.com/shcizo/package-updater/internal/agent/conn"
 	"github.com/shcizo/package-updater/internal/agent/inventory"
+	"github.com/shcizo/package-updater/internal/agent/updater"
 	"github.com/shcizo/package-updater/internal/logging"
 )
 
@@ -42,19 +43,49 @@ func run() error {
 	}
 	defer dockerCli.Close()
 
-	collector := inventory.New(dockerCli, cfg.SelfID)
+	auth, err := updater.LoadRegistryAuth(cfg.DockerConfig, logger)
+	if err != nil {
+		return fmt.Errorf("registry auth: %w", err)
+	}
+	resolver := updater.NewResolver(dockerCli, auth)
+	exec := updater.NewDispatch(
+		updater.NewComposeExecutor(dockerCli, resolver, cfg.SelfID, updater.ExecDocker, updater.ExecDockerOutput),
+		updater.NewSwarmExecutor(dockerCli, resolver, auth),
+	)
+	runner := updater.NewRunner(exec, cfg.UpdateTimeout)
+
+	collector := inventory.New(dockerCli, cfg.SelfID).
+		WithBuildCheck(updater.NewBuildChecker(dockerCli, cfg.SelfID, updater.ExecDockerOutput), logger)
 	agent := conn.New(conn.Config{
 		URL: cfg.MasterURL, Token: cfg.Token, Fingerprint: cfg.MasterKey, Version: version,
-	}, collector, logger)
+	}, collector, runner, logger)
 
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
+	logMounts(ctx, dockerCli, cfg.SelfID, logger)
+	go runner.Run(ctx)
 	go watchLoop(ctx, dockerCli, agent, logger)
-	logger.Info("agent_starting", "version", version, "master_url", cfg.MasterURL)
+	logger.Info("agent_starting", "version", version, "master_url", cfg.MasterURL,
+		"update_timeout", cfg.UpdateTimeout.String(), "registry_auth", auth != nil)
 	return agent.Run(ctx)
 }
 
+// logMounts warns at startup when Compose updates cannot work. Each job reads
+// the mounts again itself; this is only for the operator.
+func logMounts(ctx context.Context, cli updater.SelfInspector, selfID string, log *slog.Logger) {
+	roots, err := updater.MountRoots(ctx, cli, selfID)
+	if err != nil || len(roots) == 0 {
+		attrs := []any{"hint", "Compose updates will fail; mount each stacks directory at the same path, read-write"}
+		if err != nil {
+			attrs = append(attrs, "error", err)
+		}
+		log.Warn("no_stack_mounts", attrs...)
+		return
+	}
+	log.Info("stack_mounts", "roots", roots)
+}
+
 // watchLoop keeps a Docker event subscription alive. When the stream breaks
 // (daemon restart), it resubscribes and asks for a refresh, because events
 // may have been missed in between. It also asks for a full re-inventory every

cmd/master/main.go

@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strings"
 	"syscall"
 	"time"
@@ -19,9 +20,13 @@ import (
 	"github.com/shcizo/package-updater/internal/master/config"
 	"github.com/shcizo/package-updater/internal/master/events"
 	"github.com/shcizo/package-updater/internal/master/hub"
+	"github.com/shcizo/package-updater/internal/master/jobs"
+	"github.com/shcizo/package-updater/internal/master/poller"
+	"github.com/shcizo/package-updater/internal/master/registry"
 	"github.com/shcizo/package-updater/internal/master/store"
 	"github.com/shcizo/package-updater/internal/master/web"
 	"github.com/shcizo/package-updater/internal/master/webauth"
+	"github.com/shcizo/package-updater/internal/master/webhook"
 	"github.com/shcizo/package-updater/internal/pin"
 )
 
@@ -35,6 +40,39 @@ func main() {
 	}
 }
 
+// pollFirstDelay is how long the master runs before its first registry
+// polling cycle, so agents have reported their inventory.
+const pollFirstDelay = time.Minute
+
+// pollWorkers is how many registry lookups run at once.
+const pollWorkers = 4
+
+// registryCredentialsDir returns the directory the registry client reads
+// credentials from (DOCKER_CONFIG, else ~/.docker), whether a config.json is
+// in it, and the error from opening it when it exists but cannot be read (for
+// example because the master runs as another user than the file owner). The
+// file is opened and closed, never read; only the path is logged.
+func registryCredentialsDir() (dir string, found bool, readErr error) {
+	dir = os.Getenv("DOCKER_CONFIG")
+	if dir == "" {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return "", false, nil
+		}
+		dir = filepath.Join(home, ".docker")
+	}
+	path := filepath.Join(dir, "config.json")
+	if _, err := os.Stat(path); err != nil {
+		return dir, false, nil
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return dir, true, err
+	}
+	_ = f.Close()
+	return dir, true, nil
+}
+
 func run() error {
 	cfg, err := config.Load()
 	if err != nil {
@@ -66,6 +104,7 @@ func run() error {
 
 	broker := events.NewBroker()
 	agentHub := hub.New(st, broker, logger)
+	jobService := jobs.New(st, agentHub, broker, cfg.AgentImage, logger)
 
 	provider, err := webauth.NewOIDCProvider(ctx, webauth.OIDCConfig{
 		Issuer: cfg.OIDCIssuer, ClientID: cfg.OIDCClientID, ClientSecret: cfg.OIDCClientSecret,
@@ -76,8 +115,9 @@ func run() error {
 	}
 	auth := webauth.New(provider, st, cfg.AllowedGroup, strings.HasPrefix(cfg.PublicURL, "https://"), logger)
 
-	ui, err := web.New(st, agentHub, broker, web.Config{
+	ui, err := web.New(st, agentHub, jobService, broker, web.Config{
 		AgentURL: cfg.AgentURL, MasterKey: fingerprint, AgentImage: cfg.AgentImage,
+		PollInterval: cfg.PollInterval,
 	}, logger)
 	if err != nil {
 		return err
@@ -91,6 +131,14 @@ func run() error {
 		}
 		_, _ = w.Write([]byte("ok"))
 	})
+	// The webhook does its own bearer-token check, so it sits outside the
+	// OIDC login like /healthz.
+	mux.Handle("POST /update", webhook.New(st, jobService, broker, webhook.Config{
+		Token: cfg.UpdaterAPIKey, Timeout: cfg.UpdateTimeout, PublicURL: cfg.PublicURL,
+	}, logger))
+	if cfg.UpdaterAPIKey == "" {
+		logger.Warn("webhook_disabled", "reason", "UPDATER_API_KEY not set")
+	}
 	mux.HandleFunc("GET /auth/login", auth.Login)
 	mux.HandleFunc("GET /auth/callback", auth.Callback)
 	mux.Handle("POST /auth/logout", auth.Require(http.HandlerFunc(auth.Logout)))
@@ -101,8 +149,8 @@ func run() error {
 	// instead of holding Shutdown until its timeout.
 	baseContext := func(net.Listener) context.Context { return ctx }
 
-	// No WriteTimeout on either server: SSE streams and agent connections
-	// are long-lived by design.
+	// No WriteTimeout on either server: SSE streams, agent connections and
+	// POST /update (which waits up to UPDATE_TIMEOUT) are long-lived by design.
 	webSrv := &http.Server{
 		Addr: ":" + cfg.Port, Handler: mux, ReadHeaderTimeout: 10 * time.Second,
 		BaseContext: baseContext,
@@ -116,22 +164,43 @@ func run() error {
 		BaseContext: baseContext,
 	}
 
+	credDir, credFound, credReadErr := registryCredentialsDir()
+	logger.Info("registry_credentials", "docker_config", credDir, "config_json_found", credFound,
+		"config_json_readable", credFound && credReadErr == nil)
+	if credReadErr != nil {
+		logger.Warn("registry_credentials_unreadable", "docker_config", credDir, "error", credReadErr)
+	}
+	pollCtx, stopPolling := context.WithCancel(ctx)
+	defer stopPolling()
+	registryPoller := poller.New(st, jobService, registry.New("package-updater-master/"+version, logger), broker, poller.Config{
+		Interval: cfg.PollInterval, FirstDelay: pollFirstDelay, Workers: pollWorkers,
+	}, logger)
+	pollDone := make(chan struct{})
+	go func() {
+		defer close(pollDone)
+		registryPoller.Run(pollCtx)
+	}()
+
 	errCh := make(chan error, 2)
 	go func() { errCh <- webSrv.ListenAndServe() }()
 	go func() { errCh <- agentSrv.ListenAndServeTLS("", "") }()
 	logger.Info("listening", "web", webSrv.Addr, "agents", agentSrv.Addr)
 
+	var serveErr error
 	select {
 	case <-ctx.Done():
 		logger.Info("shutdown_signal")
 	case err := <-errCh:
 		if !errors.Is(err, http.ErrServerClosed) {
-			return err
+			serveErr = err
 		}
 	}
 	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	_ = webSrv.Shutdown(shutdownCtx)
 	_ = agentSrv.Shutdown(shutdownCtx)
-	return nil
+	// The poller writes to the store, which is closed when run returns.
+	stopPolling()
+	<-pollDone
+	return serveErr
 }

cmd/master/main_test.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRegistryCredentialsDir(t *testing.T) {
+	t.Run("missing file", func(t *testing.T) {
+		dir := t.TempDir()
+		t.Setenv("DOCKER_CONFIG", dir)
+		got, found, readErr := registryCredentialsDir()
+		assert.Equal(t, dir, got)
+		assert.False(t, found)
+		assert.NoError(t, readErr)
+	})
+
+	t.Run("readable file", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o600))
+		t.Setenv("DOCKER_CONFIG", dir)
+		_, found, readErr := registryCredentialsDir()
+		assert.True(t, found)
+		assert.NoError(t, readErr)
+	})
+
+	t.Run("unreadable file", func(t *testing.T) {
+		if os.Geteuid() == 0 {
+			t.Skip("root can read any file")
+		}
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o000))
+		t.Setenv("DOCKER_CONFIG", dir)
+		_, found, readErr := registryCredentialsDir()
+		assert.True(t, found)
+		assert.Error(t, readErr)
+	})
+}

cmd/server/main.go

@@ -1,144 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-
-	"github.com/docker/docker/client"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/config"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/shcizo/package-updater/internal/metrics"
-	"github.com/shcizo/package-updater/internal/updater"
-)
-
-// Build info is injected at link time via -ldflags="-X main.version=..."
-var (
-	version   = "dev"
-	commit    = "unknown"
-	buildTime = "unknown"
-)
-
-func main() {
-	if err := run(); err != nil {
-		fmt.Fprintln(os.Stderr, "fatal:", err)
-		os.Exit(1)
-	}
-}
-
-func run() error {
-	cfg, err := config.Load()
-	if err != nil {
-		return err
-	}
-
-	logger := logging.New(cfg.LogLevel)
-	logger.Info("starting",
-		"version", version, "commit", commit, "port", cfg.Port,
-		"mode", cfg.Mode, "stacks_root", cfg.StacksRoot, "opt_in_label", cfg.OptInLabel,
-	)
-
-	dockerCli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-	if err != nil {
-		return fmt.Errorf("docker client: %w", err)
-	}
-	defer dockerCli.Close()
-
-	var finder api.Finder
-	var exec updater.Executor
-	switch cfg.Mode {
-	case "swarm":
-		finder = discovery.NewSwarm(dockerCli, cfg.OptInLabel)
-		exec = updater.NewSwarmExecutor(dockerCli)
-	case "compose":
-		finder = discovery.New(dockerCli, cfg.StacksRoot, cfg.OptInLabel)
-		exec = updater.NewComposeExecutor()
-	default:
-		// Unreachable: config.Load already validates Mode, but guard
-		// here too rather than silently falling through to a nil
-		// finder/exec pair.
-		return fmt.Errorf("unknown MODE %q", cfg.Mode)
-	}
-
-	reg := prometheus.NewRegistry()
-	m := metrics.New(reg)
-	m.BuildInfo.WithLabelValues(version, commit).Set(1)
-
-	queue := updater.NewQueue(exec, m)
-	queue.Start(context.Background())
-	defer queue.Stop()
-
-	handlers := api.NewHandlers(finder, &submitterAdapter{queue: queue, timeout: cfg.UpdateTimeout}, dockerCli, version, commit, buildTime, m)
-
-	mux := http.NewServeMux()
-	mux.HandleFunc("POST /update", handlers.Update)
-	mux.HandleFunc("GET /healthz", handlers.Healthz)
-	mux.HandleFunc("GET /version", handlers.Version)
-	mux.Handle("GET /metrics", metrics.Handler(reg))
-
-	authed := api.Auth(cfg.APIKey)
-	handler := api.RequestID(api.RequestLogger(logger, m)(routeAuth(mux, authed)))
-
-	srv := &http.Server{
-		Addr:              ":" + cfg.Port,
-		Handler:           handler,
-		ReadHeaderTimeout: 10 * time.Second,
-	}
-
-	errCh := make(chan error, 1)
-	go func() {
-		logger.Info("listening", "addr", srv.Addr)
-		errCh <- srv.ListenAndServe()
-	}()
-
-	sig := make(chan os.Signal, 1)
-	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
-
-	select {
-	case s := <-sig:
-		logger.Info("shutdown_signal", "signal", s.String())
-	case err := <-errCh:
-		if err != nil && err != http.ErrServerClosed {
-			return err
-		}
-	}
-
-	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	_ = srv.Shutdown(shutdownCtx)
-	return nil
-}
-
-// routeAuth applies the bearer-token middleware to /update only.
-// /healthz, /metrics, /version are unauthenticated by design (internal
-// network only; healthcheck and Prometheus scraper need to reach them
-// without secrets).
-func routeAuth(next *http.ServeMux, mw func(http.Handler) http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/update" {
-			mw(next).ServeHTTP(w, r)
-			return
-		}
-		next.ServeHTTP(w, r)
-	})
-}
-
-// submitterAdapter wraps Queue.Submit to inject the per-request timeout
-// from config, satisfying the api.Submitter interface.
-type submitterAdapter struct {
-	queue   *updater.Queue
-	timeout time.Duration
-}
-
-func (s *submitterAdapter) Submit(ctx context.Context, jobs []discovery.Job) []updater.Result {
-	ctx, cancel := context.WithTimeout(ctx, s.timeout)
-	defer cancel()
-	return s.queue.Submit(ctx, jobs)
-}

deploy/agent.compose.yml

@@ -0,0 +1,19 @@
+# Example agent deployment, one per machine. The master UI generates this
+# snippet with MASTER_URL, MASTER_KEY and AGENT_TOKEN filled in (Agents -> Add
+# agent); the token is shown once.
+services:
+  updater-agent:
+    image: gitea.shcizo.se/shcizo/package-updater:latest
+    entrypoint: ["/usr/local/bin/updater-agent"]
+    restart: unless-stopped
+    environment:
+      MASTER_URL: wss://10.0.0.5:8090/agent
+      MASTER_KEY: sha256:<fingerprint shown in the master UI>
+      AGENT_TOKEN: <token shown once when the agent was added>
+      UPDATE_TIMEOUT: 5m
+      # Optional registry credentials (a config.json written by docker login):
+      # DOCKER_CONFIG: /docker-config
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      # - /path/to/stacks:/path/to/stacks  # mount each stacks directory at the same path, read-write
+      # - /root/.docker/config.json:/docker-config/config.json:ro

deploy/master.stack.yml

@@ -1,4 +1,4 @@
-# Example Swarm stack for the package-updater master (phase 1: inventory only).
+# Example Swarm stack for the package-updater master (inventory, updates and the CI webhook).
 # Deploy on the manager: docker stack deploy -c master.stack.yml package-updater-master
 #
 # Before deploying:
@@ -6,19 +6,39 @@
 #   2. In Authentik: create an OAuth2/OpenID provider + application with redirect URI
 #      https://updater.example.com/auth/callback, and a group "updater-admins".
 #   3. docker secret create pu_oidc_client_secret - ; docker secret create pu_database_url -
+#   4. docker secret create pu_updater_api_key -   (required by this example; it enables POST /update)
+#      To run without the webhook, remove the pu_updater_api_key lines (the UPDATER_API_KEY
+#      part of the entrypoint, the service secret and the secrets entry); the endpoint then answers 503.
+#      The reverse proxy must allow a read timeout longer than UPDATE_TIMEOUT (default 5m).
+#   5. Optional, for registry polling of private images: put a config.json written by
+#      `docker login` (inline "auths" only; no credsStore) in /path/to/docker-config and
+#      uncomment DOCKER_CONFIG and the volumes entry below. The master runs as 65534:65534, so
+#      the directory and config.json must be readable by that user while staying private:
+#      `chown -R 65534:65534 /path/to/docker-config` (keep 0700/0600), or set `user:` below
+#      to the owner's uid:gid. Do not make config.json world-readable; it holds registry
+#      passwords. The directory must exist on every manager node (the master is placed on a
+#      manager), or Swarm rejects the task. Without it registries are accessed anonymously.
 services:
   master:
     image: gitea.shcizo.se/shcizo/package-updater:latest
-    entrypoint: ["/bin/sh", "-c", "DATABASE_URL=$$(cat /run/secrets/pu_database_url) OIDC_CLIENT_SECRET=$$(cat /run/secrets/pu_oidc_client_secret) exec /usr/local/bin/updater-master"]
+    # Runs as nobody: the master needs no root (the agent does, for the Docker socket).
+    user: "65534:65534"
+    entrypoint: ["/bin/sh", "-c", "DATABASE_URL=$$(cat /run/secrets/pu_database_url) OIDC_CLIENT_SECRET=$$(cat /run/secrets/pu_oidc_client_secret) UPDATER_API_KEY=$$(cat /run/secrets/pu_updater_api_key) exec /usr/local/bin/updater-master"]
     environment:
       PUBLIC_URL: https://updater.example.com
       AGENT_URL: wss://10.0.0.5:8090/agent
       OIDC_ISSUER: https://auth.example.com/application/o/package-updater/
       OIDC_CLIENT_ID: package-updater
       ALLOWED_GROUP: updater-admins
+      # UPDATE_TIMEOUT: 5m
+      # POLL_INTERVAL: 6h
+      # DOCKER_CONFIG: /docker-config   # step 5
     secrets:
       - pu_database_url
       - pu_oidc_client_secret
+      - pu_updater_api_key
+    # volumes:                        # step 5
+    #   - /path/to/docker-config:/docker-config:ro
     ports:
       # Agents connect here directly over the internal network, not via the proxy.
       - target: 8090
@@ -35,6 +55,8 @@ secrets:
     external: true
   pu_oidc_client_secret:
     external: true
+  pu_updater_api_key:
+    external: true
 
 networks:
   proxy:

docker-compose.example.yml

@@ -1,32 +0,0 @@
-# Example deployment of package-updater itself.
-# Copy to /home/shcizo/self-hosted/package-updater/docker-compose.yml
-# and create a .env file alongside containing:
-#   UPDATER_API_KEY=<openssl rand -hex 32>
-
-services:
-  package-updater:
-    image: registry.example.com/package-updater:latest
-    container_name: package-updater
-    restart: unless-stopped
-    environment:
-      - UPDATER_API_KEY=${UPDATER_API_KEY}
-      - STACKS_ROOT=/home/shcizo/self-hosted
-      - LOG_LEVEL=info
-      - PORT=8080
-      - UPDATE_TIMEOUT=5m
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - /home/shcizo/self-hosted:/home/shcizo/self-hosted:ro
-    labels:
-      - "se.shcizo.auto-update=true"
-    networks:
-      - proxy
-    healthcheck:
-      test: ["CMD", "wget", "-q", "-O-", "http://localhost:8080/healthz"]
-      interval: 30s
-      timeout: 5s
-      retries: 3
-
-networks:
-  proxy:
-    external: true

docker-compose.local.yml

@@ -1,40 +0,0 @@
-# Local development / smoke-test compose file.
-# Unlike docker-compose.example.yml (which pulls a pre-built image from a registry),
-# this file BUILDS the image from the local Dockerfile so you can iterate without
-# pushing anywhere.
-#
-# Usage:
-#   1. Copy .env.example to .env and fill in UPDATER_API_KEY (e.g. `openssl rand -hex 32`)
-#   2. Optionally adjust STACKS_ROOT below to point at a directory you actually have
-#      compose stacks in (default points at this repo dir, which is fine for smoke tests)
-#   3. docker compose -f docker-compose.local.yml up --build
-#   4. curl -s http://localhost:8080/healthz | jq
-#   5. curl -sH "Authorization: Bearer $UPDATER_API_KEY" -d '{"image":"test"}' \
-#        http://localhost:8080/update | jq
-
-services:
-  package-updater:
-    build:
-      context: .
-      args:
-        VERSION: dev-local
-        COMMIT: ${COMMIT:-unknown}
-        BUILD_TIME: ${BUILD_TIME:-unknown}
-    container_name: package-updater-local
-    restart: unless-stopped
-    environment:
-      - UPDATER_API_KEY=${UPDATER_API_KEY}
-      - STACKS_ROOT=${STACKS_ROOT:-/tmp}
-      - LOG_LEVEL=debug
-      - PORT=8080
-      - UPDATE_TIMEOUT=5m
-    ports:
-      - "8080:8080"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - ${STACKS_ROOT:-/tmp}:${STACKS_ROOT:-/tmp}:ro
-    healthcheck:
-      test: ["CMD", "wget", "-q", "-O-", "http://localhost:8080/healthz"]
-      interval: 30s
-      timeout: 5s
-      retries: 3

docs/superpowers/plans/2026-10-12-master-agent-phase-3-webhook.md

@@ -0,0 +1,148 @@
+# Master/Agent Phase 3 — Webhook and Auto Mode Implementation Plan
+
+> This plan documents what was built for issue #8 and where it deviates from the spec. Code in "Frozen contracts" is final; tasks give the contract and the test cases.
+
+**Goal:** CI calls `POST /update` on the master after pushing an image. The master records the pushed version as *available* on every service that uses the image. Services in `auto` mode are updated at once and the request waits for the result, so CI sees a broken deploy. People switch a service between `manual` and `auto` in the UI.
+
+**Architecture:** A new package `internal/master/webhook` is an `http.Handler` mounted on the master's mux outside the OIDC login; it does its own bearer-token check. It looks up services by image repo, writes `available_*` on each match, starts a job through `jobs.Service.Start` for the `auto` ones and waits for them. Waiting subscribes to the in-process broker before any job exists, reacts to `events.Jobs`, re-reads each pending job from Postgres on every signal and on a fallback ticker, and gives up at `UPDATE_TIMEOUT`. `FinishJob` clears `available_*` in the transaction that records a successful deploy.
+
+**Tech Stack:** unchanged. No new dependencies.
+
+**Spec:** `docs/superpowers/specs/2026-10-12-master-agent-design.md`, "Webhook" section. Phase 2 plan: `docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md` (its conventions still apply).
+
+## Global Constraints
+
+- `go test ./...` passes with no Docker daemon and no Postgres. Postgres tests carry `//go:build integration` and need `DATABASE_URL`. Run `go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/`.
+- Interfaces are declared in the consuming package. Hand-written fakes, `stretchr/testify`, table-driven where there is more than one case.
+- Every exported identifier gets a GoDoc comment. Conventional Commits; every commit message ends with ` (#8)`.
+- The token (`UPDATER_API_KEY`) never appears in logs.
+- Every state-changing UI request is a POST carrying the CSRF token (the mode toggle included).
+- The HTTP server for the UI has no `WriteTimeout`, so a webhook request can wait up to `UPDATE_TIMEOUT`.
+
+## Deliberate deviations from the spec in this phase
+
+- **No `image_versions` table and no `available_digest` column.** Both belong to phase 4 (version history and the version picker). Phase 3 stores only `available_ref`, `available_source` (`webhook`) and `available_at` on `services`.
+- **The old `207` is gone.** A request with both a working and a failed service answers `500` as a whole; there is no partial-success code. The Gitea action fails on any 4xx/5xx, so CI surfaces it.
+- **`busy` and `refused` are not failures.** A service that already has an active job (`busy`, with that job's ID, not waited for) or may not be updated (`refused`, with the reason) does not make the request fail. Only `failed` and `timeout` do. A manual match is `marked`, also not a failure.
+- **A webhook with a `tag` bypasses the "follows no tag" refusal.** The target is then explicit (`<service's image_repo>:<tag>`), so a `repo@digest` service can be moved to a tag. Removed, self, built and container refusals still apply. Without a `tag`, the target is the followed tag (bare image = `:latest`); a service that follows no tag is left alone (`available_*` untouched) and is reported `refused` ("no tag given and the service follows no tag"), manual or auto. Because of this, the UI allows `auto` for a service that follows no tag (`jobs.AutoAllowed`), unlike the Update button (`jobs.Updatable`).
+- **The target ref is built from the service's own repo.** `imageref.Parse(req.Image).Repo` only selects services; it never reaches the job, so a request cannot point a service at another registry.
+- **Token is optional on the master.** Unset `UPDATER_API_KEY` means `POST /update` answers `503` and startup logs `webhook_disabled`; the master still runs.
+- **`tag` is validated** against Docker's tag grammar (`400 invalid tag`). The spec does not say; without it a tag such as `v1@sha256:x` or one with spaces would end up in the override file and fail on the agent.
+- **An oversized body (over 64 KiB) answers `400 invalid JSON body`**, not `413`.
+- **`UPDATE_TIMEOUT` exists on both sides** with the same name and parsing: on the agent it bounds one job, on the master it bounds one webhook request. The master never fails or cancels a job that outlives the request; the agent's own timeout does.
+- **Marking and starting jobs runs on `context.WithoutCancel(r.Context())`.** A client that disconnects stops only the waiting; otherwise a queued job could be left that no agent knows about.
+
+## Review Focus
+
+| Failure case | Test that covers it |
+|---|---|
+| Wrong, missing or malformed token never reaches the lookup; unset key answers 503. | webhook `TestAuth` |
+| An unknown or differently-cased image answers 404 and starts nothing. | webhook `TestNoMatch` |
+| The image string is normalised (tag, digest, registry with port) before matching. | webhook `TestImageIsNormalisedBeforeMatching` |
+| The auto job's target is built from the service's repo and the request tag, never the request's image string. | webhook `TestAutoJobTargetIsBuiltFromServiceRepo` |
+| A job that fails at start (agent offline) is reported at once, not waited for. | webhook `TestJobFailedAtStartIsNotWaitedFor` |
+| A dropped broker signal is recovered by the fallback poll. | webhook `TestDroppedSignalIsRecoveredByPolling` |
+| A timed-out job is reported `timeout` and left untouched. | webhook `TestTimeout` |
+| A client that disconnects does not abort marking or starting. | webhook `TestClientCancelStopsWaitingButNotStarting` |
+| The webhook waits for the real agent's result and answers 200 / 500. | e2e `TestWebhookWaitsForAutoJob`, `TestWebhookFailedAutoJobIs500` |
+| The available ref is cleared only by a successful job whose `to_ref` equals it and that was created at or after the suggestion; a different ref, a newer suggestion and a failed job keep it. | store `TestFinishJob_ClearsAvailableOnlyWhenThatVersionWasDeployed` |
+| Re-reporting inventory never resets a service's update mode; new services are manual. | store `TestUpsertKeepsUpdateMode` |
+| `auto` is refused for removed, self, built and container services; `manual` is always allowed. | jobs `TestAutoAllowed`; web `TestSetMode_AutoRefusedShows409`, `TestSetMode_ManualAlwaysAllowed` |
+
+## Frozen contracts
+
+### C1. Migration `internal/master/store/migrations/0003_webhook.sql` (complete)
+
+```sql
+ALTER TABLE services ADD COLUMN available_ref    text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN available_source text NOT NULL DEFAULT ''
+    CHECK (available_source IN ('', 'webhook'));
+ALTER TABLE services ADD COLUMN available_at     timestamptz;
+
+CREATE INDEX services_by_image_repo ON services (image_repo) WHERE removed_at IS NULL;
+```
+
+### C2. `internal/master/model` additions
+
+`Service` gains `AvailableRef string`, `AvailableSource string`, `AvailableAt *time.Time`. Constants `ModeManual`, `ModeAuto`, `SourceWebhook`. `type Available struct { Ref, Source string; At time.Time }`.
+
+### C3. Store methods
+
+```go
+ServicesByRepo(ctx, repo string) ([]model.Service, error)   // non-removed, same order as ListServices
+SetAvailable(ctx, id int64, a model.Available) error
+SetUpdateMode(ctx, id int64, mode string) error             // model.ErrNotFound for unknown or removed
+```
+
+`FinishJob` additionally reads the job's `created_at` and, on success, clears `available_*` in the same transaction when the job's `to_ref` (as created, without `@digest`) equals `available_ref` and `available_at <= created_at`.
+
+### C4. `internal/master/jobs`
+
+```go
+type StartRequest struct {
+    Trigger     model.JobTrigger
+    RequestedBy string
+    Ref         string // empty: the service's followed tag; set: used as is
+}
+func (s *Service) Start(ctx, svc model.Service, req StartRequest) (model.Job, error)
+func (s *Service) AutoAllowed(svc model.Service) error
+func FollowedRef(actualRef string) string // "repo:tag", or "" when no tag is followed
+```
+
+`Start` has the return contract `Trigger` had: `*model.NotUpdatableError` with no job, `model.ErrJobActive` with the existing job, and an already-failed job for offline / too-old / start failures. `Trigger(ctx, id, by)` is `GetService` plus `Start` with `TriggerManual`.
+
+### C5. `internal/master/webhook`
+
+```go
+type Config struct { Token string; Timeout time.Duration; PublicURL string }
+func New(store Store, jobs Jobs, broker Broker, cfg Config, log *slog.Logger) *Handler
+func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request)
+```
+
+Request `{"image","tag"}`; response `{"request_id","image","tag","matched","results":[{"service_id","service","kind","mode","status","job_id","job_url","to_ref","error","duration_ms"}]}`. Statuses `succeeded | failed | timeout | marked | busy | refused`; HTTP `500` when any result is `failed` or `timeout`, else `200`. Other codes: `400`, `401`, `404`, `503`. See the README for the full table.
+
+### C6. Master configuration
+
+`UPDATER_API_KEY` (optional) and `UPDATE_TIMEOUT` (default `5m`, positive Go duration, same error text style as the agent's).
+
+### C7. Web
+
+`POST /services/{id}/mode` with `mode=manual|auto` and optional `return=overview`. `303` to the service page (or `/`), `400` for a bad mode, `404` for an unknown or removed service, `409` re-rendering the service page with "Cannot enable auto: <reason>." Publishes `events.Services`.
+
+## Tasks
+
+### Task 1: Store and model
+
+Migration C1, model C2, store C3. Integration tests (`//go:build integration`): `ServicesByRepo` excludes removed rows and other repos; `SetAvailable` round trip; `SetUpdateMode` including `ErrNotFound`; new service is manual and upsert/snapshot keep the mode; `FinishJob` clears the same ref, keeps a different ref, keeps newer, keeps on failure.
+
+### Task 2: Jobs service
+
+Extract `Start` from `Trigger`; `updatable(svc, explicitRef)`; `AutoAllowed`; `FollowedRef`. Tests: webhook trigger with explicit ref is recorded; explicit ref bypasses only the no-tag refusal; `AutoAllowed` table; existing `TestTrigger*` stay green.
+
+### Task 3: Config, webhook, wiring
+
+Config C6 with tests. Handler C5 with fakes and the real `events.Broker`. `cmd/master/main.go` mounts `POST /update` directly on the mux and logs `webhook_disabled` when the key is empty.
+
+### Task 4: Web
+
+Store gains `SetUpdateMode`, Jobs gains `AutoAllowed`. Overview: mode column with a toggle form (hidden `csrf_token`, `mode`, `return=overview`; disabled with the reason as visible text when auto is not allowed) and an "update available" badge. Service page: Available row, Mode row, toggle form, 409 error. The forms are inside the `services` template that the SSE fragment re-renders, so they survive a refresh.
+
+### Task 5: End-to-end
+
+`internal/e2e/webhook_test.go`: `TestWebhookWaitsForAutoJob` (real in-process master and agent, gate executor, the handler on an `httptest` server) and `TestWebhookFailedAutoJobIs500`. The in-memory `memStore` gains `ServicesByRepo`, `SetAvailable` and the same clearing rule as the SQL.
+
+### Task 6: Documentation
+
+README (webhook section, master variables, warning about `auto`), `gitea-action/README.md` (do not change `action.yml`), `deploy/master.stack.yml` (`UPDATER_API_KEY` as a Docker secret), `CLAUDE.md`, this plan.
+
+## Final verification
+
+```bash
+gofmt -l . ; go vet ./...
+go build ./cmd/master ./cmd/agent && rm -f master agent
+go test ./...
+go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/
+DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/   # when a Postgres is available
+```
+
+Manual smoke test: set `UPDATER_API_KEY`, make a throwaway Compose service `auto`, then `curl -X POST -H "Authorization: Bearer $KEY" -d '{"image":"<repo>","tag":"<pushed tag>"}' https://<master>/update`. Expect `200` with `succeeded` after the job finishes, the job visible in the UI with trigger `webhook`, and the "update available" badge gone.

docs/superpowers/plans/2026-10-13-master-agent-phase-4-polling.md

@@ -0,0 +1,311 @@
+# Master/Agent Phase 4 — Registry Polling and Version History Implementation Plan
+
+> This plan documents what to build for issue #9 and where it deviates from the spec. Code in "Frozen contracts" is final; tasks give the contract and the test cases.
+
+**Goal:** The master records every version it sees (webhook, registry polling, running inventory) in `image_versions`. Every `POLL_INTERVAL` (default 6h) it checks each distinct followed `repo:tag` against its registry with one HEAD request. A new digest becomes the service's available suggestion, and `auto` services get a job. The Service page lists the 10 most recent versions of the service's repo. Picking one starts a manual job to exactly that reference; picking an older one is a rollback, including to a digest-pinned or tagless reference.
+
+**Architecture:**
+- New package `internal/master/registry`: thin wrapper around `go-containerregistry`'s `remote.Head` with `authn.DefaultKeychain` (reads `DOCKER_CONFIG`/`config.json`).
+- New package `internal/master/poller`: owns the cycle (list services, group by followed ref, run HEADs with bounded concurrency, then set available / start auto jobs / store poll error per service). `cmd/master` runs it as a background goroutine on the signal context.
+- Store: `image_versions` written in `SetAvailable` (webhook and poll, same transaction) and in `upsertService` (running inventory, inside the existing snapshot/upsert transactions).
+- Jobs gains `TriggerVersion`, which takes a version ID, never a ref from the form. `Hub.Route` gains a minimum protocol version so a pinned or tagless ref only goes to agents that understand it.
+- Agent: resolver passes digest-pinned refs through unchanged. Protocol version goes to 3.
+
+**Tech Stack:** adds `github.com/google/go-containerregistry` (`pkg/name`, `pkg/v1/remote`, `pkg/authn`; `pkg/registry` and `pkg/v1/random` in tests only).
+
+**Spec:** `docs/superpowers/specs/2026-10-12-master-agent-design.md`, sections "Versions: what the system has seen", "Data model", "Update", "Failure handling", "Registry polling", "UI". Phase 3 plan conventions still apply.
+
+## Global Constraints
+
+- `go test ./...` passes with no Docker, no Postgres and no network. Registry tests use `httptest` plus `ggcr/pkg/registry`. Postgres tests carry `//go:build integration`.
+- Interfaces declared at the consumer (`poller.Store`, `poller.Jobs`, `poller.Registry`). Hand-written fakes and testify, table-driven where there is more than one case.
+- GoDoc on every exported identifier. Conventional Commits, each ending with ` (#9)`.
+- Registry credentials never appear in logs or in `last_poll_error`. Error text stored truncated to 500 bytes.
+- Registry traffic is HEAD only: never `GET /v2/<repo>/manifests/...` (Docker Hub counts GETs as pulls). A test asserts this.
+- A pick from the UI is a POST with the CSRF token and a version ID. The ref is built server-side from the `image_versions` row, and the row must belong to the service's `image_repo`.
+- The poller never overlaps itself. Next cycle is scheduled after the previous one finishes.
+- `go.mod` `go` directive must stay `1.26.3` after `go get`.
+
+## Deliberate deviations from the spec in this phase
+
+1. **Webhook records `(repo, tag, '')`, no digest.** The master does not resolve the digest at webhook time (no registry round-trip/credentials in the CI path). The digest arrives later from polling or running inventory. The picker hides a digest-less row when a row with the same `(repo, tag)` and a digest was first seen at or after it.
+2. **Empty string instead of NULL** for missing tag or digest in `image_versions`, so `UNIQUE (repo, tag, digest)` works. A CHECK forbids both being empty.
+3. **`first_seen_at` is never bumped.** Re-observation is `ON CONFLICT DO NOTHING`; `source` is the first source that saw it. "10 most recent" = "10 most recently first seen". A new digest of a tag is a new row at the top; old digest stays below (enables rollback of moving tags).
+4. **Not polled:** services that follow no tag, `build:` services, services with empty `actual_digest`. UI shows the reason. Self and container services ARE polled (badge informative; jobs still refused by `jobs`).
+5. **Poller does not re-suggest the same version.** If `available_ref == ref && available_digest == D`, the badge is not written again (no `available_at` bump). The auto job is decided separately: it starts only if the service is auto, `actual_digest != D`, `D != last_tag_digest` and `desired_ref != ref@D`, on a fresh read of the row. `last_tag_digest` is set when a job row was created (not on `ErrJobActive`/`NotUpdatableError`, which retry next cycle). A failed auto job is not retried every interval; a rollback of an auto service is not undone by polling until a new digest appears. The poller also sets `last_tag_digest` to D when `actual_digest == D` (seeding services that predate the column). `FinishJob` also sets `last_tag_digest` to the pinned digest after any successful job whose `to_ref` as created has no digest (manual, webhook or poll); pinned and tagless picks leave it, so a rollback holds whoever started the update.
+6. **FinishJob clearing rule gets a digest condition** (C3). Rollback to an older digest of the same tag no longer clears the newer-version badge.
+7. **Poll auto jobs use `repo:tag`, not `repo:tag@D`.** v2 agents keep working for poll and webhook jobs. Only UI picks of digest-pinned or tagless versions need a v3 agent.
+8. **First cycle runs 1 minute after start**, then every `POLL_INTERVAL` measured from the end of the previous cycle.
+9. **No `last_polled_at` column** (YAGNI).
+10. **Overview gets two new badges: "follows no tag" and "poll failed"** (error as `title`). Service page gets a "Polling" row. "Drifted" is not in this phase.
+11. **Credentials from `DOCKER_CONFIG`** (a directory), same name and meaning as on the agent. Only inline `auths` entries supported in the container.
+
+## Implementation notes (differences found while building)
+
+- The plan's test names `TestResolve` and `TestComposeWritesPinnedRef` became `TestResolver`
+  (extended), a subtest of `TestComposeExecutor` and `TestSwarmUsesPinnedRef`.
+- Tasks 2 and 3 are one commit: changing `Hub.Route`'s signature breaks `jobs` and the e2e hub
+  wrapper until both are updated.
+- `Poller.Cycle` runs the registry lookups concurrently and then records the results
+  sequentially (on `context.WithoutCancel`).
+- `web.Config` gained `PollInterval` for the "every <interval>" text.
+- `authn.DefaultKeychain` with a `credsStore` whose helper binary is missing makes the lookup fail
+  with `error getting credentials - err: exec: "docker-credential-...": executable file not found in $PATH`
+  (covered by `TestCredsStoreWithoutHelperFailsTheLookup`); it does not fall back to anonymous access.
+
+## Review Focus
+
+| Failure case | Test that covers it |
+|---|---|
+| Registry access is HEAD only; no manifest GET ever happens. | registry `TestDigestUsesHeadOnly` |
+| Multi-arch: digest returned is the index digest (same as `RepoDigests` / Swarm spec). | registry `TestDigestOfIndexIsIndexDigest` |
+| Credentials from `$DOCKER_CONFIG/config.json` are sent; anonymous works without the file. | registry `TestCredentialsFromDockerConfig`, `TestAnonymousWithoutConfig` |
+| One HEAD per distinct followed ref (`repo:tag`, `repo:tag@d`, bare → `:latest`). | poller `TestOneHeadPerDistinctRef` |
+| No tag / built / no digest are skipped, no HEAD. | poller `TestSkipped` |
+| Error on one ref goes into its services' `last_poll_error`; others continue; later success clears it. | poller `TestErrorIsPerRef`, `TestErrorClearedOnSuccess` |
+| Already-suggested digest neither re-suggested nor re-started. | poller `TestSameSuggestionIsIdempotent` |
+| Auto gets exactly one `TriggerPoll` job to `repo:tag`; manual only gets marked. | poller `TestNewDigest` table |
+| `ErrJobActive` / `NotUpdatableError` from `Start` do not abort the cycle. | poller `TestStartErrorsDoNotAbortCycle` |
+| Concurrency within limit; cancellation stops the cycle; marking/starting on `WithoutCancel`. | poller `TestConcurrencyLimit`, `TestRunStopsOnCancel` |
+| Form cannot deploy an arbitrary ref: unknown version ID → 404; version of another repo → 404, no job. | jobs `TestTriggerVersion_*`, web `TestUpdateService_Version*` |
+| Pinned/tagless refs routed only to protocol ≥ 3 agents; old agents → failed "agent too old to deploy a pinned version (needs protocol 3)" job (`ErrAgentTooOldForPinned`, which also satisfies `errors.Is(err, ErrAgentTooOld)`). | jobs `TestTriggerVersion_RequiresPinnedProtocol`, hub `TestRoute` |
+| Agent deploys a pinned ref verbatim, no registry resolve. | updater `TestResolve`, `TestComposeWritesPinnedRef`, `TestSwarmUsesPinnedRef` |
+| Rollback to older digest of same tag keeps newer suggestion; pick of suggested digest clears it. | store `TestFinishJob_ClearsAvailableOnlyWhenThatVersionWasDeployed` (extended) |
+| Running versions recorded only with a digest; re-snapshot does not duplicate or reorder. | store `TestSnapshotRecordsRunningVersions` |
+| Digest-less webhook row hidden once the tag's digest is known; order, limit, repo filter. | store `TestRecentVersions` |
+| `available_source = 'poll'` passes the CHECK. | store `TestSetAvailable_RecordsVersion` |
+| E2E: pick of a pinned version reaches the agent as that exact ref. | e2e `TestPickPinnedVersion` |
+
+## Frozen contracts
+
+### C1. Migration `internal/master/store/migrations/0004_polling.sql`
+
+```sql
+ALTER TABLE services DROP CONSTRAINT services_available_source_check;
+ALTER TABLE services ADD CONSTRAINT services_available_source_check
+    CHECK (available_source IN ('', 'webhook', 'poll'));
+ALTER TABLE services ADD COLUMN available_digest text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN last_poll_error  text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN last_tag_digest text NOT NULL DEFAULT '';
+
+-- Every version the master has seen. '' means "no tag" / "digest unknown";
+-- never NULL, so the unique constraint deduplicates.
+CREATE TABLE image_versions (
+    id            bigserial   PRIMARY KEY,
+    repo          text        NOT NULL,
+    tag           text        NOT NULL DEFAULT '',
+    digest        text        NOT NULL DEFAULT '',
+    source        text        NOT NULL CHECK (source IN ('webhook', 'poll', 'running')),
+    first_seen_at timestamptz NOT NULL,
+    CONSTRAINT image_versions_tag_or_digest CHECK (tag <> '' OR digest <> ''),
+    CONSTRAINT image_versions_unique UNIQUE (repo, tag, digest)
+);
+CREATE INDEX image_versions_recent ON image_versions (repo, first_seen_at DESC, id DESC);
+```
+
+Confirm the existing constraint name in `0003_webhook.sql` before relying on it.
+
+### C2. `internal/master/model`
+
+```go
+// Service gains:
+AvailableDigest string // digest of AvailableRef when known (poll); "" from the webhook
+LastPollError   string // last registry error for this service's followed ref; "" when fine
+
+const SourcePoll = "poll"       // alongside SourceWebhook
+const SourceRunning = "running" // image_versions only
+
+type Available struct { Ref, Digest, Source string; At time.Time } // Digest added
+
+// ImageVersion is one row of image_versions.
+type ImageVersion struct {
+    ID          int64
+    Repo, Tag, Digest, Source string
+    FirstSeenAt time.Time
+}
+// Ref returns "repo:tag@digest", "repo:tag" or "repo@digest".
+func (v ImageVersion) Ref() string
+```
+
+### C3. Store
+
+```go
+// SetAvailable sets available_ref/_digest/_source/_at and, in the same
+// transaction, inserts image_versions(repo, tag parsed from a.Ref, a.Digest,
+// a.Source, a.At) ON CONFLICT DO NOTHING.
+SetAvailable(ctx, id int64, a model.Available) error
+
+// RecentVersions returns up to limit versions of repo, newest first_seen_at
+// first (ties: id DESC). It leaves out a digest-less row when a row with the
+// same repo and tag and a digest was first seen at or after it.
+RecentVersions(ctx, repo string, limit int) ([]model.ImageVersion, error)
+
+// GetImageVersion returns one version or model.ErrNotFound.
+GetImageVersion(ctx, id int64) (model.ImageVersion, error)
+
+// SetPollError sets last_poll_error on the given services ("" clears).
+SetPollError(ctx, ids []int64, msg string) error
+```
+
+`RecentVersions` SQL:
+
+```sql
+SELECT id, repo, tag, digest, source, first_seen_at FROM image_versions v
+WHERE v.repo = $1
+  AND NOT (v.digest = '' AND EXISTS (
+        SELECT 1 FROM image_versions d
+        WHERE d.repo = v.repo AND d.tag = v.tag AND d.digest <> ''
+          AND d.first_seen_at >= v.first_seen_at))
+ORDER BY v.first_seen_at DESC, v.id DESC
+LIMIT $2
+```
+
+`upsertService` (running inventory): after the service upsert, on the same `execer`, only when `svc.Digest` matches `^sha256:[0-9a-f]{64}$`:
+
+```sql
+INSERT INTO image_versions (repo, tag, digest, source, first_seen_at)
+VALUES ($1, $2, $3, 'running', $4) ON CONFLICT DO NOTHING
+```
+
+`repo`/`tag` from `imageref.Parse(svc.Image)`, `digest` = `svc.Digest`. A bare image (no tag, no digest in the name) is recorded with `tag = 'latest'`, like `jobs.FollowedRef`, so it dedupes with the poll row; `repo@digest` is recorded with `tag = ''`. Covers `ApplySnapshot` and `UpsertService` with no hub change.
+
+`serviceColumns` / `scanService` gain `s.available_digest, s.last_poll_error, s.last_tag_digest`. `SetLastTagDigest(ctx, id, digest)` sets the last column; inventory writes never touch it.
+
+`FinishJob` clearing rule (replaces phase 3's). With `requestedRef, requestedDigest, _ := strings.Cut(to_ref_as_created, "@")`:
+
+```sql
+available_ref = $4 AND available_at <= $3
+  AND ($5 = '' OR available_digest = $5)
+```
+
+`$5` = `requestedDigest`. A digest-pinned job does NOT clear a digest-less (webhook) suggestion: a rollback to `main@old` must not hide that a newer `main` was announced. The badge then stays until a plain tag job succeeds; that is accepted. Clearing sets `available_ref=''`, `available_digest=''`, `available_source=''`, `available_at=NULL`.
+
+### C4. `internal/master/registry`
+
+```go
+// Client looks up tag digests with HEAD requests only.
+type Client struct{ /* keychain authn.Keychain, userAgent string */ }
+// New uses authn.DefaultKeychain ($DOCKER_CONFIG/config.json, else ~/.docker/config.json).
+func New(userAgent string) *Client
+// Digest returns the digest ref ("repo:tag", any digest suffix ignored) points
+// at: the index digest for multi-arch images. HEAD /v2/<repo>/manifests/<tag> only.
+func (c *Client) Digest(ctx context.Context, ref string) (string, error)
+```
+
+Implementation: `name.ParseReference(repo+":"+tag)`; `remote.Head(r, remote.WithContext(ctx), remote.WithAuthFromKeychain(c.keychain), remote.WithUserAgent(c.userAgent))`; return `desc.Digest.String()`. Never fall back to GET if `Docker-Content-Digest` is missing.
+
+### C5. `internal/master/poller`
+
+```go
+type Store interface {
+    ListServices(ctx context.Context) ([]model.Service, error)
+    GetService(ctx context.Context, id int64) (model.Service, error)
+    SetAvailable(ctx context.Context, id int64, a model.Available) error
+    SetLastTagDigest(ctx context.Context, id int64, digest string) error
+    SetPollError(ctx context.Context, ids []int64, msg string) error
+}
+type Jobs interface {
+    Start(ctx context.Context, svc model.Service, req jobs.StartRequest) (model.Job, error)
+}
+type Registry interface { Digest(ctx context.Context, ref string) (string, error) }
+type Publisher interface { Publish(t events.Topic) }
+
+type Config struct {
+    Interval   time.Duration // POLL_INTERVAL
+    FirstDelay time.Duration // 1m in production; small in tests
+    Workers    int           // 4
+}
+func New(st Store, j Jobs, reg Registry, pub Publisher, cfg Config, log *slog.Logger) *Poller
+// Run polls until ctx is done. Blocking.
+func (p *Poller) Run(ctx context.Context)
+// Cycle runs one poll over all services (exported for tests and e2e).
+func (p *Poller) Cycle(ctx context.Context)
+// SkipReason says why svc is not polled: "follows no tag", "built from
+// source", "no registry digest", or "" when it is polled.
+func SkipReason(svc model.Service) string
+```
+
+Cycle semantics, per distinct `ref = jobs.FollowedRef(svc.ActualRef)` over non-removed, non-skipped services:
+1. `D, err := reg.Digest(ctx, ref)` with a 30s timeout per call.
+2. On error: `SetPollError(ids whose LastPollError != msg, msg)`, log `poll_failed` (ref, error).
+3. On success: `SetPollError(ids with LastPollError != "", "")`. Then per service:
+   - Suggestion: `svc.ActualDigest == D` → nothing; `svc.AvailableRef == ref && svc.AvailableDigest == D` → no write; otherwise `SetAvailable({ref, D, poll, now})`.
+   - Auto job, independent of the suggestion: if `UpdateMode == auto && ActualDigest != D && D != LastTagDigest && DesiredRef != ref+"@"+D`, re-read the service (`GetService`), re-check the same condition and removed/mode on the fresh row, then `Start(TriggerPoll, Ref: ref)`. If a job row was created, `SetLastTagDigest(D)`. `ErrJobActive` and `NotUpdatableError` are logged at debug and ignored, leaving `last_tag_digest` unset so the next cycle retries.
+
+Steps 2 and 3 use `context.WithoutCancel(ctx)`. At the end, publish `events.Services` if anything was written; log `poll_cycle` with refs, ok, failed, suggested, jobs, skipped, duration_ms.
+
+### C6. `internal/master/jobs` and hub
+
+```go
+// Hub.Route gains the minimum protocol version for this job:
+Route(svc model.Service, minProtocol int) (model.Route, error)
+
+// Store gains:
+GetImageVersion(ctx context.Context, id int64) (model.ImageVersion, error)
+
+// TriggerVersion starts a manual job to exactly version versionID of the
+// service's repo. model.ErrNotFound for an unknown service, an unknown
+// version, or a version of another repo. Otherwise what Start returns.
+func (s *Service) TriggerVersion(ctx context.Context, serviceID, versionID int64, requestedBy string) (model.Job, error)
+```
+
+- Ref = `svc.ImageRepo` + `":"+tag` if tag set + `"@"+digest` if digest set.
+- Calls `Start` with `TriggerManual` and explicit ref ("follows no tag" refusal does not apply; removed, self, built, container still do).
+- `Start` passes `protocol.MinPinnedRefVersion` to `Route` when the ref has a digest or no tag, otherwise `protocol.MinJobVersion`.
+
+### C7. Protocol and agent
+
+```go
+const Version = 3
+const MinJobVersion = 2       // unchanged: repo:tag jobs
+const MinPinnedRefVersion = 3 // jobs whose Ref has a digest or no tag
+```
+
+`JobStart` doc: "Ref is `repo:tag`, `repo:tag@digest` or `repo@digest`. For `repo:tag` the agent resolves the tag's current digest; a ref with a digest is deployed as is."
+
+`Resolver.Resolve`: digest present → return ref unchanged, no `DistributionInspect`; tag present → resolve as today; neither → error "has no tag to resolve" (unchanged). Compose writes the ref verbatim into the override, Swarm sets it as the image. Executors print `pinned <ref>` instead of `resolved <ref>` when nothing was resolved.
+
+### C8. Master configuration
+
+- `POLL_INTERVAL`: default `6h`, positive Go duration, error `"POLL_INTERVAL must be a positive duration such as 6h, got %q"`.
+- `DOCKER_CONFIG`: not parsed by `config`; `go-containerregistry` reads it. `main` logs `registry_credentials` with the path and whether `config.json` exists, never its contents.
+
+### C9. Web
+
+- `POST /services/{id}/update` gains optional form field `version` (an `image_versions.id`): empty → `Trigger` as today; non-numeric → `400 invalid version`; otherwise `TriggerVersion`. Results: `303` to the job (also on `ErrJobActive`), `404` on `ErrNotFound`, `409` re-render "Cannot update: <reason>."
+- `web.Store` gains `RecentVersions`; `web.Jobs` gains `TriggerVersion`.
+- Service page: "Versions" table, up to 10 rows: ref (shortRef, full ref as title), source, first seen, "running" marker when digest equals `ActualDigest`, a POST form with `csrf_token` and `version`. Forms shown only when `AutoBlocked == ""`. Rows with empty tag note "the service will follow no tag afterwards". When the service is `auto`, header notes auto mode will move it forward at the next new version. A "Polling" row shows `SkipReason`, or `last_poll_error`, or "every <interval>".
+- Overview: badge "follows no tag" when `FollowedRef(ActualRef) == ""`; badge "poll failed" (title = error) when `LastPollError != ""`. Template func `followsNoTag`.
+
+## Tasks
+
+### Task 1: Store and model (C1, C2, C3)
+Migration, model fields, store methods, `upsertService` running versions, `FinishJob` rule. Integration tests: SetAvailable writes digest/source poll and records version, second call keeps first_seen_at; TestSnapshotRecordsRunningVersions (tag+digest, tagless digest, no digest skipped, re-snapshot idempotent); TestRecentVersions (order, limit 10, repo filter, digest-less row hidden only when a digest row for the same tag was first seen at or after it); GetImageVersion ErrNotFound; SetPollError sets/clears, upsert does not touch last_poll_error/available_*; FinishJob table extended (tag job clears poll suggestion; pick tag@old keeps suggestion tag@new; pick tag@new clears; webhook suggestion (digest '') cleared by matching tag; tagless pick keeps it). Update the e2e `memStore` in the same commit.
+
+### Task 2: Protocol, agent and hub routing (C7, C6 Route)
+Bump protocol, MinPinnedRefVersion, docs; Resolver pass-through; compose/swarm output line; `hub.Route(svc, minProtocol)`. Tests: TestResolve table (tag resolves; tag@d and @d pass through with zero inspect calls; bare errors); compose pinned ref verbatim in override; swarm spec image equals pinned ref; route rows v2 agent + min 3 → ErrAgentTooOld, v3 → ok, swarm fallback to v3 manager when reporting manager is v2.
+
+### Task 3: Jobs (C6)
+`Start` picks min version from the ref; `TriggerVersion`. Tests: version of other repo → ErrNotFound, no CreateJob; ref construction table; min version passed to fake hub (plain Trigger → 2, pinned → 3); explicit ref bypasses no-tag but not removed/self/built/container; existing tests green.
+
+### Task 4: Registry client and poller (C4, C5)
+Registry tests (local httptest + ggcr/pkg/registry, no network): random.Index digest equals index digest; recording handler shows only HEAD on /manifests/; basic-auth handler succeeds with temp DOCKER_CONFIG (t.Setenv) and fails without; 404 → error. Verify behaviour of DefaultKeychain when config.json has credsStore but no helper binary; document. Poller tests per Review Focus, plus Run test (first cycle after delay, no overlap, returns on cancel).
+
+### Task 5: Config, wiring, web (C8, C9)
+Config POLL_INTERVAL with tests; add to setAll list. `cmd/master/main.go`: registry.New, poller, `go p.Run(ctx)` with done channel, wait for it after HTTP shutdown and before st.Close; log registry_credentials. Web handlers/templates per C9 with tests (picker ≤10 rows with CSRF and version IDs; hidden for built/self/container/removed; POST with version → TriggerVersion, without → Trigger; 400/404/409; overview badges; Polling row).
+
+### Task 6: End-to-end
+`internal/e2e/versions_test.go`: TestPickPinnedVersion; TestPollStartsAutoJob (poller.Cycle with fake registry → one TriggerPoll job to repo:tag; second Cycle starts nothing). `currentHub.Route` new signature.
+
+### Task 7: Documentation
+README (POLL_INTERVAL, DOCKER_CONFIG, "Registry polling" section: polled/skipped, credentials only inline auths, Docker Hub note, version picker & rollback, auto caveat, agent protocol 3 for pinned/tagless picks). Spec: replace "Believed not to count against Docker Hub pull limits for HEAD requests — verify during implementation" with verified statement + source link (docs.docker.com/docker-hub/usage/pulls: "Using GET emulates a real pull and counts towards the limit. Using HEAD won't."; note the separate unquantified abuse rate limit). `deploy/master.stack.yml`: mount config dir read-only, set DOCKER_CONFIG. CLAUDE.md: architecture entries for registry and poller; gotchas (HEAD only; digest-less webhook row hidden once digest known; image_versions uses '' not NULL; picks by version ID and pinned/tagless need protocol 3; FinishJob rule includes digest (replace existing bullet); rollback to repo@digest makes the service follow no tag); phase 4 plan reference. Save this plan as `docs/superpowers/plans/2026-10-13-master-agent-phase-4-polling.md`.
+
+## Final verification
+
+```bash
+gofmt -l . ; go vet ./...
+go build ./cmd/master ./cmd/agent && rm -f master agent
+go test ./...
+go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/
+DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/
+```

docs/superpowers/plans/2026-10-13-master-agent-phase-5-drift-merge.md

@@ -0,0 +1,104 @@
+# Master/Agent Phase 5 — Drift and Merge Implementation Plan
+
+> This plan documents what was built for issue #10 (backend only) and where it deviates from the spec. Code in "Frozen contracts" is final; tasks give the contract and the test cases.
+
+**Goal:** The master knows when a service runs something other than the version the last job deployed (drift), never corrects it by itself, and offers two ways out: restore the desired version or accept the running one. When a Compose service disappears from one agent and appears on another, the master suggests that it moved, and a merge moves the old row (history, mode, desired ref) to the new location. Merging can also be done by hand.
+
+**Architecture:**
+- Model: `Service.Drifted()` plus `RunsRef` and `RunningRef` decide drift and what accept writes. `Move` carries a suggestion.
+- Store: migration 0005 adds `services.awaiting_inventory`; `FinishJob` sets it, every inventory upsert clears it. New `AcceptRunning`, `MoveCandidates`, `MergeSources`, `MergeService` (the only code that deletes a `services` row).
+- Jobs: `Start` refuses poll and webhook jobs for a drifted service. `Restorable` and `RestoreDesired` deploy the recorded ref again as a manual job.
+- Web: three POST routes (`/services/{id}/restore`, `/accept`, `/merge`) and new `pageData` fields. No template or CSS changes: issue #18 (the Ledger rewrite of the Services and Agents pages) is a separate UI change; the restore, accept and merge buttons arrive with the UI follow-up, issue #28.
+
+**Tech Stack:** no new dependencies.
+
+**Spec:** `docs/superpowers/specs/2026-10-12-master-agent-design.md`, sections "Drift" and "Service identity and moves". Phase 4 plan conventions still apply.
+
+## Global Constraints
+
+- `go test ./...` passes with no Docker and no Postgres. Postgres tests carry `//go:build integration`.
+- No changes to `internal/master/web/templates/*`, `static/*`, `layout.html` or `groupServices`.
+- Interfaces declared at the consumer (`jobs.Store`, `web.Store`, `web.Jobs`). Hand-written fakes and testify, table-driven where there is more than one case.
+- GoDoc on every exported identifier. Conventional Commits, each ending with ` (#10)`.
+- Every state-changing request is a POST with a CSRF token. Refusals render the service page with the reason (409), not a bare error.
+- The invariants of earlier phases hold: one active job per service (`update_jobs_one_active`), the webhook and poller never create services, the updater's own services are never updated or merged.
+
+## Deliberate deviations from the spec in this phase
+
+1. **Drift compares what runs, not reference strings.** The spec says a service drifts when `actual_ref` differs from `desired_ref`. `desired_ref` is always `repo:tag@digest` while `actual_ref` is the raw image string, so a string comparison would flag every healthy service. `RunsRef` compares the repo exactly (no `docker.io` normalisation), then the digest when both sides know one, else the followed tag. The same digest under a different tag is therefore not drift.
+2. **`awaiting_inventory`.** The agent's upsert arrives a debounce interval after `job.result`; until then `desired_ref` is newer than `actual_ref`. The flag, set by `FinishJob` and cleared by the next upsert of the row, keeps that window from counting as drift.
+3. **Poll and webhook jobs are refused for a drifted service** (`ReasonDrifted`). Manual jobs are allowed, so a person can still update or roll back.
+4. **Move suggestion is stricter than "exactly one appears".** Besides one removed and one live row on each side, the live row must be newer, have no desired ref and no jobs, and the removed row must have something worth moving (desired ref, auto mode or jobs). Otherwise every restarted stack would produce a suggestion.
+5. **Manual merge is allowed for Compose and Swarm of the same kind**, names may differ, same agent allowed. Plain containers and the updater itself are refused.
+6. **Accept does not clear `available_*`**, and the form's `actual` field (`Service.RunningRef()`) is only a guard; the value written comes from the row.
+7. **The overview does not show moves or drift yet.** `MoveCandidates` is only read on the service page. `Service.Drifted()` is a model method, so the UI follow-up needs no further wiring.
+8. **`CreateJob` maps the foreign key violation on `update_jobs_service_id_fkey` to `ErrNotFound`**, so a job for a service that a merge just deleted fails cleanly.
+
+## Review Focus
+
+| Failure case | Test that covers it |
+|---|---|
+| Drift by digest, not string: tag+digest vs tag and reported digest, bare image, tagless, port colon, other repo, case. | model `TestRunsRef`, `TestDrifted` |
+| Accept writes `repo:tag@digest` from the running version. | model `TestRunningRef` |
+| A fresh job result is not drift until the inventory arrives. | store `TestFinishJob_SetsAwaitingInventoryUntilTheNextUpsert`; e2e `TestNoDriftAfterASuccessfulJob` |
+| Accept is guarded by the running version, refuses removed/never-updated, leaves `last_tag_digest`, `available_*`, mode alone. | store `TestAcceptRunning_*` |
+| Poll and webhook start no job for a drifted service; manual does. | jobs `TestStart_RefusesPollAndWebhookWhileDrifted`; e2e `TestDriftedAutoServiceIsNotUpdatedByPollOrWebhook` |
+| Restore deploys the recorded ref and remembers the drifted digest of the same tag. | jobs `TestRestoreDesired_*`; e2e `TestRestoreDeploysThePinnedRefOnTheAgent` |
+| Move suggestion only for a unique, new, history-less target and a removed source with history. | store `TestMoveCandidates` |
+| Merge keeps id, mode, desired ref, first seen; takes location and inventory; moves jobs; deletes the duplicate. | store `TestMergeService_MovesHistoryModeAndDesiredToTheNewLocation` |
+| Merge refusals change nothing (active job, kinds, container, self, live old, removed new, revoked agent, unknown). | store `TestMergeService_Refusals` |
+| Inventory after a merge updates the survivor; the old agent returning creates a new row. | store `TestMergeService_InventoryAfterTheMerge` |
+| A job for a deleted service is `ErrNotFound`. | store `TestCreateJob_UnknownServiceIsNotFound` |
+| Handlers: redirects, 400/404/409, signals published, survivor id in the redirect. | web `TestRestoreDesired`, `TestAcceptRunning_*`, `TestMergeService_*`, `TestServiceDetail_DriftAndMoveData` |
+
+## Frozen contracts
+
+### C1. Migration `0005_drift_merge.sql`
+`services.awaiting_inventory boolean NOT NULL DEFAULT false`; index `services_by_compose_name` on `(compose_project, compose_service) WHERE kind = 'compose'`.
+
+### C2. `internal/master/model`
+`Service.AwaitingInventory`; `Service.Drifted()`; `RunsRef(desired, actualRef, actualDigest)`; `RunningRef(actualRef, actualDigest)`; `Move{FromID, ToID, FromAgent, ToAgent}` (FromID is the removed row). `ReasonDrifted`, `ReasonNotDrifted`; `MergeRefusedError{Reason}` with the `ReasonMerge*` constants (`ReasonSelf` is reused).
+
+### C3. Store
+- `upsertService` sets `awaiting_inventory = false`; `FinishJob` success sets it to true.
+- `AcceptRunning(ctx, id, actualRef, actualDigest, desired)`: updates only a live, previously updated row that still reports that actual ref and digest. `ErrNotFound` for unknown or removed, `ErrConflict` otherwise.
+- `MoveCandidates(ctx) ([]model.Move, error)`: Compose only, not self, other agent, unique on both sides before the extra filters (live row newer, no desired ref, no jobs; removed row has a desired ref, auto mode or jobs). Agent names by `LEFT JOIN`, so a revoked agent keeps its name.
+- `MergeSources(ctx, toID)`: removed rows of the same kind (not container, not self), same name first, then `removed_at DESC`, at most 50.
+- `MergeService(ctx, fromID, toID)`: one transaction. Lock order: the live row's agent (`FOR SHARE`, `ErrRevoked`), then both rows `ORDER BY id FOR UPDATE`. Validates (`MergeRefusedError`), refuses an active job on either row (`ErrJobActive`), moves the duplicate's jobs, reads its location and inventory, **deletes it**, then updates the old row (keeps id, kind, `first_seen_at`, `update_mode`, `desired_ref`; takes agent, location, inventory columns, available suggestion and poll error; `last_tag_digest` only if the old row has none; `removed_at = NULL`, `awaiting_inventory = false`). The delete comes first because `location_key` is unique and not deferrable.
+- `CreateJob`: foreign key violation `update_jobs_service_id_fkey` becomes `ErrNotFound`.
+
+### C4. Jobs
+`Start` returns `NotUpdatableError{ReasonDrifted}` for a non-manual trigger on a drifted service. `Restorable(svc)`: drifted, then `updatable(svc, true)`, and the desired ref's repo is not the agent image repo. `RestoreDesired(ctx, id, by)`: `Start` with `Ref: svc.DesiredRef`, manual. After a created job, `SetLastTagDigest` to the drifted digest when the actual tag equals the desired tag, the digest is well formed, differs from the desired digest and from `last_tag_digest`, and `last_tag_digest` is empty or equals the desired digest (otherwise it holds back a newer digest and stays).
+
+### C5. Web
+Routes `POST /services/{id}/restore`, `/accept` (form `actual` required, the `Service.RunningRef()` the user saw, compared with the row's; optional `return=overview`), `/merge` (form `from` = removed row, `{id}` = live row; redirects to the surviving id). `pageData` gains `RestoreBlocked`, `MoveFrom`, `MoveTo`, `MergeSources`. Log events `drift_accepted` and `service_merged`.
+
+### C6. Concurrency
+An upsert for the live row's location waits on its lock, then updates the survivor. A snapshot after the merge lists the survivor's key, so it is not marked removed. A removal racing a merge can be lost; the next snapshot fixes it. `CreateJob` on the deleted row gets `ErrNotFound`; the poller and webhook re-read and log it.
+
+## Tasks
+
+### Task 1: Store and model (C1, C2, C3)
+Migration, `imageref.IsDigest` (replaces the local pattern in the store), model types and functions, store methods. Tests: model tables; integration tests listed in Review Focus.
+
+### Task 2: Jobs (C4)
+`Store.SetLastTagDigest` on the consumer interface, `Start` refusal, `Restorable`, `RestoreDesired`. Tests: refusal table, manual job to the desired ref with protocol 3, active job, old agent, last-tag-digest table.
+
+### Task 3: Web (C5)
+Handlers, interfaces, `pageData`. Tests with fakes.
+
+### Task 4: End-to-end
+`memStore.FinishJob` sets `AwaitingInventory`. `internal/e2e/drift_test.go`.
+
+### Task 5: Documentation
+README section "Drift and moved services", webhook note, CLAUDE.md design intent and gotchas, this plan.
+
+## Final verification
+
+```bash
+gofmt -l . ; go vet ./...
+go build ./cmd/master ./cmd/agent && rm -f master agent
+go test ./...
+go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/
+DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/
+```

docs/superpowers/specs/2026-10-12-master-agent-design.md

@@ -247,8 +247,12 @@ them. An agent recreating itself would kill the job it is running.
 - Mechanism: `HEAD /v2/<repo>/manifests/<tag>`; the returned digest is compared with
   `actual_digest`. Use `go-containerregistry` — it handles Docker Hub's token auth and
   multi-arch indexes.
-- Interval `POLL_INTERVAL`, default 6h. Believed not to count against Docker Hub pull
-  limits for HEAD requests — verify during implementation.
+- Interval `POLL_INTERVAL`, default 6h. Docker Hub documents that HEAD requests do not
+  count against the pull limit ("Using GET emulates a real pull and counts towards the
+  limit. Using HEAD won't."; verified against
+  https://docs.docker.com/docker-hub/usage/pulls/). The same page mentions a separate,
+  unquantified abuse rate limit for many pulls from one IP address, so the poller stays
+  HEAD-only and never downloads a manifest.
 - Credentials: a Docker `config.json` (as written by `docker login`) mounted into the
   master. No credential UI, no encryption in Postgres.
 

env.sample

@@ -1,15 +0,0 @@
-# Copy to .env (which is gitignored) and fill in real values:
-#   cp env.sample .env && $EDITOR .env
-
-# Required: bearer token clients must send as `Authorization: Bearer <value>`.
-# Generate with: openssl rand -hex 32
-UPDATER_API_KEY=
-
-# Optional: root directory the service is allowed to manage stacks under.
-# Defaults to /tmp for local smoke testing. For real use, set to your stacks dir,
-# e.g. /home/shcizo/self-hosted
-# STACKS_ROOT=/home/shcizo/self-hosted
-
-# Optional: build info baked into the binary via -ldflags. Not required.
-# COMMIT=$(git rev-parse --short HEAD)
-# BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)

gitea-action/README.md

@@ -1,6 +1,6 @@
 # Deploy via package-updater (composite action)
 
-Notifies `package-updater` to `docker compose pull` + `up -d` for the matching service(s) after a CI build.
+Notifies the `package-updater` master after a CI build. The master records the pushed version on every matching service and, for services in auto mode, updates them and waits for the result.
 
 ## Usage
 
@@ -20,7 +20,7 @@ jobs:
           token: ${{ secrets.UPDATER_TOKEN }}
 ```
 
-`UPDATER_TOKEN` should be set as an organisation-level secret in Gitea so all repos share it.
+`endpoint` is the master's `/update`, and `UPDATER_TOKEN` is the master's `UPDATER_API_KEY`. It should be set as an organisation-level secret in Gitea so all repos share it.
 
 ## Inputs
 
@@ -28,9 +28,20 @@ jobs:
 |---|---|---|---|
 | `endpoint` | yes | — | Full URL to `/update` |
 | `image` | yes | — | Image reference without tag |
-| `tag` | no | `""` | Tag that was just pushed (logged for audit) |
+| `tag` | no | `""` | Tag that was just pushed. Auto services are updated to `<their image repo>:<tag>`, so it must exist in the registry. Omit it to re-pull the tag each service already follows. |
 | `token` | yes | — | Bearer token configured in package-updater |
 
 ## Failure modes
 
-The step exits non-zero if `package-updater` returns HTTP 4xx or 5xx. This is intentional — the workflow surfaces the deploy failure to whoever pushed.
+The step exits non-zero on any HTTP 4xx or 5xx. This is intentional: the workflow surfaces the deploy failure to whoever pushed.
+
+| Code | Meaning |
+|---|---|
+| `200` | Every match was marked, updated, busy or refused. Manual services are only marked, which is not an error. A service that follows no tag is `refused` when the request has no `tag`. |
+| `401` | Wrong or missing token. |
+| `404` | No service uses the image. Check `image` for typos; matching is case-sensitive. |
+| `500` | At least one auto job failed or timed out (`UPDATE_TIMEOUT` on the master). |
+| `503` | The master has no `UPDATER_API_KEY`. |
+
+A request that matches both a working and a failed service now fails as a whole (`500`); the old single-instance updater's `207` no longer exists.
+The master waits for auto jobs, so the request can take minutes; raise any step timeout in your workflow accordingly.

go.mod

@@ -5,23 +5,25 @@ go 1.26.3
 require (
 	github.com/coder/websocket v1.8.15
 	github.com/coreos/go-oidc/v3 v3.21.0
+	github.com/distribution/reference v0.6.0
 	github.com/docker/docker v28.5.2+incompatible
+	github.com/google/go-containerregistry v0.22.1
 	github.com/jackc/pgx/v5 v5.10.0
-	github.com/prometheus/client_golang v1.23.2
-	github.com/prometheus/client_model v0.6.2
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/stretchr/testify v1.11.1
 	golang.org/x/oauth2 v0.36.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/errdefs v1.0.0 // indirect
 	github.com/containerd/errdefs/pkg v0.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/cli v29.7.2+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.9.3 // indirect
 	github.com/docker/go-connections v0.7.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -31,29 +33,24 @@ require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/klauspost/compress v1.19.2 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/sys/atomicwriter v0.1.0 // indirect
 	github.com/moby/term v0.5.2 // indirect
 	github.com/morikuni/aec v1.1.0 // indirect
-	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/sirupsen/logrus v1.9.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
 	go.opentelemetry.io/otel v1.43.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect
 	go.opentelemetry.io/otel/metric v1.43.0 // indirect
 	go.opentelemetry.io/otel/trace v1.43.0 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/sync v0.20.0 // indirect
-	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/text v0.35.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	google.golang.org/protobuf v1.36.11 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.2 // indirect
 )

go.sum

@@ -2,8 +2,6 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK
 github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -23,8 +21,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/cli v29.7.2+incompatible h1:dlkwallR8XqfeVnA2ELEhdwvb4lsSwuB4IgsG8Q9cLY=
+github.com/docker/cli v29.7.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
 github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8=
+github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo=
 github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
 github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -40,6 +42,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/go-containerregistry v0.22.1 h1:RZuuSYhTvlDvtsK+NkutoCZ//C0X2ebLK8X8l3ULs84=
+github.com/google/go-containerregistry v0.22.1/go.mod h1:bJR35SK8XgisYmhg/FMQ/5RK0S/XrOAqLBV5/LR2XE0=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
@@ -52,14 +56,12 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
 github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
+github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
@@ -70,8 +72,6 @@ github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
 github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
 github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
 github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
@@ -80,18 +80,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
-github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
-github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
-github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
-github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
-github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
-github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
-github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
-github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
-github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -117,22 +109,22 @@ go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09
 go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
 go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
 go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
-go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
+golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
+golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
 golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
 golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
-golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
-golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
-golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
-golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
 golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
+golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
 google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
 google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg=

internal/agent/config/config.go

@@ -2,10 +2,14 @@
 package config
 
 import (
+	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 )
 
 var masterKeyPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
@@ -17,8 +21,17 @@ type Config struct {
 	Token     string // AGENT_TOKEN, shown once when the agent was added
 	SelfID    string // SELF_ID, defaults to the hostname (= container ID prefix)
 	LogLevel  string // LOG_LEVEL, default info
+
+	// UpdateTimeout bounds one update job (UPDATE_TIMEOUT, default 5m).
+	UpdateTimeout time.Duration
+	// DockerConfig is the directory holding config.json with registry
+	// credentials (DOCKER_CONFIG). Empty means anonymous registry access.
+	DockerConfig string
 }
 
+// defaultUpdateTimeout is used when UPDATE_TIMEOUT is unset.
+const defaultUpdateTimeout = 5 * time.Minute
+
 // Load reads and validates the agent configuration. It fails fast so a
 // misconfigured agent never starts dialling.
 func Load() (*Config, error) {
@@ -28,6 +41,9 @@ func Load() (*Config, error) {
 		Token:     os.Getenv("AGENT_TOKEN"),
 		SelfID:    os.Getenv("SELF_ID"),
 		LogLevel:  os.Getenv("LOG_LEVEL"),
+
+		UpdateTimeout: defaultUpdateTimeout,
+		DockerConfig:  os.Getenv("DOCKER_CONFIG"),
 	}
 	switch {
 	case cfg.MasterURL == "":
@@ -41,6 +57,18 @@ func Load() (*Config, error) {
 	case cfg.Token == "":
 		return nil, errors.New("AGENT_TOKEN is required")
 	}
+	if v := os.Getenv("UPDATE_TIMEOUT"); v != "" {
+		d, err := time.ParseDuration(v)
+		if err != nil || d <= 0 {
+			return nil, fmt.Errorf("UPDATE_TIMEOUT must be a positive duration such as 5m, got %q", v)
+		}
+		cfg.UpdateTimeout = d
+	}
+	if cfg.DockerConfig != "" {
+		if err := checkDockerConfig(cfg.DockerConfig); err != nil {
+			return nil, err
+		}
+	}
 	if cfg.SelfID == "" {
 		cfg.SelfID, _ = os.Hostname()
 	}
@@ -49,3 +77,17 @@ func Load() (*Config, error) {
 	}
 	return cfg, nil
 }
+
+// checkDockerConfig fails when DOCKER_CONFIG is set but its config.json is
+// missing or not JSON, so a typo does not silently turn into anonymous pulls.
+func checkDockerConfig(dir string) error {
+	path := filepath.Join(dir, "config.json")
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("DOCKER_CONFIG: read %s: %w", path, err)
+	}
+	if !json.Valid(raw) {
+		return fmt.Errorf("DOCKER_CONFIG: %s is not valid JSON", path)
+	}
+	return nil
+}

internal/agent/config/config_test.go

@@ -2,7 +2,9 @@ package config
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -16,6 +18,8 @@ func setValid(t *testing.T) {
 	t.Setenv("AGENT_TOKEN", "pua_abc")
 	t.Setenv("SELF_ID", "")
 	t.Setenv("LOG_LEVEL", "")
+	t.Setenv("UPDATE_TIMEOUT", "")
+	t.Setenv("DOCKER_CONFIG", "")
 }
 
 func TestLoad_Valid(t *testing.T) {
@@ -25,7 +29,7 @@ func TestLoad_Valid(t *testing.T) {
 	host, _ := os.Hostname()
 	assert.Equal(t, &Config{
 		MasterURL: "wss://10.0.0.5:8090/agent", MasterKey: validKey, Token: "pua_abc",
-		SelfID: host, LogLevel: "info",
+		SelfID: host, LogLevel: "info", UpdateTimeout: 5 * time.Minute,
 	}, cfg)
 }
 
@@ -48,3 +52,45 @@ func TestLoad_Invalid(t *testing.T) {
 		})
 	}
 }
+
+func TestLoad_JobSettings(t *testing.T) {
+	withConfig := func(t *testing.T, body string) string {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(body), 0o600))
+		return dir
+	}
+	cases := []struct {
+		name      string
+		timeout   string
+		dockerCfg func(t *testing.T) string
+		wantTime  time.Duration
+		wantErr   string
+	}{
+		{name: "defaults", wantTime: 5 * time.Minute},
+		{name: "custom timeout", timeout: "90s", wantTime: 90 * time.Second},
+		{name: "zero timeout", timeout: "0", wantErr: "UPDATE_TIMEOUT"},
+		{name: "garbage timeout", timeout: "abc", wantErr: "UPDATE_TIMEOUT"},
+		{name: "docker config without file", dockerCfg: func(t *testing.T) string { return t.TempDir() }, wantErr: "DOCKER_CONFIG"},
+		{name: "docker config not json", dockerCfg: func(t *testing.T) string { return withConfig(t, "nope") }, wantErr: "not valid JSON"},
+		{name: "docker config ok", dockerCfg: func(t *testing.T) string { return withConfig(t, `{"auths":{}}`) }, wantTime: 5 * time.Minute},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			setValid(t)
+			t.Setenv("UPDATE_TIMEOUT", c.timeout)
+			dir := ""
+			if c.dockerCfg != nil {
+				dir = c.dockerCfg(t)
+				t.Setenv("DOCKER_CONFIG", dir)
+			}
+			cfg, err := Load()
+			if c.wantErr != "" {
+				assert.ErrorContains(t, err, c.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, c.wantTime, cfg.UpdateTimeout)
+			assert.Equal(t, dir, cfg.DockerConfig)
+		})
+	}
+}

internal/agent/conn/conn.go

@@ -1,65 +1,97 @@
 // Package conn keeps an agent connected to the master: it dials over pinned
-// TLS, announces itself, sends its inventory, and streams changes until the
-// connection drops, then reconnects with backoff.
+// TLS, announces itself, sends its inventory, runs the jobs the master sends
+// and streams changes and job results until the connection drops, then
+// reconnects with backoff.
 package conn
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/coder/websocket"
 	"github.com/coder/websocket/wsjson"
 
 	"github.com/shcizo/package-updater/internal/agent/inventory"
+	"github.com/shcizo/package-updater/internal/agent/updater"
 	"github.com/shcizo/package-updater/internal/pin"
 	"github.com/shcizo/package-updater/internal/protocol"
 )
 
-// readLimit caps a single incoming message. The library default (32 KiB)
-// is too small for the master's future job messages; matches the master.
+// readLimit caps a single incoming message. Matches the master.
 const readLimit = 1 << 20
 
 // dialTimeout bounds the TLS + WebSocket handshake so a master that accepts
 // TCP but never answers does not hang the agent forever.
 const dialTimeout = 30 * time.Second
 
+// writeTimeout bounds one websocket write so a stalled peer ends the session
+// instead of blocking the writer forever.
+const writeTimeout = 10 * time.Second
+
+// defaultProgressInterval is how often job output is batched to the master.
+const defaultProgressInterval = 500 * time.Millisecond
+
 // Source supplies what the agent reports. *inventory.Collector satisfies it.
 type Source interface {
 	Hello(ctx context.Context) (protocol.Hello, error)
 	Snapshot(ctx context.Context) ([]protocol.Service, error)
 }
 
+// Jobs is the agent's job queue as the connection sees it. *updater.Runner
+// satisfies it.
+type Jobs interface {
+	// Submit queues a job; a job ID already held is ignored.
+	Submit(id string, start protocol.JobStart)
+	// Ack forgets a finished job the master has stored.
+	Ack(id string)
+	// Known lists every job ID held: queued, running, or finished unacked.
+	Known() []string
+	// TakeProgress returns output produced since the previous call.
+	TakeProgress() []updater.Progress
+	// Pending returns results the master has not acknowledged yet.
+	Pending() []updater.Finished
+	// Changed receives when there is new progress or a new result.
+	Changed() <-chan struct{}
+}
+
 // Config holds connection settings. Token is sent as a bearer header and must
 // never be logged.
 type Config struct {
-	URL         string // wss://host:port/agent
-	Token       string // AGENT_TOKEN
-	Fingerprint string // MASTER_KEY
-	Version     string
-	MinBackoff  time.Duration // default 1s
-	MaxBackoff  time.Duration // default 30s
+	URL              string // wss://host:port/agent
+	Token            string // AGENT_TOKEN
+	Fingerprint      string // MASTER_KEY
+	Version          string
+	MinBackoff       time.Duration // default 1s
+	MaxBackoff       time.Duration // default 30s
+	ProgressInterval time.Duration // default 500ms
 }
 
 // Agent is the agent's side of the master connection.
 type Agent struct {
 	cfg     Config
 	src     Source
+	jobs    Jobs
 	log     *slog.Logger
 	changes chan struct{}
 }
 
 // New returns an Agent. Call Run to connect.
-func New(cfg Config, src Source, log *slog.Logger) *Agent {
+func New(cfg Config, src Source, jobs Jobs, log *slog.Logger) *Agent {
 	if cfg.MinBackoff == 0 {
 		cfg.MinBackoff = time.Second
 	}
 	if cfg.MaxBackoff == 0 {
 		cfg.MaxBackoff = 30 * time.Second
 	}
-	return &Agent{cfg: cfg, src: src, log: log, changes: make(chan struct{}, 1)}
+	if cfg.ProgressInterval == 0 {
+		cfg.ProgressInterval = defaultProgressInterval
+	}
+	return &Agent{cfg: cfg, src: src, jobs: jobs, log: log, changes: make(chan struct{}, 1)}
 }
 
 // Notify tells the agent its inventory may have changed. It never blocks;
@@ -94,6 +126,9 @@ func (a *Agent) Run(ctx context.Context) error {
 
 // session runs one connection until it fails. connected reports whether the
 // hello was delivered, so Run resets its backoff only after real progress.
+//
+// Two goroutines serve the connection: a reader, which handles what the
+// master sends, and a writer, the only goroutine that writes to the socket.
 func (a *Agent) session(parent context.Context) (connected bool, err error) {
 	httpClient := &http.Client{Transport: &http.Transport{TLSClientConfig: pin.ClientConfig(a.cfg.Fingerprint)}}
 	// The dial context only bounds the handshake; coder/websocket does not
@@ -107,59 +142,140 @@ func (a *Agent) session(parent context.Context) (connected bool, err error) {
 	if err != nil {
 		return false, fmt.Errorf("dial master: %w", err)
 	}
-	defer c.CloseNow()
 	c.SetReadLimit(readLimit)
 
-	ctx, cancel := context.WithCancel(parent)
-	defer cancel()
-	// The master sends nothing yet in this phase, but reading is how a
-	// dropped connection is noticed, so a reader runs for the whole session.
-	go func() {
-		defer cancel()
-		for {
-			if _, _, err := c.Read(ctx); err != nil {
-				return
-			}
+	ctx, cancel := context.WithCancelCause(parent)
+	out := make(chan protocol.Envelope, 64)
+	var wg sync.WaitGroup
+	defer func() {
+		cancel(nil)
+		if parent.Err() != nil {
+			_ = c.Close(websocket.StatusNormalClosure, "")
 		}
+		c.CloseNow()
+		wg.Wait()
 	}()
 
+	wg.Add(2)
+	go func() { defer wg.Done(); a.writeLoop(ctx, cancel, c, out) }()
+	go func() { defer wg.Done(); a.readLoop(ctx, cancel, c) }()
+
+	// send queues an envelope for the writer. It gives up when the session ends.
+	send := func(typ string, payload any) error {
+		env, err := protocol.Encode(typ, payload)
+		if err != nil {
+			return err
+		}
+		return enqueue(ctx, out, env)
+	}
+	sendJob := func(typ, id string, payload any) error {
+		env, err := protocol.EncodeJob(typ, id, payload)
+		if err != nil {
+			return err
+		}
+		return enqueue(ctx, out, env)
+	}
+	fail := func(connected bool, err error) (bool, error) {
+		if cause := context.Cause(ctx); cause != nil && !errors.Is(cause, context.Canceled) {
+			return connected, cause
+		}
+		return connected, err
+	}
+
 	hello, err := a.src.Hello(ctx)
 	if err != nil {
-		return false, err
+		return fail(false, err)
 	}
 	hello.AgentVersion = a.cfg.Version
-	if err := send(ctx, c, protocol.TypeHello, hello); err != nil {
-		return false, err
+	hello.ProtocolVersion = protocol.Version
+	hello.KnownJobs = a.jobs.Known()
+	if err := send(protocol.TypeHello, hello); err != nil {
+		return fail(false, err)
 	}
 
 	last, err := a.src.Snapshot(ctx)
 	if err != nil {
-		return true, err
+		return fail(true, err)
 	}
-	if err := send(ctx, c, protocol.TypeSnapshot, protocol.Snapshot{Services: last}); err != nil {
-		return true, err
+	if err := send(protocol.TypeSnapshot, protocol.Snapshot{Services: last}); err != nil {
+		return fail(true, err)
 	}
-	a.log.Info("master_connected", "services", len(last))
+	a.log.Info("master_connected", "services", len(last), "known_jobs", len(hello.KnownJobs))
 
+	sent := make(map[string]bool) // job results already sent on this connection
+	flush := func() error {
+		for _, p := range a.jobs.TakeProgress() {
+			if err := sendJob(protocol.TypeJobProgress, p.ID, protocol.JobProgress{Output: p.Output}); err != nil {
+				return err
+			}
+		}
+		for _, f := range a.jobs.Pending() {
+			if sent[f.ID] {
+				continue
+			}
+			if err := sendJob(protocol.TypeJobResult, f.ID, f.Result); err != nil {
+				return err
+			}
+			sent[f.ID] = true
+		}
+		return nil
+	}
+
+	// flush is throttled to one call per ProgressInterval; a signal that
+	// arrives early arms the timer for the remainder instead of being lost.
+	lastFlush := time.Time{}
+	var timer *time.Timer
+	var timerC <-chan time.Time
+	defer func() {
+		if timer != nil {
+			timer.Stop()
+		}
+	}()
+	maybeFlush := func() error {
+		wait := a.cfg.ProgressInterval - time.Since(lastFlush)
+		if wait > 0 {
+			if timerC == nil {
+				timer = time.NewTimer(wait)
+				timerC = timer.C
+			}
+			return nil
+		}
+		lastFlush = time.Now()
+		return flush()
+	}
+	if err := maybeFlush(); err != nil {
+		return fail(true, err)
+	}
+
+	jobsChanged := a.jobs.Changed()
 	for {
 		select {
 		case <-ctx.Done():
-			_ = c.Close(websocket.StatusNormalClosure, "")
-			return true, ctx.Err()
+			return fail(true, ctx.Err())
+		case <-jobsChanged:
+			if err := maybeFlush(); err != nil {
+				return fail(true, err)
+			}
+		case <-timerC:
+			timerC = nil
+			lastFlush = time.Now()
+			if err := flush(); err != nil {
+				return fail(true, err)
+			}
 		case <-a.changes:
 			next, err := a.src.Snapshot(ctx)
 			if err != nil {
-				return true, err
+				return fail(true, err)
 			}
 			upserts, removes := inventory.Diff(last, next)
 			for _, s := range upserts {
-				if err := send(ctx, c, protocol.TypeUpsert, protocol.Upsert{Service: s}); err != nil {
-					return true, err
+				if err := send(protocol.TypeUpsert, protocol.Upsert{Service: s}); err != nil {
+					return fail(true, err)
 				}
 			}
 			for _, k := range removes {
-				if err := send(ctx, c, protocol.TypeRemove, protocol.Remove{Key: k}); err != nil {
-					return true, err
+				if err := send(protocol.TypeRemove, protocol.Remove{Key: k}); err != nil {
+					return fail(true, err)
 				}
 			}
 			last = next
@@ -167,13 +283,58 @@ func (a *Agent) session(parent context.Context) (connected bool, err error) {
 	}
 }
 
-func send(ctx context.Context, c *websocket.Conn, typ string, payload any) error {
-	env, err := protocol.Encode(typ, payload)
-	if err != nil {
-		return err
+// enqueue hands env to the writer, or gives up when the session ends.
+func enqueue(ctx context.Context, out chan<- protocol.Envelope, env protocol.Envelope) error {
+	select {
+	case out <- env:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// writeLoop is the only goroutine that writes to c. A failed or timed-out
+// write ends the session.
+func (a *Agent) writeLoop(ctx context.Context, cancel context.CancelCauseFunc, c *websocket.Conn, out <-chan protocol.Envelope) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case env := <-out:
+			wctx, wcancel := context.WithTimeout(ctx, writeTimeout)
+			err := wsjson.Write(wctx, c, env)
+			wcancel()
+			if err != nil {
+				cancel(fmt.Errorf("send %s: %w", env.Type, err))
+				return
+			}
+		}
+	}
+}
+
+// readLoop handles what the master sends. A message it cannot parse ends the
+// session; a message type it does not know is logged and ignored, so a newer
+// master can add message types without breaking this agent.
+func (a *Agent) readLoop(ctx context.Context, cancel context.CancelCauseFunc, c *websocket.Conn) {
+	for {
+		var env protocol.Envelope
+		if err := wsjson.Read(ctx, c, &env); err != nil {
+			cancel(fmt.Errorf("read: %w", err))
+			return
+		}
+		switch env.Type {
+		case protocol.TypeJobStart:
+			start, err := protocol.Decode[protocol.JobStart](env)
+			if err != nil || env.ID == "" {
+				cancel(fmt.Errorf("malformed job.start (id %q): %v", env.ID, err))
+				return
+			}
+			a.jobs.Submit(env.ID, start)
+		case protocol.TypeJobAck:
+			// JobAck has no fields, so the payload is not inspected.
+			a.jobs.Ack(env.ID)
+		default:
+			a.log.Warn("unknown_message_type", "type", env.Type)
+		}
 	}
-	if err := wsjson.Write(ctx, c, env); err != nil {
-		return fmt.Errorf("send %s: %w", typ, err)
-	}
-	return nil
 }

internal/agent/conn/conn_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/shcizo/package-updater/internal/agent/updater"
 	"github.com/shcizo/package-updater/internal/pin"
 	"github.com/shcizo/package-updater/internal/protocol"
 )
@@ -113,12 +114,111 @@ var (
 	svcB = protocol.Service{Kind: protocol.KindContainer, ContainerName: "b", Image: "b:1", Status: "running"}
 )
 
+// fakeJobs records what the connection tells it and returns scripted state.
+type fakeJobs struct {
+	mu       sync.Mutex
+	known    []string
+	pending  []updater.Finished
+	progress []string // output chunks since the last TakeProgress
+	submits  map[string]protocol.JobStart
+	acks     []string
+	changed  chan struct{}
+}
+
+func newFakeJobs() *fakeJobs {
+	return &fakeJobs{submits: map[string]protocol.JobStart{}, changed: make(chan struct{}, 1)}
+}
+
+func (f *fakeJobs) Submit(id string, s protocol.JobStart) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.submits[id] = s
+}
+
+func (f *fakeJobs) Ack(id string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.acks = append(f.acks, id)
+	kept := f.pending[:0:0]
+	for _, p := range f.pending {
+		if p.ID != id {
+			kept = append(kept, p)
+		}
+	}
+	f.pending = kept
+}
+
+func (f *fakeJobs) Known() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.known
+}
+
+func (f *fakeJobs) TakeProgress() []updater.Progress {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if len(f.progress) == 0 {
+		return nil
+	}
+	out := updater.Progress{ID: "1", Output: strings.Join(f.progress, "")}
+	f.progress = nil
+	return []updater.Progress{out}
+}
+
+func (f *fakeJobs) Pending() []updater.Finished {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]updater.Finished(nil), f.pending...)
+}
+
+func (f *fakeJobs) Changed() <-chan struct{} { return f.changed }
+
+func (f *fakeJobs) signal() {
+	select {
+	case f.changed <- struct{}{}:
+	default:
+	}
+}
+
+func (f *fakeJobs) addProgress(s string) {
+	f.mu.Lock()
+	f.progress = append(f.progress, s)
+	f.mu.Unlock()
+	f.signal()
+}
+
+func (f *fakeJobs) addResult(id string, r protocol.JobResult) {
+	f.mu.Lock()
+	f.pending = append(f.pending, updater.Finished{ID: id, Result: r})
+	f.mu.Unlock()
+	f.signal()
+}
+
+func (f *fakeJobs) submitted(id string) (protocol.JobStart, bool) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	s, ok := f.submits[id]
+	return s, ok
+}
+
+func (f *fakeJobs) acked() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.acks...)
+}
+
 func startAgent(t *testing.T, fm *fakeMaster, fingerprint string, src Source) *Agent {
+	t.Helper()
+	return startAgentWith(t, fm, fingerprint, src, newFakeJobs(), 0)
+}
+
+func startAgentWith(t *testing.T, fm *fakeMaster, fingerprint string, src Source, jobs Jobs, progressInterval time.Duration) *Agent {
 	t.Helper()
 	a := New(Config{
 		URL: fm.url(), Token: "pua_test", Fingerprint: fingerprint, Version: "v-test",
 		MinBackoff: 10 * time.Millisecond, MaxBackoff: 50 * time.Millisecond,
-	}, src, slog.New(slog.NewTextHandler(io.Discard, nil)))
+		ProgressInterval: progressInterval,
+	}, src, jobs, slog.New(slog.NewTextHandler(io.Discard, nil)))
 	ctx, cancel := context.WithCancel(context.Background())
 	done := make(chan struct{})
 	go func() { _ = a.Run(ctx); close(done) }()
@@ -137,7 +237,7 @@ func TestAgent_SendsHelloThenSnapshotWithToken(t *testing.T) {
 	require.Equal(t, protocol.TypeHello, hello.Type)
 	h, err := protocol.Decode[protocol.Hello](hello)
 	require.NoError(t, err)
-	assert.Equal(t, protocol.Hello{Hostname: "node-2", AgentVersion: "v-test"}, h)
+	assert.Equal(t, protocol.Hello{Hostname: "node-2", AgentVersion: "v-test", ProtocolVersion: protocol.Version}, h)
 
 	snap := fm.next(t)
 	require.Equal(t, protocol.TypeSnapshot, snap.Type)
@@ -198,3 +298,212 @@ func TestAgent_RefusesMasterWithWrongKey(t *testing.T) {
 	time.Sleep(200 * time.Millisecond)
 	assert.Empty(t, fm.auth, "the token must never be sent to a server with the wrong key")
 }
+
+func sendEnv(t *testing.T, c *websocket.Conn, env protocol.Envelope) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	require.NoError(t, wsjson.Write(ctx, c, env))
+}
+
+func TestAgent_HelloCarriesVersionAndKnownJobs(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	jobs.known = []string{"3", "7"}
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, time.Millisecond)
+
+	h, err := protocol.Decode[protocol.Hello](fm.next(t))
+	require.NoError(t, err)
+	assert.Equal(t, protocol.Version, h.ProtocolVersion)
+	assert.Equal(t, []string{"3", "7"}, h.KnownJobs)
+}
+
+func TestAgent_JobStartSubmitsAndAckAcks(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, time.Millisecond)
+	fm.next(t)
+	fm.next(t)
+	c := <-fm.conns
+
+	start := protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"}
+	env, err := protocol.EncodeJob(protocol.TypeJobStart, "42", start)
+	require.NoError(t, err)
+	sendEnv(t, c, env)
+	require.Eventually(t, func() bool { _, ok := jobs.submitted("42"); return ok }, 2*time.Second, 5*time.Millisecond)
+	got, _ := jobs.submitted("42")
+	assert.Equal(t, start, got)
+
+	ack, err := protocol.EncodeJob(protocol.TypeJobAck, "42", protocol.JobAck{})
+	require.NoError(t, err)
+	sendEnv(t, c, ack)
+	require.Eventually(t, func() bool { return len(jobs.acked()) == 1 }, 2*time.Second, 5*time.Millisecond)
+	assert.Equal(t, []string{"42"}, jobs.acked())
+}
+
+func TestAgent_SendsPendingResultsAfterSnapshot(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	jobs.pending = []updater.Finished{
+		{ID: "5", Result: protocol.JobResult{Status: protocol.ResultSucceeded, Ref: "a:1@sha256:aa", Output: "ok\n"}},
+		{ID: "6", Result: protocol.JobResult{Status: protocol.ResultFailed, Error: "boom"}},
+	}
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, time.Millisecond)
+
+	assert.Equal(t, protocol.TypeHello, fm.next(t).Type)
+	assert.Equal(t, protocol.TypeSnapshot, fm.next(t).Type)
+	r5 := fm.next(t)
+	assert.Equal(t, protocol.TypeJobResult, r5.Type)
+	assert.Equal(t, "5", r5.ID)
+	res, err := protocol.Decode[protocol.JobResult](r5)
+	require.NoError(t, err)
+	assert.Equal(t, "a:1@sha256:aa", res.Ref)
+	r6 := fm.next(t)
+	assert.Equal(t, "6", r6.ID)
+}
+
+func TestAgent_ResendsUnackedResultAfterReconnect(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	jobs.addResult("5", protocol.JobResult{Status: protocol.ResultSucceeded})
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, time.Millisecond)
+
+	fm.next(t) // hello
+	fm.next(t) // snapshot
+	assert.Equal(t, "5", fm.next(t).ID)
+	(<-fm.conns).CloseNow() // dropped without ack
+
+	assert.Equal(t, protocol.TypeHello, fm.next(t).Type)
+	assert.Equal(t, protocol.TypeSnapshot, fm.next(t).Type)
+	again := fm.next(t)
+	assert.Equal(t, protocol.TypeJobResult, again.Type)
+	assert.Equal(t, "5", again.ID)
+
+	c := <-fm.conns
+	ack, err := protocol.EncodeJob(protocol.TypeJobAck, "5", protocol.JobAck{})
+	require.NoError(t, err)
+	sendEnv(t, c, ack)
+	require.Eventually(t, func() bool { return len(jobs.acked()) == 1 }, 2*time.Second, 5*time.Millisecond)
+	c.CloseNow()
+
+	assert.Equal(t, protocol.TypeHello, fm.next(t).Type)
+	assert.Equal(t, protocol.TypeSnapshot, fm.next(t).Type)
+	select {
+	case e := <-fm.envs:
+		t.Fatalf("acked result was sent again: %+v", e)
+	case <-time.After(100 * time.Millisecond):
+	}
+}
+
+func TestAgent_ProgressIsThrottled(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, 100*time.Millisecond)
+	fm.next(t)
+	fm.next(t)
+
+	for range 20 {
+		jobs.addProgress("x")
+		time.Sleep(2500 * time.Microsecond)
+	}
+	var batches int
+	var total strings.Builder
+	collect := func(d time.Duration) {
+		deadline := time.After(d)
+		for {
+			select {
+			case e := <-fm.envs:
+				if e.Type != protocol.TypeJobProgress {
+					continue
+				}
+				p, err := protocol.Decode[protocol.JobProgress](e)
+				require.NoError(t, err)
+				batches++
+				total.WriteString(p.Output)
+			case <-deadline:
+				return
+			}
+		}
+	}
+	collect(100 * time.Millisecond)
+	assert.LessOrEqual(t, batches, 2)
+	collect(400 * time.Millisecond)
+	assert.Equal(t, strings.Repeat("x", 20), total.String(), "the last chunk must be delivered")
+}
+
+func TestAgent_IgnoresUnknownType(t *testing.T) {
+	fm := newFakeMaster(t)
+	jobs := newFakeJobs()
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, jobs, time.Millisecond)
+	fm.next(t)
+	fm.next(t)
+	c := <-fm.conns
+
+	sendEnv(t, c, protocol.Envelope{Type: "future.thing", Payload: []byte(`{"a":1}`)})
+	env, err := protocol.EncodeJob(protocol.TypeJobStart, "9", protocol.JobStart{Kind: protocol.KindCompose, Ref: "a:1"})
+	require.NoError(t, err)
+	sendEnv(t, c, env)
+	require.Eventually(t, func() bool { _, ok := jobs.submitted("9"); return ok }, 2*time.Second, 5*time.Millisecond)
+	assert.Zero(t, len(fm.conns), "no reconnect expected")
+}
+
+func TestAgent_ClosesOnMalformedJobStart(t *testing.T) {
+	fm := newFakeMaster(t)
+	startAgentWith(t, fm, fm.fingerprint, &fakeSource{}, newFakeJobs(), time.Millisecond)
+	fm.next(t)
+	fm.next(t)
+	c := <-fm.conns
+
+	env, err := protocol.Encode(protocol.TypeJobStart, protocol.JobStart{Ref: "a:1"}) // no ID
+	require.NoError(t, err)
+	sendEnv(t, c, env)
+
+	select {
+	case <-fm.conns:
+	case <-time.After(2 * time.Second):
+		t.Fatal("agent did not reconnect after a malformed job.start")
+	}
+}
+
+func TestAgent_InventoryAndProgressConcurrently(t *testing.T) {
+	fm := newFakeMaster(t)
+	src := &fakeSource{}
+	src.set(svcA)
+	jobs := newFakeJobs()
+	a := startAgentWith(t, fm, fm.fingerprint, src, jobs, time.Millisecond)
+	fm.next(t)
+	fm.next(t)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := range 30 {
+			jobs.addProgress("y")
+			if i%2 == 0 {
+				src.set(svcB)
+			} else {
+				src.set(svcA)
+			}
+			a.Notify()
+			time.Sleep(time.Millisecond)
+		}
+	}()
+	<-done
+	var progress, inventory int
+	deadline := time.After(500 * time.Millisecond)
+	for {
+		select {
+		case e := <-fm.envs:
+			switch e.Type {
+			case protocol.TypeJobProgress:
+				progress++
+			case protocol.TypeUpsert, protocol.TypeRemove:
+				inventory++
+			}
+		case <-deadline:
+			assert.Positive(t, progress)
+			assert.Positive(t, inventory)
+			return
+		}
+	}
+}

internal/agent/inventory/collector.go

@@ -5,8 +5,12 @@ package inventory
 import (
 	"context"
 	"fmt"
+	"log/slog"
+	"os"
 	"sort"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/image"
@@ -18,10 +22,12 @@ import (
 )
 
 const (
-	labelProject    = "com.docker.compose.project"
-	labelService    = "com.docker.compose.service"
-	labelWorkingDir = "com.docker.compose.project.working_dir"
-	labelSwarmTask  = "com.docker.swarm.service.id"
+	labelProject     = "com.docker.compose.project"
+	labelService     = "com.docker.compose.service"
+	labelWorkingDir  = "com.docker.compose.project.working_dir"
+	labelConfigFiles = "com.docker.compose.project.config_files"
+	labelEnvFile     = "com.docker.compose.project.environment_file"
+	labelSwarmTask   = "com.docker.swarm.service.id"
 )
 
 // DockerClient is the subset of the Docker SDK the collector needs.
@@ -34,16 +40,56 @@ type DockerClient interface {
 	Info(ctx context.Context) (system.Info, error)
 }
 
+// BuildCheck finds the services of a Compose project that are built from
+// source. The arguments are labels of one of the project's containers.
+// *updater.BuildChecker satisfies it.
+type BuildCheck interface {
+	BuiltServices(ctx context.Context, project, workingDir, configFilesLabel, envFileLabel string) (map[string]bool, error)
+}
+
+// buildRetry is how long a failed build check is not repeated while the
+// project's files stay the same.
+const buildRetry = 5 * time.Minute
+
 // Collector reads the local daemon's inventory.
 type Collector struct {
 	cli    DockerClient
 	selfID string
+
+	check BuildCheck
+	log   *slog.Logger
+	now   func() time.Time
+
+	mu     sync.Mutex
+	cache  map[string]buildEntry // by Compose project
+	warned map[string]bool       // logged warnings: "<project>" (check failed) and "<project>/<service>" (service not in config)
 }
 
+// buildEntry is the cached outcome of a build check. It is valid as long as
+// key (the project's labels plus its files' sizes and mtimes) is unchanged.
+type buildEntry struct {
+	key   string
+	built map[string]bool
+	err   error
+	at    time.Time
+}
+
+// stackLabels are the Compose labels a build check needs.
+type stackLabels struct{ workingDir, configFiles, envFile string }
+
 // New returns a Collector. selfID is a prefix of the agent's own container
 // ID (inside a container, the hostname); that container is marked Self.
 func New(cli DockerClient, selfID string) *Collector {
-	return &Collector{cli: cli, selfID: selfID}
+	return &Collector{cli: cli, selfID: selfID, now: time.Now, cache: map[string]buildEntry{}, warned: map[string]bool{}}
+}
+
+// WithBuildCheck makes the collector mark Compose services that have a
+// build: section as Built. The check re-runs only when a project's files
+// change. When it fails, the project's services are reported as not built and
+// the failure is logged once. It returns c.
+func (c *Collector) WithBuildCheck(check BuildCheck, log *slog.Logger) *Collector {
+	c.check, c.log = check, log
+	return c
 }
 
 // Hello returns the machine facts sent on connect. AgentVersion is left for
@@ -80,6 +126,7 @@ func (c *Collector) Snapshot(ctx context.Context) ([]protocol.Service, error) {
 	}
 
 	byKey := make(map[string]protocol.Service)
+	stacks := make(map[string]stackLabels)
 	for _, ct := range containers {
 		if _, isTask := ct.Labels[labelSwarmTask]; isTask {
 			continue
@@ -89,12 +136,19 @@ func (c *Collector) Snapshot(ctx context.Context) ([]protocol.Service, error) {
 			return nil, err
 		}
 		svc := c.fromContainer(ct, ref, repoDigests[ct.ImageID])
+		if svc.Kind == protocol.KindCompose {
+			if _, ok := stacks[svc.ComposeProject]; !ok {
+				stacks[svc.ComposeProject] = stackLabels{svc.WorkingDir, svc.ConfigFiles, ct.Labels[labelEnvFile]}
+			}
+		}
 		if prev, ok := byKey[svc.Key()]; ok {
 			svc = mergeReplicas(prev, svc)
 		}
 		byKey[svc.Key()] = svc
 	}
 
+	c.markBuilt(ctx, byKey, stacks)
+
 	if info.Swarm.ControlAvailable {
 		services, err := c.cli.ServiceList(ctx, swarm.ServiceListOptions{Status: true})
 		if err != nil {
@@ -148,6 +202,7 @@ func (c *Collector) fromContainer(ct container.Summary, ref string, repoDigests
 		svc.ComposeProject = project
 		svc.ComposeService = service
 		svc.WorkingDir = ct.Labels[labelWorkingDir]
+		svc.ConfigFiles = ct.Labels[labelConfigFiles]
 		return svc
 	}
 	svc.Kind = protocol.KindContainer
@@ -214,3 +269,93 @@ func swarmStatus(st *swarm.ServiceStatus) string {
 		return "degraded"
 	}
 }
+
+// markBuilt sets Built on Compose services that have a build: section.
+func (c *Collector) markBuilt(ctx context.Context, byKey map[string]protocol.Service, stacks map[string]stackLabels) {
+	if c.check == nil {
+		return
+	}
+	for key, svc := range byKey {
+		if svc.Kind != protocol.KindCompose {
+			continue
+		}
+		built := c.builtServices(ctx, svc.ComposeProject, stacks[svc.ComposeProject])
+		if built == nil {
+			continue // check failed; already logged
+		}
+		isBuilt, known := built[svc.ComposeService]
+		if !known {
+			// Fail closed: a service compose does not list cannot be shown to
+			// be free of build:, so it is not offered for update.
+			c.warnOnce(svc.ComposeProject+"/"+svc.ComposeService, "compose_service_not_in_config",
+				"project", svc.ComposeProject, "service", svc.ComposeService)
+		}
+		if isBuilt || !known {
+			svc.Built = true
+			byKey[key] = svc
+		}
+	}
+}
+
+// builtServices returns the project's built services, from the cache while
+// the project's files are unchanged.
+func (c *Collector) builtServices(ctx context.Context, project string, l stackLabels) map[string]bool {
+	// The key covers the files named in the labels. Changes in files they
+	// pull in (extends:, include:, the default .env) do not invalidate the
+	// cache, so Built can be stale until another change or an agent restart.
+	// The job-time check is always fresh, so jobs stay safe.
+	key := l.workingDir + "|" + l.configFiles + "|" + l.envFile + "|" + fileStamps(l.configFiles+","+l.envFile)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if e, ok := c.cache[project]; ok && e.key == key && (e.err == nil || c.now().Sub(e.at) < buildRetry) {
+		return e.built
+	}
+	built, err := c.check.BuiltServices(ctx, project, l.workingDir, l.configFiles, l.envFile)
+	c.cache[project] = buildEntry{key: key, built: built, err: err, at: c.now()}
+	if err == nil {
+		// Forget the project's warnings so they are logged again if the
+		// problem returns. Project names cannot contain "/".
+		for k := range c.warned {
+			if k == project || strings.HasPrefix(k, project+"/") {
+				delete(c.warned, k)
+			}
+		}
+	} else {
+		c.warnOnceLocked(project, "compose_build_check_failed", "project", project, "error", err)
+	}
+	return built
+}
+
+// warnOnce logs msg the first time it is called for key. key is a project
+// name (its build check failed) or "<project>/<service>" (the service is not
+// in the project's config); a successful check of the project forgets both.
+func (c *Collector) warnOnce(key, msg string, args ...any) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.warnOnceLocked(key, msg, args...)
+}
+
+func (c *Collector) warnOnceLocked(key, msg string, args ...any) {
+	if c.warned[key] || c.log == nil {
+		return
+	}
+	c.warned[key] = true
+	c.log.Warn(msg, args...)
+}
+
+// fileStamps summarises size and mtime of the comma-separated files, so a
+// changed file changes the string.
+func fileStamps(list string) string {
+	var b strings.Builder
+	for _, f := range strings.Split(list, ",") {
+		if f = strings.TrimSpace(f); f == "" {
+			continue
+		}
+		if fi, err := os.Stat(f); err == nil {
+			fmt.Fprintf(&b, "%s:%d:%d;", f, fi.Size(), fi.ModTime().UnixNano())
+		} else {
+			fmt.Fprintf(&b, "%s:missing;", f)
+		}
+	}
+	return b.String()
+}

internal/agent/inventory/collector_test.go

@@ -1,9 +1,16 @@
 package inventory
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/image"
@@ -200,3 +207,132 @@ func TestHello(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, protocol.Hello{Hostname: "swarm-mgr", SwarmManager: true, SwarmClusterID: "cluster-1"}, got)
 }
+
+func TestSnapshot_ComposeConfigFilesIsRawLabel(t *testing.T) {
+	ct := composeContainer("abc123", "slusk", "web", "slusk:1", "sha256:img1", "running")
+	ct.Labels[labelConfigFiles] = "/s/a/compose.yaml,/s/a/compose.override.yaml"
+	got, err := New(&fakeDocker{containers: []container.Summary{ct}}, "").Snapshot(context.Background())
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, "/s/a/compose.yaml,/s/a/compose.override.yaml", got[0].ConfigFiles)
+}
+
+type fakeBuildCheck struct {
+	built map[string]bool
+	err   error
+	calls int
+}
+
+func (f *fakeBuildCheck) BuiltServices(context.Context, string, string, string, string) (map[string]bool, error) {
+	f.calls++
+	return f.built, f.err
+}
+
+func buildCollector(t *testing.T, check BuildCheck, files string) (*Collector, *fakeDocker) {
+	t.Helper()
+	ct := composeContainer("abc", "app", "web", "app-web", "sha256:img1", "running")
+	ct.Labels[labelConfigFiles] = files
+	other := composeContainer("def", "app", "db", "pg:16", "sha256:img2", "running")
+	other.Labels[labelConfigFiles] = files
+	cli := &fakeDocker{containers: []container.Summary{ct, other}}
+	return New(cli, "").WithBuildCheck(check, slog.New(slog.NewTextHandler(io.Discard, nil))), cli
+}
+
+func TestSnapshot_BuiltServiceIsMarked(t *testing.T) {
+	check := &fakeBuildCheck{built: map[string]bool{"web": true, "db": false}}
+	c, _ := buildCollector(t, check, "/s/app/compose.yaml")
+	got, err := c.Snapshot(context.Background())
+	require.NoError(t, err)
+	byName := map[string]protocol.Service{}
+	for _, s := range got {
+		byName[s.ComposeService] = s
+	}
+	assert.True(t, byName["web"].Built)
+	assert.False(t, byName["db"].Built)
+}
+
+func TestSnapshot_BuildCheckIsCachedUntilFilesChange(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "compose.yaml")
+	require.NoError(t, os.WriteFile(file, []byte("services: {}\n"), 0o644))
+	check := &fakeBuildCheck{built: map[string]bool{"web": true}}
+	c, _ := buildCollector(t, check, file)
+
+	for range 3 {
+		_, err := c.Snapshot(context.Background())
+		require.NoError(t, err)
+	}
+	assert.Equal(t, 1, check.calls, "one check per project while files are unchanged")
+
+	require.NoError(t, os.WriteFile(file, []byte("services:\n  web: {}\n"), 0o644))
+	_, err := c.Snapshot(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 2, check.calls, "a changed file re-runs the check")
+}
+
+func TestSnapshot_BuildCheckFailureIsNotFatalAndRetriesLater(t *testing.T) {
+	check := &fakeBuildCheck{err: errors.New("no mounts")}
+	c, _ := buildCollector(t, check, "/s/app/compose.yaml")
+	now := time.Now()
+	c.now = func() time.Time { return now }
+	var logs bytes.Buffer
+	c.log = slog.New(slog.NewTextHandler(&logs, nil))
+
+	got, err := c.Snapshot(context.Background())
+	require.NoError(t, err)
+	for _, s := range got {
+		assert.False(t, s.Built)
+	}
+	_, _ = c.Snapshot(context.Background())
+	assert.Equal(t, 1, check.calls, "a failure is not retried on every snapshot")
+
+	now = now.Add(buildRetry + time.Second)
+	_, _ = c.Snapshot(context.Background())
+	assert.Equal(t, 2, check.calls)
+	assert.Equal(t, 1, strings.Count(logs.String(), "compose_build_check_failed"), "logged once per project")
+}
+
+func TestSnapshot_ServiceAbsentFromConfigFailsClosed(t *testing.T) {
+	check := &fakeBuildCheck{built: map[string]bool{"db": false}} // "web" is missing
+	c, _ := buildCollector(t, check, "/s/app/compose.yaml")
+	var logs bytes.Buffer
+	c.log = slog.New(slog.NewTextHandler(&logs, nil))
+
+	for range 2 {
+		got, err := c.Snapshot(context.Background())
+		require.NoError(t, err)
+		byName := map[string]protocol.Service{}
+		for _, s := range got {
+			byName[s.ComposeService] = s
+		}
+		assert.True(t, byName["web"].Built, "an unknown service is not offered for update")
+		assert.False(t, byName["db"].Built)
+	}
+	assert.Equal(t, 1, strings.Count(logs.String(), "compose_service_not_in_config"), "logged once per project")
+}
+
+func TestSnapshot_MissingServiceIsLoggedAgainAfterItCameBack(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "compose.yaml")
+	write := func(body string) {
+		require.NoError(t, os.WriteFile(file, []byte(body), 0o644))
+	}
+	write("v1")
+	check := &fakeBuildCheck{built: map[string]bool{"db": false}} // "web" missing
+	c, _ := buildCollector(t, check, file)
+	var logs bytes.Buffer
+	c.log = slog.New(slog.NewTextHandler(&logs, nil))
+	snap := func() {
+		_, err := c.Snapshot(context.Background())
+		require.NoError(t, err)
+	}
+
+	snap()
+	write("v2 changed")
+	check.built = map[string]bool{"db": false, "web": false} // back
+	snap()
+	assert.Empty(t, c.warned, "warnings are forgotten once the project checks out")
+	write("v3 changed again")
+	check.built = map[string]bool{"db": false} // missing again
+	snap()
+
+	assert.Equal(t, 2, strings.Count(logs.String(), "compose_service_not_in_config"))
+}

internal/agent/updater/build.go

@@ -0,0 +1,117 @@
+package updater
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// OutputRunner runs "docker <args>" in dir and returns its stdout. A failure
+// carries the tail of stderr in the error.
+type OutputRunner func(ctx context.Context, dir string, args ...string) ([]byte, error)
+
+// ExecDockerOutput is the real OutputRunner. It uses the same minimal
+// environment as ExecDocker.
+func ExecDockerOutput(ctx context.Context, dir string, args ...string) ([]byte, error) {
+	cmd := exec.CommandContext(ctx, "docker", args...)
+	cmd.Dir = dir
+	cmd.Env = dockerEnv()
+	cmd.WaitDelay = 10 * time.Second
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		tail := &tailBuffer{max: errorOutputTail}
+		tail.Write(stderr.Bytes())
+		return nil, fmt.Errorf("%w (output: %s)", err, strings.TrimSpace(tail.String()))
+	}
+	return stdout.Bytes(), nil
+}
+
+// BuildChecker finds out which services of a Compose project are built from
+// source, by asking `docker compose config`. Such a service has a local
+// image that an update would replace with the upstream one, so it is never
+// updated.
+type BuildChecker struct {
+	cli    SelfInspector
+	selfID string
+	run    OutputRunner
+}
+
+// NewBuildChecker returns a BuildChecker. cli and selfID name the agent's own
+// container, whose mounts decide which files may be read; run is
+// ExecDockerOutput outside tests.
+func NewBuildChecker(cli SelfInspector, selfID string, run OutputRunner) *BuildChecker {
+	return &BuildChecker{cli: cli, selfID: selfID, run: run}
+}
+
+// BuiltServices returns every service of project (all profiles) mapped to
+// whether it has a build: section. A service that is not a key is not in the
+// project's configuration.
+// The arguments are the labels of one of the project's containers. Files that
+// are not inside the agent's read-write same-path mounts are refused.
+func (b *BuildChecker) BuiltServices(ctx context.Context, project, workingDir, configFilesLabel, envFileLabel string) (map[string]bool, error) {
+	roots, err := MountRoots(ctx, b.cli, b.selfID)
+	if err != nil {
+		return nil, err
+	}
+	files, dir, err := stackFiles(roots, configFilesLabel)
+	if err != nil {
+		return nil, err
+	}
+	if workingDir == "" {
+		workingDir = dir
+	} else if !insideAny(roots, workingDir) {
+		return nil, notMounted(workingDir)
+	}
+	envs, err := envFiles(roots, envFileLabel)
+	if err != nil {
+		return nil, err
+	}
+	return b.builtServices(ctx, project, workingDir, files, envs)
+}
+
+// builtServices runs `docker compose config --format json` on files that are
+// already checked.
+func (b *BuildChecker) builtServices(ctx context.Context, project, workingDir string, files, envs []string) (map[string]bool, error) {
+	args := composeBase(project, workingDir, files, envs)
+	// Without --profile * compose omits services whose profile is not active,
+	// and a built service behind a profile would read as absent.
+	args = append(args, "--profile", "*", "config", "--format", "json")
+	raw, err := b.run(ctx, workingDir, args...)
+	if err != nil {
+		return nil, fmt.Errorf("docker compose config: %w", err)
+	}
+	var cfg struct {
+		Services map[string]struct {
+			Build json.RawMessage `json:"build"`
+		} `json:"services"`
+	}
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return nil, fmt.Errorf("parse docker compose config: %w", err)
+	}
+	built := make(map[string]bool, len(cfg.Services))
+	for name, svc := range cfg.Services {
+		built[name] = len(svc.Build) > 0 && string(svc.Build) != "null"
+	}
+	return built, nil
+}
+
+// composeBase returns the leading `docker compose` arguments every call
+// shares. --project-directory is explicit because, with -f, compose would
+// take it from the first file, which differs for stacks started with the
+// flag.
+func composeBase(project, workingDir string, files, envs []string) []string {
+	args := []string{"compose", "-p", project, "--project-directory", workingDir}
+	for _, e := range envs {
+		args = append(args, "--env-file", e)
+	}
+	for _, f := range files {
+		args = append(args, "-f", f)
+	}
+	return args
+}

internal/agent/updater/build_test.go

@@ -0,0 +1,44 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuildChecker_BuiltServices(t *testing.T) {
+	root, _ := filepath.EvalSymlinks(t.TempDir())
+	stack := filepath.Join(root, "app")
+	require.NoError(t, os.MkdirAll(stack, 0o755))
+	self := fakeSelf{mounts: []container.MountPoint{{Type: mount.TypeBind, Source: root, Destination: root, RW: true}}}
+	files := filepath.Join(stack, "compose.yaml") + "," + filepath.Join(stack, OverrideName)
+
+	t.Run("reports only services with build", func(t *testing.T) {
+		o := &outRunner{json: `{"services":{"web":{"image":"nginx","build":{"context":"."}},"db":{"image":"pg"},"x":{"build":null}}}`}
+		got, err := NewBuildChecker(self, "me", o.run).BuiltServices(context.Background(), "app", stack, files, "")
+		require.NoError(t, err)
+		assert.Equal(t, map[string]bool{"web": true, "db": false, "x": false}, got)
+		require.Len(t, o.cmds, 1)
+		assert.Equal(t, stack, o.cmds[0].dir)
+		assert.Equal(t, []string{"compose", "-p", "app", "--project-directory", stack, "-f", filepath.Join(stack, "compose.yaml"), "--profile", "*", "config", "--format", "json"}, o.cmds[0].args)
+	})
+	t.Run("files outside the roots are refused without running docker", func(t *testing.T) {
+		o := &outRunner{json: plainConfig}
+		_, err := NewBuildChecker(self, "me", o.run).BuiltServices(context.Background(), "app", "", "/elsewhere/compose.yaml", "")
+		assert.ErrorContains(t, err, "not mounted")
+		assert.Empty(t, o.cmds)
+	})
+	t.Run("runner and parse errors", func(t *testing.T) {
+		_, err := NewBuildChecker(self, "me", (&outRunner{err: errors.New("boom")}).run).BuiltServices(context.Background(), "app", stack, files, "")
+		assert.ErrorContains(t, err, "boom")
+		_, err = NewBuildChecker(self, "me", (&outRunner{json: "not json"}).run).BuiltServices(context.Background(), "app", stack, files, "")
+		assert.ErrorContains(t, err, "parse")
+	})
+}

internal/agent/updater/compose.go

@@ -0,0 +1,175 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+const (
+	labelComposeProject = "com.docker.compose.project"
+	labelComposeService = "com.docker.compose.service"
+	labelWorkingDir     = "com.docker.compose.project.working_dir"
+	labelConfigFiles    = "com.docker.compose.project.config_files"
+	labelEnvFile        = "com.docker.compose.project.environment_file"
+)
+
+// errorOutputTail is how much of a failed command's output goes into the
+// job's error message; the full output is in the job log.
+const errorOutputTail = 2 << 10
+
+// tailWriter adapts a tailBuffer to io.Writer.
+type tailWriter struct{ b *tailBuffer }
+
+func (w tailWriter) Write(p []byte) (int, error) {
+	w.b.Write(p)
+	return len(p), nil
+}
+
+// CommandRunner runs "docker <args>" in dir with stdout and stderr to out.
+type CommandRunner func(ctx context.Context, dir string, out io.Writer, args ...string) error
+
+// ExecDocker is the real CommandRunner. When ctx ends the process is killed
+// and, after a grace period, its output pipes are closed.
+func ExecDocker(ctx context.Context, dir string, out io.Writer, args ...string) error {
+	cmd := exec.CommandContext(ctx, "docker", args...)
+	cmd.Dir = dir
+	cmd.Stdout = out
+	cmd.Stderr = out
+	cmd.Env = dockerEnv()
+	cmd.WaitDelay = 10 * time.Second
+	return cmd.Run()
+}
+
+// dockerEnvVars are the only variables from the agent's environment that
+// reach docker compose. Compose prefers process variables over the stack's
+// .env file when it interpolates, so passing the agent's own (HOSTNAME,
+// AGENT_TOKEN, ...) would change stack configuration and expose the token.
+var dockerEnvVars = []string{"PATH", "HOME", "DOCKER_HOST", "DOCKER_CONFIG", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"}
+
+// dockerEnv returns the allowed variables that are set in the agent's
+// environment, as KEY=value pairs.
+func dockerEnv() []string {
+	var env []string
+	for _, k := range dockerEnvVars {
+		if v := os.Getenv(k); v != "" {
+			env = append(env, k+"="+v)
+		}
+	}
+	return env
+}
+
+// ComposeDocker is the subset of the Docker SDK ComposeExecutor needs.
+type ComposeDocker interface {
+	SelfInspector
+	ContainerList(ctx context.Context, opts container.ListOptions) ([]container.Summary, error)
+}
+
+// ComposeExecutor updates a Compose service by writing the agent-owned
+// override file and running `docker compose pull` and `up -d`.
+type ComposeExecutor struct {
+	cli     ComposeDocker
+	resolve *Resolver
+	selfID  string
+	run     CommandRunner
+	build   *BuildChecker
+}
+
+// NewComposeExecutor returns an executor for Compose jobs. selfID names the
+// agent's own container, whose mounts decide which directories may be
+// touched; run and runOut are ExecDocker and ExecDockerOutput outside tests.
+func NewComposeExecutor(cli ComposeDocker, resolve *Resolver, selfID string, run CommandRunner, runOut OutputRunner) *ComposeExecutor {
+	return &ComposeExecutor{cli: cli, resolve: resolve, selfID: selfID, run: run, build: NewBuildChecker(cli, selfID, runOut)}
+}
+
+// Execute pins job.Ref to its current digest (a ref that has one is used as
+// is), writes it to the stack's override file and recreates the service.
+// Working directory and config files come from the service's container
+// labels on this machine, never from the job. It returns the pinned ref.
+func (e *ComposeExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	project, service := job.ComposeProject, job.ComposeService
+	cts, err := e.cli.ContainerList(ctx, container.ListOptions{
+		All: true,
+		Filters: filters.NewArgs(
+			filters.Arg("label", labelComposeProject+"="+project),
+			filters.Arg("label", labelComposeService+"="+service),
+		),
+	})
+	if err != nil {
+		return "", fmt.Errorf("list containers: %w", err)
+	}
+	if len(cts) == 0 {
+		return "", fmt.Errorf("no container for compose service %s/%s on this machine", project, service)
+	}
+	workingDir := cts[0].Labels[labelWorkingDir]
+	configFiles := cts[0].Labels[labelConfigFiles]
+	envFileLabel := cts[0].Labels[labelEnvFile]
+
+	roots, err := MountRoots(ctx, e.cli, e.selfID)
+	if err != nil {
+		return "", err
+	}
+	files, dir, err := composeFiles(roots, configFiles)
+	if err != nil {
+		return "", err
+	}
+	if workingDir == "" {
+		workingDir = dir
+	} else if !insideAny(roots, workingDir) {
+		return "", notMounted(workingDir)
+	}
+
+	envs, err := envFiles(roots, envFileLabel)
+	if err != nil {
+		return "", err
+	}
+	// Fresh check, not the inventory's cache: a build: section may have been
+	// added since. Nothing is written for such a service.
+	built, err := e.build.builtServices(ctx, project, workingDir, files[:len(files)-1], envs)
+	if err != nil {
+		return "", err
+	}
+	isBuilt, known := built[service]
+	if !known {
+		return "", fmt.Errorf("service %s not found in compose config", service)
+	}
+	if isBuilt {
+		return "", fmt.Errorf("service %s is built from source (build:), not updated", service)
+	}
+
+	pinned, err := e.resolve.Resolve(ctx, job.Ref)
+	if err != nil {
+		return "", err
+	}
+	fmt.Fprintln(out, resolveNote(job.Ref, pinned))
+
+	// Giving the override the stack's owner is best effort: a failure is only
+	// a warning, the update itself is fine.
+	_, ownerErr, err := WriteOverride(dir, service, pinned, files[0])
+	if err != nil {
+		return "", err
+	}
+	if ownerErr != nil {
+		fmt.Fprintf(out, "warning: could not give the override the stack's owner: %v\n", ownerErr)
+	}
+
+	base := composeBase(project, workingDir, files, envs)
+	for _, step := range [][]string{{"pull", service}, {"up", "-d", service}} {
+		args := append(append([]string(nil), base...), step...)
+		fmt.Fprintf(out, "$ docker %s\n", strings.Join(args, " "))
+		tail := &tailBuffer{max: errorOutputTail}
+		if err := e.run(ctx, workingDir, io.MultiWriter(out, tailWriter{tail}), args...); err != nil {
+			return "", fmt.Errorf("docker compose %s: %w (output: %s)", step[0], err, strings.TrimSpace(tail.String()))
+		}
+	}
+	return pinned, nil
+}

internal/agent/updater/compose_test.go

@@ -0,0 +1,321 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+type fakeCompose struct {
+	fakeSelf
+	containers []container.Summary
+}
+
+func (f fakeCompose) ContainerList(context.Context, container.ListOptions) ([]container.Summary, error) {
+	return f.containers, nil
+}
+
+// outRunner is a fake OutputRunner for `docker compose config`.
+type outRunner struct {
+	json string
+	err  error
+	cmds []ranCmd
+}
+
+func (o *outRunner) run(_ context.Context, dir string, args ...string) ([]byte, error) {
+	o.cmds = append(o.cmds, ranCmd{dir, args})
+	return []byte(o.json), o.err
+}
+
+const plainConfig = `{"services":{"web":{"image":"x"},"db":{"image":"y"}}}`
+
+type ranCmd struct {
+	dir  string
+	args []string
+}
+
+type recorder struct {
+	cmds       []ranCmd
+	failOn     string // second arg after -f list, e.g. "pull"
+	failOutput string
+	onFirst    func()
+}
+
+func (r *recorder) run(_ context.Context, dir string, out io.Writer, args ...string) error {
+	if len(r.cmds) == 0 && r.onFirst != nil {
+		r.onFirst()
+	}
+	r.cmds = append(r.cmds, ranCmd{dir, args})
+	if r.failOn != "" && strings.Contains(strings.Join(args, " "), " "+r.failOn+" ") {
+		io.WriteString(out, r.failOutput)
+		return errors.New("exit status 1")
+	}
+	return nil
+}
+
+func TestComposeExecutor(t *testing.T) {
+	const d = "sha256:d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1"
+	root, _ := filepath.EvalSymlinks(t.TempDir())
+	stack := filepath.Join(root, "slusk")
+	require.NoError(t, os.MkdirAll(stack, 0o755))
+	composeFile := stackFile(t, stack)
+	job := protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "gitea.shcizo.se/shcizo/slusk:1.4"}
+	pinned := job.Ref + "@" + d
+
+	ct := container.Summary{Labels: map[string]string{
+		labelWorkingDir:  stack,
+		labelConfigFiles: filepath.Join(stack, "compose.yaml"),
+	}}
+	rw := fakeSelf{mounts: []container.MountPoint{{Type: mount.TypeBind, Source: root, Destination: root, RW: true}}}
+	ro := fakeSelf{mounts: []container.MountPoint{{Type: mount.TypeBind, Source: root, Destination: root, RW: false}}}
+	wantFiles := []string{"-f", filepath.Join(stack, "compose.yaml"), "-f", filepath.Join(stack, OverrideName)}
+	cmd := func(rest ...string) []string {
+		return append(append([]string{"compose", "-p", "slusk", "--project-directory", stack}, wantFiles...), rest...)
+	}
+	overridePath := filepath.Join(stack, OverrideName)
+	cleanOverride := func() { os.Remove(overridePath) }
+
+	outr := &outRunner{json: plainConfig}
+	setup := func(cli ComposeDocker, dist *fakeDist, rec *recorder) *ComposeExecutor {
+		*outr = outRunner{json: plainConfig}
+		return NewComposeExecutor(cli, NewResolver(dist, nil), "me", rec.run, outr.run)
+	}
+
+	t.Run("happy path", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{onFirst: func() {
+			_, err := os.Stat(overridePath)
+			assert.NoError(t, err, "override must exist before the first command")
+		}}
+		var out strings.Builder
+		got, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, &out)
+		require.NoError(t, err)
+		assert.Equal(t, pinned, got)
+		require.Len(t, rec.cmds, 2)
+		assert.Equal(t, ranCmd{stack, cmd("pull", "web")}, rec.cmds[0])
+		assert.Equal(t, ranCmd{stack, cmd("up", "-d", "web")}, rec.cmds[1])
+		_, doc := readOverride(t, stack)
+		assert.Equal(t, pinned, serviceImage(doc, "web"))
+		assert.Contains(t, out.String(), "resolved "+pinned)
+		assert.NotContains(t, out.String(), "warning")
+	})
+	t.Run("an unreadable owner is a warning and the job still succeeds", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		require.NoError(t, os.Rename(composeFile, composeFile+".bak"))
+		t.Cleanup(func() { os.Rename(composeFile+".bak", composeFile) })
+		var out strings.Builder
+		got, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, &recorder{}).Execute(context.Background(), job, &out)
+		require.NoError(t, err)
+		assert.Equal(t, pinned, got)
+		assert.Contains(t, out.String(), "warning: could not give the override the stack's owner: read owner: ")
+		_, doc := readOverride(t, stack)
+		assert.Equal(t, pinned, serviceImage(doc, "web"))
+	})
+	t.Run("a pinned ref is written verbatim and not resolved", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		for _, ref := range []string{"gitea.shcizo.se/shcizo/slusk:1.3@" + d, "gitea.shcizo.se/shcizo/slusk@" + d} {
+			rec := &recorder{}
+			dist := &fakeDist{digest: "sha256:other"}
+			var out strings.Builder
+			pinnedJob := job
+			pinnedJob.Ref = ref
+			got, err := setup(fakeCompose{rw, []container.Summary{ct}}, dist, rec).Execute(context.Background(), pinnedJob, &out)
+			require.NoError(t, err)
+			assert.Equal(t, ref, got)
+			assert.Empty(t, dist.refs, "no registry lookup")
+			_, doc := readOverride(t, stack)
+			assert.Equal(t, ref, serviceImage(doc, "web"))
+			assert.Contains(t, out.String(), "pinned "+ref)
+			assert.NotContains(t, out.String(), "resolved")
+		}
+	})
+	t.Run("built from source is refused before anything is written", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{}
+		e := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec)
+		outr.json = `{"services":{"web":{"image":"nginx","build":{"context":"."}}}}`
+		_, err := e.Execute(context.Background(), job, io.Discard)
+		assert.EqualError(t, err, "service web is built from source (build:), not updated")
+		assert.Empty(t, rec.cmds, "no pull/up")
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("build check uses the job's files without the override", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		require.NoError(t, err)
+		require.Len(t, outr.cmds, 1)
+		assert.Equal(t, ranCmd{stack, []string{"compose", "-p", "slusk", "--project-directory", stack, "-f", filepath.Join(stack, "compose.yaml"), "--profile", "*", "config", "--format", "json"}}, outr.cmds[0])
+	})
+	t.Run("service missing from the config is refused", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{}
+		e := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec)
+		outr.json = `{"services":{"db":{"image":"y"}}}`
+		dist := &fakeDist{digest: d}
+		e.resolve = NewResolver(dist, nil)
+		_, err := e.Execute(context.Background(), job, io.Discard)
+		assert.EqualError(t, err, "service web not found in compose config")
+		assert.Empty(t, rec.cmds)
+		assert.Empty(t, dist.refs, "nothing resolved")
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("build check failure stops the job", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{}
+		e := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec)
+		outr.err = errors.New("invalid compose file")
+		_, err := e.Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "invalid compose file")
+		assert.Empty(t, rec.cmds)
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("env files are passed to every compose call", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		envFile := filepath.Join(stack, ".env.prod")
+		require.NoError(t, os.WriteFile(envFile, []byte("A=1\n"), 0o644))
+		t.Cleanup(func() { os.Remove(envFile); os.Remove(filepath.Join(stack, ".env.local")) })
+		envFile2 := filepath.Join(stack, ".env.local")
+		require.NoError(t, os.WriteFile(envFile2, []byte("B=2\n"), 0o644))
+		withEnv := container.Summary{Labels: map[string]string{
+			labelWorkingDir: stack, labelConfigFiles: filepath.Join(stack, "compose.yaml"), labelEnvFile: envFile + "," + envFile2,
+		}}
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{withEnv}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		require.NoError(t, err)
+		prefix := []string{"compose", "-p", "slusk", "--project-directory", stack, "--env-file", envFile, "--env-file", envFile2}
+		require.Len(t, outr.cmds, 1)
+		assert.Equal(t, prefix, outr.cmds[0].args[:len(prefix)])
+		require.Len(t, rec.cmds, 2)
+		assert.Equal(t, prefix, rec.cmds[0].args[:len(prefix)])
+		assert.Equal(t, prefix, rec.cmds[1].args[:len(prefix)])
+	})
+	t.Run("env file outside the roots is refused", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		outside := filepath.Join(t.TempDir(), "secrets.env")
+		require.NoError(t, os.WriteFile(outside, []byte("A=1\n"), 0o644))
+		bad := container.Summary{Labels: map[string]string{
+			labelWorkingDir: stack, labelConfigFiles: filepath.Join(stack, "compose.yaml"), labelEnvFile: outside,
+		}}
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{bad}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "env file")
+		assert.Empty(t, rec.cmds)
+		assert.Empty(t, outr.cmds)
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("env file symlinked outside the roots is refused", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		outside := filepath.Join(t.TempDir(), "secrets.env")
+		require.NoError(t, os.WriteFile(outside, []byte("A=1\n"), 0o644))
+		link := filepath.Join(stack, ".env.link")
+		require.NoError(t, os.Symlink(outside, link))
+		t.Cleanup(func() { os.Remove(link) })
+		bad := container.Summary{Labels: map[string]string{
+			labelWorkingDir: stack, labelConfigFiles: filepath.Join(stack, "compose.yaml"), labelEnvFile: link,
+		}}
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{bad}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "env file")
+		assert.Empty(t, rec.cmds)
+	})
+	t.Run("no matching container", func(t *testing.T) {
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, nil}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "no container for compose service slusk/web on this machine")
+		assert.Empty(t, rec.cmds)
+	})
+	t.Run("resolve fails", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{err: errors.New("denied")}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "denied")
+		assert.Empty(t, rec.cmds)
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("pull fails", func(t *testing.T) {
+		t.Cleanup(cleanOverride)
+		rec := &recorder{failOn: "pull", failOutput: "pull access denied\n"}
+		_, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "pull access denied")
+		require.Len(t, rec.cmds, 1, "up must not run")
+	})
+	t.Run("override refused", func(t *testing.T) {
+		other := filepath.Join(stack, "docker-compose.override.yml")
+		require.NoError(t, os.WriteFile(other, []byte("services: {}\n"), 0o644))
+		t.Cleanup(func() { os.Remove(other) })
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{ct}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "refused")
+		assert.Empty(t, rec.cmds)
+	})
+	t.Run("working dir outside the roots", func(t *testing.T) {
+		outside := filepath.Join(t.TempDir(), "elsewhere")
+		bad := container.Summary{Labels: map[string]string{
+			labelWorkingDir:  outside,
+			labelConfigFiles: filepath.Join(stack, "compose.yaml"),
+		}}
+		rec := &recorder{}
+		_, err := setup(fakeCompose{rw, []container.Summary{bad}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "not mounted")
+		assert.Empty(t, rec.cmds)
+		assert.NoFileExists(t, overridePath)
+	})
+	t.Run("read-only mount", func(t *testing.T) {
+		rec := &recorder{}
+		_, err := setup(fakeCompose{ro, []container.Summary{ct}}, &fakeDist{digest: d}, rec).Execute(context.Background(), job, io.Discard)
+		assert.ErrorContains(t, err, "no stack directory is mounted")
+		assert.Empty(t, rec.cmds)
+		assert.NoFileExists(t, overridePath)
+	})
+}
+
+func TestDockerEnv(t *testing.T) {
+	t.Setenv("AGENT_TOKEN", "secret")
+	t.Setenv("HOSTNAME", "abc")
+	t.Setenv("MASTER_URL", "wss://x")
+	t.Setenv("PATH", "/usr/bin")
+	t.Setenv("DOCKER_CONFIG", "/docker-config")
+	t.Setenv("DOCKER_HOST", "")
+	os.Unsetenv("DOCKER_CERT_PATH")
+
+	env := dockerEnv()
+	assert.Contains(t, env, "PATH=/usr/bin")
+	assert.Contains(t, env, "DOCKER_CONFIG=/docker-config")
+	assert.NotContains(t, env, "DOCKER_HOST=", "empty variables are not passed")
+	for _, e := range env {
+		assert.NotContains(t, e, "AGENT_TOKEN")
+		assert.NotContains(t, e, "HOSTNAME")
+		assert.NotContains(t, e, "MASTER_URL")
+		assert.False(t, strings.HasPrefix(e, "DOCKER_CERT_PATH="), "unset variables are not passed")
+	}
+}
+
+func TestExecDocker_UsesMinimalEnv(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "docker"), []byte("#!/bin/sh\nenv\n"), 0o755))
+	t.Setenv("PATH", dir+":/usr/bin:/bin")
+	t.Setenv("AGENT_TOKEN", "secret")
+	t.Setenv("DOCKER_CONFIG", "/docker-config")
+
+	var out strings.Builder
+	require.NoError(t, ExecDocker(context.Background(), dir, &out, "compose", "ps"))
+	assert.NotContains(t, out.String(), "AGENT_TOKEN")
+	assert.Contains(t, out.String(), "DOCKER_CONFIG=/docker-config")
+
+	raw, err := ExecDockerOutput(context.Background(), dir, "compose", "config")
+	require.NoError(t, err)
+	assert.NotContains(t, string(raw), "AGENT_TOKEN")
+	assert.Contains(t, string(raw), "DOCKER_CONFIG=/docker-config")
+}

internal/agent/updater/dispatch.go

@@ -0,0 +1,28 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+type dispatch struct{ compose, swarm Executor }
+
+// NewDispatch returns an Executor that picks compose or swarm by job.Kind.
+// KindContainer and unknown kinds fail.
+func NewDispatch(compose, swarm Executor) Executor {
+	return dispatch{compose: compose, swarm: swarm}
+}
+
+func (d dispatch) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	switch job.Kind {
+	case protocol.KindCompose:
+		return d.compose.Execute(ctx, job, out)
+	case protocol.KindSwarm:
+		return d.swarm.Execute(ctx, job, out)
+	default:
+		return "", fmt.Errorf("cannot update a service of kind %q", job.Kind)
+	}
+}

internal/agent/updater/dispatch_test.go

@@ -0,0 +1,31 @@
+package updater
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+func TestDispatch(t *testing.T) {
+	compose := execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "compose", nil })
+	swarm := execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "swarm", nil })
+	d := NewDispatch(compose, swarm)
+
+	got, err := d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindCompose}, io.Discard)
+	require.NoError(t, err)
+	assert.Equal(t, "compose", got)
+
+	got, err = d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindSwarm}, io.Discard)
+	require.NoError(t, err)
+	assert.Equal(t, "swarm", got)
+
+	_, err = d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindContainer}, io.Discard)
+	assert.ErrorContains(t, err, "cannot update")
+	_, err = d.Execute(context.Background(), protocol.JobStart{Kind: "weird"}, io.Discard)
+	assert.ErrorContains(t, err, "cannot update")
+}

internal/agent/updater/override.go

@@ -0,0 +1,297 @@
+package updater
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"gopkg.in/yaml.v3"
+)
+
+// OverrideName is the override file the agent owns. docker compose loads it
+// automatically next to compose.yaml or docker-compose.yml.
+const OverrideName = "compose.override.yaml"
+
+// overrideHeader marks a file the agent may rewrite. A file without it
+// belongs to the operator and is never touched.
+const overrideHeader = "# managed by package-updater — do not edit\n"
+
+// overrideNames are the override files docker compose auto-loads. It loads
+// only the first one present (in this order), so any other one next to ours
+// would either hide ours or be hidden by it.
+var overrideNames = []string{
+	"compose.override.yml", "compose.override.yaml",
+	"docker-compose.override.yml", "docker-compose.override.yaml",
+}
+
+// WriteOverride sets services.<service>.image to ref in dir/OverrideName,
+// keeping every other entry in the file, and returns the file's path. It
+// refuses when the file exists without the header, or when another
+// auto-loaded override file exists in dir. The write is atomic.
+//
+// composeFile is the stack's first compose file. The new file gets the owner
+// of the existing override, or of composeFile when there is none or the
+// existing one is owned by root (see pickOwner). The agent runs as root and the
+// rename creates a new file, so without this the stack's owner would lose
+// control of the file. ownerErr reports that the owner could not be read or
+// set; the override is still written. err means it was not written.
+func WriteOverride(dir, service, ref, composeFile string) (path string, ownerErr error, err error) {
+	overridePath := filepath.Join(dir, OverrideName)
+	return writeOverride(dir, service, ref, func() (fileOwner, error) {
+		return resolveOwner(overridePath, composeFile)
+	})
+}
+
+// fileOwner is the uid and gid of a file.
+type fileOwner struct{ uid, gid int }
+
+// readOwner returns the owner of the file at path.
+func readOwner(path string) (fileOwner, error) {
+	fi, err := os.Stat(path)
+	if err != nil {
+		return fileOwner{}, fmt.Errorf("read owner: %w", err)
+	}
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return fileOwner{}, fmt.Errorf("read owner of %s: not available on this platform", path)
+	}
+	return fileOwner{int(st.Uid), int(st.Gid)}, nil
+}
+
+// pickOwner chooses the owner of a new override: the existing override's
+// (nil when there is none), unless that is root, then the compose file's.
+// Overrides written before the agent set owners are root-owned and would
+// otherwise stay so. compose is only called when needed.
+func pickOwner(override *fileOwner, compose func() (fileOwner, error)) (fileOwner, error) {
+	if override != nil && override.uid != 0 {
+		return *override, nil
+	}
+	return compose()
+}
+
+// resolveOwner applies pickOwner to the files on disk.
+func resolveOwner(overridePath, composeFile string) (fileOwner, error) {
+	var existing *fileOwner
+	if o, err := readOwner(overridePath); err == nil {
+		existing = &o
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return fileOwner{}, err
+	}
+	return pickOwner(existing, func() (fileOwner, error) { return readOwner(composeFile) })
+}
+
+// writeOverride is WriteOverride with the owner lookup injected, for tests.
+func writeOverride(dir, service, ref string, owner func() (fileOwner, error)) (string, error, error) {
+	for _, name := range overrideNames {
+		if name == OverrideName {
+			continue
+		}
+		if _, err := os.Lstat(filepath.Join(dir, name)); err == nil {
+			return "", nil, fmt.Errorf("refused: %s exists; the agent only manages %s and never edits operator files", name, OverrideName)
+		} else if !errors.Is(err, os.ErrNotExist) {
+			return "", nil, fmt.Errorf("check %s: %w", name, err)
+		}
+	}
+	path := filepath.Join(dir, OverrideName)
+	doc := map[string]any{}
+	raw, err := os.ReadFile(path)
+	switch {
+	case errors.Is(err, os.ErrNotExist):
+	case err != nil:
+		return "", nil, fmt.Errorf("read override: %w", err)
+	case !bytes.HasPrefix(raw, []byte(overrideHeader)):
+		return "", nil, fmt.Errorf("refused: %s exists and was not written by package-updater", path)
+	default:
+		if err := yaml.Unmarshal(raw, &doc); err != nil {
+			return "", nil, fmt.Errorf("parse override: %w", err)
+		}
+		if doc == nil {
+			doc = map[string]any{}
+		}
+	}
+	services, _ := doc["services"].(map[string]any)
+	if services == nil {
+		services = map[string]any{}
+	}
+	entry, _ := services[service].(map[string]any)
+	if entry == nil {
+		entry = map[string]any{}
+	}
+	entry["image"] = ref
+	services[service] = entry
+	doc["services"] = services
+	body, err := yaml.Marshal(doc)
+	if err != nil {
+		return "", nil, fmt.Errorf("encode override: %w", err)
+	}
+	// Temp file in the same directory + rename: a crash never leaves a
+	// half-written override that compose would then load.
+	tmp, err := os.CreateTemp(dir, "."+OverrideName+".*")
+	if err != nil {
+		return "", nil, fmt.Errorf("write override: %w", err)
+	}
+	defer os.Remove(tmp.Name()) // no-op after a successful rename
+	if _, err := tmp.WriteString(overrideHeader + string(body)); err != nil {
+		tmp.Close()
+		return "", nil, fmt.Errorf("write override: %w", err)
+	}
+	if err := tmp.Chmod(0o644); err != nil {
+		tmp.Close()
+		return "", nil, fmt.Errorf("write override: %w", err)
+	}
+	// Chown the open file, not the path after the rename: a path could have
+	// been swapped for a symlink by someone with write access to the stack.
+	var ownerErr error
+	if o, err := owner(); err != nil {
+		ownerErr = err
+	} else if err := tmp.Chown(o.uid, o.gid); err != nil {
+		ownerErr = fmt.Errorf("set owner %d:%d: %w", o.uid, o.gid, err)
+	}
+	if err := tmp.Close(); err != nil {
+		return "", nil, fmt.Errorf("write override: %w", err)
+	}
+	if err := os.Rename(tmp.Name(), path); err != nil {
+		return "", nil, fmt.Errorf("write override: %w", err)
+	}
+	return path, ownerErr, nil
+}
+
+// SelfInspector is the subset of the Docker SDK MountRoots needs.
+type SelfInspector interface {
+	ContainerInspect(ctx context.Context, containerID string) (container.InspectResponse, error)
+}
+
+// errNoRoots is returned when the agent has no directory it may write stacks in.
+var errNoRoots = errors.New("refused: no stack directory is mounted into the agent at the same path (read-write); " +
+	"mount each stacks directory as /path:/path and check that SELF_ID (default: the hostname) names the agent's container")
+
+// MountRoots returns the directories the agent may update Compose stacks in:
+// its own container's read-write bind mounts whose host path equals the
+// path inside the container. Compose labels hold host paths, so only those
+// mounts make a label path usable as is.
+func MountRoots(ctx context.Context, cli SelfInspector, selfID string) ([]string, error) {
+	if selfID == "" {
+		return nil, errNoRoots
+	}
+	insp, err := cli.ContainerInspect(ctx, selfID)
+	if err != nil {
+		return nil, fmt.Errorf("inspect own container %s: %w", selfID, err)
+	}
+	var roots []string
+	for _, m := range insp.Mounts {
+		if m.Type == mount.TypeBind && m.RW && filepath.Clean(m.Source) == filepath.Clean(m.Destination) {
+			roots = append(roots, filepath.Clean(m.Destination))
+		}
+	}
+	return roots, nil
+}
+
+// composeFiles returns the -f arguments for a job: the stack's config files
+// from the container label with any copy of the override removed, then the
+// override last so it wins. Every file, and the real location of its
+// directory after resolving symlinks, must be inside one of roots.
+func composeFiles(roots []string, configFilesLabel string) (files []string, dir string, err error) {
+	files, dir, err = stackFiles(roots, configFilesLabel)
+	if err != nil {
+		return nil, "", err
+	}
+	return append(files, filepath.Join(dir, OverrideName)), dir, nil
+}
+
+// stackFiles is composeFiles without the override: the operator's own config
+// files, checked against roots. dir is the first file's directory.
+func stackFiles(roots []string, configFilesLabel string) (files []string, dir string, err error) {
+	if len(roots) == 0 {
+		return nil, "", errNoRoots
+	}
+	for _, f := range strings.Split(configFilesLabel, ",") {
+		if f = strings.TrimSpace(f); f != "" {
+			files = append(files, f)
+		}
+	}
+	if len(files) == 0 {
+		return nil, "", errors.New("container has no com.docker.compose.project.config_files label")
+	}
+	dir = filepath.Dir(files[0])
+	override := filepath.Join(dir, OverrideName)
+
+	realRoots := realDirs(roots)
+	out := files[:0]
+	for _, f := range files {
+		if !filepath.IsAbs(f) || !insideAny(roots, f) {
+			return nil, "", notMounted(filepath.Dir(f))
+		}
+		real, err := filepath.EvalSymlinks(filepath.Dir(f))
+		if err != nil {
+			return nil, "", fmt.Errorf("resolve %s: %w", filepath.Dir(f), err)
+		}
+		if !insideAny(realRoots, real) {
+			return nil, "", notMounted(filepath.Dir(f))
+		}
+		if filepath.Clean(f) != override {
+			out = append(out, f)
+		}
+	}
+	return out, dir, nil
+}
+
+// envFiles returns the --env-file paths from the container's
+// com.docker.compose.project.environment_file label (empty label: none).
+// Like config files, each must be inside one of roots, also after resolving
+// symlinks of the file itself.
+func envFiles(roots []string, label string) ([]string, error) {
+	var files []string
+	realRoots := realDirs(roots)
+	for _, f := range strings.Split(label, ",") {
+		if f = strings.TrimSpace(f); f == "" {
+			continue
+		}
+		if !filepath.IsAbs(f) || !insideAny(roots, f) {
+			return nil, fmt.Errorf("env file %s is not inside a directory mounted into the agent at the same path (read-write)", f)
+		}
+		real, err := filepath.EvalSymlinks(f)
+		if err != nil {
+			return nil, fmt.Errorf("resolve env file %s: %w", f, err)
+		}
+		if !insideAny(realRoots, real) {
+			return nil, fmt.Errorf("env file %s is not inside a directory mounted into the agent at the same path (read-write)", f)
+		}
+		files = append(files, f)
+	}
+	return files, nil
+}
+
+// realDirs returns roots with symlinks resolved. Only directories count once
+// resolved: this drops a same-path mount of a single file such as
+// /var/run/docker.sock.
+func realDirs(roots []string) []string {
+	var out []string
+	for _, r := range roots {
+		if rr, err := filepath.EvalSymlinks(r); err == nil {
+			if fi, err := os.Stat(rr); err == nil && fi.IsDir() {
+				out = append(out, rr)
+			}
+		}
+	}
+	return out
+}
+
+func insideAny(roots []string, path string) bool {
+	for _, r := range roots {
+		if IsInsideRoot(r, path) {
+			return true
+		}
+	}
+	return false
+}
+
+func notMounted(dir string) error {
+	return fmt.Errorf("stack directory %s is not mounted into the agent at the same path (read-write)", dir)
+}

internal/agent/updater/override_test.go

@@ -0,0 +1,290 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func readOverride(t *testing.T, dir string) (string, map[string]any) {
+	t.Helper()
+	raw, err := os.ReadFile(filepath.Join(dir, OverrideName))
+	require.NoError(t, err)
+	var doc map[string]any
+	require.NoError(t, yaml.Unmarshal(raw, &doc))
+	return string(raw), doc
+}
+
+func serviceImage(doc map[string]any, name string) any {
+	svcs, _ := doc["services"].(map[string]any)
+	entry, _ := svcs[name].(map[string]any)
+	return entry["image"]
+}
+
+// stackFile creates dir/compose.yaml and returns its path.
+func stackFile(t *testing.T, dir string) string {
+	t.Helper()
+	p := filepath.Join(dir, "compose.yaml")
+	require.NoError(t, os.WriteFile(p, []byte("services: {}\n"), 0o644))
+	return p
+}
+
+func TestPickOwner(t *testing.T) {
+	compose := fileOwner{1000, 1000}
+	fromCompose := func() (fileOwner, error) { return compose, nil }
+	failing := func() (fileOwner, error) { return fileOwner{}, errors.New("boom") }
+	tests := []struct {
+		name     string
+		override *fileOwner
+		compose  func() (fileOwner, error)
+		want     fileOwner
+		wantErr  bool
+	}{
+		{"no override uses the compose file", nil, fromCompose, compose, false},
+		{"override owned by a user wins", &fileOwner{1001, 1002}, fromCompose, fileOwner{1001, 1002}, false},
+		{"root-owned override uses the compose file", &fileOwner{0, 0}, fromCompose, compose, false},
+		{"compose file is not read when the override decides", &fileOwner{1001, 1002}, failing, fileOwner{1001, 1002}, false},
+		{"unreadable compose file is an error", nil, failing, fileOwner{}, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := pickOwner(tt.override, tt.compose)
+			assert.Equal(t, tt.wantErr, err != nil)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestResolveOwner(t *testing.T) {
+	me := fileOwner{os.Getuid(), os.Getgid()}
+	t.Run("no override uses the compose file", func(t *testing.T) {
+		dir := t.TempDir()
+		got, err := resolveOwner(filepath.Join(dir, OverrideName), stackFile(t, dir))
+		require.NoError(t, err)
+		assert.Equal(t, me, got)
+	})
+	t.Run("existing override is used without the compose file", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, OverrideName), nil, 0o644))
+		got, err := resolveOwner(filepath.Join(dir, OverrideName), filepath.Join(dir, "missing.yaml"))
+		require.NoError(t, err)
+		assert.Equal(t, me, got)
+	})
+	t.Run("neither file is an error", func(t *testing.T) {
+		dir := t.TempDir()
+		_, err := resolveOwner(filepath.Join(dir, OverrideName), filepath.Join(dir, "missing.yaml"))
+		assert.Error(t, err)
+	})
+}
+
+func TestWriteOverride_Owner(t *testing.T) {
+	t.Run("success sets no ownerErr", func(t *testing.T) {
+		dir := t.TempDir()
+		_, ownerErr, err := WriteOverride(dir, "web", "a:1@sha256:aa", stackFile(t, dir))
+		require.NoError(t, err)
+		assert.NoError(t, ownerErr)
+	})
+	t.Run("unreadable owner is reported and the file is still written", func(t *testing.T) {
+		dir := t.TempDir()
+		_, ownerErr, err := WriteOverride(dir, "web", "a:1@sha256:aa", filepath.Join(dir, "missing.yaml"))
+		require.NoError(t, err)
+		assert.ErrorContains(t, ownerErr, "read owner")
+		_, doc := readOverride(t, dir)
+		assert.Equal(t, "a:1@sha256:aa", serviceImage(doc, "web"))
+	})
+	t.Run("chown failure is reported and the file is still written", func(t *testing.T) {
+		if os.Getuid() == 0 {
+			t.Skip("root can chown to anyone")
+		}
+		dir := t.TempDir()
+		_, ownerErr, err := writeOverride(dir, "web", "a:1@sha256:aa", func() (fileOwner, error) { return fileOwner{0, 0}, nil })
+		require.NoError(t, err)
+		assert.ErrorContains(t, ownerErr, "set owner 0:0")
+		_, doc := readOverride(t, dir)
+		assert.Equal(t, "a:1@sha256:aa", serviceImage(doc, "web"))
+	})
+}
+
+func TestWriteOverride(t *testing.T) {
+	t.Run("new file", func(t *testing.T) {
+		dir := t.TempDir()
+		path, ownerErr, err := WriteOverride(dir, "web", "a:1@sha256:aa", stackFile(t, dir))
+		require.NoError(t, err)
+		assert.NoError(t, ownerErr)
+		assert.Equal(t, filepath.Join(dir, OverrideName), path)
+		raw, doc := readOverride(t, dir)
+		assert.True(t, len(raw) > len(overrideHeader) && raw[:len(overrideHeader)] == overrideHeader)
+		assert.Equal(t, "a:1@sha256:aa", serviceImage(doc, "web"))
+	})
+	t.Run("keeps other services", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, OverrideName), []byte(overrideHeader+"services:\n  db:\n    image: db:1\n"), 0o644))
+		_, _, err := WriteOverride(dir, "web", "a:1@sha256:aa", stackFile(t, dir))
+		require.NoError(t, err)
+		_, doc := readOverride(t, dir)
+		assert.Equal(t, "db:1", serviceImage(doc, "db"))
+		assert.Equal(t, "a:1@sha256:aa", serviceImage(doc, "web"))
+	})
+	t.Run("replaces own entry", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, OverrideName), []byte(overrideHeader+"services:\n  web:\n    image: old\n"), 0o644))
+		_, _, err := WriteOverride(dir, "web", "a:2@sha256:bb", stackFile(t, dir))
+		require.NoError(t, err)
+		_, doc := readOverride(t, dir)
+		assert.Equal(t, "a:2@sha256:bb", serviceImage(doc, "web"))
+	})
+	t.Run("operator file without header is never touched", func(t *testing.T) {
+		dir := t.TempDir()
+		const body = "services:\n  web:\n    environment: [A=1]\n"
+		require.NoError(t, os.WriteFile(filepath.Join(dir, OverrideName), []byte(body), 0o644))
+		_, _, err := WriteOverride(dir, "web", "a:1@sha256:aa", stackFile(t, dir))
+		assert.ErrorContains(t, err, "refused")
+		got, _ := os.ReadFile(filepath.Join(dir, OverrideName))
+		assert.Equal(t, body, string(got))
+	})
+	for _, other := range []string{"compose.override.yml", "docker-compose.override.yml", "docker-compose.override.yaml"} {
+		t.Run("refuses next to "+other, func(t *testing.T) {
+			dir := t.TempDir()
+			require.NoError(t, os.WriteFile(filepath.Join(dir, other), []byte("services: {}\n"), 0o644))
+			_, _, err := WriteOverride(dir, "web", "a:1@sha256:aa", stackFile(t, dir))
+			assert.ErrorContains(t, err, other)
+			_, statErr := os.Stat(filepath.Join(dir, OverrideName))
+			assert.True(t, errors.Is(statErr, os.ErrNotExist))
+		})
+	}
+	t.Run("leaves no temp files", func(t *testing.T) {
+		dir := t.TempDir()
+		_, _, err := WriteOverride(dir, "web", "a:1@sha256:aa", stackFile(t, dir))
+		require.NoError(t, err)
+		entries, err := os.ReadDir(dir)
+		require.NoError(t, err)
+		require.Len(t, entries, 2)
+		assert.Equal(t, "compose.override.yaml", entries[0].Name())
+		assert.Equal(t, "compose.yaml", entries[1].Name())
+	})
+}
+
+type fakeSelf struct {
+	mounts []container.MountPoint
+	err    error
+}
+
+func (f fakeSelf) ContainerInspect(context.Context, string) (container.InspectResponse, error) {
+	return container.InspectResponse{Mounts: f.mounts}, f.err
+}
+
+func TestMountRoots(t *testing.T) {
+	bind := func(src, dst string, rw bool) container.MountPoint {
+		return container.MountPoint{Type: mount.TypeBind, Source: src, Destination: dst, RW: rw}
+	}
+	cases := []struct {
+		name    string
+		self    fakeSelf
+		selfID  string
+		want    []string
+		wantErr string
+	}{
+		{"rw same path", fakeSelf{mounts: []container.MountPoint{bind("/srv/stacks", "/srv/stacks", true)}}, "me", []string{"/srv/stacks"}, ""},
+		{"read-only", fakeSelf{mounts: []container.MountPoint{bind("/srv/stacks", "/srv/stacks", false)}}, "me", nil, ""},
+		{"path translated", fakeSelf{mounts: []container.MountPoint{bind("/home/x/stacks", "/stacks", true)}}, "me", nil, ""},
+		{"named volume", fakeSelf{mounts: []container.MountPoint{{Type: mount.TypeVolume, Source: "/v", Destination: "/v", RW: true}}}, "me", nil, ""},
+		{"inspect error", fakeSelf{err: errors.New("no such container")}, "me", nil, "inspect own container"},
+		{"no self id", fakeSelf{}, "", nil, "no stack directory is mounted"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got, err := MountRoots(context.Background(), c.self, c.selfID)
+			if c.wantErr != "" {
+				assert.ErrorContains(t, err, c.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, c.want, got)
+		})
+	}
+}
+
+func TestComposeFiles(t *testing.T) {
+	root := t.TempDir()
+	root, err := filepath.EvalSymlinks(root) // macOS temp dirs are symlinked
+	require.NoError(t, err)
+	mk := func(p ...string) string {
+		dir := filepath.Join(append([]string{root}, p...)...)
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		return dir
+	}
+	a := mk("a")
+	nested := mk("apps", "x", "y")
+	second := t.TempDir()
+	second, _ = filepath.EvalSymlinks(second)
+	secondDir := filepath.Join(second, "s")
+	require.NoError(t, os.MkdirAll(secondDir, 0o755))
+
+	outside, _ := filepath.EvalSymlinks(t.TempDir())
+	require.NoError(t, os.Symlink(outside, filepath.Join(root, "evil")))
+
+	sock := filepath.Join(root, "docker.sock")
+	require.NoError(t, os.WriteFile(sock, nil, 0o644))
+
+	t.Run("adds override last", func(t *testing.T) {
+		files, dir, err := composeFiles([]string{root}, filepath.Join(a, "compose.yaml"))
+		require.NoError(t, err)
+		assert.Equal(t, []string{filepath.Join(a, "compose.yaml"), filepath.Join(a, OverrideName)}, files)
+		assert.Equal(t, a, dir)
+	})
+	t.Run("override in label listed once, last", func(t *testing.T) {
+		label := filepath.Join(a, OverrideName) + "," + filepath.Join(a, "compose.yaml") + "," + filepath.Join(a, "extra.yaml")
+		files, _, err := composeFiles([]string{root}, label)
+		require.NoError(t, err)
+		assert.Equal(t, []string{filepath.Join(a, "compose.yaml"), filepath.Join(a, "extra.yaml"), filepath.Join(a, OverrideName)}, files)
+	})
+	t.Run("nested", func(t *testing.T) {
+		_, dir, err := composeFiles([]string{root}, filepath.Join(nested, "compose.yaml"))
+		require.NoError(t, err)
+		assert.Equal(t, nested, dir)
+	})
+	t.Run("second root", func(t *testing.T) {
+		_, _, err := composeFiles([]string{root, second}, filepath.Join(secondDir, "compose.yaml"))
+		require.NoError(t, err)
+	})
+	t.Run("empty label", func(t *testing.T) {
+		_, _, err := composeFiles([]string{root}, "")
+		assert.ErrorContains(t, err, "config_files")
+	})
+	t.Run("no roots", func(t *testing.T) {
+		_, _, err := composeFiles(nil, filepath.Join(a, "compose.yaml"))
+		assert.ErrorContains(t, err, "no stack directory is mounted")
+	})
+	t.Run("outside every root", func(t *testing.T) {
+		_, _, err := composeFiles([]string{root}, filepath.Join(second, "s", "compose.yaml"))
+		assert.EqualError(t, err, "stack directory "+filepath.Join(second, "s")+" is not mounted into the agent at the same path (read-write)")
+	})
+	t.Run("symlink escaping the root", func(t *testing.T) {
+		_, _, err := composeFiles([]string{root}, filepath.Join(root, "evil", "compose.yaml"))
+		assert.EqualError(t, err, "stack directory "+filepath.Join(root, "evil")+" is not mounted into the agent at the same path (read-write)")
+	})
+	t.Run("second file under a symlink escaping the root", func(t *testing.T) {
+		label := filepath.Join(a, "compose.yaml") + "," + filepath.Join(root, "evil", "extra.yaml")
+		_, _, err := composeFiles([]string{root}, label)
+		assert.ErrorContains(t, err, "not mounted")
+	})
+	t.Run("prefix trick", func(t *testing.T) {
+		evil := root + "-evil"
+		require.NoError(t, os.MkdirAll(evil, 0o755))
+		t.Cleanup(func() { os.RemoveAll(evil) })
+		_, _, err := composeFiles([]string{root}, filepath.Join(evil, "compose.yaml"))
+		assert.ErrorContains(t, err, "not mounted")
+	})
+	t.Run("a file root allows nothing", func(t *testing.T) {
+		_, _, err := composeFiles([]string{sock}, filepath.Join(sock, "compose.yaml"))
+		assert.Error(t, err)
+	})
+}

internal/agent/updater/pathcheck.go

@@ -1,4 +1,4 @@
-package discovery
+package updater
 
 import (
 	"path/filepath"

internal/agent/updater/pathcheck_test.go

@@ -1,9 +1,9 @@
-package discovery_test
+package updater_test
 
 import (
 	"testing"
 
-	"github.com/shcizo/package-updater/internal/discovery"
+	"github.com/shcizo/package-updater/internal/agent/updater"
 	"github.com/stretchr/testify/require"
 )
 
@@ -26,7 +26,7 @@ func TestIsInsideRoot(t *testing.T) {
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			require.Equal(t, c.want, discovery.IsInsideRoot(c.root, c.path))
+			require.Equal(t, c.want, updater.IsInsideRoot(c.root, c.path))
 		})
 	}
 }

internal/agent/updater/registryauth.go

@@ -0,0 +1,105 @@
+package updater
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/distribution/reference"
+	"github.com/docker/docker/api/types/registry"
+)
+
+// RegistryAuth holds inline credentials from a docker config.json, already
+// encoded as X-Registry-Auth values. A nil *RegistryAuth means anonymous
+// access.
+type RegistryAuth struct {
+	byHost map[string]string
+}
+
+type dockerConfigFile struct {
+	Auths map[string]struct {
+		Auth          string `json:"auth"`
+		IdentityToken string `json:"identitytoken"`
+	} `json:"auths"`
+	CredsStore  string            `json:"credsStore"`
+	CredHelpers map[string]string `json:"credHelpers"`
+}
+
+// LoadRegistryAuth reads dockerConfigDir/config.json. An empty dir returns
+// nil, nil. Only inline `auth` and `identitytoken` entries are supported;
+// credential helpers are not available inside the agent container, which is
+// logged once.
+func LoadRegistryAuth(dockerConfigDir string, log *slog.Logger) (*RegistryAuth, error) {
+	if dockerConfigDir == "" {
+		return nil, nil
+	}
+	path := filepath.Join(dockerConfigDir, "config.json")
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	var cfg dockerConfigFile
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return nil, fmt.Errorf("parse %s: %w", path, err)
+	}
+	if (cfg.CredsStore != "" || len(cfg.CredHelpers) > 0) && log != nil {
+		log.Warn("registry_cred_helpers_unsupported", "path", path)
+	}
+	a := &RegistryAuth{byHost: make(map[string]string)}
+	for key, entry := range cfg.Auths {
+		host := normaliseRegistryKey(key)
+		ac := registry.AuthConfig{ServerAddress: host}
+		switch {
+		case entry.IdentityToken != "":
+			ac.IdentityToken = entry.IdentityToken
+		case entry.Auth != "":
+			dec, err := base64.StdEncoding.DecodeString(entry.Auth)
+			if err != nil {
+				return nil, fmt.Errorf("%s: auth for %s is not base64", path, key)
+			}
+			user, pass, ok := strings.Cut(string(dec), ":")
+			if !ok {
+				return nil, fmt.Errorf("%s: auth for %s is not user:password", path, key)
+			}
+			ac.Username, ac.Password = user, pass
+		default:
+			continue
+		}
+		enc, err := registry.EncodeAuthConfig(ac)
+		if err != nil {
+			return nil, fmt.Errorf("encode auth for %s: %w", key, err)
+		}
+		a.byHost[host] = enc
+	}
+	return a, nil
+}
+
+// normaliseRegistryKey reduces a config.json key to a bare registry host,
+// mapping every Docker Hub spelling to "docker.io".
+func normaliseRegistryKey(key string) string {
+	k := strings.TrimPrefix(strings.TrimPrefix(key, "https://"), "http://")
+	if i := strings.Index(k, "/"); i >= 0 {
+		k = k[:i]
+	}
+	switch k {
+	case "index.docker.io", "docker.io", "registry-1.docker.io":
+		return "docker.io"
+	}
+	return k
+}
+
+// For returns the X-Registry-Auth value for repo's registry, or "".
+func (a *RegistryAuth) For(repo string) string {
+	if a == nil {
+		return ""
+	}
+	named, err := reference.ParseNormalizedNamed(repo)
+	if err != nil {
+		return ""
+	}
+	return a.byHost[reference.Domain(named)]
+}

internal/agent/updater/registryauth_test.go

@@ -0,0 +1,72 @@
+package updater
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/api/types/registry"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func writeDockerConfig(t *testing.T, body string) string {
+	t.Helper()
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(body), 0o600))
+	return dir
+}
+
+func decodeAuth(t *testing.T, enc string) registry.AuthConfig {
+	t.Helper()
+	raw, err := base64.URLEncoding.DecodeString(enc)
+	require.NoError(t, err)
+	var ac registry.AuthConfig
+	require.NoError(t, json.Unmarshal(raw, &ac))
+	return ac
+}
+
+func TestRegistryAuth(t *testing.T) {
+	user := base64.StdEncoding.EncodeToString([]byte("bob:s3cret"))
+	log := slog.New(slog.NewTextHandler(io.Discard, nil))
+
+	t.Run("docker hub spellings", func(t *testing.T) {
+		for _, key := range []string{"https://index.docker.io/v1/", "index.docker.io", "docker.io", "registry-1.docker.io"} {
+			dir := writeDockerConfig(t, `{"auths":{"`+key+`":{"auth":"`+user+`"}}}`)
+			a, err := LoadRegistryAuth(dir, log)
+			require.NoError(t, err)
+			for _, repo := range []string{"gitea/gitea", "docker.io/library/nginx", "nginx"} {
+				ac := decodeAuth(t, a.For(repo))
+				assert.Equal(t, "bob", ac.Username, "%s for %s", key, repo)
+				assert.Equal(t, "s3cret", ac.Password)
+			}
+			assert.Empty(t, a.For("gitea.shcizo.se/shcizo/slusk"))
+		}
+	})
+	t.Run("custom registry with scheme", func(t *testing.T) {
+		dir := writeDockerConfig(t, `{"auths":{"https://gitea.shcizo.se":{"auth":"`+user+`"},"ghcr.io":{"identitytoken":"tok"}}}`)
+		a, err := LoadRegistryAuth(dir, log)
+		require.NoError(t, err)
+		assert.Equal(t, "bob", decodeAuth(t, a.For("gitea.shcizo.se/shcizo/slusk")).Username)
+		assert.Equal(t, "tok", decodeAuth(t, a.For("ghcr.io/x/y")).IdentityToken)
+		assert.Empty(t, a.For("registry.example.com/x"))
+	})
+	t.Run("empty dir is anonymous", func(t *testing.T) {
+		a, err := LoadRegistryAuth("", log)
+		require.NoError(t, err)
+		assert.Nil(t, a)
+		assert.Empty(t, a.For("gitea/gitea"))
+	})
+	t.Run("malformed json", func(t *testing.T) {
+		_, err := LoadRegistryAuth(writeDockerConfig(t, "{"), log)
+		assert.Error(t, err)
+	})
+	t.Run("missing file", func(t *testing.T) {
+		_, err := LoadRegistryAuth(t.TempDir(), log)
+		assert.Error(t, err)
+	})
+}

internal/agent/updater/resolve.go

@@ -0,0 +1,61 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/docker/docker/api/types/registry"
+
+	"github.com/shcizo/package-updater/internal/imageref"
+)
+
+// DistributionClient is the subset of the Docker SDK the Resolver needs.
+// *client.Client satisfies it.
+type DistributionClient interface {
+	DistributionInspect(ctx context.Context, imageRef, encodedRegistryAuth string) (registry.DistributionInspect, error)
+}
+
+// Resolver turns "repo:tag" into "repo:tag@digest" by asking the local
+// daemon, which contacts the registry. A ref that already has a digest is
+// passed through.
+type Resolver struct {
+	cli  DistributionClient
+	auth *RegistryAuth
+}
+
+// NewResolver returns a Resolver. auth may be nil (anonymous access).
+func NewResolver(cli DistributionClient, auth *RegistryAuth) *Resolver {
+	return &Resolver{cli: cli, auth: auth}
+}
+
+// Resolve returns ref pinned to the digest its tag currently points at. A ref
+// that already has a digest ("repo:tag@digest" or "repo@digest") is returned
+// unchanged without asking the daemon. It fails for a ref with neither tag nor
+// digest.
+func (r *Resolver) Resolve(ctx context.Context, ref string) (string, error) {
+	p := imageref.Parse(ref)
+	switch {
+	case p.Digest != "":
+		return ref, nil
+	case p.Tag == "":
+		return "", fmt.Errorf("image %q has no tag to resolve", ref)
+	}
+	info, err := r.cli.DistributionInspect(ctx, ref, r.auth.For(p.Repo))
+	if err != nil {
+		return "", fmt.Errorf("resolve %s: %w", ref, err)
+	}
+	if info.Descriptor.Digest == "" {
+		return "", fmt.Errorf("resolve %s: registry returned no digest", ref)
+	}
+	return ref + "@" + info.Descriptor.Digest.String(), nil
+}
+
+// resolveNote is the job-output line that says which ref is deployed:
+// "resolved" when pinned came from resolving a tag, "pinned" when ref already
+// carried its digest.
+func resolveNote(ref, pinned string) string {
+	if pinned == ref {
+		return "pinned " + pinned
+	}
+	return "resolved " + pinned
+}

internal/agent/updater/resolve_test.go

@@ -0,0 +1,45 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolver(t *testing.T) {
+	const d = "sha256:d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1"
+	auth := &RegistryAuth{byHost: map[string]string{"gitea.shcizo.se": "enc-gitea"}}
+
+	t.Run("resolves tag with registry auth", func(t *testing.T) {
+		dist := &fakeDist{digest: d}
+		got, err := NewResolver(dist, auth).Resolve(context.Background(), "gitea.shcizo.se/shcizo/slusk:1.4")
+		require.NoError(t, err)
+		assert.Equal(t, "gitea.shcizo.se/shcizo/slusk:1.4@"+d, got)
+		assert.Equal(t, []string{"gitea.shcizo.se/shcizo/slusk:1.4"}, dist.refs)
+		assert.Equal(t, []string{"enc-gitea"}, dist.auths)
+	})
+	t.Run("a ref with a digest passes through without asking the daemon", func(t *testing.T) {
+		for _, ref := range []string{"repo@sha256:x", "repo:1@sha256:x", "localhost:5000/repo:1@sha256:x"} {
+			dist := &fakeDist{digest: d}
+			got, err := NewResolver(dist, nil).Resolve(context.Background(), ref)
+			require.NoError(t, err, ref)
+			assert.Equal(t, ref, got)
+			assert.Empty(t, dist.refs)
+		}
+	})
+	t.Run("a bare ref is refused without asking the daemon", func(t *testing.T) {
+		dist := &fakeDist{digest: d}
+		_, err := NewResolver(dist, nil).Resolve(context.Background(), "repo")
+		assert.ErrorContains(t, err, "has no tag to resolve")
+		assert.Empty(t, dist.refs)
+	})
+	t.Run("daemon error is wrapped", func(t *testing.T) {
+		cause := errors.New("unauthorized")
+		_, err := NewResolver(&fakeDist{err: cause}, nil).Resolve(context.Background(), "repo:1")
+		assert.ErrorIs(t, err, cause)
+		assert.ErrorContains(t, err, "repo:1")
+	})
+}

internal/agent/updater/runner.go

@@ -0,0 +1,272 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// maxProgress caps the output buffered between two TakeProgress calls, so a
+// chatty job cannot grow a single job.progress message without bound.
+const maxProgress = 32 << 10
+
+// Executor deploys one job and returns the "repo:tag@digest" it deployed.
+// Everything it writes to out is streamed to the master as job output.
+type Executor interface {
+	Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error)
+}
+
+// Progress is output a running job produced since the previous TakeProgress.
+type Progress struct{ ID, Output string }
+
+// Finished is a job's outcome, kept until the master acknowledges it.
+type Finished struct {
+	ID     string
+	Result protocol.JobResult
+}
+
+// tailBuffer keeps the last max bytes written to it.
+type tailBuffer struct {
+	max       int
+	buf       []byte
+	truncated bool
+}
+
+func (b *tailBuffer) Write(p []byte) {
+	b.buf = append(b.buf, p...)
+	if over := len(b.buf) - b.max; over > 0 {
+		b.buf = append(b.buf[:0], b.buf[over:]...)
+		b.truncated = true
+	}
+}
+
+func (b *tailBuffer) String() string  { return string(b.buf) }
+func (b *tailBuffer) Truncated() bool { return b.truncated }
+func (b *tailBuffer) Reset()          { b.buf, b.truncated = nil, false }
+
+type jobState struct {
+	start    protocol.JobStart
+	running  bool
+	result   *protocol.JobResult
+	output   tailBuffer // everything, capped at protocol.MaxJobOutput
+	unsent   tailBuffer // since the last TakeProgress, capped at maxProgress
+	reported bool       // a Progress has been taken since the job started
+	finished time.Time  // orders Pending
+}
+
+// Runner executes jobs one at a time in the order they arrive and keeps each
+// job's result until Ack. Two docker compose calls against the same stack
+// would race, so there is deliberately exactly one worker.
+//
+// Runner satisfies conn.Jobs.
+type Runner struct {
+	exec    Executor
+	timeout time.Duration
+	now     func() time.Time
+
+	mu    sync.Mutex // guards jobs and queue
+	jobs  map[string]*jobState
+	queue []string
+
+	wake    chan struct{} // wakes the worker
+	changed chan struct{} // tells the connection there is something to send
+}
+
+// NewRunner returns a Runner that gives every job at most timeout. Call Run
+// to start the worker.
+func NewRunner(exec Executor, timeout time.Duration) *Runner {
+	return &Runner{
+		exec:    exec,
+		timeout: timeout,
+		now:     time.Now,
+		jobs:    make(map[string]*jobState),
+		wake:    make(chan struct{}, 1),
+		changed: make(chan struct{}, 1),
+	}
+}
+
+// signal does a non-blocking send so neither side ever waits on the other.
+func signal(ch chan struct{}) {
+	select {
+	case ch <- struct{}{}:
+	default:
+	}
+}
+
+// Submit queues a job. A job whose ID the runner already holds (queued,
+// running, or finished and not yet acknowledged) is ignored.
+func (r *Runner) Submit(id string, start protocol.JobStart) {
+	r.mu.Lock()
+	if _, ok := r.jobs[id]; ok {
+		r.mu.Unlock()
+		return
+	}
+	r.jobs[id] = &jobState{
+		start:  start,
+		output: tailBuffer{max: protocol.MaxJobOutput},
+		unsent: tailBuffer{max: maxProgress},
+	}
+	r.queue = append(r.queue, id)
+	r.mu.Unlock()
+	signal(r.wake)
+}
+
+// Ack forgets a finished job. Acks for unknown, queued or running jobs are
+// ignored.
+func (r *Runner) Ack(id string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if j, ok := r.jobs[id]; ok && j.result != nil {
+		delete(r.jobs, id)
+	}
+}
+
+// Known returns the sorted IDs of every job the runner holds.
+func (r *Runner) Known() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	ids := make([]string, 0, len(r.jobs))
+	for id := range r.jobs {
+		ids = append(ids, id)
+	}
+	sort.Strings(ids)
+	return ids
+}
+
+// TakeProgress returns the output running jobs produced since the previous
+// call. A job's first entry (possibly with empty Output) tells the master it
+// is running.
+func (r *Runner) TakeProgress() []Progress {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	var out []Progress
+	for id, j := range r.jobs {
+		if !j.running {
+			continue
+		}
+		s := j.unsent.String()
+		if j.reported && s == "" {
+			continue
+		}
+		j.reported = true
+		j.unsent.Reset()
+		out = append(out, Progress{ID: id, Output: s})
+	}
+	sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID })
+	return out
+}
+
+// Pending returns the results not yet acknowledged, oldest first.
+func (r *Runner) Pending() []Finished {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	var out []Finished
+	var at []time.Time
+	for id, j := range r.jobs {
+		if j.result != nil {
+			out = append(out, Finished{ID: id, Result: *j.result})
+			at = append(at, j.finished)
+		}
+	}
+	idx := make([]int, len(out))
+	for i := range idx {
+		idx[i] = i
+	}
+	sort.Slice(idx, func(a, b int) bool {
+		if !at[idx[a]].Equal(at[idx[b]]) {
+			return at[idx[a]].Before(at[idx[b]])
+		}
+		return out[idx[a]].ID < out[idx[b]].ID
+	})
+	sorted := make([]Finished, len(out))
+	for i, k := range idx {
+		sorted[i] = out[k]
+	}
+	return sorted
+}
+
+// Changed returns a channel that receives when there is new progress or a
+// new result to send.
+func (r *Runner) Changed() <-chan struct{} { return r.changed }
+
+// Run works through the queue until ctx ends. Jobs still queued then are
+// dropped; the master marks them lost on the agent's next hello.
+func (r *Runner) Run(ctx context.Context) {
+	for {
+		id, start, ok := r.next()
+		if !ok {
+			select {
+			case <-ctx.Done():
+				return
+			case <-r.wake:
+				continue
+			}
+		}
+		if ctx.Err() != nil {
+			return
+		}
+		r.runOne(ctx, id, start)
+	}
+}
+
+// next pops the next queued job and marks it running.
+func (r *Runner) next() (string, protocol.JobStart, bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if len(r.queue) == 0 {
+		return "", protocol.JobStart{}, false
+	}
+	id := r.queue[0]
+	r.queue = r.queue[1:]
+	j := r.jobs[id]
+	j.running = true
+	signal(r.changed)
+	return id, j.start, true
+}
+
+func (r *Runner) runOne(ctx context.Context, id string, start protocol.JobStart) {
+	started := r.now()
+	jobCtx, cancel := context.WithTimeout(ctx, r.timeout)
+	ref, err := r.exec.Execute(jobCtx, start, &jobWriter{r: r, id: id})
+	timedOut := errors.Is(jobCtx.Err(), context.DeadlineExceeded)
+	cancel()
+
+	res := protocol.JobResult{Status: protocol.ResultSucceeded, Ref: ref, StartedAt: started, FinishedAt: r.now()}
+	switch {
+	case timedOut:
+		res = protocol.JobResult{Status: protocol.ResultFailed, Error: fmt.Sprintf("timed out after %s", r.timeout), StartedAt: started, FinishedAt: res.FinishedAt}
+	case err != nil:
+		res = protocol.JobResult{Status: protocol.ResultFailed, Error: err.Error(), StartedAt: started, FinishedAt: res.FinishedAt}
+	}
+
+	r.mu.Lock()
+	j := r.jobs[id]
+	res.Output, res.OutputTruncated = j.output.String(), j.output.Truncated()
+	j.running, j.result, j.finished = false, &res, res.FinishedAt
+	j.unsent.Reset()
+	r.mu.Unlock()
+	signal(r.changed)
+}
+
+// jobWriter feeds an executor's output into the job's buffers.
+type jobWriter struct {
+	r  *Runner
+	id string
+}
+
+func (w *jobWriter) Write(p []byte) (int, error) {
+	w.r.mu.Lock()
+	if j, ok := w.r.jobs[w.id]; ok {
+		j.output.Write(p)
+		j.unsent.Write(p)
+	}
+	w.r.mu.Unlock()
+	signal(w.r.changed)
+	return len(p), nil
+}

internal/agent/updater/runner_test.go

@@ -0,0 +1,200 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"io"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// fakeExec writes "pulling <ref>" and then waits on gate (if set). Refs
+// starting with "bad" fail. It records the highest concurrency it saw.
+type fakeExec struct {
+	gate    chan struct{}
+	cur     atomic.Int32
+	maxSeen atomic.Int32
+	started chan string
+}
+
+func newFakeExec() *fakeExec {
+	return &fakeExec{gate: make(chan struct{}), started: make(chan string, 10)}
+}
+
+func (f *fakeExec) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	n := f.cur.Add(1)
+	defer f.cur.Add(-1)
+	for {
+		m := f.maxSeen.Load()
+		if n <= m || f.maxSeen.CompareAndSwap(m, n) {
+			break
+		}
+	}
+	f.started <- job.Ref
+	io.WriteString(out, "pulling "+job.Ref+"\n")
+	select {
+	case <-f.gate:
+	case <-ctx.Done():
+		return "", ctx.Err()
+	}
+	if strings.HasPrefix(job.Ref, "bad") {
+		return "", errors.New("boom")
+	}
+	return job.Ref + "@sha256:aa", nil
+}
+
+func startRunner(t *testing.T, exec Executor, timeout time.Duration) *Runner {
+	t.Helper()
+	r := NewRunner(exec, timeout)
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() { defer wg.Done(); r.Run(ctx) }()
+	t.Cleanup(func() { cancel(); wg.Wait() })
+	return r
+}
+
+func waitPending(t *testing.T, r *Runner, n int) []Finished {
+	t.Helper()
+	var got []Finished
+	require.Eventually(t, func() bool { got = r.Pending(); return len(got) >= n }, 2*time.Second, 5*time.Millisecond)
+	return got
+}
+
+func TestRunner_FIFOSingleWorker(t *testing.T) {
+	fe := newFakeExec()
+	r := startRunner(t, fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	r.Submit("2", protocol.JobStart{Ref: "bad:2"})
+	assert.Equal(t, "a:1", <-fe.started)
+	fe.gate <- struct{}{}
+	assert.Equal(t, "bad:2", <-fe.started)
+	fe.gate <- struct{}{}
+
+	got := waitPending(t, r, 2)
+	assert.Equal(t, int32(1), fe.maxSeen.Load())
+	require.Len(t, got, 2)
+	assert.Equal(t, "1", got[0].ID)
+	assert.Equal(t, protocol.ResultSucceeded, got[0].Result.Status)
+	assert.Equal(t, "a:1@sha256:aa", got[0].Result.Ref)
+	assert.Equal(t, "pulling a:1\n", got[0].Result.Output)
+	assert.Equal(t, "2", got[1].ID)
+	assert.Equal(t, protocol.ResultFailed, got[1].Result.Status)
+	assert.Equal(t, "boom", got[1].Result.Error)
+	assert.False(t, got[0].Result.StartedAt.IsZero())
+}
+
+func TestRunner_DuplicateSubmitIgnored(t *testing.T) {
+	fe := newFakeExec()
+	r := startRunner(t, fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	assert.Equal(t, "a:1", <-fe.started)
+	r.Submit("1", protocol.JobStart{Ref: "other:9"})
+	fe.gate <- struct{}{}
+	got := waitPending(t, r, 1)
+	assert.Equal(t, "a:1@sha256:aa", got[0].Result.Ref)
+	select {
+	case ref := <-fe.started:
+		t.Fatalf("duplicate job ran: %s", ref)
+	case <-time.After(50 * time.Millisecond):
+	}
+}
+
+func TestRunner_KnownAndAck(t *testing.T) {
+	fe := newFakeExec()
+	r := startRunner(t, fe, time.Minute)
+	r.Submit("2", protocol.JobStart{Ref: "a:2"})
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	assert.Equal(t, "a:2", <-fe.started)
+	assert.Equal(t, []string{"1", "2"}, r.Known())
+
+	r.Ack("2") // running: ignored
+	assert.Equal(t, []string{"1", "2"}, r.Known())
+
+	fe.gate <- struct{}{}
+	waitPending(t, r, 1)
+	r.Ack("2")
+	assert.Equal(t, []string{"1"}, r.Known())
+	<-fe.started
+	fe.gate <- struct{}{}
+	waitPending(t, r, 1)
+}
+
+func TestRunner_TakeProgress(t *testing.T) {
+	fe := newFakeExec()
+	r := startRunner(t, fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	<-fe.started
+	require.Eventually(t, func() bool { return len(r.Pending()) == 0 && len(r.Known()) == 1 }, time.Second, 5*time.Millisecond)
+
+	var first []Progress
+	require.Eventually(t, func() bool { first = r.TakeProgress(); return len(first) > 0 }, time.Second, 5*time.Millisecond)
+	assert.Equal(t, []Progress{{ID: "1", Output: "pulling a:1\n"}}, first)
+	assert.Empty(t, r.TakeProgress(), "nothing new since the last call")
+
+	fe.gate <- struct{}{}
+	waitPending(t, r, 1)
+	assert.Empty(t, r.TakeProgress(), "a finished job no longer reports progress")
+}
+
+func TestRunner_TakeProgressReportsStartedJobOnce(t *testing.T) {
+	gate := make(chan struct{})
+	started := make(chan struct{})
+	r := startRunner(t, execFunc(func(ctx context.Context, _ protocol.JobStart, _ io.Writer) (string, error) {
+		close(started)
+		<-gate
+		return "x", nil
+	}), time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	<-started
+	assert.Equal(t, []Progress{{ID: "1", Output: ""}}, r.TakeProgress())
+	assert.Empty(t, r.TakeProgress())
+	close(gate)
+}
+
+type execFunc func(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error)
+
+func (f execFunc) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	return f(ctx, job, out)
+}
+
+func TestRunner_Timeout(t *testing.T) {
+	r := startRunner(t, execFunc(func(ctx context.Context, _ protocol.JobStart, _ io.Writer) (string, error) {
+		<-ctx.Done()
+		return "", ctx.Err()
+	}), 20*time.Millisecond)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	got := waitPending(t, r, 1)
+	assert.Equal(t, protocol.ResultFailed, got[0].Result.Status)
+	assert.Equal(t, "timed out after 20ms", got[0].Result.Error)
+}
+
+func TestRunner_OutputCapped(t *testing.T) {
+	r := startRunner(t, execFunc(func(_ context.Context, _ protocol.JobStart, out io.Writer) (string, error) {
+		io.WriteString(out, strings.Repeat("a", 10))
+		io.WriteString(out, strings.Repeat("b", protocol.MaxJobOutput))
+		return "x", nil
+	}), time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	got := waitPending(t, r, 1)
+	assert.Equal(t, strings.Repeat("b", protocol.MaxJobOutput), got[0].Result.Output)
+	assert.True(t, got[0].Result.OutputTruncated)
+}
+
+func TestRunner_ChangedSignals(t *testing.T) {
+	r := startRunner(t, execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "x", nil }), time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	select {
+	case <-r.Changed():
+	case <-time.After(time.Second):
+		t.Fatal("no change signal")
+	}
+}

internal/agent/updater/swarm.go

@@ -0,0 +1,93 @@
+package updater
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/system"
+
+	"github.com/shcizo/package-updater/internal/imageref"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// SwarmClient is the subset of the Docker SDK SwarmExecutor depends on.
+// Defined as an interface so tests can supply a fake.
+type SwarmClient interface {
+	Info(ctx context.Context) (system.Info, error)
+	ServiceInspectWithRaw(ctx context.Context, serviceID string, opts swarm.ServiceInspectOptions) (swarm.Service, []byte, error)
+	ServiceUpdate(ctx context.Context, serviceID string, version swarm.Version, service swarm.ServiceSpec, opts swarm.ServiceUpdateOptions) (swarm.ServiceUpdateResponse, error)
+}
+
+// SwarmExecutor updates a Swarm service's image via the Docker API,
+// equivalent to `docker service update --image`.
+type SwarmExecutor struct {
+	cli     SwarmClient
+	resolve *Resolver
+	auth    *RegistryAuth
+}
+
+// NewSwarmExecutor returns an executor that drives Swarm service updates
+// through the Docker API. auth may be nil (anonymous registry access).
+func NewSwarmExecutor(cli SwarmClient, resolve *Resolver, auth *RegistryAuth) *SwarmExecutor {
+	return &SwarmExecutor{cli: cli, resolve: resolve, auth: auth}
+}
+
+// Execute checks that the local daemon is a manager of the job's cluster,
+// pins job.Ref to its current digest (a ref that has one is used as is) and
+// updates the service to it. The update is addressed by the ID and version of
+// the service that was inspected, so it changes exactly that object. Success
+// means the daemon accepted the new spec, not that the tasks have converged.
+// It returns the pinned ref.
+func (e *SwarmExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	info, err := e.cli.Info(ctx)
+	if err != nil {
+		return "", fmt.Errorf("docker info: %w", err)
+	}
+	if !info.Swarm.ControlAvailable {
+		return "", fmt.Errorf("refused: this agent's daemon is not a swarm manager")
+	}
+	actual := ""
+	if info.Swarm.Cluster != nil {
+		actual = info.Swarm.Cluster.ID
+	}
+	if actual != job.SwarmClusterID {
+		return "", fmt.Errorf("refused: job is for swarm cluster %s, this daemon is in %s", job.SwarmClusterID, actual)
+	}
+
+	svc, _, err := e.cli.ServiceInspectWithRaw(ctx, job.SwarmService, swarm.ServiceInspectOptions{})
+	if err != nil {
+		return "", fmt.Errorf("inspect service %s: %w", job.SwarmService, err)
+	}
+	if svc.Spec.Name != job.SwarmService {
+		return "", fmt.Errorf("inspect service %s: daemon returned service %q", job.SwarmService, svc.Spec.Name)
+	}
+	if svc.Spec.TaskTemplate.ContainerSpec == nil {
+		return "", fmt.Errorf("service %s has no container spec", job.SwarmService)
+	}
+
+	pinned, err := e.resolve.Resolve(ctx, job.Ref)
+	if err != nil {
+		return "", err
+	}
+	fmt.Fprintln(out, resolveNote(job.Ref, pinned))
+
+	spec := svc.Spec
+	spec.TaskTemplate.ContainerSpec.Image = pinned
+	opts := swarm.ServiceUpdateOptions{QueryRegistry: true}
+	if enc := e.auth.For(imageref.Parse(job.Ref).Repo); enc != "" {
+		opts.EncodedRegistryAuth = enc
+	} else {
+		opts.RegistryAuthFrom = swarm.RegistryAuthFromSpec
+	}
+	resp, err := e.cli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, opts)
+	if err != nil {
+		return "", fmt.Errorf("update service %s: %w", job.SwarmService, err)
+	}
+	for _, w := range resp.Warnings {
+		fmt.Fprintf(out, "warning: %s\n", w)
+	}
+	fmt.Fprintf(out, "service %s updated\n", job.SwarmService)
+	return pinned, nil
+}

internal/agent/updater/swarm_test.go

@@ -0,0 +1,158 @@
+package updater
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/docker/docker/api/types/registry"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/system"
+	"github.com/opencontainers/go-digest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+type fakeSwarm struct {
+	info       system.Info
+	svc        swarm.Service
+	inspectErr error
+	updateErr  error
+	warnings   []string
+
+	inspectCalls int
+	updateCalls  int
+	updID        string
+	updVersion   swarm.Version
+	updSpec      swarm.ServiceSpec
+	updOpts      swarm.ServiceUpdateOptions
+}
+
+func (f *fakeSwarm) Info(context.Context) (system.Info, error) { return f.info, nil }
+
+func (f *fakeSwarm) ServiceInspectWithRaw(context.Context, string, swarm.ServiceInspectOptions) (swarm.Service, []byte, error) {
+	f.inspectCalls++
+	return f.svc, nil, f.inspectErr
+}
+
+func (f *fakeSwarm) ServiceUpdate(_ context.Context, id string, v swarm.Version, spec swarm.ServiceSpec, opts swarm.ServiceUpdateOptions) (swarm.ServiceUpdateResponse, error) {
+	f.updateCalls++
+	f.updID, f.updVersion, f.updSpec, f.updOpts = id, v, spec, opts
+	return swarm.ServiceUpdateResponse{Warnings: f.warnings}, f.updateErr
+}
+
+type fakeDist struct {
+	digest string
+	err    error
+	refs   []string
+	auths  []string
+}
+
+func (f *fakeDist) DistributionInspect(_ context.Context, ref, auth string) (registry.DistributionInspect, error) {
+	f.refs = append(f.refs, ref)
+	f.auths = append(f.auths, auth)
+	var d registry.DistributionInspect
+	d.Descriptor.Digest = digest.Digest(f.digest)
+	return d, f.err
+}
+
+func managerOf(cluster string) system.Info {
+	var info system.Info
+	info.Swarm.ControlAvailable = true
+	info.Swarm.Cluster = &swarm.ClusterInfo{ID: cluster}
+	return info
+}
+
+func swarmSvc(name string) swarm.Service {
+	return swarm.Service{
+		ID:   "svc-id",
+		Meta: swarm.Meta{Version: swarm.Version{Index: 7}},
+		Spec: swarm.ServiceSpec{
+			Annotations:  swarm.Annotations{Name: name},
+			TaskTemplate: swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{Image: "gitea/gitea:1.26"}},
+		},
+	}
+}
+
+func TestSwarmUsesPinnedRef(t *testing.T) {
+	const ref = "gitea/gitea:1.26@sha256:abababababababababababababababababababababababababababababababab"
+	job := protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: ref}
+	fake := &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_gitea")}
+	dist := &fakeDist{digest: "sha256:other"}
+	var out bytes.Buffer
+
+	got, err := NewSwarmExecutor(fake, NewResolver(dist, nil), nil).Execute(context.Background(), job, &out)
+
+	require.NoError(t, err)
+	assert.Equal(t, ref, got)
+	assert.Equal(t, ref, fake.updSpec.TaskTemplate.ContainerSpec.Image)
+	assert.Empty(t, dist.refs, "no registry lookup")
+	assert.Contains(t, out.String(), "pinned "+ref)
+}
+
+func TestSwarmExecutor(t *testing.T) {
+	job := protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"}
+	workerInfo := managerOf("c1")
+	workerInfo.Swarm.ControlAvailable = false
+	noContainer := swarmSvc("gitea_gitea")
+	noContainer.Spec.TaskTemplate.ContainerSpec = nil
+
+	cases := []struct {
+		name        string
+		fake        *fakeSwarm
+		auth        *RegistryAuth
+		wantErr     string
+		wantUpdate  bool
+		wantEncAuth bool
+	}{
+		{name: "happy path, no auth", fake: &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_gitea")}, wantUpdate: true},
+		{name: "happy path, auth known", fake: &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_gitea"), warnings: []string{"w1"}},
+			auth: &RegistryAuth{byHost: map[string]string{"docker.io": "enc"}}, wantUpdate: true, wantEncAuth: true},
+		{name: "cluster mismatch", fake: &fakeSwarm{info: managerOf("other"), svc: swarmSvc("gitea_gitea")},
+			wantErr: "refused: job is for swarm cluster c1, this daemon is in other"},
+		{name: "worker node", fake: &fakeSwarm{info: workerInfo, svc: swarmSvc("gitea_gitea")}, wantErr: "not a swarm manager"},
+		{name: "other service returned", fake: &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_other")}, wantErr: "daemon returned service"},
+		{name: "no container spec", fake: &fakeSwarm{info: managerOf("c1"), svc: noContainer}, wantErr: "no container spec"},
+		{name: "inspect fails", fake: &fakeSwarm{info: managerOf("c1"), inspectErr: errors.New("gone")}, wantErr: "gone"},
+		{name: "update fails", fake: &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_gitea"), updateErr: errors.New("rejected")},
+			wantErr: "rejected", wantUpdate: true},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			dist := &fakeDist{digest: "sha256:" + string(bytes.Repeat([]byte("a"), 64))}
+			e := NewSwarmExecutor(c.fake, NewResolver(dist, c.auth), c.auth)
+			var out bytes.Buffer
+			ref, err := e.Execute(context.Background(), job, &out)
+			if c.wantErr != "" {
+				assert.ErrorContains(t, err, c.wantErr)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, "gitea/gitea:1.27@"+dist.digest, ref)
+				assert.Contains(t, out.String(), "resolved "+ref)
+			}
+			if c.wantErr == "refused: job is for swarm cluster c1, this daemon is in other" || c.wantErr == "not a swarm manager" {
+				assert.Zero(t, c.fake.inspectCalls)
+			}
+			if !c.wantUpdate {
+				assert.Zero(t, c.fake.updateCalls)
+				return
+			}
+			assert.Equal(t, 1, c.fake.updateCalls)
+			assert.Equal(t, "svc-id", c.fake.updID)
+			assert.Equal(t, uint64(7), c.fake.updVersion.Index)
+			assert.Equal(t, "gitea/gitea:1.27@"+dist.digest, c.fake.updSpec.TaskTemplate.ContainerSpec.Image)
+			assert.True(t, c.fake.updOpts.QueryRegistry)
+			if c.wantEncAuth {
+				assert.Equal(t, "enc", c.fake.updOpts.EncodedRegistryAuth)
+				assert.Empty(t, c.fake.updOpts.RegistryAuthFrom)
+				assert.Contains(t, out.String(), "warning: w1")
+			} else {
+				assert.Empty(t, c.fake.updOpts.EncodedRegistryAuth)
+				assert.Equal(t, swarm.RegistryAuthFromSpec, c.fake.updOpts.RegistryAuthFrom)
+			}
+		})
+	}
+}

internal/api/handlers.go

@@ -1,148 +0,0 @@
-package api
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-
-	"github.com/docker/docker/api/types"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/shcizo/package-updater/internal/metrics"
-	"github.com/shcizo/package-updater/internal/updater"
-)
-
-// Finder is the discovery interface used by the update handler.
-type Finder interface {
-	FindJobs(ctx context.Context, image string) ([]discovery.Job, error)
-}
-
-// Submitter is the queue interface used by the update handler.
-type Submitter interface {
-	Submit(ctx context.Context, jobs []discovery.Job) []updater.Result
-}
-
-// Pinger pings the Docker daemon for the healthcheck.
-type Pinger interface {
-	Ping(ctx context.Context) (types.Ping, error)
-}
-
-// Handlers wires the HTTP endpoints to the rest of the service.
-type Handlers struct {
-	finder    Finder
-	submitter Submitter
-	pinger    Pinger
-	version   string
-	commit    string
-	buildTime string
-	metrics   *metrics.Metrics
-}
-
-// NewHandlers constructs a Handlers value with all dependencies injected.
-// m may be nil, in which case metrics recording is silently skipped.
-func NewHandlers(f Finder, s Submitter, p Pinger, version, commit, buildTime string, m *metrics.Metrics) *Handlers {
-	return &Handlers{finder: f, submitter: s, pinger: p, version: version, commit: commit, buildTime: buildTime, metrics: m}
-}
-
-// Update implements POST /update.
-func (h *Handlers) Update(w http.ResponseWriter, r *http.Request) {
-	var req UpdateRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeJSONError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-	if req.Image == "" {
-		writeJSONError(w, http.StatusBadRequest, "image is required")
-		return
-	}
-
-	requestedImage := req.Image
-	if req.Tag != "" {
-		requestedImage = req.Image + ":" + req.Tag
-	}
-
-	jobs, err := h.finder.FindJobs(r.Context(), requestedImage)
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "discovery failed: "+err.Error())
-		return
-	}
-
-	resp := UpdateResponse{
-		RequestID: logging.RequestIDFrom(r.Context()),
-		Image:     req.Image,
-		Tag:       req.Tag,
-		Matched:   len(jobs),
-		Results:   []ResultDTO{},
-	}
-
-	if len(jobs) == 0 {
-		writeJSON(w, http.StatusOK, resp)
-		return
-	}
-
-	results := h.submitter.Submit(r.Context(), jobs)
-	updated, failed := 0, 0
-	for _, res := range results {
-		composeFile := ""
-		if len(res.Job.ConfigFiles) > 0 {
-			composeFile = res.Job.ConfigFiles[0]
-		}
-		resp.Results = append(resp.Results, ResultDTO{
-			Project:     res.Job.Project,
-			Service:     res.Job.Service,
-			ComposeFile: composeFile,
-			ServiceID:   res.Job.ServiceID,
-			Status:      string(res.Status),
-			Error:       res.Error,
-			DurationMs:  res.DurationMs,
-		})
-		if res.Status == updater.StatusUpdated {
-			updated++
-		} else {
-			failed++
-		}
-	}
-
-	switch {
-	case failed == 0:
-		writeJSON(w, http.StatusOK, resp)
-	case updated == 0:
-		writeJSON(w, http.StatusInternalServerError, resp)
-	default:
-		writeJSON(w, http.StatusMultiStatus, resp)
-	}
-}
-
-// Healthz implements GET /healthz.
-func (h *Handlers) Healthz(w http.ResponseWriter, r *http.Request) {
-	if _, err := h.pinger.Ping(r.Context()); err != nil {
-		if h.metrics != nil {
-			h.metrics.DockerPingUp.Set(0)
-		}
-		writeJSON(w, http.StatusServiceUnavailable, HealthResponse{Status: "unhealthy", Docker: "unreachable"})
-		return
-	}
-	if h.metrics != nil {
-		h.metrics.DockerPingUp.Set(1)
-	}
-	writeJSON(w, http.StatusOK, HealthResponse{Status: "ok", Docker: "ok"})
-}
-
-// Version implements GET /version.
-func (h *Handlers) Version(w http.ResponseWriter, _ *http.Request) {
-	writeJSON(w, http.StatusOK, VersionResponse{
-		Version:   h.version,
-		Commit:    h.commit,
-		BuildTime: h.buildTime,
-	})
-}
-
-func writeJSON(w http.ResponseWriter, code int, body any) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(code)
-	_ = json.NewEncoder(w).Encode(body)
-}
-
-func writeJSONError(w http.ResponseWriter, code int, msg string) {
-	writeJSON(w, code, map[string]string{"error": msg})
-}

internal/api/handlers_test.go

@@ -1,216 +0,0 @@
-package api_test
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/docker/docker/api/types"
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/updater"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeFinder struct {
-	jobs []discovery.Job
-	err  error
-
-	// gotImage records the image argument passed to FindJobs, for
-	// assertions on the exact reference the handler built.
-	gotImage string
-	// swarmShaped, when true, makes FindJobs return a Job whose Image
-	// field carries the received image argument, simulating
-	// discovery.SwarmDiscovery.FindJobs.
-	swarmShaped bool
-}
-
-func (f *fakeFinder) FindJobs(_ context.Context, image string) ([]discovery.Job, error) {
-	f.gotImage = image
-	if f.swarmShaped {
-		return []discovery.Job{{Service: "myapp_web", ServiceID: "svc1", Image: image}}, f.err
-	}
-	return f.jobs, f.err
-}
-
-type fakeSubmitter struct {
-	results []updater.Result
-}
-
-func (f *fakeSubmitter) Submit(_ context.Context, jobs []discovery.Job) []updater.Result {
-	if f.results != nil {
-		return f.results
-	}
-	out := make([]updater.Result, len(jobs))
-	for i, j := range jobs {
-		out[i] = updater.Result{Job: j, Status: updater.StatusUpdated}
-	}
-	return out
-}
-
-type fakePinger struct{ err error }
-
-func (f *fakePinger) Ping(_ context.Context) (types.Ping, error) {
-	return types.Ping{}, f.err
-}
-
-func decode[T any](t *testing.T, body io.Reader) T {
-	t.Helper()
-	var v T
-	require.NoError(t, json.NewDecoder(body).Decode(&v))
-	return v
-}
-
-func TestUpdate_ValidationError(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodPost, "/update",
-		strings.NewReader(`{"tag":"v1.2.3"}`))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusBadRequest, w.Code)
-}
-
-func TestUpdate_BadJSON(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`not json`))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusBadRequest, w.Code)
-}
-
-func TestUpdate_DiscoveryFailureReturns500(t *testing.T) {
-	finder := &fakeFinder{err: errors.New("daemon unreachable")}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusInternalServerError, w.Code)
-}
-
-func TestUpdate_ZeroMatchesReturns200(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	resp := decode[api.UpdateResponse](t, w.Body)
-	require.Equal(t, 0, resp.Matched)
-}
-
-func TestUpdate_AllSucceeded200(t *testing.T) {
-	finder := &fakeFinder{jobs: []discovery.Job{
-		{Project: "p", Service: "s", WorkingDir: "/x", ConfigFiles: []string{"/x/c.yml"}},
-	}}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x", Tag: "v1"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	resp := decode[api.UpdateResponse](t, w.Body)
-	require.Equal(t, 1, resp.Matched)
-	require.Equal(t, "updated", resp.Results[0].Status)
-	require.Equal(t, "/x/c.yml", resp.Results[0].ComposeFile)
-}
-
-func TestUpdate_MixedReturns207(t *testing.T) {
-	jobs := []discovery.Job{
-		{Project: "p1", Service: "s", ConfigFiles: []string{"/x/c.yml"}},
-		{Project: "p2", Service: "s", ConfigFiles: []string{"/y/c.yml"}},
-	}
-	finder := &fakeFinder{jobs: jobs}
-	submitter := &fakeSubmitter{results: []updater.Result{
-		{Job: jobs[0], Status: updater.StatusUpdated},
-		{Job: jobs[1], Status: updater.StatusFailed, Error: "boom"},
-	}}
-	h := api.NewHandlers(finder, submitter, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusMultiStatus, w.Code)
-}
-
-func TestUpdate_AllFailedReturns500(t *testing.T) {
-	jobs := []discovery.Job{{Project: "p", Service: "s", ConfigFiles: []string{"/x/c.yml"}}}
-	finder := &fakeFinder{jobs: jobs}
-	submitter := &fakeSubmitter{results: []updater.Result{
-		{Job: jobs[0], Status: updater.StatusFailed, Error: "boom"},
-	}}
-	h := api.NewHandlers(finder, submitter, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusInternalServerError, w.Code)
-}
-
-func TestHealthz_OKWhenDockerUp(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
-	w := httptest.NewRecorder()
-	h.Healthz(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-}
-
-func TestHealthz_503WhenDockerDown(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{err: errors.New("ping fail")}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
-	w := httptest.NewRecorder()
-	h.Healthz(w, req)
-	require.Equal(t, http.StatusServiceUnavailable, w.Code)
-}
-
-func TestUpdate_FindJobsCalledWithTaggedImage(t *testing.T) {
-	finder := &fakeFinder{}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "registry.example.com/myapp", Tag: "v2"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, "registry.example.com/myapp:v2", finder.gotImage)
-}
-
-func TestUpdate_FindJobsCalledWithBareImageWhenNoTag(t *testing.T) {
-	finder := &fakeFinder{}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "registry.example.com/myapp"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, "registry.example.com/myapp", finder.gotImage)
-}
-
-func TestUpdate_SwarmModeThreadsTagThroughDiscovery(t *testing.T) {
-	finder := &fakeFinder{swarmShaped: true}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "registry.example.com/myapp", Tag: "v2"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	// The fake Finder's returned Job.Image (as SwarmDiscovery would build
-	// it) reflects the image argument it received from the handler.
-	// Asserting on the captured argument proves the full request ->
-	// discovery flow carries the tag through to what would reach the
-	// SwarmExecutor.
-	require.Equal(t, "registry.example.com/myapp:v2", finder.gotImage)
-}
-
-func TestVersion(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v1.2.3", "abcdef", "2026-05-22T00:00:00Z", nil)
-	req := httptest.NewRequest(http.MethodGet, "/version", nil)
-	w := httptest.NewRecorder()
-	h.Version(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	resp := decode[api.VersionResponse](t, w.Body)
-	require.Equal(t, "v1.2.3", resp.Version)
-}

internal/api/middleware.go

@@ -1,112 +0,0 @@
-// Package api contains HTTP handlers, middleware, and request/response DTOs.
-package api
-
-import (
-	"crypto/rand"
-	"crypto/subtle"
-	"fmt"
-	"log/slog"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/shcizo/package-updater/internal/metrics"
-)
-
-// Auth returns middleware that requires a matching bearer token.
-// Compares with constant-time to defeat timing attacks.
-func Auth(token string) func(http.Handler) http.Handler {
-	tokenBytes := []byte(token)
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			h := r.Header.Get("Authorization")
-			const prefix = "Bearer "
-			if !strings.HasPrefix(h, prefix) {
-				writeAuthError(w)
-				return
-			}
-			provided := []byte(strings.TrimPrefix(h, prefix))
-			if len(provided) == 0 ||
-				subtle.ConstantTimeCompare(provided, tokenBytes) != 1 {
-				writeAuthError(w)
-				return
-			}
-			next.ServeHTTP(w, r)
-		})
-	}
-}
-
-func writeAuthError(w http.ResponseWriter) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(http.StatusUnauthorized)
-	_, _ = w.Write([]byte(`{"error":"unauthorized"}`))
-}
-
-// newUUID generates a random UUID v4 string using crypto/rand.
-func newUUID() string {
-	var b [16]byte
-	_, _ = rand.Read(b[:])
-	b[6] = (b[6] & 0x0f) | 0x40 // version 4
-	b[8] = (b[8] & 0x3f) | 0x80 // variant bits
-	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
-		b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
-}
-
-// RequestID middleware ensures every request has an X-Request-ID
-// header (generated if absent) and stores it in the request context.
-func RequestID(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		id := r.Header.Get("X-Request-ID")
-		if id == "" {
-			id = newUUID()
-		}
-		w.Header().Set("X-Request-ID", id)
-		ctx := logging.WithRequestID(r.Context(), id)
-		next.ServeHTTP(w, r.WithContext(ctx))
-	})
-}
-
-// RequestLogger emits a structured access-log line per request. m may be nil,
-// in which case metrics recording is silently skipped.
-func RequestLogger(base *slog.Logger, m *metrics.Metrics) func(http.Handler) http.Handler {
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			start := time.Now()
-			sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
-			next.ServeHTTP(sw, r)
-			logger := logging.FromContext(r.Context(), base)
-			logger.Info("http_request",
-				"method", r.Method,
-				"path", r.URL.Path,
-				"status", sw.status,
-				"duration_ms", time.Since(start).Milliseconds(),
-				"client_ip", clientIP(r),
-			)
-			if m != nil {
-				m.HTTPRequests.WithLabelValues(r.URL.Path, strconv.Itoa(sw.status)).Inc()
-			}
-		})
-	}
-}
-
-type statusWriter struct {
-	http.ResponseWriter
-	status int
-}
-
-func (s *statusWriter) WriteHeader(code int) {
-	s.status = code
-	s.ResponseWriter.WriteHeader(code)
-}
-
-func clientIP(r *http.Request) string {
-	if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
-		if comma := strings.Index(xff, ","); comma >= 0 {
-			return strings.TrimSpace(xff[:comma])
-		}
-		return xff
-	}
-	return r.RemoteAddr
-}

internal/api/middleware_test.go

@@ -1,92 +0,0 @@
-package api_test
-
-import (
-	"io"
-	"log/slog"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/stretchr/testify/require"
-)
-
-func newTestLogger() *slog.Logger {
-	return slog.New(slog.NewJSONHandler(io.Discard, nil))
-}
-
-func TestAuth_AllowsMatchingToken(t *testing.T) {
-	called := false
-	h := api.Auth("secret")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		called = true
-		w.WriteHeader(http.StatusOK)
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	req.Header.Set("Authorization", "Bearer secret")
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.True(t, called)
-	require.Equal(t, http.StatusOK, w.Code)
-}
-
-func TestAuth_Rejects(t *testing.T) {
-	cases := []struct{ name, header string }{
-		{"missing", ""},
-		{"wrong scheme", "Token secret"},
-		{"wrong value", "Bearer nope"},
-		{"empty bearer", "Bearer "},
-	}
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			h := api.Auth("secret")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-				t.Fatal("handler must not be called")
-			}))
-			req := httptest.NewRequest(http.MethodPost, "/update", nil)
-			if c.header != "" {
-				req.Header.Set("Authorization", c.header)
-			}
-			w := httptest.NewRecorder()
-			h.ServeHTTP(w, req)
-			require.Equal(t, http.StatusUnauthorized, w.Code)
-		})
-	}
-}
-
-func TestRequestID_GeneratesIfMissing(t *testing.T) {
-	var seenID string
-	h := api.RequestID(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
-		seenID = logging.RequestIDFrom(r.Context())
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.NotEmpty(t, seenID)
-	require.Equal(t, seenID, w.Header().Get("X-Request-ID"))
-}
-
-func TestRequestID_UsesIncoming(t *testing.T) {
-	var seenID string
-	h := api.RequestID(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
-		seenID = logging.RequestIDFrom(r.Context())
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	req.Header.Set("X-Request-ID", "given-id")
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.Equal(t, "given-id", seenID)
-}
-
-func TestRequestLogger_LogsAndDelegates(t *testing.T) {
-	logger := newTestLogger()
-	called := false
-	h := api.RequestLogger(logger, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		called = true
-		w.WriteHeader(http.StatusTeapot)
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.True(t, called)
-	require.Equal(t, http.StatusTeapot, w.Code)
-}

internal/api/swarm_result_test.go

@@ -1,48 +0,0 @@
-package api_test
-
-import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/docker/docker/api/types"
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/updater"
-	"github.com/stretchr/testify/require"
-)
-
-type swarmResultFinder struct{ jobs []discovery.Job }
-
-func (f *swarmResultFinder) FindJobs(_ context.Context, _ string) ([]discovery.Job, error) {
-	return f.jobs, nil
-}
-
-type swarmResultSubmitter struct{ results []updater.Result }
-
-func (s *swarmResultSubmitter) Submit(_ context.Context, _ []discovery.Job) []updater.Result {
-	return s.results
-}
-
-type swarmResultPinger struct{}
-
-func (swarmResultPinger) Ping(_ context.Context) (types.Ping, error) { return types.Ping{}, nil }
-
-func TestUpdate_SwarmJobPopulatesServiceID(t *testing.T) {
-	job := discovery.Job{Service: "myapp_web", ServiceID: "svc123", Image: "registry.example.com/myapp:v2"}
-	finder := &swarmResultFinder{jobs: []discovery.Job{job}}
-	submitter := &swarmResultSubmitter{results: []updater.Result{
-		{Job: job, Status: updater.StatusUpdated, DurationMs: 42},
-	}}
-	h := api.NewHandlers(finder, submitter, swarmResultPinger{}, "v", "c", "b", nil)
-
-	req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"image":"registry.example.com/myapp"}`))
-	rec := httptest.NewRecorder()
-	h.Update(rec, req)
-
-	require.Equal(t, http.StatusOK, rec.Code)
-	require.Contains(t, rec.Body.String(), `"service_id":"svc123"`)
-	require.Contains(t, rec.Body.String(), `"service":"myapp_web"`)
-}

internal/api/types.go

@@ -1,40 +0,0 @@
-package api
-
-// UpdateRequest is the body of POST /update.
-type UpdateRequest struct {
-	Image string `json:"image"`
-	Tag   string `json:"tag,omitempty"`
-}
-
-// UpdateResponse is the body of POST /update.
-type UpdateResponse struct {
-	RequestID string      `json:"request_id"`
-	Image     string      `json:"image"`
-	Tag       string      `json:"tag,omitempty"`
-	Matched   int         `json:"matched"`
-	Results   []ResultDTO `json:"results"`
-}
-
-// ResultDTO is one row in UpdateResponse.Results.
-type ResultDTO struct {
-	Project     string `json:"project"`
-	Service     string `json:"service"`
-	ComposeFile string `json:"compose_file"`
-	ServiceID   string `json:"service_id,omitempty"`
-	Status      string `json:"status"`
-	Error       string `json:"error,omitempty"`
-	DurationMs  int64  `json:"duration_ms"`
-}
-
-// HealthResponse is the body of GET /healthz.
-type HealthResponse struct {
-	Status string `json:"status"`
-	Docker string `json:"docker"`
-}
-
-// VersionResponse is the body of GET /version.
-type VersionResponse struct {
-	Version   string `json:"version"`
-	Commit    string `json:"commit"`
-	BuildTime string `json:"build_time"`
-}

internal/config/config.go

@@ -1,58 +0,0 @@
-// Package config loads service configuration from environment variables.
-package config
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"time"
-)
-
-// Config holds all runtime configuration for the service.
-type Config struct {
-	APIKey        string
-	StacksRoot    string
-	Port          string
-	LogLevel      string
-	UpdateTimeout time.Duration
-	OptInLabel    string
-	Mode          string
-}
-
-// Load reads configuration from environment variables, applies defaults,
-// and validates required fields. Returns an error if validation fails so
-// the service can fail-fast at startup.
-func Load() (*Config, error) {
-	cfg := &Config{
-		APIKey:     os.Getenv("UPDATER_API_KEY"),
-		StacksRoot: getenvDefault("STACKS_ROOT", "/home/shcizo/self-hosted"),
-		Port:       getenvDefault("PORT", "8080"),
-		LogLevel:   getenvDefault("LOG_LEVEL", "info"),
-		OptInLabel: getenvDefault("OPT_IN_LABEL", "se.shcizo.auto-update"),
-		Mode:       getenvDefault("MODE", "compose"),
-	}
-
-	if cfg.APIKey == "" {
-		return nil, errors.New("UPDATER_API_KEY is required")
-	}
-
-	if cfg.Mode != "compose" && cfg.Mode != "swarm" {
-		return nil, fmt.Errorf("MODE %q is invalid: must be %q or %q", cfg.Mode, "compose", "swarm")
-	}
-
-	timeoutStr := getenvDefault("UPDATE_TIMEOUT", "5m")
-	d, err := time.ParseDuration(timeoutStr)
-	if err != nil {
-		return nil, fmt.Errorf("UPDATE_TIMEOUT %q is not a valid duration: %w", timeoutStr, err)
-	}
-	cfg.UpdateTimeout = d
-
-	return cfg, nil
-}
-
-func getenvDefault(key, fallback string) string {
-	if v := os.Getenv(key); v != "" {
-		return v
-	}
-	return fallback
-}

internal/config/config_test.go

@@ -1,82 +0,0 @@
-package config_test
-
-import (
-	"testing"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/config"
-	"github.com/stretchr/testify/require"
-)
-
-func TestLoad_RequiresAPIKey(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "")
-	_, err := config.Load()
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "UPDATER_API_KEY")
-}
-
-func TestLoad_AppliesDefaults(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("STACKS_ROOT", "")
-	t.Setenv("PORT", "")
-	t.Setenv("LOG_LEVEL", "")
-	t.Setenv("UPDATE_TIMEOUT", "")
-	t.Setenv("OPT_IN_LABEL", "")
-
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "secret", cfg.APIKey)
-	require.Equal(t, "/home/shcizo/self-hosted", cfg.StacksRoot)
-	require.Equal(t, "8080", cfg.Port)
-	require.Equal(t, "info", cfg.LogLevel)
-	require.Equal(t, 5*time.Minute, cfg.UpdateTimeout)
-	require.Equal(t, "se.shcizo.auto-update", cfg.OptInLabel)
-}
-
-func TestLoad_OverridesViaEnv(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("STACKS_ROOT", "/srv/stacks")
-	t.Setenv("PORT", "9090")
-	t.Setenv("LOG_LEVEL", "debug")
-	t.Setenv("UPDATE_TIMEOUT", "30s")
-	t.Setenv("OPT_IN_LABEL", "io.example.update")
-
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "/srv/stacks", cfg.StacksRoot)
-	require.Equal(t, "9090", cfg.Port)
-	require.Equal(t, "debug", cfg.LogLevel)
-	require.Equal(t, 30*time.Second, cfg.UpdateTimeout)
-	require.Equal(t, "io.example.update", cfg.OptInLabel)
-}
-
-func TestLoad_InvalidTimeoutErrors(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("UPDATE_TIMEOUT", "not-a-duration")
-	_, err := config.Load()
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "UPDATE_TIMEOUT")
-}
-
-func TestLoad_DefaultsToComposeMode(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "compose", cfg.Mode)
-}
-
-func TestLoad_AcceptsSwarmMode(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("MODE", "swarm")
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "swarm", cfg.Mode)
-}
-
-func TestLoad_RejectsInvalidMode(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("MODE", "kubernetes")
-	_, err := config.Load()
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "MODE")
-}

internal/discovery/discovery.go

@@ -1,96 +0,0 @@
-package discovery
-
-import (
-	"context"
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/docker/docker/api/types/container"
-)
-
-// Job describes a single (project, service, config_files) update to execute.
-type Job struct {
-	Project     string
-	Service     string
-	WorkingDir  string
-	ConfigFiles []string
-	// Image is the full image reference the update request asked for.
-	// Compose jobs ignore it (compose.yml already pins the reference to
-	// pull); Swarm jobs need it to know what to set on the service spec.
-	Image string
-	// ServiceID is the Swarm service ID. Empty for Compose jobs.
-	ServiceID string
-	// Refused is true when the WorkingDir falls outside STACKS_ROOT.
-	// The job is returned so the caller can surface a per-job "refused"
-	// result, but it MUST NOT be executed.
-	Refused       bool
-	RefusedReason string
-}
-
-// Discovery orchestrates "given an image, what jobs should we enqueue?".
-type Discovery struct {
-	cli        DockerClient
-	stacksRoot string
-	optInLabel string
-}
-
-// New returns a Discovery bound to the given Docker client and config.
-func New(cli DockerClient, stacksRoot, optInLabel string) *Discovery {
-	return &Discovery{cli: cli, stacksRoot: stacksRoot, optInLabel: optInLabel}
-}
-
-// FindJobs lists running containers, filters by image match + opt-in label,
-// extracts Compose info, applies the path safety check, and deduplicates.
-func (d *Discovery) FindJobs(ctx context.Context, image string) ([]Job, error) {
-	all, err := d.cli.ContainerList(ctx, container.ListOptions{All: true})
-	if err != nil {
-		return nil, fmt.Errorf("docker container list: %w", err)
-	}
-
-	seen := make(map[string]struct{})
-	var jobs []Job
-
-	for _, c := range all {
-		if !ImagesMatch(image, c.Image) {
-			continue
-		}
-		if !HasOptIn(c.Labels, d.optInLabel) {
-			continue
-		}
-		cl, err := ParseComposeLabels(c.Labels)
-		if err != nil {
-			continue
-		}
-
-		refused := !IsInsideRoot(d.stacksRoot, cl.WorkingDir)
-		reason := ""
-		if refused {
-			reason = fmt.Sprintf("working_dir %q outside STACKS_ROOT %q", cl.WorkingDir, d.stacksRoot)
-		}
-
-		key := dedupKey(cl)
-		if _, dup := seen[key]; dup {
-			continue
-		}
-		seen[key] = struct{}{}
-
-		jobs = append(jobs, Job{
-			Project:       cl.Project,
-			Service:       cl.Service,
-			WorkingDir:    cl.WorkingDir,
-			ConfigFiles:   cl.ConfigFiles,
-			Image:         image,
-			Refused:       refused,
-			RefusedReason: reason,
-		})
-	}
-
-	return jobs, nil
-}
-
-func dedupKey(cl ComposeLabels) string {
-	files := append([]string(nil), cl.ConfigFiles...)
-	sort.Strings(files)
-	return cl.Project + "|" + cl.Service + "|" + strings.Join(files, ",")
-}

internal/discovery/discovery_test.go

@@ -1,172 +0,0 @@
-package discovery_test
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeDockerClient struct {
-	containers []types.Container
-	err        error
-}
-
-func (f *fakeDockerClient) ContainerList(_ context.Context, _ container.ListOptions) ([]types.Container, error) {
-	return f.containers, f.err
-}
-
-func (f *fakeDockerClient) Ping(_ context.Context) (types.Ping, error) {
-	return types.Ping{}, nil
-}
-
-func mkContainer(image string, labels map[string]string) types.Container {
-	return types.Container{Image: image, Labels: labels}
-}
-
-func mkComposeLabels(project, service, workingDir, configFile string, optIn bool) map[string]string {
-	m := map[string]string{
-		"com.docker.compose.project":              project,
-		"com.docker.compose.service":              service,
-		"com.docker.compose.project.working_dir":  workingDir,
-		"com.docker.compose.project.config_files": configFile,
-	}
-	if optIn {
-		m["se.shcizo.auto-update"] = "true"
-	}
-	return m
-}
-
-func TestFindJobs_MatchAndOptIn(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			true,
-		)),
-		mkContainer("registry.example.com/other:v1", mkComposeLabels(
-			"other", "web",
-			"/home/shcizo/self-hosted/other",
-			"/home/shcizo/self-hosted/other/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.Equal(t, "myapp-prod", jobs[0].Project)
-	require.Equal(t, "web", jobs[0].Service)
-	require.Equal(t, "/home/shcizo/self-hosted/myapp-prod", jobs[0].WorkingDir)
-}
-
-func TestFindJobs_SkipsWithoutOptIn(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			false,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}
-
-func TestFindJobs_DedupReplicas(t *testing.T) {
-	labels := mkComposeLabels(
-		"myapp", "web",
-		"/home/shcizo/self-hosted/myapp",
-		"/home/shcizo/self-hosted/myapp/docker-compose.yml",
-		true,
-	)
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", labels),
-		mkContainer("registry.example.com/myapp:v1", labels),
-		mkContainer("registry.example.com/myapp:v1", labels),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-}
-
-func TestFindJobs_OutsideRootProducesRefusedJob(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp", "web",
-			"/opt/elsewhere/myapp",
-			"/opt/elsewhere/myapp/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.True(t, jobs[0].Refused)
-}
-
-func TestFindJobs_MultipleStacksSameImage(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			true,
-		)),
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-staging", "web",
-			"/home/shcizo/self-hosted/myapp-staging",
-			"/home/shcizo/self-hosted/myapp-staging/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 2)
-}
-
-func TestFindJobs_DockerError(t *testing.T) {
-	fake := &fakeDockerClient{err: errors.New("connection refused")}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	_, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.Error(t, err)
-}
-
-func TestFindJobs_NonComposeContainerIsSkipped(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", map[string]string{
-			"se.shcizo.auto-update": "true",
-		}),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}
-
-func TestFindJobs_PopulatesImageField(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.Equal(t, "registry.example.com/myapp", jobs[0].Image)
-	require.Empty(t, jobs[0].ServiceID)
-}

internal/discovery/docker_client.go

@@ -1,15 +0,0 @@
-package discovery
-
-import (
-	"context"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-)
-
-// DockerClient is the subset of the Docker SDK we depend on.
-// Defined as an interface so tests can supply a fake.
-type DockerClient interface {
-	ContainerList(ctx context.Context, opts container.ListOptions) ([]types.Container, error)
-	Ping(ctx context.Context) (types.Ping, error)
-}

internal/discovery/labels.go

@@ -1,71 +0,0 @@
-package discovery
-
-import (
-	"fmt"
-	"strings"
-)
-
-// ComposeLabels captures the four Compose-managed labels we need
-// to drive a `docker compose pull`/`up -d` against the right stack.
-type ComposeLabels struct {
-	Project     string
-	Service     string
-	WorkingDir  string
-	ConfigFiles []string
-}
-
-const (
-	labelProject     = "com.docker.compose.project"
-	labelService     = "com.docker.compose.service"
-	labelWorkingDir  = "com.docker.compose.project.working_dir"
-	labelConfigFiles = "com.docker.compose.project.config_files"
-)
-
-// ParseComposeLabels extracts the Compose labels we need. Returns an
-// error naming the missing label if any required field is absent —
-// in practice this should only happen if the container was not
-// started by Compose.
-func ParseComposeLabels(labels map[string]string) (ComposeLabels, error) {
-	get := func(key string) (string, error) {
-		v, ok := labels[key]
-		if !ok || v == "" {
-			return "", fmt.Errorf("missing required label: %s", key)
-		}
-		return v, nil
-	}
-
-	project, err := get(labelProject)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-	service, err := get(labelService)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-	workingDir, err := get(labelWorkingDir)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-	configFilesRaw, err := get(labelConfigFiles)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-
-	files := strings.Split(configFilesRaw, ",")
-	for i, f := range files {
-		files[i] = strings.TrimSpace(f)
-	}
-
-	return ComposeLabels{
-		Project:     project,
-		Service:     service,
-		WorkingDir:  workingDir,
-		ConfigFiles: files,
-	}, nil
-}
-
-// HasOptIn reports whether the labels include the opt-in marker with
-// value "true" (exact, case-sensitive — anything else is excluded).
-func HasOptIn(labels map[string]string, key string) bool {
-	return labels[key] == "true"
-}

internal/discovery/labels_test.go

@@ -1,88 +0,0 @@
-package discovery_test
-
-import (
-	"testing"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-func TestParseComposeLabels_Success(t *testing.T) {
-	labels := map[string]string{
-		"com.docker.compose.project":                  "myapp-prod",
-		"com.docker.compose.service":                  "web",
-		"com.docker.compose.project.working_dir":      "/home/shcizo/self-hosted/myapp-prod",
-		"com.docker.compose.project.config_files":     "/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-		"se.shcizo.auto-update":                       "true",
-	}
-	got, err := discovery.ParseComposeLabels(labels)
-	require.NoError(t, err)
-	require.Equal(t, "myapp-prod", got.Project)
-	require.Equal(t, "web", got.Service)
-	require.Equal(t, "/home/shcizo/self-hosted/myapp-prod", got.WorkingDir)
-	require.Equal(t, []string{"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml"}, got.ConfigFiles)
-}
-
-func TestParseComposeLabels_MultipleConfigFiles(t *testing.T) {
-	labels := map[string]string{
-		"com.docker.compose.project":                  "myapp",
-		"com.docker.compose.service":                  "web",
-		"com.docker.compose.project.working_dir":      "/srv/myapp",
-		"com.docker.compose.project.config_files":     "/srv/myapp/docker-compose.yml,/srv/myapp/docker-compose.prod.yml",
-	}
-	got, err := discovery.ParseComposeLabels(labels)
-	require.NoError(t, err)
-	require.Equal(t, []string{
-		"/srv/myapp/docker-compose.yml",
-		"/srv/myapp/docker-compose.prod.yml",
-	}, got.ConfigFiles)
-}
-
-func TestParseComposeLabels_MissingFieldsError(t *testing.T) {
-	cases := []struct {
-		name    string
-		missing string
-		labels  map[string]string
-	}{
-		{"project", "com.docker.compose.project", map[string]string{
-			"com.docker.compose.service":              "web",
-			"com.docker.compose.project.working_dir":  "/x",
-			"com.docker.compose.project.config_files": "/x/y.yml",
-		}},
-		{"service", "com.docker.compose.service", map[string]string{
-			"com.docker.compose.project":              "p",
-			"com.docker.compose.project.working_dir":  "/x",
-			"com.docker.compose.project.config_files": "/x/y.yml",
-		}},
-		{"working_dir", "com.docker.compose.project.working_dir", map[string]string{
-			"com.docker.compose.project":              "p",
-			"com.docker.compose.service":              "web",
-			"com.docker.compose.project.config_files": "/x/y.yml",
-		}},
-		{"config_files", "com.docker.compose.project.config_files", map[string]string{
-			"com.docker.compose.project":             "p",
-			"com.docker.compose.service":             "web",
-			"com.docker.compose.project.working_dir": "/x",
-		}},
-	}
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			_, err := discovery.ParseComposeLabels(c.labels)
-			require.Error(t, err)
-			require.Contains(t, err.Error(), c.missing)
-		})
-	}
-}
-
-func TestHasOptIn(t *testing.T) {
-	require.True(t, discovery.HasOptIn(map[string]string{
-		"se.shcizo.auto-update": "true",
-	}, "se.shcizo.auto-update"))
-	require.False(t, discovery.HasOptIn(map[string]string{
-		"se.shcizo.auto-update": "false",
-	}, "se.shcizo.auto-update"))
-	require.False(t, discovery.HasOptIn(map[string]string{
-		"se.shcizo.auto-update": "TRUE",
-	}, "se.shcizo.auto-update"))
-	require.False(t, discovery.HasOptIn(map[string]string{}, "se.shcizo.auto-update"))
-}

internal/discovery/matching.go

@@ -1,33 +0,0 @@
-package discovery
-
-import "strings"
-
-// NormaliseImage strips the tag and digest from an image reference,
-// returning the bare repository name.
-//
-// Tricky case: "localhost:5000/foo:v1" — the first colon is a port,
-// not a tag. We disambiguate by splitting on "/" first and only
-// treating colons in the last segment as tag separators.
-func NormaliseImage(ref string) string {
-	if at := strings.Index(ref, "@"); at >= 0 {
-		ref = ref[:at]
-	}
-	slash := strings.LastIndex(ref, "/")
-	if slash < 0 {
-		if colon := strings.Index(ref, ":"); colon >= 0 {
-			return ref[:colon]
-		}
-		return ref
-	}
-	prefix, last := ref[:slash], ref[slash+1:]
-	if colon := strings.Index(last, ":"); colon >= 0 {
-		last = last[:colon]
-	}
-	return prefix + "/" + last
-}
-
-// ImagesMatch reports whether two image references resolve to the same
-// repository, ignoring tag and digest. Case-sensitive per spec section 5.2.
-func ImagesMatch(a, b string) bool {
-	return NormaliseImage(a) == NormaliseImage(b)
-}

internal/discovery/matching_test.go

@@ -1,51 +0,0 @@
-package discovery_test
-
-import (
-	"testing"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNormaliseImage(t *testing.T) {
-	cases := []struct {
-		in   string
-		want string
-	}{
-		{"registry.example.com/myapp", "registry.example.com/myapp"},
-		{"registry.example.com/myapp:v1.2.3", "registry.example.com/myapp"},
-		{"registry.example.com/myapp:latest", "registry.example.com/myapp"},
-		{"registry.example.com/myapp@sha256:abc123", "registry.example.com/myapp"},
-		{"registry.example.com/myapp:v1.2.3@sha256:abc123", "registry.example.com/myapp"},
-		{"nginx", "nginx"},
-		{"nginx:1.25-alpine", "nginx"},
-		{"library/nginx:latest", "library/nginx"},
-		{"gcr.io/proj/svc:tag", "gcr.io/proj/svc"},
-		{"localhost:5000/myimg:v1", "localhost:5000/myimg"},
-	}
-	for _, c := range cases {
-		t.Run(c.in, func(t *testing.T) {
-			require.Equal(t, c.want, discovery.NormaliseImage(c.in))
-		})
-	}
-}
-
-func TestImagesMatch(t *testing.T) {
-	require.True(t, discovery.ImagesMatch(
-		"registry.example.com/myapp",
-		"registry.example.com/myapp:v1.2.3",
-	))
-	require.True(t, discovery.ImagesMatch(
-		"registry.example.com/myapp:v1.0.0",
-		"registry.example.com/myapp:v9.9.9",
-	))
-	require.False(t, discovery.ImagesMatch(
-		"registry.example.com/myapp",
-		"registry.example.com/otherapp",
-	))
-	// Case-sensitive per spec section 5.2
-	require.False(t, discovery.ImagesMatch(
-		"registry.example.com/MyApp",
-		"registry.example.com/myapp",
-	))
-}

internal/discovery/swarm.go

@@ -1,60 +0,0 @@
-package discovery
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/docker/docker/api/types/swarm"
-)
-
-// SwarmDockerClient is the subset of the Docker SDK SwarmDiscovery depends
-// on. Defined as an interface so tests can supply a fake.
-type SwarmDockerClient interface {
-	ServiceList(ctx context.Context, opts swarm.ServiceListOptions) ([]swarm.Service, error)
-}
-
-// SwarmDiscovery orchestrates "given an image, which Swarm services should
-// we update?". Unlike Discovery, it has no filesystem path to check — the
-// opt-in label on the service itself is the only gate, since Swarm services
-// have no local compose file to anchor a STACKS_ROOT check against.
-type SwarmDiscovery struct {
-	cli        SwarmDockerClient
-	optInLabel string
-}
-
-// NewSwarm returns a SwarmDiscovery bound to the given Docker client and
-// opt-in label.
-func NewSwarm(cli SwarmDockerClient, optInLabel string) *SwarmDiscovery {
-	return &SwarmDiscovery{cli: cli, optInLabel: optInLabel}
-}
-
-// FindJobs lists Swarm services, filters by image match + opt-in label, and
-// returns one Job per matching service. Signature matches Discovery.FindJobs
-// so both satisfy api.Finder.
-func (d *SwarmDiscovery) FindJobs(ctx context.Context, image string) ([]Job, error) {
-	services, err := d.cli.ServiceList(ctx, swarm.ServiceListOptions{})
-	if err != nil {
-		return nil, fmt.Errorf("docker service list: %w", err)
-	}
-
-	var jobs []Job
-	for _, svc := range services {
-		if svc.Spec.TaskTemplate.ContainerSpec == nil {
-			continue
-		}
-		if !ImagesMatch(image, svc.Spec.TaskTemplate.ContainerSpec.Image) {
-			continue
-		}
-		if !HasOptIn(svc.Spec.Labels, d.optInLabel) {
-			continue
-		}
-
-		jobs = append(jobs, Job{
-			Service:   svc.Spec.Name,
-			ServiceID: svc.ID,
-			Image:     image,
-		})
-	}
-
-	return jobs, nil
-}

internal/discovery/swarm_test.go

@@ -1,86 +0,0 @@
-package discovery_test
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/docker/docker/api/types/swarm"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeSwarmDockerClient struct {
-	services []swarm.Service
-	err      error
-}
-
-func (f *fakeSwarmDockerClient) ServiceList(_ context.Context, _ swarm.ServiceListOptions) ([]swarm.Service, error) {
-	return f.services, f.err
-}
-
-func mkService(id, name, image string, labels map[string]string) swarm.Service {
-	return swarm.Service{
-		ID: id,
-		Spec: swarm.ServiceSpec{
-			Annotations: swarm.Annotations{Name: name, Labels: labels},
-			TaskTemplate: swarm.TaskSpec{
-				ContainerSpec: &swarm.ContainerSpec{Image: image},
-			},
-		},
-	}
-}
-
-func TestSwarmFindJobs_MatchAndOptIn(t *testing.T) {
-	fake := &fakeSwarmDockerClient{services: []swarm.Service{
-		mkService("svc-myapp", "myapp_web", "registry.example.com/myapp:v1", map[string]string{
-			"se.shcizo.auto-update": "true",
-		}),
-		mkService("svc-other", "other_web", "registry.example.com/other:v1", map[string]string{
-			"se.shcizo.auto-update": "true",
-		}),
-	}}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.Equal(t, "myapp_web", jobs[0].Service)
-	require.Equal(t, "svc-myapp", jobs[0].ServiceID)
-	require.Equal(t, "registry.example.com/myapp", jobs[0].Image)
-	require.False(t, jobs[0].Refused)
-	require.Empty(t, jobs[0].WorkingDir)
-	require.Empty(t, jobs[0].ConfigFiles)
-}
-
-func TestSwarmFindJobs_SkipsWithoutOptIn(t *testing.T) {
-	fake := &fakeSwarmDockerClient{services: []swarm.Service{
-		mkService("svc-myapp", "myapp_web", "registry.example.com/myapp:v1", nil),
-	}}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}
-
-func TestSwarmFindJobs_DockerError(t *testing.T) {
-	fake := &fakeSwarmDockerClient{err: errors.New("connection refused")}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	_, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.Error(t, err)
-}
-
-func TestSwarmFindJobs_NoContainerSpecIsSkipped(t *testing.T) {
-	fake := &fakeSwarmDockerClient{services: []swarm.Service{
-		{
-			ID: "svc-weird",
-			Spec: swarm.ServiceSpec{
-				Annotations:  swarm.Annotations{Name: "weird", Labels: map[string]string{"se.shcizo.auto-update": "true"}},
-				TaskTemplate: swarm.TaskSpec{ContainerSpec: nil},
-			},
-		},
-	}}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}

internal/e2e/agent_master_test.go

@@ -11,6 +11,7 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -21,7 +22,9 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/shcizo/package-updater/internal/agent/conn"
+	"github.com/shcizo/package-updater/internal/agent/updater"
 	"github.com/shcizo/package-updater/internal/agenttoken"
+	"github.com/shcizo/package-updater/internal/imageref"
 	"github.com/shcizo/package-updater/internal/master/events"
 	"github.com/shcizo/package-updater/internal/master/hub"
 	"github.com/shcizo/package-updater/internal/master/model"
@@ -35,6 +38,224 @@ type memStore struct {
 	lookups   int
 	snapshots int
 	services  map[string]protocol.Service
+
+	// Jobs: in-memory twins of the store methods in C5, with the same
+	// ownership and "only active jobs change" rules.
+	svcRows       map[int64]model.Service
+	jobs          map[int64]*model.Job
+	nextJob       int64
+	finishApplied int // FinishJob calls that returned true
+	versions      map[int64]model.ImageVersion
+}
+
+// seedVersion adds an image_versions row that GetImageVersion returns.
+func (m *memStore) seedVersion(v model.ImageVersion) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.versions == nil {
+		m.versions = map[int64]model.ImageVersion{}
+	}
+	m.versions[v.ID] = v
+}
+
+func (m *memStore) GetImageVersion(_ context.Context, id int64) (model.ImageVersion, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	v, ok := m.versions[id]
+	if !ok {
+		return model.ImageVersion{}, model.ErrNotFound
+	}
+	return v, nil
+}
+
+func (m *memStore) ListServices(_ context.Context) ([]model.Service, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	var out []model.Service
+	for _, svc := range m.svcRows {
+		if svc.RemovedAt == nil {
+			out = append(out, svc)
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
+	return out, nil
+}
+
+func (m *memStore) SetPollError(_ context.Context, ids []int64, msg string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for _, id := range ids {
+		svc := m.svcRows[id]
+		svc.LastPollError = msg
+		m.svcRows[id] = svc
+	}
+	return nil
+}
+
+func (m *memStore) SetLastTagDigest(_ context.Context, id int64, digest string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	svc, ok := m.svcRows[id]
+	if !ok {
+		return model.ErrNotFound
+	}
+	svc.LastTagDigest = digest
+	m.svcRows[id] = svc
+	return nil
+}
+
+// seedService adds a service row that jobs can be triggered for.
+func (m *memStore) seedService(svc model.Service) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.svcRows == nil {
+		m.svcRows = map[int64]model.Service{}
+	}
+	m.svcRows[svc.ID] = svc
+}
+
+func (m *memStore) GetService(_ context.Context, id int64) (model.Service, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	svc, ok := m.svcRows[id]
+	if !ok {
+		return model.Service{}, model.ErrNotFound
+	}
+	return svc, nil
+}
+
+// ServicesByRepo returns the seeded services whose image repo equals repo,
+// ordered by ID.
+func (m *memStore) ServicesByRepo(_ context.Context, repo string) ([]model.Service, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	var out []model.Service
+	for _, svc := range m.svcRows {
+		if svc.ImageRepo == repo && svc.RemovedAt == nil {
+			out = append(out, svc)
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
+	return out, nil
+}
+
+func (m *memStore) SetAvailable(_ context.Context, id int64, a model.Available) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	svc, ok := m.svcRows[id]
+	if !ok {
+		return model.ErrNotFound
+	}
+	svc.AvailableRef, svc.AvailableDigest, svc.AvailableSource, svc.AvailableAt = a.Ref, a.Digest, a.Source, &a.At
+	m.svcRows[id] = svc
+	return nil
+}
+
+func (m *memStore) CreateJob(_ context.Context, nj model.NewJob) (model.Job, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.jobs == nil {
+		m.jobs = map[int64]*model.Job{}
+	}
+	for _, j := range m.jobs {
+		if j.ServiceID == nj.ServiceID && j.Active() {
+			return *j, model.ErrJobActive
+		}
+	}
+	m.nextJob++
+	j := &model.Job{ID: m.nextJob, ServiceID: nj.ServiceID, AgentID: nj.AgentID, FromRef: nj.FromRef, ToRef: nj.ToRef,
+		Trigger: nj.Trigger, RequestedBy: nj.RequestedBy, Status: model.JobQueued, CreatedAt: nj.CreatedAt}
+	m.jobs[j.ID] = j
+	return *j, nil
+}
+
+func (m *memStore) FailJob(_ context.Context, id int64, msg string, at time.Time) (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	j, ok := m.jobs[id]
+	if !ok || !j.Active() {
+		return false, nil
+	}
+	j.Status, j.Error, j.FinishedAt = model.JobFailed, msg, &at
+	return true, nil
+}
+
+func (m *memStore) GetJob(_ context.Context, id int64) (model.Job, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	j, ok := m.jobs[id]
+	if !ok {
+		return model.Job{}, model.ErrNotFound
+	}
+	return *j, nil
+}
+
+func (m *memStore) FailLostJobs(_ context.Context, agentID int64, known []int64, at time.Time) ([]int64, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	var lost []int64
+	for _, j := range m.jobs {
+		if j.AgentID == nil || *j.AgentID != agentID || !j.Active() {
+			continue
+		}
+		isKnown := false
+		for _, k := range known {
+			isKnown = isKnown || k == j.ID
+		}
+		if !isKnown {
+			j.Status, j.Error, j.FinishedAt = model.JobFailed, "lost: the agent did not know this job when it reconnected", &at
+			lost = append(lost, j.ID)
+		}
+	}
+	return lost, nil
+}
+
+func (m *memStore) JobProgress(_ context.Context, agentID, jobID int64, chunk string, at time.Time) (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	j, ok := m.jobs[jobID]
+	if !ok || j.AgentID == nil || *j.AgentID != agentID || !j.Active() {
+		return false, nil
+	}
+	j.Status, j.Output = model.JobRunning, j.Output+chunk
+	if j.StartedAt == nil {
+		j.StartedAt = &at
+	}
+	return true, nil
+}
+
+func (m *memStore) FinishJob(_ context.Context, agentID, jobID int64, f model.JobFinish) (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	j, ok := m.jobs[jobID]
+	if !ok || j.AgentID == nil || *j.AgentID != agentID || !j.Active() {
+		return false, nil
+	}
+	j.Status, j.Error, j.Output, j.OutputTruncated, j.FinishedAt = f.Status, f.Error, f.Output, f.OutputTruncated, &f.FinishedAt
+	requestedRef := j.ToRef
+	if f.Ref != "" {
+		j.ToRef = f.Ref
+	}
+	if j.StartedAt == nil {
+		j.StartedAt = &f.StartedAt
+	}
+	if f.Status == model.JobSucceeded {
+		svc := m.svcRows[j.ServiceID]
+		svc.DesiredRef = f.Ref
+		svc.AwaitingInventory = true
+		if _, d, _ := strings.Cut(f.Ref, "@"); !strings.Contains(requestedRef, "@") && imageref.IsDigest(d) {
+			svc.LastTagDigest = d
+		}
+		// Same rule as the SQL: the job was created for the suggested ref after it was reported.
+		requested, digest, _ := strings.Cut(requestedRef, "@")
+		if svc.AvailableAt != nil && svc.AvailableRef == requested && !svc.AvailableAt.After(j.CreatedAt) &&
+			(digest == "" || svc.AvailableDigest == digest) {
+			svc.AvailableRef, svc.AvailableDigest, svc.AvailableSource, svc.AvailableAt = "", "", "", nil
+		}
+		m.svcRows[j.ServiceID] = svc
+	}
+	m.finishApplied++
+	return true, nil
 }
 
 func (m *memStore) AgentByTokenHash(_ context.Context, hash string) (model.Agent, error) {
@@ -112,8 +333,23 @@ func (s *staticSource) set(v ...protocol.Service) {
 type testMaster struct {
 	url         string
 	fingerprint string
-	hub         *hub.Hub
+	hub         *currentHub   // the hub behind /agent; follows restart()
 	attempts    *atomic.Int64 // TCP connections accepted by the server
+	broker      *events.Broker
+	restart     func() // replaces the hub and drops every connection
+}
+
+// currentHub forwards to whichever hub is live, so a master restart can swap
+// the hub without the mux or the jobs service noticing.
+type currentHub struct{ p atomic.Pointer[hub.Hub] }
+
+func (c *currentHub) ServeHTTP(w http.ResponseWriter, r *http.Request) { c.p.Load().ServeHTTP(w, r) }
+func (c *currentHub) Disconnect(id int64)                              { c.p.Load().Disconnect(id) }
+func (c *currentHub) Route(svc model.Service, minProtocol int) (model.Route, error) {
+	return c.p.Load().Route(svc, minProtocol)
+}
+func (c *currentHub) StartJob(r model.Route, id int64, s protocol.JobStart) error {
+	return c.p.Load().StartJob(r, id, s)
 }
 
 func startMaster(t *testing.T, store *memStore) testMaster {
@@ -125,7 +361,10 @@ func startMaster(t *testing.T, store *memStore) testMaster {
 	fingerprint, err := pin.Fingerprint(key.Public())
 	require.NoError(t, err)
 
-	h := hub.New(store, events.NewBroker(), slog.New(slog.NewTextHandler(io.Discard, nil)))
+	quiet := slog.New(slog.NewTextHandler(io.Discard, nil))
+	broker := events.NewBroker()
+	h := &currentHub{}
+	h.p.Store(hub.New(store, broker, quiet))
 	mux := http.NewServeMux()
 	mux.Handle("GET /agent", h)
 	attempts := &atomic.Int64{}
@@ -139,14 +378,27 @@ func startMaster(t *testing.T, store *memStore) testMaster {
 	srv.TLS = &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13}
 	srv.StartTLS()
 	t.Cleanup(srv.Close)
-	return testMaster{url: "wss" + strings.TrimPrefix(srv.URL, "https") + "/agent", fingerprint: fingerprint, hub: h, attempts: attempts}
+	restart := func() {
+		// The hub holds no job state, so a new hub over the same store is a
+		// faithful master restart.
+		old := h.p.Swap(hub.New(store, broker, quiet))
+		// CloseClientConnections skips hijacked (WebSocket) connections, so
+		// cut each agent through the old hub.
+		store.mu.Lock()
+		defer store.mu.Unlock()
+		for _, a := range store.agents {
+			old.Disconnect(a.ID)
+		}
+	}
+	return testMaster{url: "wss" + strings.TrimPrefix(srv.URL, "https") + "/agent", fingerprint: fingerprint,
+		hub: h, attempts: attempts, broker: broker, restart: restart}
 }
 
 func runAgent(t *testing.T, url, fingerprint, token string, src conn.Source) *conn.Agent {
 	t.Helper()
 	a := conn.New(conn.Config{URL: url, Token: token, Fingerprint: fingerprint, Version: "e2e",
 		MinBackoff: 10 * time.Millisecond, MaxBackoff: 50 * time.Millisecond,
-	}, src, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	}, src, updater.NewRunner(nil, time.Minute), slog.New(slog.NewTextHandler(io.Discard, nil)))
 	ctx, cancel := context.WithCancel(context.Background())
 	done := make(chan struct{})
 	go func() { _ = a.Run(ctx); close(done) }()

internal/e2e/drift_test.go

@@ -0,0 +1,113 @@
+package e2e
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/master/poller"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+const (
+	driftDesired = "sha256:" + "d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1"
+	driftOther   = "sha256:" + "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"
+)
+
+// seedDrifted replaces the environment's service with an auto service that was
+// updated to slusk:1.4.2 at driftDesired but runs driftOther now.
+func (e *jobEnv) seedDrifted() {
+	e.store.seedService(model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
+		AgentID: ptr(agentID), ImageRepo: slusk, UpdateMode: model.ModeAuto,
+		DesiredRef: slusk + ":1.4.2@" + driftDesired,
+		ActualRef:  slusk + ":1.4.2", ActualDigest: driftOther})
+}
+
+// inventoryWrite is what the store does for an agent's upsert of the service.
+func (e *jobEnv) inventoryWrite(id int64, actualRef, actualDigest string) {
+	e.store.mu.Lock()
+	defer e.store.mu.Unlock()
+	svc := e.store.svcRows[id]
+	svc.ActualRef, svc.ActualDigest, svc.AwaitingInventory = actualRef, actualDigest, false
+	e.store.svcRows[id] = svc
+}
+
+func TestDriftedAutoServiceIsNotUpdatedByPollOrWebhook(t *testing.T) {
+	e := newJobEnv(t)
+	e.seedDrifted()
+	reg := &fixedRegistry{digests: map[string]string{slusk + ":1.4.2": "sha256:new"}}
+	p := poller.New(e.store, e.jobs, reg, e.m.broker, poller.Config{Workers: 1}, slog.New(slog.NewTextHandler(io.Discard, nil)))
+
+	p.Cycle(context.Background())
+
+	assert.Equal(t, 1, reg.calls)
+	assert.Zero(t, e.store.jobCount(), "polling starts no job for a drifted service")
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, slusk+":1.4.2", svc.AvailableRef, "the suggestion is still recorded")
+
+	code, body := callWebhook(t, e.webhookServer(t), `{"image":"`+slusk+`:ignored","tag":"2.0.0"}`)
+
+	assert.Equal(t, 200, code)
+	require.Len(t, body.Results, 1)
+	assert.Equal(t, "refused", body.Results[0].Status)
+	assert.Contains(t, body.Results[0].Error, "drifted")
+	assert.Zero(t, e.store.jobCount(), "the webhook starts no job either")
+}
+
+func TestRestoreDeploysThePinnedRefOnTheAgent(t *testing.T) {
+	e := newJobEnv(t)
+	e.seedDrifted()
+	close(e.exec.release)
+
+	job, err := e.jobs.RestoreDesired(context.Background(), 1, "alice")
+
+	require.NoError(t, err)
+	assert.Equal(t, model.TriggerManual, job.Trigger)
+	select {
+	case ref := <-e.exec.started:
+		assert.Equal(t, slusk+":1.4.2@"+driftDesired, ref, "the agent receives exactly the desired ref")
+	case <-time.After(waitFor):
+		t.Fatal("the agent never received the job")
+	}
+	done := e.waitStatus(t, job.ID, model.JobSucceeded)
+	assert.Equal(t, slusk+":1.4.2@"+driftDesired, done.ToRef)
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, driftOther, svc.LastTagDigest, "the drifted digest is remembered so polling does not redeploy it")
+	assert.False(t, svc.Drifted(), "the inventory has not caught up yet")
+
+	e.inventoryWrite(1, slusk+":1.4.2", driftDesired)
+	svc, err = e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.False(t, svc.Drifted())
+}
+
+func TestNoDriftAfterASuccessfulJob(t *testing.T) {
+	e := newJobEnv(t)
+	close(e.exec.release)
+
+	job, err := e.jobs.Trigger(context.Background(), 1, "alice")
+
+	require.NoError(t, err)
+	e.waitStatus(t, job.ID, model.JobSucceeded)
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, slusk+":1.4.2@"+e2eDigest, svc.DesiredRef)
+	assert.Equal(t, slusk+":1.4.2@sha256:old", svc.ActualRef, "the agent's inventory write has not arrived")
+	assert.False(t, svc.Drifted(), "between job.result and the inventory write the row is not drifted")
+
+	e.inventoryWrite(1, slusk+":1.4.2", e2eDigest)
+	svc, _ = e.store.GetService(context.Background(), 1)
+	assert.False(t, svc.Drifted())
+
+	e.inventoryWrite(1, slusk+":1.4.2", driftOther)
+	svc, _ = e.store.GetService(context.Background(), 1)
+	assert.True(t, svc.Drifted(), "a later inventory write of another digest is drift")
+}

internal/e2e/jobs_test.go

@@ -0,0 +1,263 @@
+package e2e
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coder/websocket"
+	"github.com/coder/websocket/wsjson"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/agent/conn"
+	"github.com/shcizo/package-updater/internal/agent/updater"
+	"github.com/shcizo/package-updater/internal/agenttoken"
+	"github.com/shcizo/package-updater/internal/master/jobs"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+const (
+	slusk       = "gitea.shcizo.se/shcizo/slusk"
+	agentID     = int64(2)
+	waitFor     = 3 * time.Second
+	pollEvery   = 10 * time.Millisecond
+	lostMessage = "lost: the agent did not know this job when it reconnected"
+)
+
+// e2eDigest is the digest the fake executor pins a tag to.
+const e2eDigest = "sha256:" + "e2e0e2e0e2e0e2e0e2e0e2e0e2e0e2e0e2e0e2e0e2e0e2e0e2e0e2e0e2e0e2e0"
+
+// gateExecutor writes a line of output, then blocks until release is closed
+// or the job is cancelled.
+type gateExecutor struct {
+	started chan string // receives the job's Ref when it begins
+	release chan struct{}
+}
+
+func newGateExecutor() *gateExecutor {
+	return &gateExecutor{started: make(chan string, 8), release: make(chan struct{})}
+}
+
+func (g *gateExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+	_, _ = io.WriteString(out, "pulling "+job.Ref+"\n")
+	g.started <- job.Ref
+	select {
+	case <-g.release:
+	case <-ctx.Done():
+		return "", ctx.Err()
+	}
+	_, _ = io.WriteString(out, "recreated\n")
+	if strings.Contains(job.Ref, "@") {
+		return job.Ref, nil
+	}
+	return job.Ref + "@" + e2eDigest, nil
+}
+
+// jobAgent is a connected agent with a real runner and its two lifetimes: the
+// connection and the runner can be stopped separately.
+type jobAgent struct {
+	runner   *updater.Runner
+	stopConn func()
+	stopRun  func()
+}
+
+func startJobAgent(t *testing.T, m testMaster, token string, exec updater.Executor) *jobAgent {
+	t.Helper()
+	src := &staticSource{}
+	src.set(web)
+	runner := updater.NewRunner(exec, time.Minute)
+	runCtx, stopRun := context.WithCancel(context.Background())
+	runDone := make(chan struct{})
+	go func() { runner.Run(runCtx); close(runDone) }()
+
+	a := conn.New(conn.Config{URL: m.url, Token: token, Fingerprint: m.fingerprint, Version: "e2e",
+		MinBackoff: 10 * time.Millisecond, MaxBackoff: 50 * time.Millisecond, ProgressInterval: 10 * time.Millisecond,
+	}, src, runner, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	connCtx, cancelConn := context.WithCancel(context.Background())
+	connDone := make(chan struct{})
+	go func() { _ = a.Run(connCtx); close(connDone) }()
+
+	ja := &jobAgent{
+		runner:   runner,
+		stopConn: func() { cancelConn(); <-connDone },
+		stopRun:  func() { stopRun(); <-runDone },
+	}
+	t.Cleanup(func() { ja.stopConn(); ja.stopRun() })
+	return ja
+}
+
+// jobEnv is a master with one agent (ID 2, token "pua_ok") that owns a Compose
+// service, and a jobs service wired to the master's current hub.
+type jobEnv struct {
+	store *memStore
+	m     testMaster
+	jobs  *jobs.Service
+	exec  *gateExecutor
+	agent *jobAgent
+}
+
+func newJobEnv(t *testing.T) *jobEnv {
+	t.Helper()
+	store := &memStore{agents: map[string]model.Agent{agenttoken.Hash("pua_ok"): {ID: agentID, Name: "node-2"}}}
+	store.seedService(model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
+		AgentID: ptr(agentID), ImageRepo: slusk, ActualRef: slusk + ":1.4.2@sha256:old"})
+	m := startMaster(t, store)
+	exec := newGateExecutor()
+	env := &jobEnv{store: store, m: m, exec: exec,
+		jobs:  jobs.New(store, m.hub, m.broker, "gitea.shcizo.se/shcizo/package-updater:latest", slog.New(slog.NewTextHandler(io.Discard, nil))),
+		agent: startJobAgent(t, m, "pua_ok", exec)}
+	env.waitRoutable(t)
+	return env
+}
+
+func ptr[T any](v T) *T { return &v }
+
+// waitRoutable waits until the master can send jobs to the agent.
+func (e *jobEnv) waitRoutable(t *testing.T) {
+	t.Helper()
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { _, err := e.m.hub.Route(svc, protocol.MinJobVersion); return err == nil }, waitFor, pollEvery)
+}
+
+func (e *jobEnv) job(t *testing.T, id int64) model.Job {
+	t.Helper()
+	j, err := e.store.GetJob(context.Background(), id)
+	require.NoError(t, err)
+	return j
+}
+
+func (e *jobEnv) waitStatus(t *testing.T, id int64, want model.JobStatus) model.Job {
+	t.Helper()
+	require.Eventually(t, func() bool { return e.job(t, id).Status == want }, waitFor, pollEvery,
+		"job %d should become %s, is %s (%s)", id, want, e.job(t, id).Status, e.job(t, id).Error)
+	return e.job(t, id)
+}
+
+func (e *jobEnv) waitStarted(t *testing.T) {
+	t.Helper()
+	select {
+	case <-e.exec.started:
+	case <-time.After(waitFor):
+		t.Fatal("the agent never started the job")
+	}
+}
+
+func (e *jobEnv) trigger(t *testing.T) model.Job {
+	t.Helper()
+	j, err := e.jobs.Trigger(context.Background(), 1, "sub-1")
+	require.NoError(t, err)
+	return j
+}
+
+func TestManualJobFlowsToAgentAndBack(t *testing.T) {
+	e := newJobEnv(t)
+	close(e.exec.release)
+
+	j := e.trigger(t)
+	done := e.waitStatus(t, j.ID, model.JobSucceeded)
+
+	assert.Equal(t, slusk+":1.4.2@"+e2eDigest, done.ToRef)
+	assert.Contains(t, done.Output, "pulling "+slusk+":1.4.2")
+	assert.Contains(t, done.Output, "recreated")
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, slusk+":1.4.2@"+e2eDigest, svc.DesiredRef)
+	require.Eventually(t, func() bool { return len(e.agent.runner.Known()) == 0 }, waitFor, pollEvery, "the result must be acked")
+}
+
+func TestJobFinishedWhileDisconnectedIsRecordedOnce(t *testing.T) {
+	e := newJobEnv(t)
+	j := e.trigger(t)
+	e.waitStarted(t)
+
+	e.m.restart() // drops the agent's connection while the executor blocks
+	close(e.exec.release)
+
+	done := e.waitStatus(t, j.ID, model.JobSucceeded)
+	assert.NotEqual(t, lostMessage, done.Error)
+	assert.GreaterOrEqual(t, e.m.attempts.Load(), int64(2), "the agent must have reconnected")
+	require.Eventually(t, func() bool { return len(e.agent.runner.Known()) == 0 }, waitFor, pollEvery)
+	e.store.mu.Lock()
+	defer e.store.mu.Unlock()
+	assert.Equal(t, 1, e.store.finishApplied)
+}
+
+func TestMasterRestartKeepsJobTheAgentKnows(t *testing.T) {
+	e := newJobEnv(t)
+	j := e.trigger(t)
+	e.waitStarted(t)
+
+	e.m.restart()
+	e.waitRoutable(t) // the agent reconnected and said hello with its known jobs
+	assert.True(t, e.job(t, j.ID).Active(), "the agent still has the job, so it is not lost")
+	close(e.exec.release)
+
+	done := e.waitStatus(t, j.ID, model.JobSucceeded)
+	assert.Empty(t, done.Error)
+}
+
+func TestAgentRestartMarksJobLost(t *testing.T) {
+	e := newJobEnv(t)
+	j := e.trigger(t)
+	e.waitStarted(t)
+
+	// The agent process dies: connection first, so the cancelled job cannot
+	// report a result, then the runner.
+	e.agent.stopConn()
+	e.agent.stopRun()
+	fresh := startJobAgent(t, e.m, "pua_ok", newGateExecutor())
+	_ = fresh
+
+	lost := e.waitStatus(t, j.ID, model.JobFailed)
+	assert.Equal(t, lostMessage, lost.Error)
+}
+
+func TestPhaseOneAgentGetsNoJobs(t *testing.T) {
+	store := &memStore{agents: map[string]model.Agent{agenttoken.Hash("pua_old"): {ID: agentID, Name: "node-2"}}}
+	svcRow := model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
+		AgentID: ptr(agentID), ActualRef: slusk + ":1.4.2@sha256:old"}
+	store.seedService(svcRow)
+	m := startMaster(t, store)
+	js := jobs.New(store, m.hub, m.broker, "gitea.shcizo.se/shcizo/package-updater:latest", slog.New(slog.NewTextHandler(io.Discard, nil)))
+
+	// A raw client speaking the phase-1 protocol: its hello has no
+	// protocol_version.
+	ctx := context.Background()
+	c, _, err := websocket.Dial(ctx, m.url, &websocket.DialOptions{
+		HTTPHeader: http.Header{"Authorization": {"Bearer pua_old"}},
+		HTTPClient: &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}},
+	})
+	require.NoError(t, err)
+	defer c.CloseNow()
+	hello, err := protocol.Encode(protocol.TypeHello, map[string]any{"agent_version": "1", "hostname": "node-2", "swarm_manager": false})
+	require.NoError(t, err)
+	require.NoError(t, wsjson.Write(ctx, c, hello))
+	snap, err := protocol.Encode(protocol.TypeSnapshot, protocol.Snapshot{Services: []protocol.Service{web}})
+	require.NoError(t, err)
+	require.NoError(t, wsjson.Write(ctx, c, snap))
+
+	require.Eventually(t, func() bool {
+		_, err := m.hub.Route(svcRow, protocol.MinJobVersion)
+		return errors.Is(err, model.ErrAgentTooOld)
+	}, waitFor, pollEvery)
+
+	j, err := js.Trigger(ctx, 1, "sub-1")
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, j.Status)
+	assert.Equal(t, "agent too old for updates", j.Error)
+
+	rctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
+	defer cancel()
+	var env protocol.Envelope
+	err = wsjson.Read(rctx, c, &env)
+	assert.True(t, err != nil && strings.Contains(err.Error(), "deadline"), "no job.start expected, got %q (err %v)", env.Type, err)
+}

internal/e2e/versions_test.go

@@ -0,0 +1,133 @@
+package e2e
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/jobs"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/master/poller"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// jobCount is the number of jobs created so far.
+func (m *memStore) jobCount() int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return len(m.jobs)
+}
+
+// fixedRegistry answers every lookup from a fixed table.
+type fixedRegistry struct {
+	digests map[string]string
+	calls   int
+}
+
+func (f *fixedRegistry) Digest(_ context.Context, ref string) (string, error) {
+	f.calls++
+	return f.digests[ref], nil
+}
+
+func TestPickPinnedVersion(t *testing.T) {
+	env := newJobEnv(t)
+	env.store.seedVersion(model.ImageVersion{ID: 5, Repo: slusk, Tag: "1.3.0", Digest: "sha256:old130", Source: model.SourceRunning})
+
+	job, err := env.jobs.TriggerVersion(context.Background(), 1, 5, "alice")
+
+	require.NoError(t, err)
+	assert.Equal(t, slusk+":1.3.0@sha256:old130", job.ToRef)
+	select {
+	case ref := <-env.exec.started:
+		assert.Equal(t, slusk+":1.3.0@sha256:old130", ref, "the agent receives exactly the picked version")
+	case <-time.After(waitFor):
+		t.Fatal("the agent never received the job")
+	}
+	close(env.exec.release)
+	done := env.waitStatus(t, job.ID, model.JobSucceeded)
+	assert.Equal(t, model.TriggerManual, done.Trigger)
+	assert.Equal(t, "alice", done.RequestedBy)
+}
+
+func TestPickVersionOfAnotherRepoIsRefused(t *testing.T) {
+	env := newJobEnv(t)
+	env.store.seedVersion(model.ImageVersion{ID: 6, Repo: "evil/other", Tag: "1", Digest: "sha256:x", Source: model.SourcePoll})
+
+	_, err := env.jobs.TriggerVersion(context.Background(), 1, 6, "alice")
+
+	assert.ErrorIs(t, err, model.ErrNotFound)
+	assert.Zero(t, env.store.jobCount(), "no job was created")
+}
+
+func TestPollStartsAutoJob(t *testing.T) {
+	env := newJobEnv(t)
+	env.store.seedService(model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
+		AgentID: ptr(agentID), ImageRepo: slusk, ActualRef: slusk + ":1.4.2@sha256:old", ActualDigest: "sha256:old",
+		UpdateMode: model.ModeAuto})
+	reg := &fixedRegistry{digests: map[string]string{slusk + ":1.4.2": "sha256:new"}}
+	p := poller.New(env.store, env.jobs, reg, env.m.broker, poller.Config{Workers: 1}, slog.New(slog.NewTextHandler(io.Discard, nil)))
+
+	p.Cycle(context.Background())
+
+	select {
+	case ref := <-env.exec.started:
+		assert.Equal(t, slusk+":1.4.2", ref, "the agent gets repo:tag and resolves the digest itself")
+	case <-time.After(waitFor):
+		t.Fatal("the agent never received the job")
+	}
+	require.Equal(t, 1, env.store.jobCount())
+	job := env.job(t, 1)
+	assert.Equal(t, model.TriggerPoll, job.Trigger)
+	assert.Equal(t, slusk+":1.4.2", job.ToRef)
+	svc, err := env.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, slusk+":1.4.2", svc.AvailableRef)
+	assert.Equal(t, "sha256:new", svc.AvailableDigest)
+	assert.Equal(t, model.SourcePoll, svc.AvailableSource)
+	assert.Equal(t, "sha256:new", svc.LastTagDigest)
+
+	p.Cycle(context.Background())
+
+	assert.Equal(t, 2, reg.calls, "the registry is asked again")
+	assert.Equal(t, 1, env.store.jobCount(), "the same digest does not start another job")
+	close(env.exec.release)
+	env.waitStatus(t, job.ID, model.JobSucceeded)
+}
+
+// An update started by the webhook, then a rollback by hand: polling must not
+// redo the update while the registry still shows the digest that was deployed.
+func TestPollKeepsARollbackAfterAWebhookUpdate(t *testing.T) {
+	env := newJobEnv(t)
+	ctx := context.Background()
+	close(env.exec.release)
+	env.store.seedService(model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
+		AgentID: ptr(agentID), ImageRepo: slusk, ActualRef: slusk + ":main@sha256:old", ActualDigest: "sha256:old",
+		UpdateMode: model.ModeAuto})
+	env.store.seedVersion(model.ImageVersion{ID: 5, Repo: slusk, Tag: "main", Digest: "sha256:old", Source: model.SourceRunning})
+	svc, err := env.store.GetService(ctx, 1)
+	require.NoError(t, err)
+
+	hook, err := env.jobs.Start(ctx, svc, jobs.StartRequest{Trigger: model.TriggerWebhook, Ref: slusk + ":main"})
+	require.NoError(t, err)
+	env.waitStatus(t, hook.ID, model.JobSucceeded)
+	svc, err = env.store.GetService(ctx, 1)
+	require.NoError(t, err)
+	assert.Equal(t, e2eDigest, svc.LastTagDigest, "the deployed digest of the tag is remembered")
+
+	pick, err := env.jobs.TriggerVersion(ctx, 1, 5, "alice")
+	require.NoError(t, err)
+	env.waitStatus(t, pick.ID, model.JobSucceeded)
+	jobsBefore := env.store.jobCount()
+
+	reg := &fixedRegistry{digests: map[string]string{slusk + ":main": e2eDigest}}
+	p := poller.New(env.store, env.jobs, reg, env.m.broker, poller.Config{Workers: 1}, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	p.Cycle(ctx)
+
+	assert.Equal(t, 1, reg.calls)
+	assert.Equal(t, jobsBefore, env.store.jobCount(), "the rollback stays")
+}

internal/e2e/webhook_test.go

@@ -0,0 +1,129 @@
+package e2e
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/master/webhook"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// webhookServer mounts the webhook handler on a test server, wired to the
+// job environment's store, jobs service and broker.
+func (e *jobEnv) webhookServer(t *testing.T) *httptest.Server {
+	t.Helper()
+	h := webhook.New(e.store, e.jobs, e.m.broker, webhook.Config{
+		Token: "ci-key", Timeout: 5 * time.Second, PublicURL: "https://updater.example.com",
+	}, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	srv := httptest.NewServer(h)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func (e *jobEnv) setAuto(id int64) {
+	e.store.mu.Lock()
+	defer e.store.mu.Unlock()
+	svc := e.store.svcRows[id]
+	svc.UpdateMode = model.ModeAuto
+	e.store.svcRows[id] = svc
+}
+
+type webhookResponse struct {
+	Matched int `json:"matched"`
+	Results []struct {
+		Status string `json:"status"`
+		JobID  int64  `json:"job_id"`
+		ToRef  string `json:"to_ref"`
+		Error  string `json:"error"`
+	} `json:"results"`
+}
+
+func callWebhook(t *testing.T, srv *httptest.Server, body string) (int, webhookResponse) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodPost, srv.URL, strings.NewReader(body))
+	require.NoError(t, err)
+	req.Header.Set("Authorization", "Bearer ci-key")
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	var out webhookResponse
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&out))
+	return resp.StatusCode, out
+}
+
+func TestWebhookWaitsForAutoJob(t *testing.T) {
+	e := newJobEnv(t)
+	e.setAuto(1)
+	srv := e.webhookServer(t)
+
+	type reply struct {
+		code int
+		body webhookResponse
+	}
+	done := make(chan reply, 1)
+	go func() {
+		code, body := callWebhook(t, srv, `{"image":"`+slusk+`:ignored","tag":"2.0.0"}`)
+		done <- reply{code, body}
+	}()
+
+	e.waitStarted(t)
+	select {
+	case <-done:
+		t.Fatal("the webhook answered before the job finished")
+	case <-time.After(100 * time.Millisecond):
+	}
+	close(e.exec.release)
+
+	var r reply
+	select {
+	case r = <-done:
+	case <-time.After(waitFor):
+		t.Fatal("the webhook never answered")
+	}
+	require.Equal(t, http.StatusOK, r.code)
+	require.Len(t, r.body.Results, 1)
+	assert.Equal(t, "succeeded", r.body.Results[0].Status)
+	assert.Equal(t, slusk+":2.0.0", r.body.Results[0].ToRef)
+
+	job := e.job(t, r.body.Results[0].JobID)
+	assert.Equal(t, model.TriggerWebhook, job.Trigger)
+	assert.Equal(t, model.JobSucceeded, job.Status)
+	assert.Equal(t, slusk+":2.0.0@"+e2eDigest, job.ToRef)
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Empty(t, svc.AvailableRef, "the job deployed the suggestion, so it is cleared")
+}
+
+func TestWebhookFailedAutoJobIs500(t *testing.T) {
+	e := newJobEnv(t)
+	e.setAuto(1)
+	srv := e.webhookServer(t)
+	e.agent.stopConn() // no agent connection: the job fails at once with "agent offline"
+	require.Eventually(t, func() bool {
+		svc, _ := e.store.GetService(context.Background(), 1)
+		_, err := e.m.hub.Route(svc, protocol.MinJobVersion)
+		return err != nil
+	}, waitFor, pollEvery)
+
+	code, body := callWebhook(t, srv, `{"image":"`+slusk+`","tag":"2.0.0"}`)
+
+	assert.Equal(t, http.StatusInternalServerError, code)
+	require.Len(t, body.Results, 1)
+	assert.Equal(t, "failed", body.Results[0].Status)
+	assert.Equal(t, "agent offline", body.Results[0].Error)
+	assert.Equal(t, model.JobFailed, e.job(t, body.Results[0].JobID).Status)
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, slusk+":2.0.0", svc.AvailableRef, "a failed job keeps the suggestion")
+}

internal/imageref/imageref.go

@@ -2,7 +2,10 @@
 // and digest.
 package imageref
 
-import "strings"
+import (
+	"regexp"
+	"strings"
+)
 
 // Ref is an image reference split into its parts. Tag and Digest are empty
 // when the reference has none.
@@ -41,3 +44,8 @@ func (r Ref) String() string {
 	}
 	return s
 }
+
+var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
+
+// IsDigest reports whether d is a well-formed sha256 digest.
+func IsDigest(d string) bool { return digestPattern.MatchString(d) }

internal/imageref/imageref_test.go

@@ -1,6 +1,7 @@
 package imageref
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -27,3 +28,19 @@ func TestParse(t *testing.T) {
 		})
 	}
 }
+
+func TestIsDigest(t *testing.T) {
+	cases := []struct {
+		in   string
+		want bool
+	}{
+		{"sha256:" + strings.Repeat("a", 64), true},
+		{"sha256:" + strings.Repeat("a", 63), false},
+		{"sha256:" + strings.Repeat("A", 64), false},
+		{"sha512:" + strings.Repeat("a", 64), false},
+		{"", false},
+	}
+	for _, c := range cases {
+		assert.Equal(t, c.want, IsDigest(c.in), c.in)
+	}
+}

internal/master/config/config.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 )
 
 // Config holds the master's runtime configuration.
@@ -20,8 +21,24 @@ type Config struct {
 	Port             string
 	AgentPort        string
 	LogLevel         string
+
+	// UpdaterAPIKey is the bearer token for POST /update (UPDATER_API_KEY).
+	// Empty disables the webhook.
+	UpdaterAPIKey string
+	// UpdateTimeout bounds how long one webhook request waits for its auto
+	// jobs (UPDATE_TIMEOUT, default 5m).
+	UpdateTimeout time.Duration
+	// PollInterval is the time between registry polling cycles
+	// (POLL_INTERVAL, default 6h).
+	PollInterval time.Duration
 }
 
+// defaultUpdateTimeout is used when UPDATE_TIMEOUT is unset.
+const defaultUpdateTimeout = 5 * time.Minute
+
+// defaultPollInterval is used when POLL_INTERVAL is unset.
+const defaultPollInterval = 6 * time.Hour
+
 // Load reads and validates the master configuration, failing fast on
 // anything missing so a half-configured master never starts.
 func Load() (*Config, error) {
@@ -37,6 +54,9 @@ func Load() (*Config, error) {
 		Port:             getenvDefault("PORT", "8080"),
 		AgentPort:        getenvDefault("AGENT_PORT", "8090"),
 		LogLevel:         getenvDefault("LOG_LEVEL", "info"),
+		UpdaterAPIKey:    os.Getenv("UPDATER_API_KEY"),
+		UpdateTimeout:    defaultUpdateTimeout,
+		PollInterval:     defaultPollInterval,
 	}
 	for _, r := range []struct{ name, value string }{
 		{"DATABASE_URL", cfg.DatabaseURL},
@@ -57,6 +77,20 @@ func Load() (*Config, error) {
 	if !strings.HasPrefix(cfg.AgentURL, "wss://") {
 		return nil, fmt.Errorf("AGENT_URL must start with wss://")
 	}
+	if v := os.Getenv("UPDATE_TIMEOUT"); v != "" {
+		d, err := time.ParseDuration(v)
+		if err != nil || d <= 0 {
+			return nil, fmt.Errorf("UPDATE_TIMEOUT must be a positive duration such as 5m, got %q", v)
+		}
+		cfg.UpdateTimeout = d
+	}
+	if v := os.Getenv("POLL_INTERVAL"); v != "" {
+		d, err := time.ParseDuration(v)
+		if err != nil || d <= 0 {
+			return nil, fmt.Errorf("POLL_INTERVAL must be a positive duration such as 6h, got %q", v)
+		}
+		cfg.PollInterval = d
+	}
 	return cfg, nil
 }
 

internal/master/config/config_test.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -21,7 +22,7 @@ func setAll(t *testing.T) {
 	for k, v := range required {
 		t.Setenv(k, v)
 	}
-	for _, k := range []string{"PORT", "AGENT_PORT", "AGENT_IMAGE", "LOG_LEVEL"} {
+	for _, k := range []string{"PORT", "AGENT_PORT", "AGENT_IMAGE", "LOG_LEVEL", "UPDATER_API_KEY", "UPDATE_TIMEOUT", "POLL_INTERVAL"} {
 		t.Setenv(k, "")
 	}
 }
@@ -35,6 +36,49 @@ func TestLoad_Defaults(t *testing.T) {
 	assert.Equal(t, "gitea.shcizo.se/shcizo/package-updater:latest", cfg.AgentImage)
 	assert.Equal(t, "info", cfg.LogLevel)
 	assert.Equal(t, "updater-admins", cfg.AllowedGroup)
+	assert.Empty(t, cfg.UpdaterAPIKey)
+	assert.Equal(t, 5*time.Minute, cfg.UpdateTimeout)
+	assert.Equal(t, 6*time.Hour, cfg.PollInterval)
+}
+
+func TestLoad_PollInterval(t *testing.T) {
+	setAll(t)
+	t.Setenv("POLL_INTERVAL", "30m")
+	cfg, err := Load()
+	require.NoError(t, err)
+	assert.Equal(t, 30*time.Minute, cfg.PollInterval)
+}
+
+func TestLoad_InvalidPollInterval(t *testing.T) {
+	for _, v := range []string{"abc", "0s", "-1h", "6"} {
+		t.Run(v, func(t *testing.T) {
+			setAll(t)
+			t.Setenv("POLL_INTERVAL", v)
+			_, err := Load()
+			assert.ErrorContains(t, err, "POLL_INTERVAL must be a positive duration such as 6h, got")
+		})
+	}
+}
+
+func TestLoad_WebhookSettings(t *testing.T) {
+	setAll(t)
+	t.Setenv("UPDATER_API_KEY", "s3cret")
+	t.Setenv("UPDATE_TIMEOUT", "90s")
+	cfg, err := Load()
+	require.NoError(t, err)
+	assert.Equal(t, "s3cret", cfg.UpdaterAPIKey)
+	assert.Equal(t, 90*time.Second, cfg.UpdateTimeout)
+}
+
+func TestLoad_InvalidUpdateTimeout(t *testing.T) {
+	for _, v := range []string{"abc", "0s", "-5m", "30"} {
+		t.Run(v, func(t *testing.T) {
+			setAll(t)
+			t.Setenv("UPDATE_TIMEOUT", v)
+			_, err := Load()
+			assert.ErrorContains(t, err, "UPDATE_TIMEOUT must be a positive duration")
+		})
+	}
 }
 
 func TestLoad_EachRequiredVariable(t *testing.T) {

internal/master/events/broker.go

@@ -2,7 +2,10 @@
 // open browser streams.
 package events
 
-import "sync"
+import (
+	"strconv"
+	"sync"
+)
 
 // Topic names what changed. Subscribers refetch that part of the UI.
 type Topic string
@@ -11,8 +14,13 @@ type Topic string
 const (
 	Services Topic = "services"
 	Agents   Topic = "agents"
+	Jobs     Topic = "jobs" // a job was created or changed status
 )
 
+// JobTopic is signalled when job id gets new output or changes status. The
+// name avoids ":" so it is safe inside hx-trigger="sse:job-42".
+func JobTopic(id int64) Topic { return Topic("job-" + strconv.FormatInt(id, 10)) }
+
 // Broker is an in-process publish/subscribe hub.
 type Broker struct {
 	mu   sync.Mutex

internal/master/events/broker_test.go

@@ -46,3 +46,7 @@ func TestBroker_SlowSubscriberDoesNotBlockPublish(t *testing.T) {
 		t.Fatal("Publish blocked on a slow subscriber")
 	}
 }
+
+func TestJobTopic(t *testing.T) {
+	assert.Equal(t, Topic("job-42"), JobTopic(42))
+}

internal/master/hub/hub.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/coder/websocket"
@@ -25,6 +26,13 @@ import (
 // small for a full snapshot from a busy machine.
 const readLimit = 1 << 20
 
+// writeTimeout bounds one websocket write. A peer that stops reading is cut
+// off after this long.
+const writeTimeout = 10 * time.Second
+
+// outQueue is how many messages may wait for a connection's writer.
+const outQueue = 64
+
 // Store is the storage the hub writes to. *store.Store satisfies it.
 type Store interface {
 	AgentByTokenHash(ctx context.Context, hash string) (model.Agent, error)
@@ -33,6 +41,9 @@ type Store interface {
 	UpsertService(ctx context.Context, agentID int64, clusterID string, svc protocol.Service, at time.Time) error
 	RemoveService(ctx context.Context, agentID int64, clusterID, agentKey string, at time.Time) error
 	MarkSeen(ctx context.Context, agentID int64, at time.Time) error
+	FailLostJobs(ctx context.Context, agentID int64, known []int64, at time.Time) ([]int64, error)
+	JobProgress(ctx context.Context, agentID, jobID int64, chunk string, at time.Time) (bool, error)
+	FinishJob(ctx context.Context, agentID, jobID int64, f model.JobFinish) (bool, error)
 }
 
 // Publisher receives "something changed" signals. *events.Broker satisfies it.
@@ -47,13 +58,57 @@ type Hub struct {
 	log   *slog.Logger
 	now   func() time.Time
 
+	connSeq atomic.Uint64
+
 	mu    sync.Mutex
-	conns map[int64]*websocket.Conn
+	conns map[int64]*agentConn
+}
+
+// agentConn is one live agent connection. Its writer goroutine is the only
+// code that writes to ws; everything else queues envelopes on out.
+type agentConn struct {
+	ws     *websocket.Conn
+	id     uint64
+	out    chan protocol.Envelope
+	ctx    context.Context
+	cancel context.CancelFunc
+	hello  *protocol.Hello // guarded by Hub.mu; nil until the hello is reconciled
+}
+
+// send queues env for the writer. It returns false if the connection ended
+// before the message could be queued.
+func (c *agentConn) send(env protocol.Envelope) bool {
+	select {
+	case c.out <- env:
+		return true
+	case <-c.ctx.Done():
+		return false
+	}
+}
+
+// writeLoop writes queued envelopes until the connection's context ends. A
+// failed or slow write closes the connection.
+func (c *agentConn) writeLoop() {
+	for {
+		select {
+		case <-c.ctx.Done():
+			return
+		case env := <-c.out:
+			wctx, cancel := context.WithTimeout(c.ctx, writeTimeout)
+			err := wsjson.Write(wctx, c.ws, env)
+			cancel()
+			if err != nil {
+				c.ws.CloseNow()
+				c.cancel()
+				return
+			}
+		}
+	}
 }
 
 // New returns a Hub. Mount it at GET /agent on the TLS listener.
 func New(store Store, pub Publisher, log *slog.Logger) *Hub {
-	return &Hub{store: store, pub: pub, log: log, now: time.Now, conns: make(map[int64]*websocket.Conn)}
+	return &Hub{store: store, pub: pub, log: log, now: time.Now, conns: make(map[int64]*agentConn)}
 }
 
 // ServeHTTP authenticates the agent's token and runs its connection. The token
@@ -78,8 +133,12 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	c.SetReadLimit(readLimit)
-	h.register(agent, r.RemoteAddr, c)
-	defer h.unregister(agent.ID, c)
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
+	ac := &agentConn{ws: c, id: h.connSeq.Add(1), out: make(chan protocol.Envelope, outQueue), ctx: ctx, cancel: cancel}
+	go ac.writeLoop()
+	h.register(agent, r.RemoteAddr, ac)
+	defer h.unregister(agent.ID, ac)
 
 	// Revoking commits revoked_at before calling Disconnect. Re-checking after
 	// register closes the window where a revoke lands between the first lookup
@@ -90,8 +149,8 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	s := &session{hub: h, agent: agent}
-	err = s.run(r.Context(), c)
+	s := &session{hub: h, agent: agent, conn: ac}
+	err = s.run(ctx, c)
 	h.log.Info("agent_disconnected", "agent", agent.Name, "reason", err)
 }
 
@@ -110,25 +169,25 @@ func (h *Hub) Disconnect(agentID int64) {
 	c := h.conns[agentID]
 	h.mu.Unlock()
 	if c != nil {
-		c.CloseNow()
+		c.ws.CloseNow()
 	}
 }
 
 // register makes c the agent's live connection, closing any older one. A
 // replacement is logged: it is normal after a network blip, but repeated
 // replacements mean two agents share a token.
-func (h *Hub) register(agent model.Agent, remoteAddr string, c *websocket.Conn) {
+func (h *Hub) register(agent model.Agent, remoteAddr string, c *agentConn) {
 	h.mu.Lock()
 	old := h.conns[agent.ID]
 	h.conns[agent.ID] = c
 	h.mu.Unlock()
 	if old != nil {
 		h.log.Warn("agent_replaced", "agent_id", agent.ID, "agent", agent.Name, "remote_addr", remoteAddr)
-		old.CloseNow()
+		old.ws.CloseNow()
 	}
 }
 
-func (h *Hub) unregister(id int64, c *websocket.Conn) {
+func (h *Hub) unregister(id int64, c *agentConn) {
 	h.mu.Lock()
 	current := h.conns[id] == c
 	if current {
@@ -148,6 +207,7 @@ func (h *Hub) unregister(id int64, c *websocket.Conn) {
 type session struct {
 	hub   *Hub
 	agent model.Agent
+	conn  *agentConn
 	hello *protocol.Hello
 }
 
@@ -188,7 +248,25 @@ func (s *session) handle(ctx context.Context, env protocol.Envelope) error {
 		if err := h.store.RecordHello(ctx, s.agent.ID, v, h.now()); err != nil {
 			return err
 		}
+		// Reconcile jobs before the connection becomes routable, so a job is
+		// never sent to an agent whose old jobs have not been settled yet.
+		known := parseJobIDs(v.KnownJobs, func(id string) {
+			h.log.Warn("known_job_not_numeric", "agent", s.agent.Name, "job_id", id)
+		})
+		lost, err := h.store.FailLostJobs(ctx, s.agent.ID, known, h.now())
+		if err != nil {
+			return err
+		}
+		if len(lost) > 0 {
+			h.pub.Publish(events.Jobs)
+			for _, id := range lost {
+				h.pub.Publish(events.JobTopic(id))
+			}
+		}
 		s.hello = &v
+		h.mu.Lock()
+		s.conn.hello = &v
+		h.mu.Unlock()
 		h.log.Info("agent_connected", "agent", s.agent.Name, "hostname", v.Hostname, "swarm_manager", v.SwarmManager)
 		h.pub.Publish(events.Agents)
 	case protocol.TypeSnapshot:
@@ -224,8 +302,12 @@ func (s *session) handle(ctx context.Context, env protocol.Envelope) error {
 			return err
 		}
 		h.pub.Publish(events.Services)
+	case protocol.TypeJobProgress:
+		return s.handleJobProgress(ctx, env)
+	case protocol.TypeJobResult:
+		return s.handleJobResult(ctx, env)
 	default:
-		return fmt.Errorf("unknown message type %q", env.Type)
+		h.log.Warn("unknown_message_type", "agent", s.agent.Name, "type", env.Type)
 	}
 	return nil
 }

internal/master/hub/hub_test.go

@@ -3,6 +3,7 @@ package hub
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
@@ -35,6 +36,18 @@ type fakeStore struct {
 	// revokeFromLookup, when > 0, makes AgentByTokenHash return the agent as
 	// revoked from that (1-based) lookup onwards.
 	revokeFromLookup int
+
+	// Job calls, recorded; the *Result fields script the answers.
+	lostCalls    [][]int64
+	lostResult   []int64
+	lostErr      error
+	lostGate     chan struct{} // when set, FailLostJobs blocks until it is closed
+	progress     []string      // "agent|job|chunk"
+	progressOK   bool
+	finishes     []model.JobFinish
+	finishAgents []int64
+	finishOK     bool
+	finishErr    error
 }
 
 func newFakeStore(agents ...model.Agent) *fakeStore {
@@ -94,6 +107,31 @@ func (f *fakeStore) MarkSeen(_ context.Context, id int64, _ time.Time) error {
 	return nil
 }
 
+func (f *fakeStore) FailLostJobs(_ context.Context, _ int64, known []int64, _ time.Time) ([]int64, error) {
+	if f.lostGate != nil {
+		<-f.lostGate
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.lostCalls = append(f.lostCalls, known)
+	return f.lostResult, f.lostErr
+}
+
+func (f *fakeStore) JobProgress(_ context.Context, agentID, jobID int64, chunk string, _ time.Time) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.progress = append(f.progress, fmt.Sprintf("%d|%d|%s", agentID, jobID, chunk))
+	return f.progressOK, nil
+}
+
+func (f *fakeStore) FinishJob(_ context.Context, agentID, _ int64, fin model.JobFinish) (bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.finishAgents = append(f.finishAgents, agentID)
+	f.finishes = append(f.finishes, fin)
+	return f.finishOK, f.finishErr
+}
+
 func (f *fakeStore) count(get func(*fakeStore) int) func() int {
 	return func() int {
 		f.mu.Lock()

internal/master/hub/jobs.go

@@ -0,0 +1,133 @@
+package hub
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	"github.com/shcizo/package-updater/internal/master/events"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// StartJob sends job.start over exactly the connection r names. It returns
+// model.ErrAgentOffline if that connection is gone, was replaced by a newer
+// one, or has not finished its hello.
+func (h *Hub) StartJob(r model.Route, jobID int64, start protocol.JobStart) error {
+	h.mu.Lock()
+	c := h.conns[r.AgentID]
+	usable := c != nil && c.id == r.ConnID && c.hello != nil
+	h.mu.Unlock()
+	if !usable {
+		return model.ErrAgentOffline
+	}
+	env, err := protocol.EncodeJob(protocol.TypeJobStart, strconv.FormatInt(jobID, 10), start)
+	if err != nil {
+		return err
+	}
+	// A job whose write later fails stays queued; the agent's next hello
+	// settles it as lost.
+	if !c.send(env) {
+		return model.ErrAgentOffline
+	}
+	return nil
+}
+
+// parseJobIDs converts the agent's job IDs to integers, calling skip for each
+// one that is not a number.
+func parseJobIDs(ids []string, skip func(id string)) []int64 {
+	out := make([]int64, 0, len(ids))
+	for _, s := range ids {
+		id, err := strconv.ParseInt(s, 10, 64)
+		if err != nil {
+			skip(s)
+			continue
+		}
+		out = append(out, id)
+	}
+	return out
+}
+
+// cleanOutput drops NUL bytes, which Postgres text columns reject. Without
+// this a result containing one could never be stored and would be resent
+// forever.
+func cleanOutput(s string) string { return strings.ReplaceAll(s, "\x00", "") }
+
+func (s *session) handleJobProgress(ctx context.Context, env protocol.Envelope) error {
+	h := s.hub
+	v, err := protocol.Decode[protocol.JobProgress](env)
+	if err != nil {
+		return err
+	}
+	id, err := strconv.ParseInt(env.ID, 10, 64)
+	if err != nil {
+		h.log.Warn("job_progress_ignored", "agent", s.agent.Name, "job_id", env.ID, "reason", "not a number")
+		return nil
+	}
+	applied, err := h.store.JobProgress(ctx, s.agent.ID, id, cleanOutput(v.Output), h.now())
+	if err != nil {
+		return err
+	}
+	if applied {
+		h.pub.Publish(events.Jobs)
+		h.pub.Publish(events.JobTopic(id))
+	}
+	return nil
+}
+
+func (s *session) handleJobResult(ctx context.Context, env protocol.Envelope) error {
+	h := s.hub
+	v, err := protocol.Decode[protocol.JobResult](env)
+	if err != nil {
+		return err
+	}
+	ack, err := protocol.EncodeJob(protocol.TypeJobAck, env.ID, protocol.JobAck{})
+	if err != nil {
+		return err
+	}
+	id, err := strconv.ParseInt(env.ID, 10, 64)
+	if err != nil {
+		h.log.Warn("job_result_ignored", "agent", s.agent.Name, "job_id", env.ID, "reason", "not a number")
+		s.conn.send(ack)
+		return nil
+	}
+
+	fin := model.JobFinish{
+		Status:          model.JobStatus(v.Status),
+		Ref:             v.Ref,
+		Error:           v.Error,
+		Output:          cleanOutput(v.Output),
+		OutputTruncated: v.OutputTruncated,
+		StartedAt:       v.StartedAt,
+		FinishedAt:      v.FinishedAt,
+	}
+	if v.Status != protocol.ResultSucceeded && v.Status != protocol.ResultFailed {
+		fin.Status = model.JobFailed
+		fin.Ref = ""
+		fin.Error = "agent sent invalid status " + strconv.Quote(v.Status)
+	}
+	if fin.StartedAt.IsZero() {
+		fin.StartedAt = h.now()
+	}
+	if fin.FinishedAt.IsZero() {
+		fin.FinishedAt = h.now()
+	}
+
+	// A store error closes the connection without an ack, so the agent
+	// resends the result after it reconnects.
+	applied, err := h.store.FinishJob(ctx, s.agent.ID, id, fin)
+	if err != nil {
+		return err
+	}
+	if applied {
+		h.pub.Publish(events.Jobs)
+		h.pub.Publish(events.JobTopic(id))
+		if fin.Status == model.JobSucceeded {
+			h.pub.Publish(events.Services)
+		}
+	} else {
+		h.log.Warn("job_result_ignored", "agent", s.agent.Name, "job_id", env.ID, "reason", "no such active job for this agent")
+	}
+	s.conn.send(ack)
+	return nil
+}

internal/master/hub/jobs_test.go

@@ -0,0 +1,274 @@
+package hub
+
+import (
+	"context"
+	"errors"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/coder/websocket"
+	"github.com/coder/websocket/wsjson"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/events"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+var t0 = time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
+
+func sendJob(t *testing.T, c *websocket.Conn, typ, id string, payload any) {
+	t.Helper()
+	env, err := protocol.EncodeJob(typ, id, payload)
+	require.NoError(t, err)
+	require.NoError(t, wsjson.Write(context.Background(), c, env))
+}
+
+func readEnv(t *testing.T, c *websocket.Conn) protocol.Envelope {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	var env protocol.Envelope
+	require.NoError(t, wsjson.Read(ctx, c, &env))
+	return env
+}
+
+func waitRoute(t *testing.T, h *Hub) model.Route {
+	t.Helper()
+	svc := model.Service{Kind: protocol.KindCompose, AgentID: ptr(node2.ID)}
+	var r model.Route
+	require.Eventually(t, func() bool {
+		var err error
+		r, err = h.Route(svc, protocol.MinJobVersion)
+		return err == nil
+	}, 2*time.Second, 5*time.Millisecond)
+	return r
+}
+
+func TestHub_ReconcilesBeforeRoutable(t *testing.T) {
+	store := newFakeStore(node2)
+	store.lostGate = make(chan struct{})
+	store.lostResult = []int64{5}
+	h, pub, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+
+	send(t, c, protocol.TypeHello, protocol.Hello{Hostname: "node-2", ProtocolVersion: 2, KnownJobs: []string{"7", "9", "x"}})
+	svc := model.Service{Kind: protocol.KindCompose, AgentID: ptr(node2.ID)}
+	time.Sleep(50 * time.Millisecond) // hello is now blocked inside FailLostJobs
+	_, err = h.Route(svc, protocol.MinJobVersion)
+	assert.ErrorIs(t, err, model.ErrAgentOffline)
+
+	close(store.lostGate)
+	waitRoute(t, h)
+
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	assert.Equal(t, [][]int64{{7, 9}}, store.lostCalls)
+	assert.True(t, pub.has(events.Jobs))
+	assert.True(t, pub.has(events.JobTopic(5)))
+}
+
+func TestHub_StartJobSendsEnvelope(t *testing.T) {
+	store := newFakeStore(node2)
+	h, _, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	r := waitRoute(t, h)
+
+	start := protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "r:1"}
+	require.NoError(t, h.StartJob(r, 42, start))
+
+	env := readEnv(t, c)
+	assert.Equal(t, protocol.TypeJobStart, env.Type)
+	assert.Equal(t, "42", env.ID)
+	got, err := protocol.Decode[protocol.JobStart](env)
+	require.NoError(t, err)
+	assert.Equal(t, start, got)
+}
+
+func TestHub_StartJobOnReplacedConnectionFails(t *testing.T) {
+	store := newFakeStore(node2)
+	h, _, srv := newHub(t, store)
+	first, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer first.CloseNow()
+	send(t, first, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	old := waitRoute(t, h)
+
+	second, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer second.CloseNow()
+
+	// dial returns before the hub has registered the connection, so wait for
+	// the replacement to take effect: the old route is then refused. Until the
+	// new connection says hello, nothing is routable.
+	require.Eventually(t, func() bool {
+		return errors.Is(h.StartJob(old, 1, protocol.JobStart{}), model.ErrAgentOffline)
+	}, 2*time.Second, 5*time.Millisecond)
+	send(t, second, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	fresh := waitRoute(t, h)
+	require.NotEqual(t, old.ConnID, fresh.ConnID)
+	assert.ErrorIs(t, h.StartJob(old, 1, protocol.JobStart{}), model.ErrAgentOffline)
+
+	require.NoError(t, h.StartJob(fresh, 2, protocol.JobStart{Ref: "r:2"}))
+	env := readEnv(t, second)
+	assert.Equal(t, "2", env.ID, "the new connection got only the job sent to it")
+}
+
+func TestHub_ProgressAndResult(t *testing.T) {
+	store := newFakeStore(node2)
+	store.progressOK, store.finishOK = true, true
+	_, pub, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+
+	sendJob(t, c, protocol.TypeJobProgress, "42", protocol.JobProgress{Output: "out"})
+	require.Eventually(t, func() bool { return pub.has(events.JobTopic(42)) }, time.Second, 5*time.Millisecond)
+	store.mu.Lock()
+	assert.Equal(t, []string{"2|42|out"}, store.progress)
+	store.mu.Unlock()
+
+	pub.reset()
+	res := protocol.JobResult{Status: protocol.ResultSucceeded, Ref: "r:1@sha256:d", Output: "done", StartedAt: t0, FinishedAt: t0}
+	sendJob(t, c, protocol.TypeJobResult, "42", res)
+	ack := readEnv(t, c)
+	assert.Equal(t, protocol.TypeJobAck, ack.Type)
+	assert.Equal(t, "42", ack.ID)
+	assert.True(t, pub.has(events.Services))
+	assert.True(t, pub.has(events.JobTopic(42)))
+	store.mu.Lock()
+	require.Len(t, store.finishes, 1)
+	assert.Equal(t, model.JobSucceeded, store.finishes[0].Status)
+	assert.Equal(t, "r:1@sha256:d", store.finishes[0].Ref)
+	store.mu.Unlock()
+
+	// Not applied (already recorded, or not this agent's): still acked.
+	store.mu.Lock()
+	store.finishOK = false
+	store.mu.Unlock()
+	pub.reset()
+	sendJob(t, c, protocol.TypeJobResult, "43", res)
+	assert.Equal(t, "43", readEnv(t, c).ID)
+	assert.False(t, pub.has(events.JobTopic(43)))
+
+	// Non-numeric ID: acked, no store call.
+	store.mu.Lock()
+	before := len(store.finishes)
+	store.mu.Unlock()
+	sendJob(t, c, protocol.TypeJobResult, "abc", res)
+	assert.Equal(t, "abc", readEnv(t, c).ID)
+	store.mu.Lock()
+	assert.Len(t, store.finishes, before)
+	store.mu.Unlock()
+
+	// Unknown status is stored as a failure.
+	store.mu.Lock()
+	store.finishOK = true
+	store.mu.Unlock()
+	sendJob(t, c, protocol.TypeJobResult, "44", protocol.JobResult{Status: "weird", Ref: "r:9"})
+	assert.Equal(t, "44", readEnv(t, c).ID)
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	last := store.finishes[len(store.finishes)-1]
+	assert.Equal(t, model.JobFailed, last.Status)
+	assert.Equal(t, `agent sent invalid status "weird"`, last.Error)
+	assert.Empty(t, last.Ref)
+}
+
+func TestHub_ResultStoreErrorClosesConnection(t *testing.T) {
+	store := newFakeStore(node2)
+	store.finishErr = assert.AnError
+	_, _, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	sendJob(t, c, protocol.TypeJobResult, "42", protocol.JobResult{Status: protocol.ResultFailed})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	_, _, err = c.Read(ctx)
+	assert.Equal(t, websocket.StatusPolicyViolation, websocket.CloseStatus(err), "no ack, connection closed")
+}
+
+func TestHub_IgnoresUnknownType(t *testing.T) {
+	store := newFakeStore(node2)
+	_, _, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	require.NoError(t, wsjson.Write(context.Background(), c, protocol.Envelope{Type: "future.thing"}))
+	send(t, c, protocol.TypeUpsert, protocol.Upsert{Service: protocol.Service{Kind: protocol.KindContainer, ContainerName: "y"}})
+
+	require.Eventually(t, func() bool { return store.count(func(f *fakeStore) int { return len(f.upserts) })() == 1 }, time.Second, 5*time.Millisecond)
+}
+
+func TestHub_LargeResultArrives(t *testing.T) {
+	store := newFakeStore(node2)
+	store.finishOK = true
+	_, _, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+
+	out := strings.Repeat("\x1b", protocol.MaxJobOutput)
+	sendJob(t, c, protocol.TypeJobResult, "42", protocol.JobResult{Status: protocol.ResultFailed, Output: out})
+	assert.Equal(t, "42", readEnv(t, c).ID)
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	require.Len(t, store.finishes, 1)
+	assert.Equal(t, out, store.finishes[0].Output)
+}
+
+func TestHub_ConcurrentWritesAreSerialised(t *testing.T) {
+	const n = 50
+	store := newFakeStore(node2)
+	store.finishOK = true
+	h, _, srv := newHub(t, store)
+	c, _, err := dial(t, srv, "pua_node-2")
+	require.NoError(t, err)
+	defer c.CloseNow()
+	send(t, c, protocol.TypeHello, protocol.Hello{ProtocolVersion: 2})
+	r := waitRoute(t, h)
+
+	var wg sync.WaitGroup
+	for i := 0; i < n; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			assert.NoError(t, h.StartJob(r, int64(1000+i), protocol.JobStart{Ref: "r:1"}))
+		}()
+	}
+	go func() {
+		for i := 0; i < n; i++ {
+			sendJob(t, c, protocol.TypeJobResult, strconv.Itoa(i), protocol.JobResult{Status: protocol.ResultFailed})
+		}
+	}()
+
+	starts, acks := 0, 0
+	for starts+acks < 2*n {
+		switch env := readEnv(t, c); env.Type {
+		case protocol.TypeJobStart:
+			starts++
+		case protocol.TypeJobAck:
+			acks++
+		default:
+			t.Fatalf("unexpected %q", env.Type)
+		}
+	}
+	wg.Wait()
+	assert.Equal(t, n, starts)
+	assert.Equal(t, n, acks)
+}

internal/master/hub/route.go

@@ -0,0 +1,74 @@
+package hub
+
+import (
+	"sort"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// candidate is a connection that has finished its hello and may get jobs.
+type candidate struct {
+	agentID int64
+	connID  uint64
+	hello   protocol.Hello
+}
+
+// Route picks the connection a job for svc goes to, among agents that speak
+// at least protocol version minProtocol. It returns model.ErrAgentOffline or
+// model.ErrAgentTooOld (model.ErrAgentTooOldForPinned when minProtocol asks
+// for a pinned ref) when there is none.
+func (h *Hub) Route(svc model.Service, minProtocol int) (model.Route, error) {
+	h.mu.Lock()
+	conns := make([]candidate, 0, len(h.conns))
+	for id, c := range h.conns {
+		if c.hello != nil {
+			conns = append(conns, candidate{agentID: id, connID: c.id, hello: *c.hello})
+		}
+	}
+	h.mu.Unlock()
+	return route(svc, conns, minProtocol)
+}
+
+// route picks the connection a job for svc goes to. Compose services and
+// containers can only run on the agent that reported them. Swarm services
+// run on the reporting agent if it qualifies, else on any other connected
+// manager of the same cluster (lowest agent ID first). Too-old wins over
+// offline only when a connection would otherwise have qualified.
+func route(svc model.Service, conns []candidate, minProtocol int) (model.Route, error) {
+	sort.Slice(conns, func(a, b int) bool { return conns[a].agentID < conns[b].agentID })
+	tooOld := false
+	try := func(c candidate) bool {
+		if svc.Kind == protocol.KindSwarm {
+			if !c.hello.SwarmManager || c.hello.SwarmClusterID == "" || c.hello.SwarmClusterID != svc.SwarmClusterID {
+				return false
+			}
+		}
+		if c.hello.ProtocolVersion < minProtocol {
+			tooOld = true
+			return false
+		}
+		return true
+	}
+	if svc.AgentID != nil {
+		for _, c := range conns {
+			if c.agentID == *svc.AgentID && try(c) {
+				return model.Route{AgentID: c.agentID, ConnID: c.connID}, nil
+			}
+		}
+	}
+	if svc.Kind == protocol.KindSwarm {
+		for _, c := range conns {
+			if try(c) {
+				return model.Route{AgentID: c.agentID, ConnID: c.connID}, nil
+			}
+		}
+	}
+	if tooOld {
+		if minProtocol >= protocol.MinPinnedRefVersion {
+			return model.Route{}, model.ErrAgentTooOldForPinned
+		}
+		return model.Route{}, model.ErrAgentTooOld
+	}
+	return model.Route{}, model.ErrAgentOffline
+}

internal/master/hub/route_test.go

@@ -0,0 +1,59 @@
+package hub
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+func TestRoute(t *testing.T) {
+	composeSvc := model.Service{Kind: protocol.KindCompose, AgentID: ptr(int64(2))}
+	swarmSvc := model.Service{Kind: protocol.KindSwarm, AgentID: ptr(int64(1)), SwarmClusterID: "c1"}
+	v2 := protocol.Hello{ProtocolVersion: 2}
+	v3 := protocol.Hello{ProtocolVersion: 3}
+	mgrV2 := protocol.Hello{ProtocolVersion: 2, SwarmManager: true, SwarmClusterID: "c1"}
+	mgrV3 := protocol.Hello{ProtocolVersion: 3, SwarmManager: true, SwarmClusterID: "c1"}
+	jobs, pinned := protocol.MinJobVersion, protocol.MinPinnedRefVersion
+
+	tests := []struct {
+		name  string
+		svc   model.Service
+		conns []candidate
+		min   int
+		want  model.Route
+		err   error
+	}{
+		{"compose on own agent", composeSvc, []candidate{{2, 20, v2}}, jobs, model.Route{AgentID: 2, ConnID: 20}, nil},
+		{"compose, own agent too old", composeSvc, []candidate{{2, 20, protocol.Hello{}}}, jobs, model.Route{}, model.ErrAgentTooOld},
+		{"compose, own agent missing", composeSvc, nil, jobs, model.Route{}, model.ErrAgentOffline},
+		{"compose, only another agent online", composeSvc, []candidate{{5, 50, v2}}, jobs, model.Route{}, model.ErrAgentOffline},
+		{"compose, v2 agent and a pinned ref", composeSvc, []candidate{{2, 20, v2}}, pinned, model.Route{}, model.ErrAgentTooOldForPinned},
+		{"compose, v3 agent and a pinned ref", composeSvc, []candidate{{2, 20, v3}}, pinned, model.Route{AgentID: 2, ConnID: 20}, nil},
+		{"swarm on reporting agent", swarmSvc, []candidate{{1, 10, mgrV2}, {3, 30, mgrV2}}, jobs, model.Route{AgentID: 1, ConnID: 10}, nil},
+		{"swarm, reporting agent offline, other manager", swarmSvc, []candidate{{3, 30, mgrV2}}, jobs, model.Route{AgentID: 3, ConnID: 30}, nil},
+		{"swarm, lowest agent id first", swarmSvc, []candidate{{4, 40, mgrV2}, {3, 30, mgrV2}}, jobs, model.Route{AgentID: 3, ConnID: 30}, nil},
+		{"swarm, other manager in another cluster", swarmSvc,
+			[]candidate{{3, 30, protocol.Hello{ProtocolVersion: 2, SwarmManager: true, SwarmClusterID: "c2"}}}, jobs, model.Route{}, model.ErrAgentOffline},
+		{"swarm, only a v0 manager of the cluster", swarmSvc,
+			[]candidate{{1, 10, protocol.Hello{SwarmManager: true, SwarmClusterID: "c1"}}}, jobs, model.Route{}, model.ErrAgentTooOld},
+		{"swarm, v0 manager but a v2 manager exists", swarmSvc,
+			[]candidate{{1, 10, protocol.Hello{SwarmManager: true, SwarmClusterID: "c1"}}, {3, 30, mgrV2}}, jobs, model.Route{AgentID: 3, ConnID: 30}, nil},
+		{"swarm, worker of the cluster", swarmSvc,
+			[]candidate{{1, 10, protocol.Hello{ProtocolVersion: 2, SwarmClusterID: "c1"}}}, jobs, model.Route{}, model.ErrAgentOffline},
+		{"swarm, manager with empty cluster id", model.Service{Kind: protocol.KindSwarm},
+			[]candidate{{1, 10, protocol.Hello{ProtocolVersion: 2, SwarmManager: true}}}, jobs, model.Route{}, model.ErrAgentOffline},
+		{"swarm, pinned ref falls back to a v3 manager when the reporting one is v2", swarmSvc,
+			[]candidate{{1, 10, mgrV2}, {3, 30, mgrV3}}, pinned, model.Route{AgentID: 3, ConnID: 30}, nil},
+		{"swarm, pinned ref and only v2 managers", swarmSvc, []candidate{{1, 10, mgrV2}}, pinned, model.Route{}, model.ErrAgentTooOldForPinned},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := route(tt.svc, tt.conns, tt.min)
+			assert.ErrorIs(t, err, tt.err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

internal/master/jobs/drift_test.go

@@ -0,0 +1,237 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+const (
+	driftRepo = "gitea.shcizo.se/shcizo/slusk"
+	driftD1   = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
+	driftD2   = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
+	driftD3   = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
+)
+
+// driftedSvc was updated to 1.4.2 at D1 but now runs 1.4.2 at D2.
+func driftedSvc() model.Service {
+	svc := composeSvc
+	svc.ImageRepo = driftRepo
+	svc.DesiredRef = driftRepo + ":1.4.2@" + driftD1
+	svc.ActualRef = driftRepo + ":1.4.2"
+	svc.ActualDigest = driftD2
+	return svc
+}
+
+func notUpdatableReason(t *testing.T, err error) string {
+	t.Helper()
+	var nu *model.NotUpdatableError
+	require.True(t, errors.As(err, &nu), "got %v", err)
+	return nu.Reason
+}
+
+func TestRestorable(t *testing.T) {
+	tests := []struct {
+		name   string
+		mutate func(*model.Service)
+		reason string // "" means restorable
+	}{
+		{"drifted", func(*model.Service) {}, ""},
+		{"drifted service that follows no tag", func(s *model.Service) {
+			s.ActualRef = driftRepo + "@" + driftD2
+		}, ""},
+		{"not drifted", func(s *model.Service) { s.ActualDigest = driftD1 }, model.ReasonNotDrifted},
+		{"never updated", func(s *model.Service) { s.DesiredRef = "" }, model.ReasonNotDrifted},
+		{"awaiting inventory", func(s *model.Service) { s.AwaitingInventory = true }, model.ReasonNotDrifted},
+		{"removed", func(s *model.Service) { s.RemovedAt = ptr(time.Now()) }, model.ReasonRemoved},
+		{"flagged self", func(s *model.Service) { s.Self = true }, model.ReasonSelf},
+		{"built", func(s *model.Service) { s.Built = true }, model.ReasonBuilt},
+		{"container", func(s *model.Service) { s.Kind = protocol.KindContainer }, model.ReasonContainer},
+		{"desired ref is the agent image", func(s *model.Service) {
+			s.DesiredRef = selfImage + "@" + driftD1
+		}, model.ReasonSelf},
+	}
+	s, _ := newService(&fakeStore{}, &fakeHub{})
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			svc := driftedSvc()
+			tt.mutate(&svc)
+			err := s.Restorable(svc)
+			if tt.reason == "" {
+				assert.NoError(t, err)
+				return
+			}
+			assert.Equal(t, tt.reason, notUpdatableReason(t, err))
+		})
+	}
+}
+
+func TestRestoreDesired_StartsAManualJobToTheDesiredRef(t *testing.T) {
+	svc := driftedSvc()
+	st := &fakeStore{services: map[int64]model.Service{1: svc}, jobs: map[int64]model.Job{}}
+	h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
+	s, _ := newService(st, h)
+
+	job, err := s.RestoreDesired(context.Background(), 1, "sub-1")
+
+	require.NoError(t, err)
+	assert.Equal(t, model.JobQueued, job.Status)
+	require.Len(t, st.created, 1)
+	assert.Equal(t, svc.DesiredRef, st.created[0].ToRef)
+	assert.Equal(t, svc.ActualRef, st.created[0].FromRef)
+	assert.Equal(t, "sub-1", st.created[0].RequestedBy)
+	assert.Equal(t, model.TriggerManual, st.created[0].Trigger)
+	assert.Equal(t, []int{protocol.MinPinnedRefVersion}, h.mins)
+	require.Len(t, h.starts, 1)
+	assert.Equal(t, svc.DesiredRef, h.starts[0].start.Ref)
+}
+
+func TestRestoreDesired_Refusals(t *testing.T) {
+	tests := []struct {
+		name   string
+		mutate func(*model.Service)
+		reason string
+	}{
+		{"not drifted", func(s *model.Service) { s.ActualDigest = driftD1 }, model.ReasonNotDrifted},
+		{"self", func(s *model.Service) { s.Self = true }, model.ReasonSelf},
+		{"built", func(s *model.Service) { s.Built = true }, model.ReasonBuilt},
+		{"container", func(s *model.Service) { s.Kind = protocol.KindContainer }, model.ReasonContainer},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			svc := driftedSvc()
+			tt.mutate(&svc)
+			st := &fakeStore{services: map[int64]model.Service{1: svc}, jobs: map[int64]model.Job{}}
+			h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
+			s, pub := newService(st, h)
+
+			_, err := s.RestoreDesired(context.Background(), 1, "sub-1")
+
+			assert.Equal(t, tt.reason, notUpdatableReason(t, err))
+			assert.Empty(t, st.created)
+			assert.Empty(t, h.starts)
+			assert.Empty(t, pub.topics)
+			assert.Empty(t, st.lastTag, "last_tag_digest untouched")
+		})
+	}
+}
+
+func TestRestoreDesired_UnknownService(t *testing.T) {
+	s, _ := newService(&fakeStore{services: map[int64]model.Service{}}, &fakeHub{})
+
+	_, err := s.RestoreDesired(context.Background(), 9, "sub-1")
+
+	assert.ErrorIs(t, err, model.ErrNotFound)
+}
+
+func TestRestoreDesired_ActiveJobReturnsItAndMarksNothing(t *testing.T) {
+	activeJob := model.Job{ID: 99, Status: model.JobRunning}
+	st := &fakeStore{services: map[int64]model.Service{1: driftedSvc()}, jobs: map[int64]model.Job{99: activeJob}, active: &activeJob}
+	s, _ := newService(st, &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}})
+
+	job, err := s.RestoreDesired(context.Background(), 1, "sub-1")
+
+	assert.ErrorIs(t, err, model.ErrJobActive)
+	assert.Equal(t, int64(99), job.ID)
+	assert.Empty(t, st.lastTag)
+}
+
+func TestRestoreDesired_OldAgentGivesAFailedJob(t *testing.T) {
+	st := &fakeStore{services: map[int64]model.Service{1: driftedSvc()}, jobs: map[int64]model.Job{}}
+	h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}, agentProtocol: 2}
+	s, _ := newService(st, h)
+
+	job, err := s.RestoreDesired(context.Background(), 1, "sub-1")
+
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, job.Status)
+	assert.Equal(t, model.ErrAgentTooOldForPinned.Error(), job.Error)
+	assert.Empty(t, h.starts)
+}
+
+func TestRestoreDesired_LastTagDigest(t *testing.T) {
+	tests := []struct {
+		name   string
+		mutate func(*model.Service)
+		want   string // "" means SetLastTagDigest is not called
+	}{
+		{"drifted to another digest of the same tag", func(*model.Service) {}, driftD2},
+		{"digest in the actual ref", func(s *model.Service) {
+			s.ActualRef, s.ActualDigest = driftRepo+":1.4.2@"+driftD3, ""
+		}, driftD3},
+		{"last tag digest holds back a newer one", func(s *model.Service) {
+			s.DesiredRef = driftRepo + ":1.4.2@" + driftD1
+			s.LastTagDigest = driftD3
+		}, ""},
+		{"last tag digest equals the desired digest", func(s *model.Service) { s.LastTagDigest = driftD1 }, driftD2},
+		{"already remembered", func(s *model.Service) { s.LastTagDigest = driftD2 }, ""},
+		{"different tag", func(s *model.Service) { s.ActualRef = driftRepo + ":1.5.0" }, ""},
+		{"actual digest malformed", func(s *model.Service) { s.ActualDigest = "sha256:abc" }, ""},
+		{"desired follows no tag", func(s *model.Service) { s.DesiredRef = driftRepo + "@" + driftD1 }, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			svc := driftedSvc()
+			tt.mutate(&svc)
+			st := &fakeStore{services: map[int64]model.Service{1: svc}, jobs: map[int64]model.Job{}}
+			s, _ := newService(st, &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}})
+
+			_, err := s.RestoreDesired(context.Background(), 1, "sub-1")
+
+			require.NoError(t, err)
+			if tt.want == "" {
+				assert.Empty(t, st.lastTag)
+				return
+			}
+			assert.Equal(t, map[int64]string{1: tt.want}, st.lastTag)
+		})
+	}
+}
+
+func TestStart_RefusesPollAndWebhookWhileDrifted(t *testing.T) {
+	tests := []struct {
+		trigger model.JobTrigger
+		refused bool
+	}{
+		{model.TriggerPoll, true},
+		{model.TriggerWebhook, true},
+		{model.TriggerManual, false},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.trigger), func(t *testing.T) {
+			svc := driftedSvc()
+			st := &fakeStore{services: map[int64]model.Service{1: svc}, jobs: map[int64]model.Job{}}
+			h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
+			s, _ := newService(st, h)
+
+			_, err := s.Start(context.Background(), svc, StartRequest{Trigger: tt.trigger, Ref: driftRepo + ":2.0.0"})
+
+			if tt.refused {
+				assert.Equal(t, model.ReasonDrifted, notUpdatableReason(t, err))
+				assert.Empty(t, st.created)
+				return
+			}
+			require.NoError(t, err)
+			assert.Len(t, st.created, 1)
+		})
+	}
+}
+
+func TestStart_PollAllowedWhenNotDrifted(t *testing.T) {
+	svc := driftedSvc()
+	svc.AwaitingInventory = true
+	st := &fakeStore{services: map[int64]model.Service{1: svc}, jobs: map[int64]model.Job{}}
+	s, _ := newService(st, &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}})
+
+	_, err := s.Start(context.Background(), svc, StartRequest{Trigger: model.TriggerPoll})
+
+	require.NoError(t, err)
+	assert.Len(t, st.created, 1)
+}

internal/master/jobs/jobs.go

@@ -0,0 +1,298 @@
+// Package jobs decides whether a service may be updated and starts update
+// jobs on the right agent.
+package jobs
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"time"
+
+	"github.com/shcizo/package-updater/internal/imageref"
+	"github.com/shcizo/package-updater/internal/master/events"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// Store is the storage jobs needs. *store.Store satisfies it.
+type Store interface {
+	GetService(ctx context.Context, id int64) (model.Service, error)
+	CreateJob(ctx context.Context, j model.NewJob) (model.Job, error)
+	FailJob(ctx context.Context, id int64, msg string, at time.Time) (bool, error)
+	GetJob(ctx context.Context, id int64) (model.Job, error)
+	GetImageVersion(ctx context.Context, id int64) (model.ImageVersion, error)
+	SetLastTagDigest(ctx context.Context, id int64, digest string) error
+}
+
+// Hub routes jobs to agent connections. *hub.Hub satisfies it.
+type Hub interface {
+	// Route picks the connection for svc among agents speaking at least
+	// protocol version minProtocol: model.ErrAgentOffline or
+	// model.ErrAgentTooOld when there is none.
+	Route(svc model.Service, minProtocol int) (model.Route, error)
+	// StartJob sends job.start over exactly the routed connection.
+	// model.ErrAgentOffline if that connection is gone or was replaced.
+	StartJob(r model.Route, jobID int64, start protocol.JobStart) error
+}
+
+// Publisher receives change signals. *events.Broker satisfies it.
+type Publisher interface{ Publish(t events.Topic) }
+
+// Service starts update jobs.
+type Service struct {
+	store    Store
+	hub      Hub
+	pub      Publisher
+	selfRepo string
+	log      *slog.Logger
+	now      func() time.Time
+}
+
+// New returns a Service. agentImage is the master's AGENT_IMAGE; services
+// whose image repo equals its repo are the updater itself and are refused.
+func New(store Store, hub Hub, pub Publisher, agentImage string, log *slog.Logger) *Service {
+	return &Service{
+		store: store, hub: hub, pub: pub, log: log, now: time.Now,
+		selfRepo: imageref.Parse(agentImage).Repo,
+	}
+}
+
+// target splits a service's actual ref into the repo and tag to deploy. A bare
+// image ("postgres", no tag and no digest) means "latest", as it does for
+// Docker. A ref with a digest but no tag follows no tag (ok is false).
+func target(actualRef string) (repo, tag string, ok bool) {
+	ref := imageref.Parse(actualRef)
+	switch {
+	case ref.Tag != "":
+		return ref.Repo, ref.Tag, true
+	case ref.Digest == "":
+		return ref.Repo, "latest", true
+	}
+	return ref.Repo, "", false
+}
+
+// FollowedRef returns "repo:tag" for the tag a service with this actual ref
+// follows (a bare image follows "latest"), or "" when it follows no tag.
+func FollowedRef(actualRef string) string {
+	repo, tag, ok := target(actualRef)
+	if !ok {
+		return ""
+	}
+	return repo + ":" + tag
+}
+
+// Updatable returns nil or a *model.NotUpdatableError. It looks only at the
+// service row, so the UI can call it for every page render. It answers for an
+// update to the service's own followed tag (the Update button).
+func (s *Service) Updatable(svc model.Service) error {
+	return s.updatable(svc, false)
+}
+
+// AutoAllowed returns nil or a *model.NotUpdatableError saying why the
+// service can never be updated automatically. Unlike Updatable it does not
+// refuse a service that follows no tag: a webhook that names a tag can still
+// update it.
+func (s *Service) AutoAllowed(svc model.Service) error {
+	return s.updatable(svc, true)
+}
+
+// updatable checks the service row. explicitRef means the caller supplies the
+// target ref, so a service that follows no tag is fine.
+func (s *Service) updatable(svc model.Service, explicitRef bool) error {
+	switch {
+	case svc.RemovedAt != nil:
+		return &model.NotUpdatableError{Reason: model.ReasonRemoved}
+	case svc.Self || imageref.Parse(svc.ActualRef).Repo == s.selfRepo:
+		return &model.NotUpdatableError{Reason: model.ReasonSelf}
+	case svc.Built:
+		return &model.NotUpdatableError{Reason: model.ReasonBuilt}
+	case svc.Kind == protocol.KindContainer:
+		return &model.NotUpdatableError{Reason: model.ReasonContainer}
+	}
+	if _, _, ok := target(svc.ActualRef); !ok && !explicitRef {
+		return &model.NotUpdatableError{Reason: model.ReasonNoTag}
+	}
+	return nil
+}
+
+// Restorable returns nil or a *model.NotUpdatableError saying why the service
+// cannot be restored to its desired version: it must have drifted, and the
+// desired version is deployed like an explicit ref, so a service that follows
+// no tag is fine. The service page calls it for every render.
+func (s *Service) Restorable(svc model.Service) error {
+	if svc.RemovedAt != nil {
+		return &model.NotUpdatableError{Reason: model.ReasonRemoved}
+	}
+	if !svc.Drifted() {
+		return &model.NotUpdatableError{Reason: model.ReasonNotDrifted}
+	}
+	if err := s.updatable(svc, true); err != nil {
+		return err
+	}
+	if imageref.Parse(svc.DesiredRef).Repo == s.selfRepo {
+		return &model.NotUpdatableError{Reason: model.ReasonSelf}
+	}
+	return nil
+}
+
+// RestoreDesired starts a manual job that deploys the drifted service's
+// desired ref again. The ref is read from the row, never from the caller. It
+// returns model.ErrNotFound for an unknown service and otherwise what Start
+// returns, with a *model.NotUpdatableError from Restorable if it cannot be
+// restored.
+//
+// When the service had drifted to another digest of the same tag, that digest
+// becomes last_tag_digest, so the poller does not deploy it again before the
+// registry shows a newer one (the same rule as for any rollback). It is not
+// written when last_tag_digest holds back a newer digest, that is when it is
+// set and differs from the digest the desired ref names.
+func (s *Service) RestoreDesired(ctx context.Context, serviceID int64, requestedBy string) (model.Job, error) {
+	svc, err := s.store.GetService(ctx, serviceID)
+	if err != nil {
+		return model.Job{}, err
+	}
+	if err := s.Restorable(svc); err != nil {
+		return model.Job{}, err
+	}
+	job, err := s.Start(ctx, svc, StartRequest{Trigger: model.TriggerManual, RequestedBy: requestedBy, Ref: svc.DesiredRef})
+	if err != nil {
+		return job, err
+	}
+	if d := driftedTagDigest(svc); d != "" && d != svc.LastTagDigest && holdsNothingNewer(svc) {
+		if err := s.store.SetLastTagDigest(ctx, svc.ID, d); err != nil {
+			s.log.Warn("last_tag_digest_not_set", "service_id", svc.ID, "error", err)
+		}
+	}
+	return job, nil
+}
+
+// holdsNothingNewer reports whether last_tag_digest may be overwritten by a
+// restore: it is empty or equals the digest the desired ref names. Anything
+// else is a digest the poller must keep holding back.
+func holdsNothingNewer(svc model.Service) bool {
+	return svc.LastTagDigest == "" || svc.LastTagDigest == imageref.Parse(svc.DesiredRef).Digest
+}
+
+// driftedTagDigest returns the digest a drifted service runs when it runs the
+// same tag as desired at another digest, else "".
+func driftedTagDigest(svc model.Service) string {
+	tag := FollowedRef(svc.DesiredRef)
+	if tag == "" || tag != FollowedRef(svc.ActualRef) {
+		return ""
+	}
+	d := imageref.Parse(svc.ActualRef).Digest
+	if d == "" {
+		d = svc.ActualDigest
+	}
+	if !imageref.IsDigest(d) || d == imageref.Parse(svc.DesiredRef).Digest {
+		return ""
+	}
+	return d
+}
+
+// StartRequest says who asks for an update and what to deploy.
+type StartRequest struct {
+	Trigger     model.JobTrigger
+	RequestedBy string // OIDC subject for manual jobs; "" otherwise
+	// Ref is the ref to deploy ("repo:tag", "repo:tag@digest" or
+	// "repo@digest"). Empty means the service's followed tag, derived from
+	// its actual ref.
+	Ref string
+}
+
+// Trigger starts a manual update of the service to its current tag. It
+// returns what Start returns, and model.ErrNotFound for an unknown service.
+func (s *Service) Trigger(ctx context.Context, serviceID int64, requestedBy string) (model.Job, error) {
+	svc, err := s.store.GetService(ctx, serviceID)
+	if err != nil {
+		return model.Job{}, err
+	}
+	return s.Start(ctx, svc, StartRequest{Trigger: model.TriggerManual, RequestedBy: requestedBy})
+}
+
+// TriggerVersion starts a manual update of the service to exactly the image
+// version versionID, which must be a version of the service's own repo; the
+// ref is built from the stored version, never taken from the caller. It
+// returns model.ErrNotFound for an unknown service, an unknown version or a
+// version of another repo, and otherwise what Start returns. A version
+// without a tag leaves the service following no tag.
+func (s *Service) TriggerVersion(ctx context.Context, serviceID, versionID int64, requestedBy string) (model.Job, error) {
+	svc, err := s.store.GetService(ctx, serviceID)
+	if err != nil {
+		return model.Job{}, err
+	}
+	v, err := s.store.GetImageVersion(ctx, versionID)
+	if err != nil {
+		return model.Job{}, err
+	}
+	if v.Repo != svc.ImageRepo {
+		return model.Job{}, model.ErrNotFound
+	}
+	return s.Start(ctx, svc, StartRequest{Trigger: model.TriggerManual, RequestedBy: requestedBy, Ref: v.Ref()})
+}
+
+// Start starts an update of svc. It returns the job, which may already be
+// failed ("agent offline", "agent too old for updates", or "agent too old to
+// deploy a pinned version" when the ref has a digest or no tag and the agent
+// speaks protocol version 2); a
+// *model.NotUpdatableError when the service cannot be updated (no job
+// created); the active job and model.ErrJobActive when one is queued or
+// running. Poll and webhook jobs are refused for a drifted service; only a
+// person starts one then.
+func (s *Service) Start(ctx context.Context, svc model.Service, req StartRequest) (model.Job, error) {
+	if req.Trigger != model.TriggerManual && svc.Drifted() {
+		return model.Job{}, &model.NotUpdatableError{Reason: model.ReasonDrifted}
+	}
+	if err := s.updatable(svc, req.Ref != ""); err != nil {
+		return model.Job{}, err
+	}
+	ref := req.Ref
+	if ref == "" {
+		ref = FollowedRef(svc.ActualRef) // updatable checked the tag
+	}
+
+	minProtocol := protocol.MinJobVersion
+	if p := imageref.Parse(ref); p.Digest != "" || p.Tag == "" {
+		minProtocol = protocol.MinPinnedRefVersion
+	}
+	r, routeErr := s.hub.Route(svc, minProtocol)
+	nj := model.NewJob{
+		ServiceID: svc.ID, FromRef: svc.ActualRef, ToRef: ref,
+		Trigger: req.Trigger, RequestedBy: req.RequestedBy, CreatedAt: s.now(),
+	}
+	if routeErr == nil {
+		nj.AgentID = &r.AgentID
+	}
+	job, err := s.store.CreateJob(ctx, nj)
+	if errors.Is(err, model.ErrAgentOffline) {
+		// The routed agent was revoked after Route. Record the attempt as a
+		// failed job that was never sent anywhere.
+		routeErr, nj.AgentID = err, nil
+		job, err = s.store.CreateJob(ctx, nj)
+	}
+	if err != nil {
+		return job, err
+	}
+
+	failure := routeErr
+	if failure == nil {
+		start := protocol.JobStart{Kind: svc.Kind, Ref: ref}
+		if svc.Kind == protocol.KindSwarm {
+			start.SwarmClusterID, start.SwarmService = svc.SwarmClusterID, svc.SwarmService
+		} else {
+			start.ComposeProject, start.ComposeService = svc.ComposeProject, svc.ComposeService
+		}
+		if err := s.hub.StartJob(r, job.ID, start); err != nil {
+			s.log.Warn("job_start_failed", "job_id", job.ID, "error", err)
+			failure = model.ErrAgentOffline
+		}
+	}
+	if failure != nil {
+		if _, err := s.store.FailJob(ctx, job.ID, failure.Error(), s.now()); err != nil {
+			return model.Job{}, err
+		}
+	}
+	s.pub.Publish(events.Jobs)
+	s.pub.Publish(events.JobTopic(job.ID))
+	return s.store.GetJob(ctx, job.ID)
+}

internal/master/jobs/jobs_test.go

@@ -0,0 +1,502 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/events"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+type fakeStore struct {
+	services map[int64]model.Service
+	jobs     map[int64]model.Job
+	created  []model.NewJob
+	active   *model.Job // returned with ErrJobActive when set
+	versions map[int64]model.ImageVersion
+
+	revokedAgents map[int64]bool   // CreateJob returns ErrAgentOffline for these
+	lastTag       map[int64]string // SetLastTagDigest calls: service ID to digest
+}
+
+func (f *fakeStore) GetService(_ context.Context, id int64) (model.Service, error) {
+	svc, ok := f.services[id]
+	if !ok {
+		return model.Service{}, model.ErrNotFound
+	}
+	return svc, nil
+}
+
+func (f *fakeStore) CreateJob(_ context.Context, nj model.NewJob) (model.Job, error) {
+	if f.active != nil {
+		return *f.active, model.ErrJobActive
+	}
+	if nj.AgentID != nil && f.revokedAgents[*nj.AgentID] {
+		return model.Job{}, model.ErrAgentOffline
+	}
+	f.created = append(f.created, nj)
+	j := model.Job{ID: int64(len(f.created)), ServiceID: nj.ServiceID, AgentID: nj.AgentID, FromRef: nj.FromRef,
+		ToRef: nj.ToRef, Trigger: nj.Trigger, RequestedBy: nj.RequestedBy, Status: model.JobQueued, CreatedAt: nj.CreatedAt}
+	f.jobs[j.ID] = j
+	return j, nil
+}
+
+func (f *fakeStore) FailJob(_ context.Context, id int64, msg string, at time.Time) (bool, error) {
+	j := f.jobs[id]
+	j.Status, j.Error, j.FinishedAt = model.JobFailed, msg, &at
+	f.jobs[id] = j
+	return true, nil
+}
+
+func (f *fakeStore) GetImageVersion(_ context.Context, id int64) (model.ImageVersion, error) {
+	v, ok := f.versions[id]
+	if !ok {
+		return model.ImageVersion{}, model.ErrNotFound
+	}
+	return v, nil
+}
+
+func (f *fakeStore) SetLastTagDigest(_ context.Context, id int64, digest string) error {
+	if f.lastTag == nil {
+		f.lastTag = map[int64]string{}
+	}
+	f.lastTag[id] = digest
+	return nil
+}
+
+func (f *fakeStore) GetJob(_ context.Context, id int64) (model.Job, error) { return f.jobs[id], nil }
+
+type startCall struct {
+	route model.Route
+	id    int64
+	start protocol.JobStart
+}
+
+type fakeHub struct {
+	route         model.Route
+	routeErr      error
+	startErr      error
+	starts        []startCall
+	agentProtocol int   // when set, Route fails with ErrAgentTooOld for a higher minProtocol
+	mins          []int // minProtocol of every Route call
+}
+
+func (h *fakeHub) Route(_ model.Service, minProtocol int) (model.Route, error) {
+	h.mins = append(h.mins, minProtocol)
+	if h.agentProtocol != 0 && minProtocol > h.agentProtocol {
+		if minProtocol >= protocol.MinPinnedRefVersion {
+			return model.Route{}, model.ErrAgentTooOldForPinned
+		}
+		return model.Route{}, model.ErrAgentTooOld
+	}
+	return h.route, h.routeErr
+}
+
+func (h *fakeHub) StartJob(r model.Route, id int64, s protocol.JobStart) error {
+	h.starts = append(h.starts, startCall{r, id, s})
+	return h.startErr
+}
+
+type fakePub struct{ topics []events.Topic }
+
+func (p *fakePub) Publish(t events.Topic) { p.topics = append(p.topics, t) }
+
+const selfImage = "gitea.shcizo.se/shcizo/package-updater:latest"
+
+var (
+	composeSvc = model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
+		AgentID: ptr(int64(7)), ActualRef: "gitea.shcizo.se/shcizo/slusk:1.4.2@sha256:d1"}
+	bareSvc = model.Service{ID: 3, Kind: protocol.KindCompose, ComposeProject: "db", ComposeService: "pg",
+		AgentID: ptr(int64(7)), ActualRef: "postgres"}
+	swarmSvc = model.Service{ID: 2, Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea",
+		ActualRef: "gitea/gitea:1.27.3@sha256:87a6"}
+)
+
+func ptr[T any](v T) *T { return &v }
+
+func newService(st *fakeStore, h *fakeHub) (*Service, *fakePub) {
+	pub := &fakePub{}
+	return New(st, h, pub, selfImage, slog.New(slog.NewTextHandler(io.Discard, nil))), pub
+}
+
+func TestUpdatable(t *testing.T) {
+	tests := []struct {
+		name   string
+		svc    model.Service
+		reason string
+	}{
+		{"removed", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", RemovedAt: ptr(time.Now())}, model.ReasonRemoved},
+		{"flagged self", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, model.ReasonSelf},
+		{"same repo as the agent image", model.Service{Kind: protocol.KindCompose,
+			ActualRef: "gitea.shcizo.se/shcizo/package-updater:1.2.0@sha256:abc"}, model.ReasonSelf},
+		{"built from source", model.Service{Kind: protocol.KindCompose, ActualRef: "slusk:1", Built: true}, model.ReasonBuilt},
+		{"container", model.Service{Kind: protocol.KindContainer, ActualRef: "redis:7"}, model.ReasonContainer},
+		{"digest without tag", model.Service{Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}, model.ReasonNoTag},
+		{"bare image means latest", model.Service{Kind: protocol.KindCompose, ActualRef: "postgres"}, ""},
+		{"bare image on a registry with port", model.Service{Kind: protocol.KindCompose, ActualRef: "localhost:5000/foo"}, ""},
+		{"tag", model.Service{Kind: protocol.KindCompose, ActualRef: "postgres:16"}, ""},
+		{"tag and digest", model.Service{Kind: protocol.KindCompose, ActualRef: "postgres:17.10@sha256:abc"}, ""},
+		{"registry with port and tag", model.Service{Kind: protocol.KindSwarm, ActualRef: "localhost:5000/foo:v1"}, ""},
+	}
+	s, _ := newService(&fakeStore{}, &fakeHub{})
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := s.Updatable(tt.svc)
+			if tt.reason == "" {
+				assert.NoError(t, err)
+				return
+			}
+			var nu *model.NotUpdatableError
+			require.True(t, errors.As(err, &nu))
+			assert.Equal(t, tt.reason, nu.Reason)
+		})
+	}
+}
+
+func TestTrigger(t *testing.T) {
+	ok := model.Route{AgentID: 7, ConnID: 70}
+	activeJob := model.Job{ID: 99, Status: model.JobRunning}
+
+	tests := []struct {
+		name       string
+		serviceID  int64
+		hub        *fakeHub
+		active     *model.Job
+		wantErr    error
+		wantStatus model.JobStatus
+		wantError  string
+		wantStart  *protocol.JobStart
+		wantCreate bool
+	}{
+		{name: "compose happy path", serviceID: 1, hub: &fakeHub{route: ok}, wantStatus: model.JobQueued, wantCreate: true,
+			wantStart: &protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "gitea.shcizo.se/shcizo/slusk:1.4.2"}},
+		{name: "swarm", serviceID: 2, hub: &fakeHub{route: ok}, wantStatus: model.JobQueued, wantCreate: true,
+			wantStart: &protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27.3"}},
+		{name: "bare image deploys latest", serviceID: 3, hub: &fakeHub{route: ok}, wantStatus: model.JobQueued, wantCreate: true,
+			wantStart: &protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "db", ComposeService: "pg", Ref: "postgres:latest"}},
+		{name: "active job", serviceID: 1, hub: &fakeHub{route: ok}, active: &activeJob, wantErr: model.ErrJobActive, wantStatus: model.JobRunning},
+		{name: "agent offline", serviceID: 1, hub: &fakeHub{routeErr: model.ErrAgentOffline}, wantStatus: model.JobFailed,
+			wantError: "agent offline", wantCreate: true},
+		{name: "agent too old", serviceID: 1, hub: &fakeHub{routeErr: model.ErrAgentTooOld}, wantStatus: model.JobFailed,
+			wantError: "agent too old for updates", wantCreate: true},
+		{name: "start fails", serviceID: 1, hub: &fakeHub{route: ok, startErr: model.ErrAgentOffline}, wantStatus: model.JobFailed,
+			wantError: "agent offline", wantCreate: true,
+			wantStart: &protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "gitea.shcizo.se/shcizo/slusk:1.4.2"}},
+		{name: "unknown service", serviceID: 5, hub: &fakeHub{route: ok}, wantErr: model.ErrNotFound},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			st := &fakeStore{
+				services: map[int64]model.Service{1: composeSvc, 2: swarmSvc, 3: bareSvc},
+				jobs:     map[int64]model.Job{99: activeJob},
+				active:   tt.active,
+			}
+			s, pub := newService(st, tt.hub)
+
+			job, err := s.Trigger(context.Background(), tt.serviceID, "sub-1")
+
+			if tt.wantErr != nil {
+				assert.ErrorIs(t, err, tt.wantErr)
+			} else {
+				require.NoError(t, err)
+			}
+			if tt.wantErr == nil || tt.active != nil {
+				assert.Equal(t, tt.wantStatus, job.Status)
+				assert.Equal(t, tt.wantError, job.Error)
+			}
+			if tt.wantStart == nil {
+				assert.Empty(t, tt.hub.starts)
+			} else {
+				require.Len(t, tt.hub.starts, 1)
+				assert.Equal(t, *tt.wantStart, tt.hub.starts[0].start)
+				assert.Equal(t, ok, tt.hub.starts[0].route)
+				assert.Equal(t, job.ID, tt.hub.starts[0].id)
+			}
+			if tt.wantCreate {
+				require.Len(t, st.created, 1)
+				assert.Equal(t, "sub-1", st.created[0].RequestedBy)
+				assert.Equal(t, model.TriggerManual, st.created[0].Trigger)
+				assert.Equal(t, st.services[tt.serviceID].ActualRef, st.created[0].FromRef)
+				assert.Contains(t, pub.topics, events.Jobs)
+				assert.Contains(t, pub.topics, events.JobTopic(job.ID))
+			} else {
+				assert.Empty(t, st.created)
+				assert.Empty(t, pub.topics)
+			}
+		})
+	}
+}
+
+// The routed agent can be revoked between Route and CreateJob; the store then
+// refuses the job. Trigger records a failed job that was never sent.
+func TestTrigger_AgentRevokedAfterRoute(t *testing.T) {
+	st := &fakeStore{
+		services:      map[int64]model.Service{1: composeSvc},
+		jobs:          map[int64]model.Job{},
+		revokedAgents: map[int64]bool{7: true},
+	}
+	h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
+	s, _ := newService(st, h)
+
+	job, err := s.Trigger(context.Background(), 1, "sub-1")
+
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, job.Status)
+	assert.Equal(t, "agent offline", job.Error)
+	assert.Nil(t, job.AgentID)
+	assert.Empty(t, h.starts)
+}
+
+func TestTrigger_NotUpdatableCreatesNoJob(t *testing.T) {
+	st := &fakeStore{
+		services: map[int64]model.Service{1: {ID: 1, Kind: protocol.KindContainer, ActualRef: "redis:7"}},
+		jobs:     map[int64]model.Job{},
+	}
+	h := &fakeHub{}
+	s, pub := newService(st, h)
+
+	_, err := s.Trigger(context.Background(), 1, "sub-1")
+
+	var nu *model.NotUpdatableError
+	require.True(t, errors.As(err, &nu))
+	assert.Empty(t, st.created)
+	assert.Empty(t, h.starts)
+	assert.Empty(t, pub.topics)
+}
+
+func TestStart_WebhookTriggerWithExplicitRef(t *testing.T) {
+	st := &fakeStore{services: map[int64]model.Service{1: composeSvc}, jobs: map[int64]model.Job{}}
+	h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
+	s, _ := newService(st, h)
+
+	job, err := s.Start(context.Background(), composeSvc, StartRequest{
+		Trigger: model.TriggerWebhook, Ref: "gitea.shcizo.se/shcizo/slusk:2.0.0"})
+
+	require.NoError(t, err)
+	assert.Equal(t, model.JobQueued, job.Status)
+	require.Len(t, st.created, 1)
+	assert.Equal(t, model.TriggerWebhook, st.created[0].Trigger)
+	assert.Empty(t, st.created[0].RequestedBy)
+	assert.Equal(t, "gitea.shcizo.se/shcizo/slusk:2.0.0", st.created[0].ToRef)
+	require.Len(t, h.starts, 1)
+	assert.Equal(t, "gitea.shcizo.se/shcizo/slusk:2.0.0", h.starts[0].start.Ref)
+}
+
+func TestStart_ExplicitRefBypassesOnlyTheNoTagRefusal(t *testing.T) {
+	tests := []struct {
+		name   string
+		svc    model.Service
+		reason string // "" means the job starts
+	}{
+		{"digest only", model.Service{Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}, ""},
+		{"removed", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", RemovedAt: ptr(time.Now())}, model.ReasonRemoved},
+		{"self", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, model.ReasonSelf},
+		{"built", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Built: true}, model.ReasonBuilt},
+		{"container", model.Service{Kind: protocol.KindContainer, ActualRef: "redis:7"}, model.ReasonContainer},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.svc.ID = 1
+			st := &fakeStore{services: map[int64]model.Service{1: tt.svc}, jobs: map[int64]model.Job{}}
+			h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
+			s, _ := newService(st, h)
+
+			_, err := s.Start(context.Background(), tt.svc, StartRequest{Trigger: model.TriggerWebhook, Ref: "x/y:2"})
+
+			if tt.reason == "" {
+				require.NoError(t, err)
+				assert.Len(t, st.created, 1)
+				return
+			}
+			var nu *model.NotUpdatableError
+			require.True(t, errors.As(err, &nu))
+			assert.Equal(t, tt.reason, nu.Reason)
+			assert.Empty(t, st.created)
+		})
+	}
+}
+
+func TestStart_WithoutRefStillRefusesNoTag(t *testing.T) {
+	svc := model.Service{ID: 1, Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}
+	s, _ := newService(&fakeStore{jobs: map[int64]model.Job{}}, &fakeHub{})
+
+	_, err := s.Start(context.Background(), svc, StartRequest{Trigger: model.TriggerWebhook})
+
+	var nu *model.NotUpdatableError
+	require.True(t, errors.As(err, &nu))
+	assert.Equal(t, model.ReasonNoTag, nu.Reason)
+}
+
+func TestAutoAllowed(t *testing.T) {
+	tests := []struct {
+		name   string
+		svc    model.Service
+		reason string
+	}{
+		{"removed", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", RemovedAt: ptr(time.Now())}, model.ReasonRemoved},
+		{"self", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, model.ReasonSelf},
+		{"agent image repo", model.Service{Kind: protocol.KindCompose, ActualRef: selfImage}, model.ReasonSelf},
+		{"built", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Built: true}, model.ReasonBuilt},
+		{"container", model.Service{Kind: protocol.KindContainer, ActualRef: "redis:7"}, model.ReasonContainer},
+		{"digest without tag is allowed", model.Service{Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}, ""},
+		{"tag", model.Service{Kind: protocol.KindCompose, ActualRef: "postgres:16"}, ""},
+	}
+	s, _ := newService(&fakeStore{}, &fakeHub{})
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := s.AutoAllowed(tt.svc)
+			if tt.reason == "" {
+				assert.NoError(t, err)
+				return
+			}
+			var nu *model.NotUpdatableError
+			require.True(t, errors.As(err, &nu))
+			assert.Equal(t, tt.reason, nu.Reason)
+		})
+	}
+}
+
+func TestFollowedRef(t *testing.T) {
+	assert.Equal(t, "postgres:16", FollowedRef("postgres:16@sha256:abc"))
+	assert.Equal(t, "postgres:latest", FollowedRef("postgres"))
+	assert.Equal(t, "localhost:5000/foo:latest", FollowedRef("localhost:5000/foo"))
+	assert.Empty(t, FollowedRef("redis@sha256:abc"))
+}
+
+func TestStart_RoutesWithTheMinimumProtocolOfTheRef(t *testing.T) {
+	tests := []struct {
+		name string
+		ref  string // "" means the followed tag
+		want int
+	}{
+		{"followed tag", "", protocol.MinJobVersion},
+		{"repo:tag", "x/y:2", protocol.MinJobVersion},
+		{"repo:tag@digest", "x/y:2@sha256:a", protocol.MinPinnedRefVersion},
+		{"repo@digest", "x/y@sha256:a", protocol.MinPinnedRefVersion},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			st := &fakeStore{services: map[int64]model.Service{1: composeSvc}, jobs: map[int64]model.Job{}}
+			h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
+			s, _ := newService(st, h)
+
+			_, err := s.Start(context.Background(), composeSvc, StartRequest{Trigger: model.TriggerManual, Ref: tt.ref})
+
+			require.NoError(t, err)
+			assert.Equal(t, []int{tt.want}, h.mins)
+		})
+	}
+}
+
+func TestTriggerVersion_Ref(t *testing.T) {
+	const repo = "gitea.shcizo.se/shcizo/slusk"
+	noTag := model.Service{ID: 4, Kind: protocol.KindCompose, ComposeProject: "n", ComposeService: "n",
+		AgentID: ptr(int64(7)), ImageRepo: repo, ActualRef: repo + "@sha256:d1"}
+	svc := composeSvc
+	svc.ImageRepo = repo
+	tests := []struct {
+		name    string
+		svc     model.Service
+		version model.ImageVersion
+		wantRef string
+		wantMin int
+	}{
+		{"tag and digest", svc, model.ImageVersion{ID: 10, Repo: repo, Tag: "1.3.0", Digest: "sha256:old"}, repo + ":1.3.0@sha256:old", protocol.MinPinnedRefVersion},
+		{"tag only", svc, model.ImageVersion{ID: 10, Repo: repo, Tag: "main"}, repo + ":main", protocol.MinJobVersion},
+		{"digest only", svc, model.ImageVersion{ID: 10, Repo: repo, Digest: "sha256:old"}, repo + "@sha256:old", protocol.MinPinnedRefVersion},
+		{"service that follows no tag", noTag, model.ImageVersion{ID: 10, Repo: repo, Tag: "1.3.0", Digest: "sha256:old"}, repo + ":1.3.0@sha256:old", protocol.MinPinnedRefVersion},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			st := &fakeStore{
+				services: map[int64]model.Service{tt.svc.ID: tt.svc},
+				jobs:     map[int64]model.Job{},
+				versions: map[int64]model.ImageVersion{10: tt.version},
+			}
+			h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
+			s, _ := newService(st, h)
+
+			job, err := s.TriggerVersion(context.Background(), tt.svc.ID, 10, "sub-1")
+
+			require.NoError(t, err)
+			assert.Equal(t, model.JobQueued, job.Status)
+			require.Len(t, st.created, 1)
+			assert.Equal(t, model.TriggerManual, st.created[0].Trigger)
+			assert.Equal(t, "sub-1", st.created[0].RequestedBy)
+			assert.Equal(t, tt.wantRef, st.created[0].ToRef)
+			require.Len(t, h.starts, 1)
+			assert.Equal(t, tt.wantRef, h.starts[0].start.Ref)
+			assert.Equal(t, []int{tt.wantMin}, h.mins)
+		})
+	}
+}
+
+func TestTriggerVersion_NotFound(t *testing.T) {
+	const repo = "gitea.shcizo.se/shcizo/slusk"
+	svc := composeSvc
+	svc.ImageRepo = repo
+	tests := []struct {
+		name      string
+		serviceID int64
+		versionID int64
+	}{
+		{"unknown service", 9, 10},
+		{"unknown version", 1, 99},
+		{"version of another repo", 1, 11},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			st := &fakeStore{
+				services: map[int64]model.Service{1: svc},
+				jobs:     map[int64]model.Job{},
+				versions: map[int64]model.ImageVersion{
+					10: {ID: 10, Repo: repo, Tag: "1"},
+					11: {ID: 11, Repo: "evil/other", Tag: "1"},
+				},
+			}
+			h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
+			s, pub := newService(st, h)
+
+			_, err := s.TriggerVersion(context.Background(), tt.serviceID, tt.versionID, "sub-1")
+
+			assert.ErrorIs(t, err, model.ErrNotFound)
+			assert.Empty(t, st.created, "no job")
+			assert.Empty(t, h.starts)
+			assert.Empty(t, pub.topics)
+		})
+	}
+}
+
+func TestTriggerVersion_RequiresPinnedProtocol(t *testing.T) {
+	const repo = "gitea.shcizo.se/shcizo/slusk"
+	svc := composeSvc
+	svc.ImageRepo = repo
+	st := &fakeStore{
+		services: map[int64]model.Service{1: svc},
+		jobs:     map[int64]model.Job{},
+		versions: map[int64]model.ImageVersion{
+			10: {ID: 10, Repo: repo, Tag: "1.3.0", Digest: "sha256:old"},
+			11: {ID: 11, Repo: repo, Tag: "main"},
+		},
+	}
+	h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}, agentProtocol: 2}
+	s, _ := newService(st, h)
+
+	pinned, err := s.TriggerVersion(context.Background(), 1, 10, "sub-1")
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, pinned.Status)
+	assert.Equal(t, "agent too old to deploy a pinned version (needs protocol 3)", pinned.Error)
+	assert.Empty(t, h.starts, "nothing sent to the old agent")
+
+	plain, err := s.TriggerVersion(context.Background(), 1, 11, "sub-1")
+	require.NoError(t, err)
+	assert.Equal(t, model.JobQueued, plain.Status)
+}

internal/master/model/job.go

@@ -0,0 +1,150 @@
+package model
+
+import (
+	"errors"
+	"time"
+)
+
+// Sentinel errors for jobs.
+var (
+	// ErrJobActive means the service already has a queued or running job.
+	// CreateJob and jobs.Service.Trigger return it together with that job.
+	ErrJobActive = errors.New("an update is already in progress for this service")
+	// ErrAgentOffline means no connected agent can run the job.
+	ErrAgentOffline = errors.New("agent offline")
+	// ErrAgentTooOld means the only agent that could run the job speaks a
+	// protocol version without jobs.
+	ErrAgentTooOld = errors.New("agent too old for updates")
+	// ErrAgentTooOldForPinned means the only agent that could run the job
+	// speaks protocol version 2, which cannot deploy a digest-pinned or
+	// tagless ref. errors.Is(err, ErrAgentTooOld) is true for it as well.
+	ErrAgentTooOldForPinned error = agentTooOldForPinned{}
+)
+
+type agentTooOldForPinned struct{}
+
+func (agentTooOldForPinned) Error() string {
+	return "agent too old to deploy a pinned version (needs protocol 3)"
+}
+
+func (agentTooOldForPinned) Is(target error) bool { return target == ErrAgentTooOld }
+
+// NotUpdatableError says why a service cannot be updated. Reason is shown
+// in the UI as is.
+type NotUpdatableError struct {
+	Reason string
+}
+
+func (e *NotUpdatableError) Error() string { return "not updatable: " + e.Reason }
+
+// Reasons a service cannot be updated.
+const (
+	ReasonSelf      = "updated manually outside the system"
+	ReasonContainer = "plain containers cannot be updated"
+	ReasonNoTag     = "follows no tag"
+	ReasonRemoved   = "service no longer exists"
+	ReasonBuilt     = "built from source (build:)"
+
+	ReasonDrifted    = "drifted: restore or accept the running version first"
+	ReasonNotDrifted = "runs its desired version"
+)
+
+// MergeRefusedError says why two services cannot be merged. Reason is shown
+// in the UI as is.
+type MergeRefusedError struct {
+	Reason string
+}
+
+func (e *MergeRefusedError) Error() string { return "cannot merge: " + e.Reason }
+
+// Reasons two services cannot be merged. ReasonSelf is reused for the
+// updater itself.
+const (
+	ReasonMergeSame      = "a service cannot be merged into itself"
+	ReasonMergeLiveOld   = "the old service still exists"
+	ReasonMergeGoneNew   = "the new service no longer exists"
+	ReasonMergeKind      = "only services of the same kind can be merged"
+	ReasonMergeContainer = "plain containers cannot be merged"
+)
+
+// JobStatus is the state of an update job.
+type JobStatus string
+
+// Job statuses. A job moves queued -> running -> succeeded|failed, or
+// straight from queued to failed.
+const (
+	JobQueued    JobStatus = "queued"
+	JobRunning   JobStatus = "running"
+	JobSucceeded JobStatus = "succeeded"
+	JobFailed    JobStatus = "failed"
+)
+
+// JobTrigger says what created a job. TriggerManual comes from the UI, TriggerWebhook from POST /update.
+type JobTrigger string
+
+// Job triggers.
+const (
+	TriggerManual  JobTrigger = "manual"
+	TriggerWebhook JobTrigger = "webhook"
+	TriggerPoll    JobTrigger = "poll"
+)
+
+// Job is one update of one service.
+type Job struct {
+	ID              int64
+	ServiceID       int64
+	ServiceName     string // Service.DisplayName() of the job's service
+	AgentID         *int64 // agent the job was sent to; nil if it never was
+	AgentName       string
+	FromRef         string
+	ToRef           string // "repo:tag" when created; "repo:tag@digest" after success
+	Trigger         JobTrigger
+	RequestedBy     string // OIDC subject for manual jobs
+	Status          JobStatus
+	Output          string // empty in ListJobs results
+	OutputTruncated bool
+	Error           string
+	CreatedAt       time.Time
+	StartedAt       *time.Time
+	FinishedAt      *time.Time
+}
+
+// Active reports whether the job is queued or running.
+func (j Job) Active() bool { return j.Status == JobQueued || j.Status == JobRunning }
+
+// NewJob is what CreateJob needs to insert a queued job.
+type NewJob struct {
+	ServiceID   int64
+	AgentID     *int64
+	FromRef     string
+	ToRef       string
+	Trigger     JobTrigger
+	RequestedBy string
+	CreatedAt   time.Time
+}
+
+// JobFinish is a job's outcome as reported by the agent.
+type JobFinish struct {
+	Status          JobStatus // JobSucceeded or JobFailed
+	Ref             string    // deployed "repo:tag@digest"; set on success
+	Error           string
+	Output          string
+	OutputTruncated bool
+	StartedAt       time.Time // agent clock
+	FinishedAt      time.Time // agent clock
+}
+
+// JobFilter selects jobs for ListJobs. ServiceID 0 means all services.
+// Limit 0 means 100.
+type JobFilter struct {
+	ServiceID int64
+	Limit     int
+}
+
+// Route is the hub's choice of agent connection for a job. ConnID identifies
+// the connection, so a job is never sent over a connection that replaced the
+// one chosen.
+type Route struct {
+	AgentID int64
+	ConnID  uint64
+}

internal/master/model/model.go

@@ -8,12 +8,15 @@ import (
 	"strings"
 	"time"
 
+	"github.com/shcizo/package-updater/internal/imageref"
 	"github.com/shcizo/package-updater/internal/protocol"
 )
 
 // Sentinel errors returned by storage.
 var (
 	ErrNotFound = errors.New("not found")
+	// ErrConflict means the row already exists, or that it changed meanwhile
+	// (AcceptRunning, MergeService).
 	ErrConflict = errors.New("already exists")
 	ErrRevoked  = errors.New("agent revoked")
 )
@@ -41,17 +44,141 @@ type Service struct {
 	ComposeProject string
 	ComposeService string
 	WorkingDir     string
+	ConfigFiles    string // raw config_files label, paths joined by ","
 	SwarmClusterID string
 	SwarmService   string
 	ContainerName  string
 	ImageRepo      string
+	DesiredRef     string // set by the last successful job; "" if never updated
 	ActualRef      string
 	ActualDigest   string
 	Status         string
 	Self           bool
-	UpdateMode     string
-	FirstSeenAt    time.Time
-	UpdatedAt      time.Time
+	Built          bool   // Compose service with a build: section; never updated
+	UpdateMode     string // ModeManual or ModeAuto
+	// AvailableRef is the newest ref a webhook or poll reported for this
+	// service; "" if none, or if a later job already deployed it.
+	AvailableRef    string
+	AvailableDigest string // digest of AvailableRef when known (poll); "" from the webhook
+	AvailableSource string // SourceWebhook or SourcePoll; "" when AvailableRef is empty
+	AvailableAt     *time.Time
+	LastPollError   string // last registry error for the followed ref; "" when fine
+	// LastTagDigest is the newest digest of the followed tag that was deployed
+	// or started: set when the poller starts an auto job, when it sees the service
+	// already running that digest, and when any plain tag job succeeds (the
+	// digest the agent pinned); "" if never. The poller does
+	// not start another job for the same digest, so a rollback sticks until the
+	// registry shows a different one.
+	LastTagDigest string
+	// AwaitingInventory is true from a successful job until the next inventory
+	// write for the row. Until then DesiredRef is newer than ActualRef without
+	// the service having drifted.
+	AwaitingInventory bool
+	FirstSeenAt       time.Time
+	UpdatedAt         time.Time
+	RemovedAt         *time.Time
+}
+
+// Update modes of a service.
+const (
+	ModeManual = "manual"
+	ModeAuto   = "auto"
+)
+
+// Sources of an available ref and of an ImageVersion.
+const (
+	SourceWebhook = "webhook"
+	SourcePoll    = "poll"
+	// SourceRunning is used by image_versions only: an agent reported the
+	// version as running.
+	SourceRunning = "running"
+)
+
+// Available is a newer ref reported for a service.
+type Available struct {
+	Ref    string
+	Digest string // "" when unknown (webhook)
+	Source string // SourceWebhook or SourcePoll
+	At     time.Time
+}
+
+// ImageVersion is one version of an image repo the master has seen. Tag and
+// Digest are "" when the version has no tag or its digest is unknown; they
+// are never both empty.
+type ImageVersion struct {
+	ID          int64
+	Repo        string
+	Tag         string
+	Digest      string
+	Source      string // SourceWebhook, SourcePoll or SourceRunning: the first source that saw it
+	FirstSeenAt time.Time
+}
+
+// Ref returns "repo:tag@digest", "repo:tag" or "repo@digest".
+func (v ImageVersion) Ref() string {
+	return imageref.Ref{Repo: v.Repo, Tag: v.Tag, Digest: v.Digest}.String()
+}
+
+// Drifted reports whether the service runs something other than the version
+// the last job deployed. A service the updater never touched, a removed one
+// and one whose inventory has not caught up with the last job do not drift.
+func (s Service) Drifted() bool {
+	return s.DesiredRef != "" && s.RemovedAt == nil && !s.AwaitingInventory &&
+		!RunsRef(s.DesiredRef, s.ActualRef, s.ActualDigest)
+}
+
+// RunsRef reports whether a service reporting actualRef and actualDigest runs
+// the desired ref. The digest decides when both sides know one, so
+// "repo:tag@D" is run by "repo:tag" reporting digest D, and by "repo@D".
+// Otherwise the repo (compared exactly, without registry normalisation) and
+// the followed tag must be equal.
+func RunsRef(desired, actualRef, actualDigest string) bool {
+	d, a := imageref.Parse(desired), imageref.Parse(actualRef)
+	if d.Repo != a.Repo {
+		return false
+	}
+	ad := a.Digest
+	if ad == "" {
+		ad = actualDigest
+	}
+	if d.Digest != "" && ad != "" {
+		return d.Digest == ad
+	}
+	return followedTag(d) == followedTag(a)
+}
+
+// followedTag is the tag a ref follows: a bare image follows "latest" and a
+// ref with a digest but no tag follows nothing.
+func followedTag(r imageref.Ref) string {
+	if r.Tag == "" && r.Digest == "" {
+		return "latest"
+	}
+	return r.Tag
+}
+
+// RunningRef is the ref to record as the desired one when the running version
+// is accepted: actualRef itself if it carries a digest, "repo:tag@digest" if
+// only actualDigest is known (a bare image is "latest"), else actualRef.
+func RunningRef(actualRef, actualDigest string) string {
+	r := imageref.Parse(actualRef)
+	if r.Digest != "" || !imageref.IsDigest(actualDigest) {
+		return actualRef
+	}
+	r.Tag, r.Digest = followedTag(r), actualDigest
+	return r.String()
+}
+
+// RunningRef returns the ref accepting the running version records as the
+// desired one; see the function RunningRef. Templates post it as the guard of
+// the accept form.
+func (s Service) RunningRef() string { return RunningRef(s.ActualRef, s.ActualDigest) }
+
+// Move says that the removed service FromID has probably moved to the live
+// service ToID on another agent. The names are the agents' own; a revoked
+// agent keeps its name.
+type Move struct {
+	FromID, ToID       int64
+	FromAgent, ToAgent string
 }
 
 // DisplayName is the human-facing name of the service.

internal/master/model/model_test.go

@@ -2,6 +2,7 @@ package model
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 
@@ -39,3 +40,116 @@ func TestDisplayName(t *testing.T) {
 		})
 	}
 }
+
+func TestImageVersionRef(t *testing.T) {
+	cases := []struct {
+		name string
+		v    ImageVersion
+		want string
+	}{
+		{"tag and digest", ImageVersion{Repo: "r/x", Tag: "1", Digest: "sha256:a"}, "r/x:1@sha256:a"},
+		{"tag only", ImageVersion{Repo: "r/x", Tag: "1"}, "r/x:1"},
+		{"digest only", ImageVersion{Repo: "r/x", Digest: "sha256:a"}, "r/x@sha256:a"},
+		{"registry port", ImageVersion{Repo: "localhost:5000/x", Tag: "1"}, "localhost:5000/x:1"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			assert.Equal(t, c.want, c.v.Ref())
+		})
+	}
+}
+
+const (
+	dA = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	dB = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+)
+
+func TestRunsRef(t *testing.T) {
+	cases := []struct {
+		name                          string
+		desired, actualRef, actualDig string
+		want                          bool
+	}{
+		{"same string", "repo:v1@" + dA, "repo:v1@" + dA, "", true},
+		{"tag and digest, actual reports it separately", "repo:tag@" + dA, "repo:tag", dA, true},
+		{"bare image reports latest digest", "postgres:latest@" + dA, "postgres", dA, true},
+		{"tagless actual with the same digest", "repo:tag@" + dA, "repo@" + dA, "", true},
+		{"same digest under another tag", "repo:v1@" + dA, "repo:v2@" + dA, "", true},
+		{"same digest, actual tag differs, digest reported", "repo:v1@" + dA, "repo:v2", dA, true},
+		{"same tag, actual digest unknown", "repo:tag@" + dA, "repo:tag", "", true},
+		{"desired without digest, same tag", "repo:tag", "repo:tag@" + dA, "", true},
+		{"bare desired, latest actual", "repo", "repo:latest", "", true},
+		{"registry with port", "localhost:5000/foo:v1@" + dA, "localhost:5000/foo:v1", dA, true},
+		{"registry with port, other digest", "localhost:5000/foo:v1@" + dA, "localhost:5000/foo:v1", dB, false},
+		{"same tag, other digest", "repo:tag@" + dA, "repo:tag", dB, false},
+		{"other digest in the ref", "repo:tag@" + dA, "repo:tag@" + dB, "", false},
+		{"other tag, digest unknown", "repo:v1@" + dA, "repo:v2", "", false},
+		{"tagless desired, tagged actual, digest unknown", "repo@" + dA, "repo:v1", "", false},
+		{"other repo", "repo:tag@" + dA, "other:tag", dA, false},
+		{"no docker.io normalisation", "nginx:1@" + dA, "docker.io/library/nginx:1", dA, false},
+		{"case-sensitive repo", "Repo:1@" + dA, "repo:1", dA, false},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			assert.Equal(t, c.want, RunsRef(c.desired, c.actualRef, c.actualDig))
+		})
+	}
+}
+
+func TestDrifted(t *testing.T) {
+	now := time.Now()
+	drifted := Service{DesiredRef: "repo:tag@" + dA, ActualRef: "repo:tag", ActualDigest: dB}
+	cases := []struct {
+		name   string
+		mutate func(*Service)
+		want   bool
+	}{
+		{"runs another digest", func(*Service) {}, true},
+		{"never updated", func(s *Service) { s.DesiredRef = "" }, false},
+		{"removed", func(s *Service) { s.RemovedAt = &now }, false},
+		{"awaiting inventory", func(s *Service) { s.AwaitingInventory = true }, false},
+		{"runs the desired digest", func(s *Service) { s.ActualDigest = dA }, false},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			svc := drifted
+			c.mutate(&svc)
+			assert.Equal(t, c.want, svc.Drifted())
+		})
+	}
+}
+
+func TestRunningRef(t *testing.T) {
+	cases := []struct {
+		name, actualRef, actualDigest, want string
+	}{
+		{"ref with digest is kept", "repo:v1@" + dB, dA, "repo:v1@" + dB},
+		{"tag plus reported digest", "repo:v1", dA, "repo:v1@" + dA},
+		{"bare image becomes latest", "postgres", dA, "postgres:latest@" + dA},
+		{"registry with port", "localhost:5000/foo:v1", dA, "localhost:5000/foo:v1@" + dA},
+		{"no digest known", "repo:v1", "", "repo:v1"},
+		{"malformed digest", "repo:v1", "sha256:abc", "repo:v1"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			assert.Equal(t, c.want, RunningRef(c.actualRef, c.actualDigest))
+		})
+	}
+}
+
+func TestJob_Active(t *testing.T) {
+	cases := []struct {
+		status JobStatus
+		want   bool
+	}{
+		{JobQueued, true},
+		{JobRunning, true},
+		{JobSucceeded, false},
+		{JobFailed, false},
+	}
+	for _, c := range cases {
+		t.Run(string(c.status), func(t *testing.T) {
+			assert.Equal(t, c.want, Job{Status: c.status}.Active())
+		})
+	}
+}

internal/master/poller/poller.go

@@ -0,0 +1,320 @@
+// Package poller checks the registry for new digests of the tags the
+// services follow. A new digest becomes the service's available version, and
+// a service in auto mode gets an update job.
+package poller
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/shcizo/package-updater/internal/master/events"
+	"github.com/shcizo/package-updater/internal/master/jobs"
+	"github.com/shcizo/package-updater/internal/master/model"
+)
+
+// lookupTimeout bounds one registry lookup.
+const lookupTimeout = 30 * time.Second
+
+// maxErrorBytes caps the registry error stored in services.last_poll_error.
+const maxErrorBytes = 500
+
+// Store is the storage the poller needs. *store.Store satisfies it.
+type Store interface {
+	ListServices(ctx context.Context) ([]model.Service, error)
+	GetService(ctx context.Context, id int64) (model.Service, error)
+	SetAvailable(ctx context.Context, id int64, a model.Available) error
+	SetLastTagDigest(ctx context.Context, id int64, digest string) error
+	SetPollError(ctx context.Context, ids []int64, msg string) error
+}
+
+// Jobs starts updates. *jobs.Service satisfies it.
+type Jobs interface {
+	Start(ctx context.Context, svc model.Service, req jobs.StartRequest) (model.Job, error)
+}
+
+// Registry looks up the digest a "repo:tag" ref points at. *registry.Client
+// satisfies it.
+type Registry interface {
+	Digest(ctx context.Context, ref string) (string, error)
+}
+
+// Publisher receives change signals. *events.Broker satisfies it.
+type Publisher interface{ Publish(t events.Topic) }
+
+// Config holds the poller settings.
+type Config struct {
+	Interval   time.Duration // time between the end of a cycle and the start of the next (POLL_INTERVAL)
+	FirstDelay time.Duration // wait before the first cycle; 1 minute in production
+	Workers    int           // registry lookups in flight at once; 4 in production
+}
+
+// Poller runs polling cycles.
+type Poller struct {
+	store Store
+	jobs  Jobs
+	reg   Registry
+	pub   Publisher
+	cfg   Config
+	log   *slog.Logger
+	now   func() time.Time
+}
+
+// New returns a Poller.
+func New(st Store, j Jobs, reg Registry, pub Publisher, cfg Config, log *slog.Logger) *Poller {
+	if cfg.Workers < 1 {
+		cfg.Workers = 1
+	}
+	return &Poller{store: st, jobs: j, reg: reg, pub: pub, cfg: cfg, log: log, now: time.Now}
+}
+
+// SkipReason says why svc is not polled: "follows no tag", "built from
+// source", "no registry digest", or "" when it is polled.
+func SkipReason(svc model.Service) string {
+	switch {
+	case jobs.FollowedRef(svc.ActualRef) == "":
+		return model.ReasonNoTag
+	case svc.Built:
+		return "built from source"
+	case svc.ActualDigest == "":
+		return "no registry digest"
+	}
+	return ""
+}
+
+// Run polls until ctx is done. The first cycle starts Config.FirstDelay after
+// the call and each following one Config.Interval after the previous cycle
+// ended, so cycles never overlap. It blocks.
+func (p *Poller) Run(ctx context.Context) {
+	timer := time.NewTimer(p.cfg.FirstDelay)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+		}
+		p.Cycle(ctx)
+		timer.Reset(p.cfg.Interval)
+	}
+}
+
+// lookup is the registry's answer for one followed ref.
+type lookup struct {
+	ref    string
+	digest string
+	err    error
+}
+
+// cycleStats is what one cycle did, for the poll_cycle log line.
+type cycleStats struct {
+	refs, ok, failed, suggested, jobs, skipped int
+	wrote                                      bool
+}
+
+// Cycle runs one poll over all services. It looks up every distinct followed
+// ref once, with at most Config.Workers lookups at a time, then records the
+// results. Writing results and starting jobs is not cut short by ctx ending
+// during the lookups, so a job that was marked is always started.
+func (p *Poller) Cycle(ctx context.Context) {
+	began := p.now()
+	svcs, err := p.store.ListServices(ctx)
+	if err != nil {
+		p.log.Error("poll_list_failed", "error", err)
+		return
+	}
+	var stats cycleStats
+	byRef := map[string][]model.Service{}
+	var refs []string
+	for _, svc := range svcs {
+		if svc.RemovedAt != nil {
+			continue
+		}
+		if SkipReason(svc) != "" {
+			stats.skipped++
+			continue
+		}
+		ref := jobs.FollowedRef(svc.ActualRef)
+		if _, seen := byRef[ref]; !seen {
+			refs = append(refs, ref)
+		}
+		byRef[ref] = append(byRef[ref], svc)
+	}
+	stats.refs = len(refs)
+
+	write := context.WithoutCancel(ctx)
+	for _, l := range p.lookupAll(ctx, refs) {
+		if l.err != nil && ctx.Err() != nil {
+			continue // cancelled mid-lookup: not the registry's fault
+		}
+		p.record(write, l, byRef[l.ref], &stats)
+	}
+	if stats.wrote {
+		p.pub.Publish(events.Services)
+	}
+	p.log.Info("poll_cycle", "refs", stats.refs, "ok", stats.ok, "failed", stats.failed,
+		"suggested", stats.suggested, "jobs", stats.jobs, "skipped", stats.skipped,
+		"duration_ms", p.now().Sub(began).Milliseconds())
+}
+
+// lookupAll asks the registry for every ref, Config.Workers at a time. It
+// returns the answers in ref order and stops starting new lookups once ctx
+// is done; refs never looked up are missing from the result.
+func (p *Poller) lookupAll(ctx context.Context, refs []string) []lookup {
+	results := make([]*lookup, len(refs))
+	next := make(chan int)
+	var wg sync.WaitGroup
+	for w := 0; w < p.cfg.Workers; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := range next {
+				lctx, cancel := context.WithTimeout(ctx, lookupTimeout)
+				d, err := p.reg.Digest(lctx, refs[i])
+				cancel()
+				results[i] = &lookup{ref: refs[i], digest: d, err: err}
+			}
+		}()
+	}
+feed:
+	for i := range refs {
+		select {
+		case next <- i:
+		case <-ctx.Done():
+			break feed
+		}
+	}
+	close(next)
+	wg.Wait()
+	var out []lookup
+	for _, r := range results {
+		if r != nil {
+			out = append(out, *r)
+		}
+	}
+	return out
+}
+
+// record stores one lookup result for the services that follow l.ref.
+func (p *Poller) record(ctx context.Context, l lookup, svcs []model.Service, stats *cycleStats) {
+	if l.err != nil {
+		stats.failed++
+		msg := truncate(l.err.Error())
+		p.log.Warn("poll_failed", "ref", l.ref, "error", msg)
+		p.setPollError(ctx, svcs, msg, stats, func(s model.Service) bool { return s.LastPollError != msg })
+		return
+	}
+	stats.ok++
+	p.setPollError(ctx, svcs, "", stats, func(s model.Service) bool { return s.LastPollError != "" })
+	for _, svc := range svcs {
+		if !p.suggest(ctx, svc, l, stats) {
+			continue
+		}
+		if autoWanted(svc, l.ref, l.digest) {
+			p.startAuto(ctx, svc.ID, l.ref, l.digest, stats)
+		}
+	}
+}
+
+// suggest marks l.digest as the available version of svc unless the service
+// already runs it or already has exactly that suggestion. A service that
+// runs l.digest also gets it as last_tag_digest. It returns false only when
+// writing the suggestion failed.
+func (p *Poller) suggest(ctx context.Context, svc model.Service, l lookup, stats *cycleStats) bool {
+	if svc.ActualDigest == l.digest && svc.LastTagDigest != l.digest {
+		// Seed last_tag_digest for a service that was not updated by a tag job
+		// (it predates the column or was updated by hand), so that a later
+		// rollback is not undone.
+		if err := p.store.SetLastTagDigest(ctx, svc.ID, l.digest); err != nil {
+			p.log.Error("poll_set_last_tag_digest_failed", "service_id", svc.ID, "error", err)
+		}
+	}
+	if svc.ActualDigest == l.digest || (svc.AvailableRef == l.ref && svc.AvailableDigest == l.digest) {
+		return true
+	}
+	err := p.store.SetAvailable(ctx, svc.ID, model.Available{Ref: l.ref, Digest: l.digest, Source: model.SourcePoll, At: p.now()})
+	if err != nil {
+		p.log.Error("poll_set_available_failed", "service_id", svc.ID, "ref", l.ref, "error", err)
+		return false
+	}
+	stats.wrote = true
+	stats.suggested++
+	return true
+}
+
+// autoWanted reports whether an update of svc to ref at digest should be
+// started automatically: the service is in auto mode, not removed, does not
+// already run that digest, was not already updated automatically to it
+// (otherwise a rollback to an older version would be undone at the next
+// cycle), and its last job did not already deploy it (desired_ref holds the
+// digest until the agent reports the new inventory).
+func autoWanted(svc model.Service, ref, digest string) bool {
+	return svc.RemovedAt == nil &&
+		svc.UpdateMode == model.ModeAuto &&
+		svc.ActualDigest != digest &&
+		svc.LastTagDigest != digest &&
+		svc.DesiredRef != ref+"@"+digest
+}
+
+// startAuto starts the update job of an auto service to ref and remembers
+// digest as services.last_tag_digest, so the same digest is not started
+// again, for instance after the user rolled back. It re-reads the service
+// first, because a cycle can run for a long time and the user may have
+// switched the service to manual meanwhile. A service that already has a
+// job or cannot be updated is not an error for the cycle, and last_tag_digest
+// is left alone so the next cycle tries again.
+func (p *Poller) startAuto(ctx context.Context, id int64, ref, digest string, stats *cycleStats) {
+	svc, err := p.store.GetService(ctx, id)
+	if err != nil {
+		p.log.Warn("poll_get_service_failed", "service_id", id, "error", err)
+		return
+	}
+	if !autoWanted(svc, ref, digest) {
+		return
+	}
+	_, err = p.jobs.Start(ctx, svc, jobs.StartRequest{Trigger: model.TriggerPoll, Ref: ref})
+	var notUpdatable *model.NotUpdatableError
+	switch {
+	case err == nil:
+		stats.jobs++
+		if err := p.store.SetLastTagDigest(ctx, id, digest); err != nil {
+			p.log.Error("poll_set_last_tag_digest_failed", "service_id", id, "error", err)
+		}
+	case errors.Is(err, model.ErrJobActive), errors.As(err, &notUpdatable):
+		p.log.Debug("poll_job_not_started", "service_id", id, "ref", ref, "reason", err.Error())
+	default:
+		p.log.Warn("poll_job_failed", "service_id", id, "ref", ref, "error", err)
+	}
+}
+
+// setPollError sets msg as last_poll_error on the services for which
+// changed returns true; with none, it writes nothing.
+func (p *Poller) setPollError(ctx context.Context, svcs []model.Service, msg string, stats *cycleStats, changed func(model.Service) bool) {
+	var ids []int64
+	for _, svc := range svcs {
+		if changed(svc) {
+			ids = append(ids, svc.ID)
+		}
+	}
+	if len(ids) == 0 {
+		return
+	}
+	if err := p.store.SetPollError(ctx, ids, msg); err != nil {
+		p.log.Error("poll_set_error_failed", "error", err)
+		return
+	}
+	stats.wrote = true
+}
+
+// truncate cuts msg to maxErrorBytes without leaving a broken UTF-8
+// sequence, and drops NUL bytes, which Postgres text cannot hold.
+func truncate(msg string) string {
+	msg = strings.ReplaceAll(msg, "\x00", "")
+	if len(msg) > maxErrorBytes {
+		msg = strings.ToValidUTF8(msg[:maxErrorBytes], "")
+	}
+	return msg
+}

internal/master/poller/poller_test.go

@@ -0,0 +1,651 @@
+package poller
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/events"
+	"github.com/shcizo/package-updater/internal/master/jobs"
+	"github.com/shcizo/package-updater/internal/master/model"
+)
+
+type availableCall struct {
+	id int64
+	a  model.Available
+}
+
+type pollErrorCall struct {
+	ids []int64
+	msg string
+}
+
+type lastTagCall struct {
+	id     int64
+	digest string
+}
+
+type fakeStore struct {
+	mu         sync.Mutex
+	services   []model.Service
+	fresh      map[int64]model.Service // what GetService returns instead of the listed row
+	lastTag    []lastTagCall
+	listErr    error
+	available  []availableCall
+	pollErrors []pollErrorCall
+	ctxErrs    []error // ctx.Err() seen by every write
+}
+
+func (f *fakeStore) ListServices(context.Context) ([]model.Service, error) {
+	return f.services, f.listErr
+}
+
+func (f *fakeStore) GetService(_ context.Context, id int64) (model.Service, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if s, ok := f.fresh[id]; ok {
+		return s, nil
+	}
+	for _, s := range f.services {
+		if s.ID == id {
+			return s, nil
+		}
+	}
+	return model.Service{}, model.ErrNotFound
+}
+
+// SetLastTagDigest records the call and, like the database, changes the
+// row the next cycle lists.
+func (f *fakeStore) SetLastTagDigest(_ context.Context, id int64, digest string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.lastTag = append(f.lastTag, lastTagCall{id, digest})
+	for i := range f.services {
+		if f.services[i].ID == id {
+			f.services[i].LastTagDigest = digest
+		}
+	}
+	return nil
+}
+
+func (f *fakeStore) SetAvailable(ctx context.Context, id int64, a model.Available) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.ctxErrs = append(f.ctxErrs, ctx.Err())
+	f.available = append(f.available, availableCall{id, a})
+	return nil
+}
+
+func (f *fakeStore) SetPollError(ctx context.Context, ids []int64, msg string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.ctxErrs = append(f.ctxErrs, ctx.Err())
+	f.pollErrors = append(f.pollErrors, pollErrorCall{ids, msg})
+	return nil
+}
+
+type startCall struct {
+	svcID int64
+	req   jobs.StartRequest
+}
+
+type fakeJobs struct {
+	mu      sync.Mutex
+	starts  []startCall
+	errFor  map[int64]error // by service ID
+	ctxErrs []error
+}
+
+func (f *fakeJobs) Start(ctx context.Context, svc model.Service, req jobs.StartRequest) (model.Job, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.ctxErrs = append(f.ctxErrs, ctx.Err())
+	f.starts = append(f.starts, startCall{svc.ID, req})
+	return model.Job{ID: int64(len(f.starts))}, f.errFor[svc.ID]
+}
+
+type fakeRegistry struct {
+	mu       sync.Mutex
+	digests  map[string]string // ref -> digest
+	errs     map[string]error  // ref -> error
+	calls    []string
+	inFlight int
+	maxSeen  int
+	delay    time.Duration
+	started  chan struct{} // receives a value whenever a lookup starts, if set
+	release  chan struct{} // lookups wait for it to be closed, if set
+	onResult func()        // called after a lookup computed its answer
+}
+
+func (f *fakeRegistry) Digest(ctx context.Context, ref string) (string, error) {
+	f.mu.Lock()
+	f.calls = append(f.calls, ref)
+	f.inFlight++
+	if f.inFlight > f.maxSeen {
+		f.maxSeen = f.inFlight
+	}
+	f.mu.Unlock()
+	defer func() {
+		f.mu.Lock()
+		f.inFlight--
+		f.mu.Unlock()
+	}()
+
+	if f.started != nil {
+		f.started <- struct{}{}
+	}
+	if f.release != nil {
+		select {
+		case <-f.release:
+		case <-ctx.Done():
+			return "", ctx.Err()
+		}
+	}
+	if f.delay > 0 {
+		time.Sleep(f.delay)
+	}
+	if f.onResult != nil {
+		f.onResult()
+	}
+	if err := f.errs[ref]; err != nil {
+		return "", err
+	}
+	return f.digests[ref], nil
+}
+
+func (f *fakeRegistry) sortedCalls() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	out := append([]string(nil), f.calls...)
+	sort.Strings(out)
+	return out
+}
+
+type fakePub struct {
+	mu     sync.Mutex
+	topics []events.Topic
+}
+
+func (p *fakePub) Publish(t events.Topic) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.topics = append(p.topics, t)
+}
+
+const (
+	repoA = "gitea.shcizo.se/shcizo/slusk"
+	repoB = "gitea/gitea"
+)
+
+// svc builds a polled service whose running version has digest sha256:old.
+func svc(id int64, actualRef string) model.Service {
+	return model.Service{ID: id, Kind: "compose", ActualRef: actualRef, ActualDigest: "sha256:old", UpdateMode: model.ModeManual}
+}
+
+func newPoller(st *fakeStore, j *fakeJobs, reg *fakeRegistry, cfg Config) (*Poller, *fakePub) {
+	pub := &fakePub{}
+	if cfg.Workers == 0 {
+		cfg.Workers = 4
+	}
+	p := New(st, j, reg, pub, cfg, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	p.now = func() time.Time { return time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) }
+	return p, pub
+}
+
+func TestSkipReason(t *testing.T) {
+	tests := []struct {
+		name string
+		svc  model.Service
+		want string
+	}{
+		{"tag and digest", model.Service{ActualRef: "r:1", ActualDigest: "sha256:a"}, ""},
+		{"bare image follows latest", model.Service{ActualRef: "r", ActualDigest: "sha256:a"}, ""},
+		{"digest only follows no tag", model.Service{ActualRef: "r@sha256:a", ActualDigest: "sha256:a"}, model.ReasonNoTag},
+		{"built", model.Service{ActualRef: "r:1", ActualDigest: "sha256:a", Built: true}, "built from source"},
+		{"no digest", model.Service{ActualRef: "r:1"}, "no registry digest"},
+		{"self and container are polled", model.Service{ActualRef: "r:1", ActualDigest: "sha256:a", Self: true, Kind: "container"}, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, SkipReason(tt.svc))
+		})
+	}
+}
+
+func TestOneHeadPerDistinctRef(t *testing.T) {
+	st := &fakeStore{services: []model.Service{
+		svc(1, repoA+":1.4"),
+		svc(2, repoA+":1.4@sha256:old"),
+		svc(3, repoB),
+		svc(4, repoB+":latest"),
+		svc(5, repoB+":1.27"),
+	}}
+	reg := &fakeRegistry{digests: map[string]string{}}
+	p, _ := newPoller(st, &fakeJobs{}, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	assert.Equal(t, []string{repoA + ":1.4", repoB + ":1.27", repoB + ":latest"}, reg.sortedCalls())
+}
+
+func TestSkipped(t *testing.T) {
+	removed := svc(4, "r:1")
+	removed.RemovedAt = ptr(time.Now())
+	st := &fakeStore{services: []model.Service{
+		{ID: 1, ActualRef: "r@sha256:a", ActualDigest: "sha256:a"},
+		{ID: 2, ActualRef: "r:1", ActualDigest: "sha256:a", Built: true},
+		{ID: 3, ActualRef: "r:1"},
+		removed,
+	}}
+	reg := &fakeRegistry{}
+	j := &fakeJobs{}
+	p, pub := newPoller(st, j, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	assert.Empty(t, reg.calls, "no lookups")
+	assert.Empty(t, st.available)
+	assert.Empty(t, st.pollErrors)
+	assert.Empty(t, j.starts)
+	assert.Empty(t, pub.topics)
+}
+
+func TestNewDigest(t *testing.T) {
+	ref := repoA + ":1.4"
+	tests := []struct {
+		name      string
+		mode      string
+		wantStart bool
+	}{
+		{"auto gets one poll job to repo:tag", model.ModeAuto, true},
+		{"manual is only marked", model.ModeManual, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := svc(1, ref+"@sha256:old")
+			s.UpdateMode = tt.mode
+			st := &fakeStore{services: []model.Service{s}}
+			reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
+			j := &fakeJobs{}
+			p, pub := newPoller(st, j, reg, Config{})
+
+			p.Cycle(context.Background())
+
+			require.Len(t, st.available, 1)
+			assert.Equal(t, int64(1), st.available[0].id)
+			assert.Equal(t, model.Available{Ref: ref, Digest: "sha256:new", Source: model.SourcePoll,
+				At: time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)}, st.available[0].a)
+			if tt.wantStart {
+				assert.Equal(t, []startCall{{1, jobs.StartRequest{Trigger: model.TriggerPoll, Ref: ref}}}, j.starts)
+				assert.Equal(t, []lastTagCall{{1, "sha256:new"}}, st.lastTag)
+			} else {
+				assert.Empty(t, j.starts)
+				assert.Empty(t, st.lastTag)
+			}
+			assert.Equal(t, []events.Topic{events.Services}, pub.topics)
+		})
+	}
+}
+
+func TestSameSuggestionIsIdempotent(t *testing.T) {
+	ref := repoA + ":1.4"
+	running := svc(1, ref+"@sha256:old")
+	running.UpdateMode = model.ModeAuto
+	running.ActualDigest = "sha256:new" // already on the registry's digest
+	suggested := svc(2, ref)
+	suggested.UpdateMode = model.ModeAuto
+	suggested.AvailableRef, suggested.AvailableDigest = ref, "sha256:new"
+	suggested.LastTagDigest = "sha256:new" // its job was started in an earlier cycle
+	staleSuggestion := svc(3, ref)
+	staleSuggestion.AvailableRef, staleSuggestion.AvailableDigest = ref, "sha256:older"
+	st := &fakeStore{services: []model.Service{running, suggested, staleSuggestion}}
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
+	j := &fakeJobs{}
+	p, _ := newPoller(st, j, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	require.Len(t, st.available, 1, "only the service with an older suggestion is updated")
+	assert.Equal(t, int64(3), st.available[0].id)
+	assert.Empty(t, j.starts, "no new job for an already suggested digest")
+}
+
+func TestStartErrorsDoNotAbortCycle(t *testing.T) {
+	ref := repoA + ":1.4"
+	var svcs []model.Service
+	for id := int64(1); id <= 3; id++ {
+		s := svc(id, ref)
+		s.UpdateMode = model.ModeAuto
+		svcs = append(svcs, s)
+	}
+	st := &fakeStore{services: svcs}
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
+	j := &fakeJobs{errFor: map[int64]error{
+		1: model.ErrJobActive,
+		2: &model.NotUpdatableError{Reason: model.ReasonSelf},
+		3: errors.New("database down"),
+	}}
+	p, _ := newPoller(st, j, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	assert.Len(t, st.available, 3)
+	assert.Len(t, j.starts, 3)
+	assert.Empty(t, st.lastTag, "a job that was not created is retried next cycle")
+}
+
+func TestRollbackOfAutoServiceIsNotUndone(t *testing.T) {
+	ref := repoA + ":main"
+	// Updated to sha256:new automatically, then rolled back to sha256:old by hand.
+	s := svc(1, ref+"@sha256:old")
+	s.UpdateMode = model.ModeAuto
+	s.LastTagDigest = "sha256:new"
+	s.DesiredRef = ref + "@sha256:old"
+	st := &fakeStore{services: []model.Service{s}}
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
+	j := &fakeJobs{}
+	p, _ := newPoller(st, j, reg, Config{})
+
+	p.Cycle(context.Background())
+	p.Cycle(context.Background())
+
+	assert.Empty(t, j.starts, "same registry digest: the rollback stays")
+	assert.Equal(t, "sha256:new", st.available[0].a.Digest, "the newer version is still offered")
+
+	reg.digests[ref] = "sha256:newer"
+	p.Cycle(context.Background())
+
+	assert.Equal(t, []startCall{{1, jobs.StartRequest{Trigger: model.TriggerPoll, Ref: ref}}}, j.starts, "a new digest moves it again")
+	assert.Equal(t, []lastTagCall{{1, "sha256:newer"}}, st.lastTag)
+}
+
+func TestActiveJobIsRetriedNextCycle(t *testing.T) {
+	ref := repoA + ":1.4"
+	s := svc(1, ref)
+	s.UpdateMode = model.ModeAuto
+	st := &fakeStore{services: []model.Service{s}}
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
+	j := &fakeJobs{errFor: map[int64]error{1: model.ErrJobActive}}
+	p, _ := newPoller(st, j, reg, Config{})
+
+	p.Cycle(context.Background())
+	require.Len(t, j.starts, 1)
+	assert.Empty(t, st.lastTag)
+
+	j.errFor = nil
+	p.Cycle(context.Background())
+
+	assert.Len(t, j.starts, 2)
+	assert.Equal(t, []lastTagCall{{1, "sha256:new"}}, st.lastTag)
+
+	p.Cycle(context.Background())
+	assert.Len(t, j.starts, 2, "started once the job exists")
+}
+
+func TestJobThatAlreadyDeployedTheDigestIsNotStartedAgain(t *testing.T) {
+	ref := repoA + ":1.4"
+	// FinishJob set desired_ref; the agent has not reported the new digest yet.
+	s := svc(1, ref+"@sha256:old")
+	s.UpdateMode = model.ModeAuto
+	s.DesiredRef = ref + "@sha256:new"
+	st := &fakeStore{services: []model.Service{s}}
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
+	j := &fakeJobs{}
+	p, _ := newPoller(st, j, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	assert.Empty(t, j.starts)
+}
+
+func TestServiceSwitchedToManualDuringTheCycleIsNotStarted(t *testing.T) {
+	ref := repoA + ":1.4"
+	s := svc(1, ref)
+	s.UpdateMode = model.ModeAuto
+	manual := s
+	manual.UpdateMode = model.ModeManual
+	st := &fakeStore{services: []model.Service{s}, fresh: map[int64]model.Service{1: manual}}
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
+	j := &fakeJobs{}
+	p, _ := newPoller(st, j, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	assert.Len(t, st.available, 1, "still suggested")
+	assert.Empty(t, j.starts)
+	assert.Empty(t, st.lastTag)
+}
+
+func TestErrorIsPerRef(t *testing.T) {
+	bad, good := repoA+":1.4", repoB+":1.27"
+	badOld := svc(1, bad)
+	badSame := svc(2, bad)
+	badSame.LastPollError = "head: 401 unauthorized"
+	goodSvc := svc(3, good)
+	st := &fakeStore{services: []model.Service{badOld, badSame, goodSvc}}
+	reg := &fakeRegistry{
+		errs:    map[string]error{bad: errors.New("head: 401 unauthorized")},
+		digests: map[string]string{good: "sha256:new"},
+	}
+	p, pub := newPoller(st, &fakeJobs{}, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	assert.Equal(t, []pollErrorCall{{[]int64{1}, "head: 401 unauthorized"}}, st.pollErrors,
+		"only the service whose stored error differs is written")
+	require.Len(t, st.available, 1, "the other ref is still processed")
+	assert.Equal(t, int64(3), st.available[0].id)
+	assert.Equal(t, []events.Topic{events.Services}, pub.topics)
+}
+
+func TestErrorTextIsTruncated(t *testing.T) {
+	ref := repoA + ":1.4"
+	st := &fakeStore{services: []model.Service{svc(1, ref)}}
+	reg := &fakeRegistry{errs: map[string]error{ref: errors.New(strings.Repeat("é", 400) + "\x00")}}
+	p, _ := newPoller(st, &fakeJobs{}, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	require.Len(t, st.pollErrors, 1)
+	msg := st.pollErrors[0].msg
+	assert.LessOrEqual(t, len(msg), maxErrorBytes)
+	assert.Equal(t, strings.Repeat("é", 250), msg, "cut on a character boundary, NUL removed")
+}
+
+func TestErrorClearedOnSuccess(t *testing.T) {
+	ref := repoA + ":1.4"
+	failing := svc(1, ref)
+	failing.LastPollError = "head: boom"
+	fine := svc(2, ref)
+	failing.ActualDigest, fine.ActualDigest = "sha256:new", "sha256:new"
+	st := &fakeStore{services: []model.Service{failing, fine}}
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
+	p, pub := newPoller(st, &fakeJobs{}, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	assert.Equal(t, []pollErrorCall{{[]int64{1}, ""}}, st.pollErrors)
+	assert.Empty(t, st.available)
+	assert.Equal(t, []events.Topic{events.Services}, pub.topics)
+}
+
+func TestNothingWrittenNothingPublished(t *testing.T) {
+	ref := repoA + ":1.4"
+	s := svc(1, ref)
+	s.ActualDigest = "sha256:new"
+	st := &fakeStore{services: []model.Service{s}}
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
+	p, pub := newPoller(st, &fakeJobs{}, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	assert.Empty(t, st.pollErrors)
+	assert.Empty(t, st.available)
+	assert.Empty(t, pub.topics)
+}
+
+func TestListErrorEndsTheCycle(t *testing.T) {
+	st := &fakeStore{listErr: errors.New("db down")}
+	reg := &fakeRegistry{}
+	p, _ := newPoller(st, &fakeJobs{}, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	assert.Empty(t, reg.calls)
+}
+
+func TestConcurrencyLimit(t *testing.T) {
+	const workers = 3
+	var svcs []model.Service
+	for i := 0; i < 8; i++ {
+		svcs = append(svcs, svc(int64(i+1), fmt.Sprintf("%s%d:1", repoA, i)))
+	}
+	st := &fakeStore{services: svcs}
+	reg := &fakeRegistry{release: make(chan struct{}), started: make(chan struct{}, 8)}
+	p, _ := newPoller(st, &fakeJobs{}, reg, Config{Workers: workers})
+	done := make(chan struct{})
+	go func() { p.Cycle(context.Background()); close(done) }()
+
+	for i := 0; i < workers; i++ {
+		select {
+		case <-reg.started:
+		case <-time.After(2 * time.Second):
+			t.Fatalf("only %d lookups started, want %d", i, workers)
+		}
+	}
+	select {
+	case <-reg.started:
+		t.Fatal("more lookups in flight than Workers")
+	case <-time.After(100 * time.Millisecond):
+	}
+	close(reg.release)
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Cycle did not finish")
+	}
+	assert.Len(t, reg.calls, 8)
+}
+
+func TestCycleStopsOnCancel(t *testing.T) {
+	var svcs []model.Service
+	for i := 0; i < 6; i++ {
+		svcs = append(svcs, svc(int64(i+1), fmt.Sprintf("%s%d:1", repoA, i)))
+	}
+	st := &fakeStore{services: svcs}
+	reg := &fakeRegistry{release: make(chan struct{}), started: make(chan struct{}, 6)}
+	p, pub := newPoller(st, &fakeJobs{}, reg, Config{Workers: 2})
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { p.Cycle(ctx); close(done) }()
+	<-reg.started
+	<-reg.started
+
+	cancel()
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Cycle did not return after cancel")
+	}
+	assert.Len(t, reg.calls, 2, "no new lookups after cancel")
+	assert.Empty(t, st.pollErrors, "cancelled lookups are not recorded as registry errors")
+	assert.Empty(t, st.available)
+	assert.Empty(t, pub.topics)
+}
+
+func TestMarkedJobsAreStartedEvenIfCancelledDuringTheCycle(t *testing.T) {
+	ref := repoA + ":1.4"
+	s := svc(1, ref)
+	s.UpdateMode = model.ModeAuto
+	st := &fakeStore{services: []model.Service{s}}
+	ctx, cancel := context.WithCancel(context.Background())
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}, onResult: cancel}
+	j := &fakeJobs{}
+	p, _ := newPoller(st, j, reg, Config{})
+
+	p.Cycle(ctx)
+
+	require.Len(t, st.available, 1)
+	require.Len(t, j.starts, 1)
+	assert.NoError(t, st.ctxErrs[0], "writes do not see the cancelled context")
+	assert.NoError(t, j.ctxErrs[0])
+}
+
+func TestRun(t *testing.T) {
+	ref := repoA + ":1.4"
+	st := &fakeStore{services: []model.Service{svc(1, ref)}}
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:old"}, delay: 30 * time.Millisecond}
+	p, _ := newPoller(st, &fakeJobs{}, reg, Config{Interval: 5 * time.Millisecond, FirstDelay: 60 * time.Millisecond, Workers: 1})
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	began := time.Now()
+	go func() { p.Run(ctx); close(done) }()
+
+	require.Eventually(t, func() bool {
+		reg.mu.Lock()
+		defer reg.mu.Unlock()
+		return len(reg.calls) >= 3
+	}, 5*time.Second, 5*time.Millisecond)
+	cancel()
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Run did not return after cancel")
+	}
+	assert.GreaterOrEqual(t, time.Since(began), 60*time.Millisecond+2*30*time.Millisecond, "first cycle waits FirstDelay")
+	assert.Equal(t, 1, reg.maxSeen, "cycles never overlap")
+}
+
+func TestRunReturnsDuringFirstDelay(t *testing.T) {
+	p, _ := newPoller(&fakeStore{}, &fakeJobs{}, &fakeRegistry{}, Config{Interval: time.Hour, FirstDelay: time.Hour})
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { p.Run(ctx); close(done) }()
+
+	cancel()
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Run did not return after cancel")
+	}
+}
+
+func ptr[T any](v T) *T { return &v }
+
+func TestSeedsTheTagDigestOfARunningService(t *testing.T) {
+	ref := repoA + ":main"
+	s := svc(1, ref+"@sha256:new")
+	s.UpdateMode = model.ModeAuto
+	s.ActualDigest = "sha256:new" // updated outside the updater: last_tag_digest is empty
+	st := &fakeStore{services: []model.Service{s}}
+	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
+	j := &fakeJobs{}
+	p, _ := newPoller(st, j, reg, Config{})
+
+	p.Cycle(context.Background())
+
+	assert.Equal(t, []lastTagCall{{1, "sha256:new"}}, st.lastTag)
+	assert.Empty(t, j.starts)
+
+	// Rolled back by hand to the older digest.
+	st.services[0].ActualRef, st.services[0].ActualDigest = ref+"@sha256:old", "sha256:old"
+	st.services[0].DesiredRef = ref + "@sha256:old"
+	p.Cycle(context.Background())
+
+	assert.Empty(t, j.starts, "the rollback stays")
+	assert.Len(t, st.lastTag, 1, "nothing more to seed")
+}

internal/master/registry/registry.go

@@ -0,0 +1,114 @@
+// Package registry looks up the digest a tag points at in a container
+// registry, using HEAD requests only.
+package registry
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
+
+	"github.com/shcizo/package-updater/internal/imageref"
+)
+
+// Client looks up tag digests with HEAD requests only. Docker Hub counts a
+// GET of a manifest as a pull against the rate limit and a HEAD as nothing,
+// so Client never issues a manifest GET.
+type Client struct {
+	keychain  authn.Keychain
+	userAgent string
+	log       *slog.Logger
+}
+
+// New returns a Client that finds credentials the way the Docker CLI does:
+// in $DOCKER_CONFIG/config.json, else ~/.docker/config.json. Registries
+// without an entry are accessed anonymously. userAgent is added to the
+// User-Agent header of every request. log receives the full error of a
+// failed lookup, which Digest itself returns only in a sanitised form.
+func New(userAgent string, log *slog.Logger) *Client {
+	return &Client{keychain: authn.DefaultKeychain, userAgent: userAgent, log: log}
+}
+
+// Digest returns the digest that "repo:tag" points at, as the registry
+// reports it: for a multi-arch image that is the digest of the index, the
+// same one `docker inspect` shows in RepoDigests. A digest suffix in ref is
+// ignored. It sends HEAD /v2/<repo>/manifests/<tag> and nothing else to the
+// manifest endpoint; it does not fall back to GET when the registry omits
+// the digest header.
+func (c *Client) Digest(ctx context.Context, ref string) (string, error) {
+	p := imageref.Parse(ref)
+	if p.Tag == "" {
+		return "", fmt.Errorf("image %q has no tag", ref)
+	}
+	r, err := name.ParseReference(p.Repo + ":" + p.Tag)
+	if err != nil {
+		return "", fmt.Errorf("parse %s: %w", ref, err)
+	}
+	// Credentials are resolved here, not by ggcr, so that a broken credential
+	// helper keeps its own message: that text comes from the local config.
+	auth, err := c.keychain.Resolve(r.Context())
+	if err != nil {
+		return "", fmt.Errorf("lookup %s: %w", r, err)
+	}
+	desc, err := remote.Head(r,
+		remote.WithContext(ctx),
+		remote.WithAuth(auth),
+		remote.WithUserAgent(c.userAgent))
+	if err != nil {
+		var terr *transport.Error
+		if errors.As(err, &terr) {
+			return "", fmt.Errorf("lookup %s: %s", r, describe(terr))
+		}
+		c.log.Warn("registry_lookup_failed", "ref", ref, "error", err)
+		return "", fmt.Errorf("lookup %s: %s", r, classify(err))
+	}
+	return desc.Digest.String(), nil
+}
+
+// classify returns a fixed text for an error that is not a registry answer.
+// ggcr copies registry-controlled text (token responses, authentication
+// challenges, redirect URLs, certificate names) into such errors, so none of
+// it is passed on.
+func classify(err error) string {
+	var netErr net.Error
+	var urlErr *url.Error
+	switch {
+	case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
+		return "timeout"
+	case errors.As(err, &netErr), errors.As(err, &urlErr):
+		return "cannot reach registry"
+	}
+	return "lookup failed"
+}
+
+// errorCodePattern matches the registry error codes of the distribution
+// spec ("MANIFEST_UNKNOWN"). Anything else is dropped from error text.
+var errorCodePattern = regexp.MustCompile(`^[A-Z_]{1,40}$`)
+
+// describe returns the HTTP status and the structured error codes of a
+// registry error, without the messages, details or raw body that ggcr copies
+// into its error text. The registry host comes from an image name, so
+// whatever it answers must not end up in the UI.
+func describe(e *transport.Error) string {
+	msg := fmt.Sprintf("registry answered %d %s", e.StatusCode, http.StatusText(e.StatusCode))
+	var codes []string
+	for _, d := range e.Errors {
+		if errorCodePattern.MatchString(string(d.Code)) {
+			codes = append(codes, string(d.Code))
+		}
+	}
+	if len(codes) > 0 {
+		msg += " (" + strings.Join(codes, ", ") + ")"
+	}
+	return msg
+}

internal/master/registry/registry_test.go

@@ -0,0 +1,336 @@
+package registry
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	ggcrregistry "github.com/google/go-containerregistry/pkg/registry"
+	"github.com/google/go-containerregistry/pkg/v1/random"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var discardLog = slog.New(slog.NewTextHandler(io.Discard, nil))
+
+func newClient() *Client { return New("test", discardLog) }
+
+// request is one request the test registry received.
+type request struct{ method, path string }
+
+// testRegistry is an in-memory registry that records every request.
+type testRegistry struct {
+	srv  *httptest.Server
+	host string // "127.0.0.1:port", the registry part of a ref
+
+	mu       sync.Mutex
+	requests []request
+}
+
+// newTestRegistry starts a registry. When user is set every request needs
+// those basic-auth credentials.
+func newTestRegistry(t *testing.T, user, pass string) *testRegistry {
+	t.Helper()
+	backend := ggcrregistry.New(ggcrregistry.Logger(log.New(io.Discard, "", 0)))
+	tr := &testRegistry{}
+	tr.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tr.mu.Lock()
+		tr.requests = append(tr.requests, request{r.Method, r.URL.Path})
+		tr.mu.Unlock()
+		if user != "" {
+			if u, p, ok := r.BasicAuth(); !ok || u != user || p != pass {
+				w.Header().Set("WWW-Authenticate", `Basic realm="test"`)
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+		}
+		backend.ServeHTTP(w, r)
+	}))
+	t.Cleanup(tr.srv.Close)
+	tr.host = strings.TrimPrefix(tr.srv.URL, "http://")
+	return tr
+}
+
+// pushIndex stores a random multi-arch index as repo:tag and returns its
+// digest. user and pass are used when the registry requires them.
+func (tr *testRegistry) pushIndex(t *testing.T, repo, tag string, opts ...remote.Option) string {
+	t.Helper()
+	idx, err := random.Index(64, 1, 2)
+	require.NoError(t, err)
+	ref, err := name.ParseReference(tr.host + "/" + repo + ":" + tag)
+	require.NoError(t, err)
+	require.NoError(t, remote.WriteIndex(ref, idx, opts...))
+	d, err := idx.Digest()
+	require.NoError(t, err)
+	return d.String()
+}
+
+// manifestRequests returns the requests that touched a manifest.
+func (tr *testRegistry) manifestRequests() []request {
+	tr.mu.Lock()
+	defer tr.mu.Unlock()
+	var out []request
+	for _, r := range tr.requests {
+		if strings.Contains(r.path, "/manifests/") {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
+// reset forgets the requests received so far.
+func (tr *testRegistry) reset() {
+	tr.mu.Lock()
+	defer tr.mu.Unlock()
+	tr.requests = nil
+}
+
+// dockerConfig points DOCKER_CONFIG at a fresh directory holding config.json
+// with the given content; "" leaves the directory empty.
+func dockerConfig(t *testing.T, configJSON string) {
+	t.Helper()
+	dir := t.TempDir()
+	if configJSON != "" {
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(configJSON), 0o600))
+	}
+	t.Setenv("DOCKER_CONFIG", dir)
+}
+
+func TestDigestOfIndexIsIndexDigest(t *testing.T) {
+	dockerConfig(t, "")
+	reg := newTestRegistry(t, "", "")
+	want := reg.pushIndex(t, "team/app", "1.2")
+
+	got, err := newClient().Digest(context.Background(), reg.host+"/team/app:1.2")
+
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestDigestIgnoresDigestSuffix(t *testing.T) {
+	dockerConfig(t, "")
+	reg := newTestRegistry(t, "", "")
+	want := reg.pushIndex(t, "team/app", "1.2")
+
+	got, err := newClient().Digest(context.Background(), reg.host+"/team/app:1.2@sha256:"+strings.Repeat("0", 64))
+
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestDigestUsesHeadOnly(t *testing.T) {
+	dockerConfig(t, "")
+	reg := newTestRegistry(t, "", "")
+	reg.pushIndex(t, "team/app", "1.2")
+	reg.reset()
+
+	_, err := newClient().Digest(context.Background(), reg.host+"/team/app:1.2")
+
+	require.NoError(t, err)
+	assert.Equal(t, []request{{http.MethodHead, "/v2/team/app/manifests/1.2"}}, reg.manifestRequests())
+	for _, r := range reg.requests {
+		if !strings.Contains(r.path, "/manifests/") {
+			assert.Equal(t, request{http.MethodGet, "/v2/"}, r, "the only other request is the API ping")
+		}
+	}
+}
+
+func TestDigestSendsUserAgent(t *testing.T) {
+	dockerConfig(t, "")
+	var ua string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ua = r.Header.Get("User-Agent")
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	t.Cleanup(srv.Close)
+
+	_, _ = New("package-updater-test/1", discardLog).Digest(context.Background(), strings.TrimPrefix(srv.URL, "http://")+"/a/b:1")
+
+	assert.Contains(t, ua, "package-updater-test/1")
+}
+
+func TestCredentialsFromDockerConfig(t *testing.T) {
+	reg := newTestRegistry(t, "bot", "s3cret")
+	want := reg.pushIndex(t, "team/app", "1.2", remote.WithAuth(&authn.Basic{Username: "bot", Password: "s3cret"}))
+	auth := base64.StdEncoding.EncodeToString([]byte("bot:s3cret"))
+	dockerConfig(t, `{"auths":{"`+reg.host+`":{"auth":"`+auth+`"}}}`)
+
+	got, err := newClient().Digest(context.Background(), reg.host+"/team/app:1.2")
+
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestWrongCredentialsFail(t *testing.T) {
+	reg := newTestRegistry(t, "bot", "s3cret")
+	reg.pushIndex(t, "team/app", "1.2", remote.WithAuth(&authn.Basic{Username: "bot", Password: "s3cret"}))
+	auth := base64.StdEncoding.EncodeToString([]byte("bot:wrong"))
+	dockerConfig(t, `{"auths":{"`+reg.host+`":{"auth":"`+auth+`"}}}`)
+
+	_, err := newClient().Digest(context.Background(), reg.host+"/team/app:1.2")
+
+	require.Error(t, err)
+	assert.NotContains(t, err.Error(), "wrong", "credentials do not leak into errors")
+	assert.NotContains(t, err.Error(), auth)
+}
+
+func TestAnonymousWithoutConfig(t *testing.T) {
+	dockerConfig(t, "")
+	open := newTestRegistry(t, "", "")
+	want := open.pushIndex(t, "team/app", "1.2")
+	protected := newTestRegistry(t, "bot", "s3cret")
+
+	got, err := newClient().Digest(context.Background(), open.host+"/team/app:1.2")
+	require.NoError(t, err)
+	assert.Equal(t, want, got, "anonymous access works without a config file")
+
+	_, err = newClient().Digest(context.Background(), protected.host+"/team/app:1.2")
+	assert.Error(t, err, "a registry that needs credentials fails without them")
+}
+
+func TestUnknownTagIsAnError(t *testing.T) {
+	dockerConfig(t, "")
+	reg := newTestRegistry(t, "", "")
+	reg.pushIndex(t, "team/app", "1.2")
+
+	_, err := newClient().Digest(context.Background(), reg.host+"/team/app:nope")
+
+	assert.Error(t, err)
+}
+
+func TestRefWithoutTagIsAnError(t *testing.T) {
+	_, err := newClient().Digest(context.Background(), "team/app@sha256:"+strings.Repeat("a", 64))
+
+	assert.ErrorContains(t, err, "has no tag")
+}
+
+// A config.json whose credsStore names a helper that is not installed makes
+// the lookup fail for every registry, including ones that would work
+// anonymously. The master container has no credential helpers, so only
+// inline "auths" entries are supported there.
+func TestCredsStoreWithoutHelperFailsTheLookup(t *testing.T) {
+	reg := newTestRegistry(t, "", "")
+	reg.pushIndex(t, "team/app", "1.2")
+	dockerConfig(t, `{"credsStore":"no-such-helper-for-package-updater-tests"}`)
+
+	_, err := newClient().Digest(context.Background(), reg.host+"/team/app:1.2")
+
+	assert.ErrorContains(t, err, "error getting credentials")
+	assert.ErrorContains(t, err, "docker-credential-no-such-helper-for-package-updater-tests")
+}
+
+// A registry that leaves out Docker-Content-Digest must not make the client
+// retry with a manifest download, which Docker Hub counts as a pull.
+func TestMissingDigestHeaderDoesNotFallBackToDownload(t *testing.T) {
+	dockerConfig(t, "")
+	var mu sync.Mutex
+	var methods []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.URL.Path, "/manifests/") {
+			return
+		}
+		mu.Lock()
+		methods = append(methods, r.Method)
+		mu.Unlock()
+		w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
+		w.Header().Set("Content-Length", "10")
+	}))
+	t.Cleanup(srv.Close)
+
+	_, err := newClient().Digest(context.Background(), strings.TrimPrefix(srv.URL, "http://")+"/a/b:1")
+
+	assert.Error(t, err)
+	assert.Equal(t, []string{http.MethodHead}, methods)
+}
+
+// ggcr copies up to 64 KiB of the registry's response into its errors. The
+// registry host comes from an image name, so that text must not reach the
+// stored poll error.
+func TestRegistryErrorBodyDoesNotLeak(t *testing.T) {
+	dockerConfig(t, "")
+	tests := []struct {
+		name string
+		body string
+		want string
+	}{
+		{"raw body", "internal-secret-body", "registry answered 400 Bad Request"},
+		{"structured errors", `{"errors":[{"code":"MANIFEST_UNKNOWN","message":"internal-secret-body","detail":"internal-secret-body"},{"code":"internal-secret-body"}]}`,
+			"registry answered 400 Bad Request (MANIFEST_UNKNOWN)"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				// HEAD answers carry no body, so the leak is in the GET /v2/ ping.
+				w.WriteHeader(http.StatusBadRequest)
+				_, _ = w.Write([]byte(tt.body))
+			}))
+			t.Cleanup(srv.Close)
+			host := strings.TrimPrefix(srv.URL, "http://")
+
+			_, err := newClient().Digest(context.Background(), host+"/a/b:1")
+
+			require.Error(t, err)
+			assert.NotContains(t, err.Error(), "internal-secret-body")
+			assert.Equal(t, "lookup "+host+"/a/b:1: "+tt.want, strings.Replace(err.Error(), "index.docker.io/", "", 1))
+		})
+	}
+}
+
+// ggcr puts the token server's answer into its error when the answer has no
+// token, and the token server is named by the registry's challenge.
+func TestNonRegistryErrorsAreFixedText(t *testing.T) {
+	dockerConfig(t, "")
+	tokenSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"detail":"internal-secret-body"}`))
+	}))
+	t.Cleanup(tokenSrv.Close)
+	reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("WWW-Authenticate", `Bearer realm="`+tokenSrv.URL+`/internal-secret-realm",service="internal-secret-service"`)
+		w.WriteHeader(http.StatusUnauthorized)
+	}))
+	t.Cleanup(reg.Close)
+	host := strings.TrimPrefix(reg.URL, "http://")
+
+	_, err := newClient().Digest(context.Background(), host+"/a/b:1")
+
+	require.Error(t, err)
+	assert.NotContains(t, err.Error(), "internal-secret")
+	assert.Equal(t, "lookup "+host+"/a/b:1: lookup failed", err.Error())
+}
+
+func TestClassify(t *testing.T) {
+	refused := &url.Error{Op: "Get", URL: "http://internal-secret-host/v2/", Err: errors.New("connection refused")}
+	tests := []struct {
+		name string
+		err  error
+		want string
+	}{
+		{"deadline", fmt.Errorf("x: %w", context.DeadlineExceeded), "timeout"},
+		{"canceled", context.Canceled, "timeout"},
+		{"url error", refused, "cannot reach registry"},
+		{"net error", &net.DNSError{Name: "internal-secret-host"}, "cannot reach registry"},
+		{"deadline inside url error", &url.Error{Op: "Get", URL: "http://x", Err: context.DeadlineExceeded}, "timeout"},
+		{"anything else", errors.New("internal-secret"), "lookup failed"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, classify(tt.err))
+		})
+	}
+}

internal/master/store/agents.go

@@ -63,30 +63,39 @@ func (s *Store) AgentByTokenHash(ctx context.Context, hash string) (model.Agent,
 	return scanAgent(s.pool.QueryRow(ctx, `SELECT `+agentColumns+` FROM agents WHERE token_hash = $1`, hash))
 }
 
-// RevokeAgent marks the agent revoked and its Compose and plain-container
-// services removed, in one transaction. Swarm rows are shared with other
-// managers and left alone. Its token stops working immediately; closing a
-// live connection is the hub's job.
-func (s *Store) RevokeAgent(ctx context.Context, id int64, at time.Time) error {
+// RevokeAgent marks the agent revoked, its Compose and plain-container
+// services removed and its queued and running jobs failed, in one
+// transaction, and returns the IDs of the jobs it failed. Swarm rows are
+// shared with other managers and left alone. Its token stops working
+// immediately; closing a live connection is the hub's job.
+func (s *Store) RevokeAgent(ctx context.Context, id int64, at time.Time) ([]int64, error) {
 	tx, err := s.pool.Begin(ctx)
 	if err != nil {
-		return fmt.Errorf("revoke agent: begin: %w", err)
+		return nil, fmt.Errorf("revoke agent: begin: %w", err)
 	}
 	defer func() { _ = tx.Rollback(ctx) }()
 
 	if _, err := tx.Exec(ctx,
 		`UPDATE agents SET revoked_at = $2 WHERE id = $1 AND revoked_at IS NULL`, id, at); err != nil {
-		return fmt.Errorf("revoke agent: %w", err)
+		return nil, fmt.Errorf("revoke agent: %w", err)
 	}
 	if _, err := tx.Exec(ctx,
 		`UPDATE services SET removed_at = $2
 		 WHERE agent_id = $1 AND kind IN ('compose', 'container') AND removed_at IS NULL`, id, at); err != nil {
-		return fmt.Errorf("revoke agent: remove services: %w", err)
+		return nil, fmt.Errorf("revoke agent: remove services: %w", err)
+	}
+	rows, err := tx.Query(ctx, failActiveJobsSQL, id, "agent revoked", at)
+	if err != nil {
+		return nil, fmt.Errorf("revoke agent: fail jobs: %w", err)
+	}
+	failed, err := pgx.CollectRows(rows, pgx.RowTo[int64])
+	if err != nil {
+		return nil, fmt.Errorf("revoke agent: fail jobs: %w", err)
 	}
 	if err := tx.Commit(ctx); err != nil {
-		return fmt.Errorf("revoke agent: commit: %w", err)
+		return nil, fmt.Errorf("revoke agent: commit: %w", err)
 	}
-	return nil
+	return failed, nil
 }
 
 // RecordHello stores what the agent reported about itself on connect.

internal/master/store/agents_integration_test.go

@@ -46,7 +46,8 @@ func TestAgents_CreateLookupRevoke(t *testing.T) {
 	_, err = s.AgentByTokenHash(ctx, "nope")
 	assert.ErrorIs(t, err, model.ErrNotFound)
 
-	require.NoError(t, s.RevokeAgent(ctx, created.ID, time.Now()))
+	_, err = s.RevokeAgent(ctx, created.ID, time.Now())
+	require.NoError(t, err)
 	found, err = s.AgentByTokenHash(ctx, "hash-2")
 	require.NoError(t, err)
 	assert.NotNil(t, found.RevokedAt)
@@ -88,7 +89,8 @@ func TestAgents_RevokedNameCanBeReused(t *testing.T) {
 	ctx := context.Background()
 	old, err := s.CreateAgent(ctx, "node-2", "h1")
 	require.NoError(t, err)
-	require.NoError(t, s.RevokeAgent(ctx, old.ID, time.Now()))
+	_, err = s.RevokeAgent(ctx, old.ID, time.Now())
+	require.NoError(t, err)
 
 	fresh, err := s.CreateAgent(ctx, "node-2", "h2")
 	require.NoError(t, err)
@@ -104,7 +106,8 @@ func TestAgents_RevokeRemovesOwnComposeAndContainerServices(t *testing.T) {
 	id := newAgent(t, s, "swarm-mgr")
 	require.NoError(t, s.ApplySnapshot(ctx, id, manager, []protocol.Service{slusk, adhoc, gitea}, t0))
 
-	require.NoError(t, s.RevokeAgent(ctx, id, t0.Add(time.Hour)))
+	_, err := s.RevokeAgent(ctx, id, t0.Add(time.Hour))
+	require.NoError(t, err)
 
 	got, err := s.ListServices(ctx)
 	require.NoError(t, err)
@@ -115,7 +118,8 @@ func TestRevokedAgent_InventoryWritesRefused(t *testing.T) {
 	s := openTestStore(t)
 	ctx := context.Background()
 	id := newAgent(t, s, "node-2")
-	require.NoError(t, s.RevokeAgent(ctx, id, t0))
+	_, err := s.RevokeAgent(ctx, id, t0)
+	require.NoError(t, err)
 
 	assert.ErrorIs(t, s.ApplySnapshot(ctx, id, worker, []protocol.Service{slusk}, t0.Add(time.Minute)), model.ErrRevoked)
 	assert.ErrorIs(t, s.UpsertService(ctx, id, "", adhoc, t0.Add(time.Minute)), model.ErrRevoked)
@@ -130,7 +134,8 @@ func TestRevokedAgent_UpsertDoesNotResurrectRemovedRow(t *testing.T) {
 	ctx := context.Background()
 	id := newAgent(t, s, "node-2")
 	require.NoError(t, s.ApplySnapshot(ctx, id, worker, []protocol.Service{slusk}, t0))
-	require.NoError(t, s.RevokeAgent(ctx, id, t0.Add(time.Hour)))
+	_, err := s.RevokeAgent(ctx, id, t0.Add(time.Hour))
+	require.NoError(t, err)
 
 	assert.ErrorIs(t, s.UpsertService(ctx, id, "", slusk, t0.Add(2*time.Hour)), model.ErrRevoked)
 
@@ -138,3 +143,26 @@ func TestRevokedAgent_UpsertDoesNotResurrectRemovedRow(t *testing.T) {
 	require.NoError(t, err)
 	assert.Empty(t, got, "the revoked agent's removed row must stay removed")
 }
+
+func TestRevokeAgent_FailsActiveJobs(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svcA := jobFixture(t, s, "a")
+	b, svcB := jobFixture(t, s, "b")
+	ja, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	jb, err := s.CreateJob(ctx, newJob(svcB, b))
+	require.NoError(t, err)
+
+	failed, err := s.RevokeAgent(ctx, a, t0)
+	require.NoError(t, err)
+	assert.Equal(t, []int64{ja.ID}, failed, "the failed job IDs are returned so the UI can signal them")
+
+	got, err := s.GetJob(ctx, ja.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, got.Status)
+	assert.Equal(t, "agent revoked", got.Error)
+	got, err = s.GetJob(ctx, jb.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobQueued, got.Status)
+}

internal/master/store/drift_merge_integration_test.go

@@ -0,0 +1,550 @@
+//go:build integration
+
+package store
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+var (
+	digNew     = "sha256:" + strings.Repeat("e", 64)
+	digOld     = "sha256:" + strings.Repeat("f", 64)
+	sluskDrift = "gitea.shcizo.se/shcizo/slusk"
+)
+
+// serviceID returns the ID of the service the agent reports under svc's key.
+func serviceID(t *testing.T, s *Store, agentID int64, svc protocol.Service) int64 {
+	t.Helper()
+	var id int64
+	require.NoError(t, s.pool.QueryRow(context.Background(),
+		`SELECT id FROM services WHERE location_key = $1`, model.LocationKey(agentID, "", svc.Key())).Scan(&id))
+	return id
+}
+
+// finishJob runs a job for the service to success with the deployed ref.
+func finishJob(t *testing.T, s *Store, agentID, svcID int64, toRef, deployed string) {
+	t.Helper()
+	ctx := context.Background()
+	nj := newJob(svcID, agentID)
+	nj.ToRef = toRef
+	j, err := s.CreateJob(ctx, nj)
+	require.NoError(t, err)
+	ok, err := s.FinishJob(ctx, agentID, j.ID, model.JobFinish{Status: model.JobSucceeded, Ref: deployed, StartedAt: t0, FinishedAt: t0})
+	require.NoError(t, err)
+	require.True(t, ok)
+}
+
+func mustExec(t *testing.T, s *Store, sql string, args ...any) {
+	t.Helper()
+	_, err := s.pool.Exec(context.Background(), sql, args...)
+	require.NoError(t, err)
+}
+
+func TestFinishJob_SetsAwaitingInventoryUntilTheNextUpsert(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	agent, svc := jobFixture(t, s, "a")
+
+	got, err := s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.False(t, got.AwaitingInventory)
+
+	finishJob(t, s, agent, svc, "r:1", "r:1@"+digNew)
+	got, err = s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.True(t, got.AwaitingInventory, "set by the successful job")
+	assert.False(t, got.Drifted(), "desired is ahead of actual only until the inventory arrives")
+
+	require.NoError(t, s.UpsertService(ctx, agent, "", slusk, t0.Add(time.Minute)))
+	got, err = s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.False(t, got.AwaitingInventory, "cleared by the agent's upsert")
+	assert.True(t, got.Drifted(), "the agent still reports the old version")
+}
+
+func TestFinishJob_FailedJobDoesNotSetAwaitingInventory(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	agent, svc := jobFixture(t, s, "a")
+	j, err := s.CreateJob(ctx, newJob(svc, agent))
+	require.NoError(t, err)
+
+	ok, err := s.FinishJob(ctx, agent, j.ID, model.JobFinish{Status: model.JobFailed, Error: "boom", StartedAt: t0, FinishedAt: t0})
+	require.NoError(t, err)
+	require.True(t, ok)
+
+	got, err := s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.False(t, got.AwaitingInventory)
+}
+
+func TestCreateJob_UnknownServiceIsNotFound(t *testing.T) {
+	s := openTestStore(t)
+	agent := newAgent(t, s, "a")
+
+	_, err := s.CreateJob(context.Background(), newJob(9999, agent))
+
+	assert.ErrorIs(t, err, model.ErrNotFound)
+}
+
+// driftedFixture is a service updated to digNew whose agent now reports digD1.
+func driftedFixture(t *testing.T, s *Store) (agent, svc int64) {
+	t.Helper()
+	agent, svc = jobFixture(t, s, "a")
+	finishJob(t, s, agent, svc, "r:1", sluskDrift+":1.4.2@"+digNew)
+	require.NoError(t, s.UpsertService(context.Background(), agent, "", slusk, t0.Add(time.Minute)))
+	return agent, svc
+}
+
+func TestAcceptRunning_WritesTheDesiredRef(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	_, svc := driftedFixture(t, s)
+	require.NoError(t, s.SetLastTagDigest(ctx, svc, "sha256:ltd"))
+	require.NoError(t, s.SetAvailable(ctx, svc, model.Available{Ref: sluskDrift + ":1.5.0", Source: model.SourcePoll, At: t0}))
+	require.NoError(t, s.SetUpdateMode(ctx, svc, model.ModeAuto))
+	before, err := s.GetService(ctx, svc)
+	require.NoError(t, err)
+	require.True(t, before.Drifted())
+
+	running := model.RunningRef(before.ActualRef, before.ActualDigest)
+	require.NoError(t, s.AcceptRunning(ctx, svc, before.ActualRef, before.ActualDigest, running))
+
+	got, err := s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, sluskDrift+":1.4.2@"+digD1, got.DesiredRef)
+	assert.False(t, got.Drifted())
+	assert.False(t, got.AwaitingInventory)
+	assert.Equal(t, "sha256:ltd", got.LastTagDigest, "last_tag_digest untouched")
+	assert.Equal(t, sluskDrift+":1.5.0", got.AvailableRef, "available suggestion untouched")
+	assert.Equal(t, model.ModeAuto, got.UpdateMode)
+}
+
+func TestAcceptRunning_Refusals(t *testing.T) {
+	tests := []struct {
+		name      string
+		prepare   func(t *testing.T, s *Store, agent, svc int64) (id int64, actualRef, actualDigest string)
+		wantError error
+	}{
+		{"running version changed", func(t *testing.T, s *Store, agent, svc int64) (int64, string, string) {
+			return svc, sluskDrift + ":1.4.2", digOld
+		}, model.ErrConflict},
+		{"actual ref changed", func(t *testing.T, s *Store, agent, svc int64) (int64, string, string) {
+			return svc, sluskDrift + ":1.3.0", digD1
+		}, model.ErrConflict},
+		{"never updated", func(t *testing.T, s *Store, agent, svc int64) (int64, string, string) {
+			mustExec(t, s, `UPDATE services SET desired_ref = '' WHERE id = $1`, svc)
+			return svc, sluskDrift + ":1.4.2", digD1
+		}, model.ErrConflict},
+		{"removed", func(t *testing.T, s *Store, agent, svc int64) (int64, string, string) {
+			require.NoError(t, s.RemoveService(context.Background(), agent, "", slusk.Key(), t0.Add(time.Hour)))
+			return svc, sluskDrift + ":1.4.2", digD1
+		}, model.ErrNotFound},
+		{"unknown", func(t *testing.T, s *Store, agent, svc int64) (int64, string, string) {
+			return 9999, sluskDrift + ":1.4.2", digD1
+		}, model.ErrNotFound},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := openTestStore(t)
+			agent, svc := driftedFixture(t, s)
+			id, ref, digest := tt.prepare(t, s, agent, svc)
+
+			err := s.AcceptRunning(context.Background(), id, ref, digest, "x:1@"+digOld)
+
+			assert.ErrorIs(t, err, tt.wantError)
+			got, gerr := s.GetService(context.Background(), svc)
+			require.NoError(t, gerr)
+			assert.NotEqual(t, "x:1@"+digOld, got.DesiredRef, "nothing written")
+		})
+	}
+}
+
+// moveFixture has agent "a" with slusk removed after it was updated (history,
+// auto mode) and agent "b" reporting a new slusk row.
+type moveFixture struct {
+	a, b     int64
+	from, to int64
+}
+
+func newMoveFixture(t *testing.T, s *Store) moveFixture {
+	t.Helper()
+	ctx := context.Background()
+	f := moveFixture{a: newAgent(t, s, "a"), b: newAgent(t, s, "b")}
+	require.NoError(t, s.ApplySnapshot(ctx, f.a, worker, []protocol.Service{slusk}, t0))
+	f.from = serviceID(t, s, f.a, slusk)
+	finishJob(t, s, f.a, f.from, "r:1", sluskDrift+":1.4.2@"+digNew)
+	require.NoError(t, s.SetUpdateMode(ctx, f.from, model.ModeAuto))
+	require.NoError(t, s.ApplySnapshot(ctx, f.a, worker, nil, t0.Add(time.Hour)))
+	require.NoError(t, s.ApplySnapshot(ctx, f.b, worker, []protocol.Service{slusk}, t0.Add(2*time.Hour)))
+	f.to = serviceID(t, s, f.b, slusk)
+	return f
+}
+
+func TestMoveCandidates(t *testing.T) {
+	tests := []struct {
+		name   string
+		mutate func(t *testing.T, s *Store, f moveFixture)
+		want   bool
+		from   string // agent name of the removed row
+	}{
+		{"one removed and one live", func(*testing.T, *Store, moveFixture) {}, true, "a"},
+		{"removed row's agent was revoked", func(t *testing.T, s *Store, f moveFixture) {
+			_, err := s.RevokeAgent(context.Background(), f.a, t0.Add(3*time.Hour))
+			require.NoError(t, err)
+		}, true, "a"},
+		{"two live targets", func(t *testing.T, s *Store, f moveFixture) {
+			c := newAgent(t, s, "c")
+			require.NoError(t, s.ApplySnapshot(context.Background(), c, worker, []protocol.Service{slusk}, t0.Add(3*time.Hour)))
+		}, false, ""},
+		{"two removed sources", func(t *testing.T, s *Store, f moveFixture) {
+			ctx := context.Background()
+			d := newAgent(t, s, "d")
+			require.NoError(t, s.ApplySnapshot(ctx, d, worker, []protocol.Service{slusk}, t0))
+			require.NoError(t, s.ApplySnapshot(ctx, d, worker, nil, t0.Add(time.Hour)))
+		}, false, ""},
+		{"target has history", func(t *testing.T, s *Store, f moveFixture) {
+			finishJob(t, s, f.b, f.to, "r:1", sluskDrift+":1.4.2@"+digNew)
+		}, false, ""},
+		{"target has a desired ref", func(t *testing.T, s *Store, f moveFixture) {
+			mustExec(t, s, `UPDATE services SET desired_ref = 'r:1@x' WHERE id = $1`, f.to)
+		}, false, ""},
+		{"source has no history", func(t *testing.T, s *Store, f moveFixture) {
+			mustExec(t, s, `DELETE FROM update_jobs WHERE service_id = $1`, f.from)
+			mustExec(t, s, `UPDATE services SET desired_ref = '', update_mode = 'manual' WHERE id = $1`, f.from)
+		}, false, ""},
+		{"source only has auto mode", func(t *testing.T, s *Store, f moveFixture) {
+			mustExec(t, s, `DELETE FROM update_jobs WHERE service_id = $1`, f.from)
+			mustExec(t, s, `UPDATE services SET desired_ref = '' WHERE id = $1`, f.from)
+		}, true, "a"},
+		{"source only has jobs", func(t *testing.T, s *Store, f moveFixture) {
+			mustExec(t, s, `UPDATE services SET desired_ref = '', update_mode = 'manual' WHERE id = $1`, f.from)
+		}, true, "a"},
+		{"target is older than the source", func(t *testing.T, s *Store, f moveFixture) {
+			mustExec(t, s, `UPDATE services SET first_seen_at = $2 WHERE id = $1`, f.to, t0.Add(-time.Hour))
+		}, false, ""},
+		{"plain containers", func(t *testing.T, s *Store, f moveFixture) {
+			mustExec(t, s, `UPDATE services SET kind = 'container' WHERE id IN ($1, $2)`, f.from, f.to)
+		}, false, ""},
+		{"swarm services", func(t *testing.T, s *Store, f moveFixture) {
+			mustExec(t, s, `UPDATE services SET kind = 'swarm' WHERE id IN ($1, $2)`, f.from, f.to)
+		}, false, ""},
+		{"the updater itself (target)", func(t *testing.T, s *Store, f moveFixture) {
+			mustExec(t, s, `UPDATE services SET self = true WHERE id = $1`, f.to)
+		}, false, ""},
+		{"the updater itself (source)", func(t *testing.T, s *Store, f moveFixture) {
+			mustExec(t, s, `UPDATE services SET self = true WHERE id = $1`, f.from)
+		}, false, ""},
+		{"live row also removed", func(t *testing.T, s *Store, f moveFixture) {
+			require.NoError(t, s.RemoveService(context.Background(), f.b, "", slusk.Key(), t0.Add(3*time.Hour)))
+		}, false, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := openTestStore(t)
+			f := newMoveFixture(t, s)
+			tt.mutate(t, s, f)
+
+			got, err := s.MoveCandidates(context.Background())
+
+			require.NoError(t, err)
+			if !tt.want {
+				assert.Empty(t, got)
+				return
+			}
+			assert.Equal(t, []model.Move{{FromID: f.from, ToID: f.to, FromAgent: tt.from, ToAgent: "b"}}, got)
+		})
+	}
+}
+
+func TestMergeService_MovesHistoryModeAndDesiredToTheNewLocation(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	f := newMoveFixture(t, s)
+	finishJob(t, s, f.b, f.to, "r:2", sluskDrift+":2.0.0@"+digOld)
+	require.NoError(t, s.SetLastTagDigest(ctx, f.to, "sha256:fromto"))
+	require.NoError(t, s.SetAvailable(ctx, f.to, model.Available{Ref: sluskDrift + ":3", Digest: digNew, Source: model.SourcePoll, At: t0}))
+	require.NoError(t, s.SetPollError(ctx, []int64{f.to}, "registry down"))
+	// A job on the old row is finished, and awaiting_inventory is set on both.
+	before, err := s.GetService(ctx, f.from)
+	require.NoError(t, err)
+
+	require.NoError(t, s.MergeService(ctx, f.from, f.to))
+
+	got, err := s.GetService(ctx, f.from)
+	require.NoError(t, err)
+	assert.Equal(t, f.from, got.ID, "the old id survives")
+	assert.Nil(t, got.RemovedAt)
+	assert.False(t, got.AwaitingInventory)
+	assert.Equal(t, "b", got.AgentName)
+	assert.Equal(t, model.LocationKey(f.b, "", slusk.Key()), got.LocationKey)
+	assert.Equal(t, before.FirstSeenAt, got.FirstSeenAt, "first_seen_at kept")
+	assert.Equal(t, model.ModeAuto, got.UpdateMode, "mode kept")
+	assert.Equal(t, before.DesiredRef, got.DesiredRef, "desired ref kept")
+	assert.Equal(t, slusk.Image, got.ActualRef, "inventory from the duplicate")
+	assert.Equal(t, digD1, got.ActualDigest)
+	assert.Equal(t, "running", got.Status)
+	assert.Equal(t, sluskDrift+":3", got.AvailableRef)
+	assert.Equal(t, digNew, got.AvailableDigest)
+	assert.Equal(t, "registry down", got.LastPollError)
+	assert.Equal(t, before.LastTagDigest, got.LastTagDigest, "an existing last_tag_digest wins")
+
+	_, err = s.GetService(ctx, f.to)
+	assert.ErrorIs(t, err, model.ErrNotFound, "the duplicate is deleted")
+	jobs, err := s.ListJobs(ctx, model.JobFilter{ServiceID: f.from})
+	require.NoError(t, err)
+	assert.Len(t, jobs, 2, "the duplicate's job moved to the survivor")
+	live, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"slusk/web"}, names(live))
+}
+
+func TestMergeService_TakesLastTagDigestOfTheDuplicateWhenTheOldRowHasNone(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	f := newMoveFixture(t, s)
+	mustExec(t, s, `UPDATE services SET last_tag_digest = '' WHERE id = $1`, f.from)
+	require.NoError(t, s.SetLastTagDigest(ctx, f.to, "sha256:fromto"))
+
+	require.NoError(t, s.MergeService(ctx, f.from, f.to))
+
+	got, err := s.GetService(ctx, f.from)
+	require.NoError(t, err)
+	assert.Equal(t, "sha256:fromto", got.LastTagDigest)
+}
+
+func TestMergeService_ManualMergeOfAnyRemovedServiceOfTheSameKind(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	f := newMoveFixture(t, s)
+	other := protocol.Service{Kind: protocol.KindCompose, ComposeProject: "renamed", ComposeService: "app", Image: "x/y:1", Status: "running"}
+	require.NoError(t, s.UpsertService(ctx, f.b, "", other, t0.Add(3*time.Hour)))
+	target := serviceID(t, s, f.b, other)
+
+	require.NoError(t, s.MergeService(ctx, f.from, target))
+
+	got, err := s.GetService(ctx, f.from)
+	require.NoError(t, err)
+	assert.Equal(t, "renamed/app", got.DisplayName(), "names may differ")
+}
+
+func TestMergeService_SameAgentIsAllowed(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	f := newMoveFixture(t, s)
+	other := protocol.Service{Kind: protocol.KindCompose, ComposeProject: "renamed", ComposeService: "app", Image: "x/y:1", Status: "running"}
+	require.NoError(t, s.UpsertService(ctx, f.a, "", other, t0.Add(3*time.Hour)))
+
+	require.NoError(t, s.MergeService(ctx, f.from, serviceID(t, s, f.a, other)))
+
+	got, err := s.GetService(ctx, f.from)
+	require.NoError(t, err)
+	assert.Equal(t, "a", got.AgentName)
+}
+
+func TestMergeService_Swarm(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	m1 := newAgent(t, s, "m1")
+	require.NoError(t, s.ApplySnapshot(ctx, m1, manager, []protocol.Service{gitea}, t0))
+	oldID := serviceIDByName(t, s, "gitea_gitea")
+	require.NoError(t, s.ApplySnapshot(ctx, m1, manager, nil, t0.Add(time.Hour)))
+	renamed := gitea
+	renamed.SwarmService = "gitea2_gitea"
+	require.NoError(t, s.ApplySnapshot(ctx, m1, manager, []protocol.Service{renamed}, t0.Add(2*time.Hour)))
+	newID := serviceIDByName(t, s, "gitea2_gitea")
+
+	require.NoError(t, s.MergeService(ctx, oldID, newID))
+
+	got, err := s.GetService(ctx, oldID)
+	require.NoError(t, err)
+	assert.Equal(t, "gitea2_gitea", got.SwarmService)
+	assert.Nil(t, got.RemovedAt)
+}
+
+func serviceIDByName(t *testing.T, s *Store, swarmService string) int64 {
+	t.Helper()
+	var id int64
+	require.NoError(t, s.pool.QueryRow(context.Background(),
+		`SELECT id FROM services WHERE swarm_service = $1`, swarmService).Scan(&id))
+	return id
+}
+
+func TestMergeService_Refusals(t *testing.T) {
+	refused := func(reason string) func(*testing.T, error) {
+		return func(t *testing.T, err error) {
+			var mr *model.MergeRefusedError
+			require.True(t, errors.As(err, &mr), "got %v", err)
+			assert.Equal(t, reason, mr.Reason)
+		}
+	}
+	is := func(want error) func(*testing.T, error) {
+		return func(t *testing.T, err error) { assert.ErrorIs(t, err, want) }
+	}
+	tests := []struct {
+		name   string
+		run    func(t *testing.T, s *Store, f moveFixture) error
+		assert func(*testing.T, error)
+	}{
+		{"into itself", func(t *testing.T, s *Store, f moveFixture) error {
+			return s.MergeService(context.Background(), f.to, f.to)
+		}, refused(model.ReasonMergeSame)},
+		{"the old service still exists", func(t *testing.T, s *Store, f moveFixture) error {
+			mustExec(t, s, `UPDATE services SET removed_at = NULL WHERE id = $1`, f.from)
+			return s.MergeService(context.Background(), f.from, f.to)
+		}, refused(model.ReasonMergeLiveOld)},
+		{"the new service was removed", func(t *testing.T, s *Store, f moveFixture) error {
+			require.NoError(t, s.RemoveService(context.Background(), f.b, "", slusk.Key(), t0.Add(3*time.Hour)))
+			return s.MergeService(context.Background(), f.from, f.to)
+		}, refused(model.ReasonMergeGoneNew)},
+		{"different kinds", func(t *testing.T, s *Store, f moveFixture) error {
+			mustExec(t, s, `UPDATE services SET kind = 'swarm' WHERE id = $1`, f.to)
+			return s.MergeService(context.Background(), f.from, f.to)
+		}, refused(model.ReasonMergeKind)},
+		{"plain containers", func(t *testing.T, s *Store, f moveFixture) error {
+			mustExec(t, s, `UPDATE services SET kind = 'container' WHERE id IN ($1, $2)`, f.from, f.to)
+			return s.MergeService(context.Background(), f.from, f.to)
+		}, refused(model.ReasonMergeContainer)},
+		{"the updater itself (new)", func(t *testing.T, s *Store, f moveFixture) error {
+			mustExec(t, s, `UPDATE services SET self = true WHERE id = $1`, f.to)
+			return s.MergeService(context.Background(), f.from, f.to)
+		}, refused(model.ReasonSelf)},
+		{"the updater itself (old)", func(t *testing.T, s *Store, f moveFixture) error {
+			mustExec(t, s, `UPDATE services SET self = true WHERE id = $1`, f.from)
+			return s.MergeService(context.Background(), f.from, f.to)
+		}, refused(model.ReasonSelf)},
+		{"active job on the old service", func(t *testing.T, s *Store, f moveFixture) error {
+			_, err := s.CreateJob(context.Background(), newJob(f.from, f.a))
+			require.NoError(t, err)
+			return s.MergeService(context.Background(), f.from, f.to)
+		}, is(model.ErrJobActive)},
+		{"active job on the new service", func(t *testing.T, s *Store, f moveFixture) error {
+			_, err := s.CreateJob(context.Background(), newJob(f.to, f.b))
+			require.NoError(t, err)
+			return s.MergeService(context.Background(), f.from, f.to)
+		}, is(model.ErrJobActive)},
+		{"unknown old service", func(t *testing.T, s *Store, f moveFixture) error {
+			return s.MergeService(context.Background(), 9999, f.to)
+		}, is(model.ErrNotFound)},
+		{"unknown new service", func(t *testing.T, s *Store, f moveFixture) error {
+			return s.MergeService(context.Background(), f.from, 9999)
+		}, is(model.ErrNotFound)},
+		{"the new service's agent is revoked", func(t *testing.T, s *Store, f moveFixture) error {
+			_, err := s.RevokeAgent(context.Background(), f.b, t0.Add(3*time.Hour))
+			require.NoError(t, err)
+			return s.MergeService(context.Background(), f.from, f.to)
+		}, is(model.ErrRevoked)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := openTestStore(t)
+			f := newMoveFixture(t, s)
+			var rowsBefore int
+			require.NoError(t, s.pool.QueryRow(context.Background(), `SELECT count(*) FROM services`).Scan(&rowsBefore))
+
+			err := tt.run(t, s, f)
+
+			tt.assert(t, err)
+			var rowsAfter int
+			require.NoError(t, s.pool.QueryRow(context.Background(), `SELECT count(*) FROM services`).Scan(&rowsAfter))
+			assert.Equal(t, rowsBefore, rowsAfter, "nothing is deleted when the merge is refused")
+		})
+	}
+}
+
+func TestMergeService_InventoryAfterTheMerge(t *testing.T) {
+	setup := func(t *testing.T) (*Store, moveFixture) {
+		s := openTestStore(t)
+		f := newMoveFixture(t, s)
+		require.NoError(t, s.MergeService(context.Background(), f.from, f.to))
+		return s, f
+	}
+
+	t.Run("an upsert for the new location updates the survivor", func(t *testing.T) {
+		s, f := setup(t)
+		ctx := context.Background()
+		moved := slusk
+		moved.Digest = digOld
+		require.NoError(t, s.UpsertService(ctx, f.b, "", moved, t0.Add(4*time.Hour)))
+
+		live, err := s.ListServices(ctx)
+		require.NoError(t, err)
+		require.Len(t, live, 1)
+		assert.Equal(t, f.from, live[0].ID)
+		assert.Equal(t, digOld, live[0].ActualDigest)
+	})
+
+	t.Run("a snapshot keeps the survivor", func(t *testing.T) {
+		s, f := setup(t)
+		ctx := context.Background()
+		require.NoError(t, s.ApplySnapshot(ctx, f.b, worker, []protocol.Service{slusk}, t0.Add(4*time.Hour)))
+
+		got, err := s.GetService(ctx, f.from)
+		require.NoError(t, err)
+		assert.Nil(t, got.RemovedAt)
+		live, err := s.ListServices(ctx)
+		require.NoError(t, err)
+		assert.Len(t, live, 1)
+	})
+
+	t.Run("the old agent returning with the stack creates a new row", func(t *testing.T) {
+		s, f := setup(t)
+		ctx := context.Background()
+		require.NoError(t, s.ApplySnapshot(ctx, f.a, worker, []protocol.Service{slusk}, t0.Add(4*time.Hour)))
+
+		returned := serviceID(t, s, f.a, slusk)
+		assert.NotEqual(t, f.from, returned)
+		got, err := s.GetService(ctx, returned)
+		require.NoError(t, err)
+		assert.Equal(t, model.ModeManual, got.UpdateMode)
+		assert.Empty(t, got.DesiredRef)
+		live, err := s.ListServices(ctx)
+		require.NoError(t, err)
+		assert.Len(t, live, 2)
+	})
+}
+
+func TestMergeSources(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	f := newMoveFixture(t, s)
+	// A removed service with another name, removed later than the matching one.
+	other := protocol.Service{Kind: protocol.KindCompose, ComposeProject: "renamed", ComposeService: "app", Image: "x/y:1", Status: "running"}
+	require.NoError(t, s.UpsertService(ctx, f.a, "", other, t0))
+	require.NoError(t, s.RemoveService(ctx, f.a, "", other.Key(), t0.Add(5*time.Hour)))
+	otherID := serviceID(t, s, f.a, other)
+	// Other kinds, the updater itself and a live service never show up.
+	require.NoError(t, s.ApplySnapshot(ctx, f.a, worker, []protocol.Service{adhoc}, t0.Add(6*time.Hour)))
+	require.NoError(t, s.ApplySnapshot(ctx, f.a, worker, nil, t0.Add(7*time.Hour)))
+	selfSvc := protocol.Service{Kind: protocol.KindCompose, ComposeProject: "pu", ComposeService: "agent", Image: "pu:1", Status: "running", Self: true}
+	require.NoError(t, s.UpsertService(ctx, f.a, "", selfSvc, t0))
+	require.NoError(t, s.RemoveService(ctx, f.a, "", selfSvc.Key(), t0.Add(8*time.Hour)))
+
+	got, err := s.MergeSources(ctx, f.to)
+
+	require.NoError(t, err)
+	var ids []int64
+	for _, m := range got {
+		ids = append(ids, m.ID)
+	}
+	assert.Equal(t, []int64{f.from, otherID}, ids, "same name first, then the most recently removed")
+	assert.Equal(t, "a", got[0].AgentName)
+
+	_, err = s.MergeSources(ctx, 9999)
+	assert.ErrorIs(t, err, model.ErrNotFound)
+
+	containerID := serviceID(t, s, f.a, adhoc)
+	got, err = s.MergeSources(ctx, containerID)
+	require.NoError(t, err)
+	assert.Empty(t, got, "plain containers cannot be merged")
+}

internal/master/store/jobs.go

@@ -0,0 +1,257 @@
+package store
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
+
+	"github.com/shcizo/package-updater/internal/imageref"
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+const lostJobMessage = "lost: the agent did not know this job when it reconnected"
+
+// failActiveJobsSQL fails every queued or running job of agent $1 with
+// message $2 at time $3.
+const failActiveJobsSQL = `
+UPDATE update_jobs SET status = 'failed', error = $2, finished_at = $3
+WHERE agent_id = $1 AND status IN ('queued', 'running')
+RETURNING id`
+
+const jobSelectSQL = `
+SELECT j.id, j.service_id, s.kind, s.compose_project, s.compose_service, s.swarm_service, s.container_name,
+       j.agent_id, COALESCE(a.name, ''), j.from_ref, j.to_ref, j.trigger, j.requested_by, j.status,
+       %s, j.output_truncated, j.error, j.created_at, j.started_at, j.finished_at
+FROM update_jobs j
+JOIN services s ON s.id = j.service_id
+LEFT JOIN agents a ON a.id = j.agent_id`
+
+func scanJob(row pgx.Row) (model.Job, error) {
+	var j model.Job
+	var svc model.Service
+	var kind, trigger, status string
+	err := row.Scan(&j.ID, &j.ServiceID, &kind, &svc.ComposeProject, &svc.ComposeService, &svc.SwarmService, &svc.ContainerName,
+		&j.AgentID, &j.AgentName, &j.FromRef, &j.ToRef, &trigger, &j.RequestedBy, &status,
+		&j.Output, &j.OutputTruncated, &j.Error, &j.CreatedAt, &j.StartedAt, &j.FinishedAt)
+	svc.Kind = protocol.Kind(kind)
+	j.ServiceName = svc.DisplayName()
+	j.Trigger = model.JobTrigger(trigger)
+	j.Status = model.JobStatus(status)
+	return j, err
+}
+
+func (s *Store) getJob(ctx context.Context, id int64) (model.Job, error) {
+	j, err := scanJob(s.pool.QueryRow(ctx, fmt.Sprintf(jobSelectSQL, "j.output")+` WHERE j.id = $1`, id))
+	if errors.Is(err, pgx.ErrNoRows) {
+		return model.Job{}, model.ErrNotFound
+	}
+	if err != nil {
+		return model.Job{}, fmt.Errorf("get job: %w", err)
+	}
+	return j, nil
+}
+
+// CreateJob inserts a queued job. If the service already has a queued or
+// running job, it returns that job and model.ErrJobActive (detected via the
+// unique index, so it holds under concurrent calls). A job for a revoked
+// agent is refused with model.ErrAgentOffline: the agent's revoke fails its
+// jobs once, so a job created afterwards would never be settled and would
+// block the service. The share lock on the agent row makes a concurrent
+// revoke wait for this insert, so its job cleanup sees the new row.
+func (s *Store) CreateJob(ctx context.Context, j model.NewJob) (model.Job, error) {
+	// The active job can finish between the unique violation and the lookup
+	// below; then the insert is simply tried once more.
+	for attempt := 0; attempt < 2; attempt++ {
+		var id int64
+		err := s.pool.QueryRow(ctx, `
+			INSERT INTO update_jobs (service_id, agent_id, from_ref, to_ref, trigger, requested_by, status, created_at)
+			SELECT $1::bigint, $2::bigint, $3::text, $4::text, $5::text, $6::text, 'queued', $7::timestamptz
+			WHERE $2::bigint IS NULL
+			   OR EXISTS (SELECT 1 FROM agents WHERE id = $2 AND revoked_at IS NULL FOR SHARE)
+			RETURNING id`,
+			j.ServiceID, j.AgentID, j.FromRef, j.ToRef, string(j.Trigger), j.RequestedBy, j.CreatedAt).Scan(&id)
+		if errors.Is(err, pgx.ErrNoRows) {
+			return model.Job{}, model.ErrAgentOffline
+		}
+		var pgErr *pgconn.PgError
+		if errors.As(err, &pgErr) && pgErr.Code == "23505" && pgErr.ConstraintName == "update_jobs_one_active" {
+			active, err := scanJob(s.pool.QueryRow(ctx, fmt.Sprintf(jobSelectSQL, "j.output")+
+				` WHERE j.service_id = $1 AND j.status IN ('queued', 'running')`, j.ServiceID))
+			if errors.Is(err, pgx.ErrNoRows) {
+				continue
+			}
+			if err != nil {
+				return model.Job{}, fmt.Errorf("find active job: %w", err)
+			}
+			return active, model.ErrJobActive
+		}
+		if errors.As(err, &pgErr) && pgErr.Code == "23503" && pgErr.ConstraintName == "update_jobs_service_id_fkey" {
+			return model.Job{}, model.ErrNotFound
+		}
+		if err != nil {
+			return model.Job{}, fmt.Errorf("create job: %w", err)
+		}
+		return s.getJob(ctx, id)
+	}
+	return model.Job{}, fmt.Errorf("create job: active job vanished twice")
+}
+
+// FailJob sets a queued or running job to failed with msg and finished_at =
+// at. Returns false (no error) if the job is no longer active.
+func (s *Store) FailJob(ctx context.Context, id int64, msg string, at time.Time) (bool, error) {
+	tag, err := s.pool.Exec(ctx, `
+		UPDATE update_jobs SET status = 'failed', error = $2, finished_at = $3
+		WHERE id = $1 AND status IN ('queued', 'running')`, id, msg, at)
+	if err != nil {
+		return false, fmt.Errorf("fail job: %w", err)
+	}
+	return tag.RowsAffected() == 1, nil
+}
+
+// GetJob returns one job with its output. Returns model.ErrNotFound if missing.
+func (s *Store) GetJob(ctx context.Context, id int64) (model.Job, error) {
+	return s.getJob(ctx, id)
+}
+
+// ListJobs returns jobs newest first, without Output.
+func (s *Store) ListJobs(ctx context.Context, f model.JobFilter) ([]model.Job, error) {
+	limit := f.Limit
+	if limit <= 0 {
+		limit = 100
+	}
+	rows, err := s.pool.Query(ctx, fmt.Sprintf(jobSelectSQL, "''")+`
+		WHERE ($1 = 0 OR j.service_id = $1)
+		ORDER BY j.id DESC
+		LIMIT $2`, f.ServiceID, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list jobs: %w", err)
+	}
+	defer rows.Close()
+	var out []model.Job
+	for rows.Next() {
+		j, err := scanJob(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan job: %w", err)
+		}
+		out = append(out, j)
+	}
+	return out, rows.Err()
+}
+
+// FailLostJobs fails every queued or running job sent to agentID whose ID is
+// not in known, with a "lost" error. Returns the IDs it failed.
+func (s *Store) FailLostJobs(ctx context.Context, agentID int64, known []int64, at time.Time) ([]int64, error) {
+	if known == nil {
+		known = []int64{}
+	}
+	rows, err := s.pool.Query(ctx, `
+		UPDATE update_jobs SET status = 'failed', error = $3, finished_at = $4
+		WHERE agent_id = $1 AND status IN ('queued', 'running') AND NOT (id = ANY($2))
+		RETURNING id`, agentID, known, lostJobMessage, at)
+	if err != nil {
+		return nil, fmt.Errorf("fail lost jobs: %w", err)
+	}
+	defer rows.Close()
+	var ids []int64
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			return nil, fmt.Errorf("scan lost job: %w", err)
+		}
+		ids = append(ids, id)
+	}
+	return ids, rows.Err()
+}
+
+// JobProgress appends chunk to the output of a queued or running job sent to
+// agentID, keeping the last protocol.MaxJobOutput characters, and moves a
+// queued job to running (started_at = at if unset). Returns false if no such
+// active job belongs to agentID.
+func (s *Store) JobProgress(ctx context.Context, agentID, jobID int64, chunk string, at time.Time) (bool, error) {
+	tag, err := s.pool.Exec(ctx, `
+		UPDATE update_jobs SET
+			status           = 'running',
+			started_at       = COALESCE(started_at, $4),
+			output_truncated = output_truncated OR length(output) + length($3::text) > $5,
+			output           = right(output || $3::text, $5)
+		WHERE id = $1 AND agent_id = $2 AND status IN ('queued', 'running')`,
+		jobID, agentID, chunk, at, protocol.MaxJobOutput)
+	if err != nil {
+		return false, fmt.Errorf("job progress: %w", err)
+	}
+	return tag.RowsAffected() == 1, nil
+}
+
+// FinishJob records the agent's result for a queued or running job sent to
+// agentID: status, error, output (replacing what progress assembled),
+// started_at (the agent's, or the existing one if the agent sent none),
+// finished_at; to_ref = f.Ref when set. On success it also sets the
+// service's desired_ref = f.Ref and clears its available ref when that is
+// the ref the job was created for and was reported no later than the job was
+// created (that version is now running), in the same transaction. A job
+// created for "repo:tag@digest" clears only a suggestion with that same
+// digest, so rolling back to an older digest keeps a newer suggestion (and a
+// suggestion without a digest, such as a webhook's). Returns false if no
+// such active job belongs to agentID.
+func (s *Store) FinishJob(ctx context.Context, agentID, jobID int64, f model.JobFinish) (bool, error) {
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return false, fmt.Errorf("finish job: begin: %w", err)
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+
+	// The agent's clock wins when it sent a start time; otherwise the time the
+	// first progress arrived stays.
+	var startedAt *time.Time
+	if !f.StartedAt.IsZero() {
+		startedAt = &f.StartedAt
+	}
+	var serviceID int64
+	var createdAt time.Time
+	var requestedRef string
+	// requested is the job's to_ref as it was created, before f.Ref overwrites it.
+	err = tx.QueryRow(ctx, `
+		UPDATE update_jobs j SET
+			status = $3, error = $4, output = $5, output_truncated = $6,
+			to_ref = CASE WHEN $7::text <> '' THEN $7::text ELSE j.to_ref END,
+			started_at = COALESCE($8, j.started_at), finished_at = $9
+		FROM (SELECT id, to_ref FROM update_jobs WHERE id = $1) requested
+		WHERE j.id = requested.id AND j.agent_id = $2 AND j.status IN ('queued', 'running')
+		RETURNING j.service_id, j.created_at, requested.to_ref`,
+		jobID, agentID, string(f.Status), f.Error, f.Output, f.OutputTruncated, f.Ref, startedAt, f.FinishedAt).Scan(&serviceID, &createdAt, &requestedRef)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("finish job: %w", err)
+	}
+	if f.Status == model.JobSucceeded {
+		requestedRef, requestedDigest, _ := strings.Cut(requestedRef, "@")
+		// A plain tag job, whoever started it, deployed the tag at the digest
+		// the agent pinned; remember it. Pinned and tagless picks do not.
+		tagDigest := ""
+		if _, d, _ := strings.Cut(f.Ref, "@"); requestedDigest == "" && imageref.IsDigest(d) {
+			tagDigest = d
+		}
+		if _, err := tx.Exec(ctx, `
+			UPDATE services SET desired_ref = $2, awaiting_inventory = true,
+				last_tag_digest  = CASE WHEN $6 <> '' THEN $6 ELSE last_tag_digest END,
+				available_ref    = CASE WHEN available_ref = $4 AND available_at <= $3 AND ($5 = '' OR available_digest = $5) THEN '' ELSE available_ref END,
+				available_digest = CASE WHEN available_ref = $4 AND available_at <= $3 AND ($5 = '' OR available_digest = $5) THEN '' ELSE available_digest END,
+				available_source = CASE WHEN available_ref = $4 AND available_at <= $3 AND ($5 = '' OR available_digest = $5) THEN '' ELSE available_source END,
+				available_at     = CASE WHEN available_ref = $4 AND available_at <= $3 AND ($5 = '' OR available_digest = $5) THEN NULL ELSE available_at END
+			WHERE id = $1`, serviceID, f.Ref, createdAt, requestedRef, requestedDigest, tagDigest); err != nil {
+			return false, fmt.Errorf("finish job: desired ref: %w", err)
+		}
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return false, fmt.Errorf("finish job: commit: %w", err)
+	}
+	return true, nil
+}

internal/master/store/jobs_integration_test.go

@@ -0,0 +1,379 @@
+//go:build integration
+
+package store
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// jobFixture seeds one agent with one Compose service and returns their IDs.
+func jobFixture(t *testing.T, s *Store, agentName string) (agentID, serviceID int64) {
+	t.Helper()
+	ctx := context.Background()
+	agentID = newAgent(t, s, agentName)
+	require.NoError(t, s.ApplySnapshot(ctx, agentID, worker, []protocol.Service{slusk}, t0))
+	svcs, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	for _, svc := range svcs {
+		if svc.AgentID != nil && *svc.AgentID == agentID {
+			return agentID, svc.ID
+		}
+	}
+	t.Fatal("service not found")
+	return
+}
+
+func newJob(serviceID, agentID int64) model.NewJob {
+	return model.NewJob{
+		ServiceID: serviceID, AgentID: &agentID,
+		FromRef: "r:1@sha256:a", ToRef: "r:1", Trigger: model.TriggerManual,
+		RequestedBy: "sub", CreatedAt: t0,
+	}
+}
+
+func TestCreateJob_OneActivePerService(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	agentID, svcID := jobFixture(t, s, "a")
+
+	var wg sync.WaitGroup
+	jobs := make([]model.Job, 2)
+	errs := make([]error, 2)
+	for i := range jobs {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			jobs[i], errs[i] = s.CreateJob(ctx, newJob(svcID, agentID))
+		}()
+	}
+	wg.Wait()
+
+	assert.Equal(t, jobs[0].ID, jobs[1].ID)
+	var failures int
+	for _, err := range errs {
+		if err != nil {
+			assert.ErrorIs(t, err, model.ErrJobActive)
+			failures++
+		}
+	}
+	assert.Equal(t, 1, failures)
+
+	ok, err := s.FailJob(ctx, jobs[0].ID, "x", t0)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	ok, err = s.FailJob(ctx, jobs[0].ID, "x", t0)
+	require.NoError(t, err)
+	assert.False(t, ok)
+
+	next, err := s.CreateJob(ctx, newJob(svcID, agentID))
+	require.NoError(t, err)
+	assert.NotEqual(t, jobs[0].ID, next.ID)
+}
+
+func TestFailLostJobs(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svcA := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	require.NoError(t, s.ApplySnapshot(ctx, b, worker, []protocol.Service{adhoc}, t0))
+	all, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	var svcB int64
+	for _, svc := range all {
+		if svc.ID != svcA {
+			svcB = svc.ID
+		}
+	}
+
+	// Only one active job per service, so A's jobs go through the lifecycle.
+	j3, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	ok, err := s.FinishJob(ctx, a, j3.ID, model.JobFinish{Status: model.JobSucceeded, Ref: "r:1@sha256:d", StartedAt: t0, FinishedAt: t0})
+	require.NoError(t, err)
+	require.True(t, ok)
+	j1, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	j4, err := s.CreateJob(ctx, newJob(svcB, b))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, b, j4.ID, "x", t0)
+	require.NoError(t, err)
+
+	ids, err := s.FailLostJobs(ctx, a, []int64{j3.ID}, t0)
+	require.NoError(t, err)
+	assert.Equal(t, []int64{j1.ID}, ids)
+
+	got, err := s.GetJob(ctx, j1.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, got.Status)
+	assert.Contains(t, got.Error, "lost")
+	got, err = s.GetJob(ctx, j3.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobSucceeded, got.Status)
+	got, err = s.GetJob(ctx, j4.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobRunning, got.Status)
+
+	// A running job the agent still knows is kept.
+	j5, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j5.ID, "", t0)
+	require.NoError(t, err)
+	ids, err = s.FailLostJobs(ctx, a, []int64{j5.ID}, t0)
+	require.NoError(t, err)
+	assert.Empty(t, ids)
+}
+
+func TestJobProgress_OwnerOnly(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+
+	started := t0.Add(time.Second)
+	ok, err := s.JobProgress(ctx, a, j.ID, "hello ", started)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobRunning, got.Status)
+	require.NotNil(t, got.StartedAt)
+	assert.True(t, started.Equal(*got.StartedAt))
+
+	ok, err = s.JobProgress(ctx, b, j.ID, "evil", t0.Add(time.Minute))
+	require.NoError(t, err)
+	assert.False(t, ok)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "hello ", got.Output)
+
+	ok, err = s.JobProgress(ctx, a, j.ID, "world", t0.Add(time.Minute))
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "hello world", got.Output)
+	assert.True(t, started.Equal(*got.StartedAt))
+
+	_, err = s.FailJob(ctx, j.ID, "x", t0)
+	require.NoError(t, err)
+	ok, err = s.JobProgress(ctx, a, j.ID, "late", t0)
+	require.NoError(t, err)
+	assert.False(t, ok)
+}
+
+func TestJobProgress_KeepsTail(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+
+	_, err = s.JobProgress(ctx, a, j.ID, strings.Repeat("a", protocol.MaxJobOutput), t0)
+	require.NoError(t, err)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.False(t, got.OutputTruncated)
+
+	_, err = s.JobProgress(ctx, a, j.ID, "TAIL", t0)
+	require.NoError(t, err)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.True(t, got.OutputTruncated)
+	assert.Len(t, got.Output, protocol.MaxJobOutput)
+	assert.True(t, strings.HasSuffix(got.Output, "aaTAIL"))
+}
+
+func TestFinishJob_OnlyOnceAndOnlyByOwner(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j.ID, "partial", t0)
+	require.NoError(t, err)
+
+	fin := model.JobFinish{Status: model.JobSucceeded, Ref: "r:1@sha256:d", Output: "full", StartedAt: t0.Add(time.Hour), FinishedAt: t0.Add(2 * time.Hour)}
+	ok, err := s.FinishJob(ctx, b, j.ID, fin)
+	require.NoError(t, err)
+	assert.False(t, ok)
+
+	ok, err = s.FinishJob(ctx, a, j.ID, fin)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobSucceeded, got.Status)
+	assert.Equal(t, "r:1@sha256:d", got.ToRef)
+	assert.Equal(t, "full", got.Output)
+	assert.True(t, fin.StartedAt.Equal(*got.StartedAt), "the agent's started_at wins over the master's progress clock")
+	assert.True(t, fin.FinishedAt.Equal(*got.FinishedAt))
+	service, err := s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, "r:1@sha256:d", service.DesiredRef)
+
+	again := fin
+	again.Ref = "r:2@sha256:e"
+	again.Output = "other"
+	ok, err = s.FinishJob(ctx, a, j.ID, again)
+	require.NoError(t, err)
+	assert.False(t, ok)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "full", got.Output)
+	service, err = s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, "r:1@sha256:d", service.DesiredRef)
+
+	j2, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+	ok, err = s.FinishJob(ctx, a, j2.ID, model.JobFinish{Status: model.JobFailed, Error: "boom", Output: "o", StartedAt: t0, FinishedAt: t0})
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err = s.GetJob(ctx, j2.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, got.Status)
+	assert.Equal(t, "boom", got.Error)
+	assert.Equal(t, "r:1", got.ToRef)
+	service, err = s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, "r:1@sha256:d", service.DesiredRef)
+}
+
+func TestGetJobAndListJobs(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svcA := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	require.NoError(t, s.ApplySnapshot(ctx, b, worker, []protocol.Service{adhoc}, t0))
+	all, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	var svcB int64
+	for _, svc := range all {
+		if svc.ID != svcA {
+			svcB = svc.ID
+		}
+	}
+
+	j1, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j1.ID, "out", t0)
+	require.NoError(t, err)
+	_, err = s.FailJob(ctx, j1.ID, "x", t0)
+	require.NoError(t, err)
+	j2, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	j3, err := s.CreateJob(ctx, newJob(svcB, b))
+	require.NoError(t, err)
+
+	got, err := s.GetJob(ctx, j1.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "slusk/web", got.ServiceName)
+	assert.Equal(t, "a", got.AgentName)
+	assert.Equal(t, "out", got.Output)
+
+	_, err = s.GetJob(ctx, 99999)
+	assert.ErrorIs(t, err, model.ErrNotFound)
+
+	list, err := s.ListJobs(ctx, model.JobFilter{})
+	require.NoError(t, err)
+	require.Len(t, list, 3)
+	assert.Equal(t, []int64{j3.ID, j2.ID, j1.ID}, []int64{list[0].ID, list[1].ID, list[2].ID})
+	for _, j := range list {
+		assert.Empty(t, j.Output)
+	}
+
+	list, err = s.ListJobs(ctx, model.JobFilter{ServiceID: svcA})
+	require.NoError(t, err)
+	assert.Len(t, list, 2)
+
+	list, err = s.ListJobs(ctx, model.JobFilter{Limit: 1})
+	require.NoError(t, err)
+	assert.Len(t, list, 1)
+}
+
+func TestFinishJob_ZeroStartedAtKeepsProgressClock(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j.ID, "x", t0)
+	require.NoError(t, err)
+
+	ok, err := s.FinishJob(ctx, a, j.ID, model.JobFinish{Status: model.JobFailed, FinishedAt: t0.Add(time.Hour)})
+	require.NoError(t, err)
+	require.True(t, ok)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.True(t, t0.Equal(*got.StartedAt))
+}
+
+func TestCreateJob_RefusesRevokedAgent(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	_, err := s.RevokeAgent(ctx, a, t0)
+	require.NoError(t, err)
+
+	_, err = s.CreateJob(ctx, newJob(svc, a))
+	assert.ErrorIs(t, err, model.ErrAgentOffline)
+	list, err := s.ListJobs(ctx, model.JobFilter{})
+	require.NoError(t, err)
+	assert.Empty(t, list, "no job is left behind to block the service")
+
+	nj := newJob(svc, a)
+	nj.AgentID = nil
+	_, err = s.CreateJob(ctx, nj)
+	assert.NoError(t, err, "a job that was never routed has no agent to check")
+}
+
+// CreateJob must wait for a revoke that is still in flight. Without the share
+// lock it would read the agent as active, insert the job, and the revoke's
+// job cleanup (which ran before the insert committed) would never fail it.
+func TestCreateJob_WaitsForConcurrentRevoke(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+
+	tx, err := s.pool.Begin(ctx)
+	require.NoError(t, err)
+	defer func() { _ = tx.Rollback(ctx) }()
+	_, err = tx.Exec(ctx, `UPDATE agents SET revoked_at = now() WHERE id = $1`, a)
+	require.NoError(t, err)
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := s.CreateJob(ctx, newJob(svc, a))
+		done <- err
+	}()
+
+	select {
+	case err := <-done:
+		t.Fatalf("CreateJob returned (%v) while the revoke was uncommitted", err)
+	case <-time.After(500 * time.Millisecond):
+	}
+
+	require.NoError(t, tx.Commit(ctx))
+	select {
+	case err := <-done:
+		assert.ErrorIs(t, err, model.ErrAgentOffline)
+	case <-time.After(5 * time.Second):
+		t.Fatal("CreateJob did not return after the revoke committed")
+	}
+	list, err := s.ListJobs(ctx, model.JobFilter{})
+	require.NoError(t, err)
+	assert.Empty(t, list)
+}

internal/master/store/merge.go

@@ -0,0 +1,249 @@
+package store
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/jackc/pgx/v5"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// moveCandidatesSQL pairs a removed Compose service with a live one of the
+// same project and service name on another agent. A pair counts only when it
+// is the single one for both rows, and the live row is new (newer than the
+// removed one, never updated, no jobs) while the removed one has something
+// worth moving (a desired ref, auto mode or jobs).
+const moveCandidatesSQL = `
+WITH pairs AS (
+	SELECT f.id AS from_id, t.id AS to_id
+	FROM services f
+	JOIN services t ON t.kind = 'compose' AND t.removed_at IS NULL AND NOT t.self
+	 AND t.compose_project = f.compose_project AND t.compose_service = f.compose_service
+	 AND t.agent_id IS DISTINCT FROM f.agent_id
+	WHERE f.kind = 'compose' AND f.removed_at IS NOT NULL AND NOT f.self
+), uniq AS (
+	SELECT from_id, to_id FROM pairs p
+	WHERE (SELECT count(*) FROM pairs q WHERE q.from_id = p.from_id) = 1
+	  AND (SELECT count(*) FROM pairs q WHERE q.to_id   = p.to_id)   = 1
+)
+SELECT u.from_id, u.to_id, COALESCE(fa.name, ''), COALESCE(ta.name, '')
+FROM uniq u
+JOIN services f ON f.id = u.from_id
+JOIN services t ON t.id = u.to_id
+LEFT JOIN agents fa ON fa.id = f.agent_id
+LEFT JOIN agents ta ON ta.id = t.agent_id
+WHERE t.first_seen_at > f.first_seen_at
+  AND t.desired_ref = ''
+  AND NOT EXISTS (SELECT 1 FROM update_jobs j WHERE j.service_id = t.id)
+  AND (f.desired_ref <> '' OR f.update_mode = 'auto'
+       OR EXISTS (SELECT 1 FROM update_jobs j WHERE j.service_id = f.id))
+ORDER BY u.to_id`
+
+// MoveCandidates returns the removed Compose services that look like they
+// moved to another agent; see model.Move. It only suggests: nothing is merged.
+func (s *Store) MoveCandidates(ctx context.Context) ([]model.Move, error) {
+	rows, err := s.pool.Query(ctx, moveCandidatesSQL)
+	if err != nil {
+		return nil, fmt.Errorf("move candidates: %w", err)
+	}
+	defer rows.Close()
+	var out []model.Move
+	for rows.Next() {
+		var m model.Move
+		if err := rows.Scan(&m.FromID, &m.ToID, &m.FromAgent, &m.ToAgent); err != nil {
+			return nil, fmt.Errorf("scan move: %w", err)
+		}
+		out = append(out, m)
+	}
+	return out, rows.Err()
+}
+
+// MergeSources returns up to 50 removed services that could be merged into
+// the service toID: the same kind (Compose or Swarm), not the updater itself,
+// those with the same name first, then the most recently removed. Returns
+// model.ErrNotFound if toID does not exist.
+func (s *Store) MergeSources(ctx context.Context, toID int64) ([]model.Service, error) {
+	to, err := s.GetService(ctx, toID)
+	if err != nil {
+		return nil, err
+	}
+	if to.Kind == protocol.KindContainer {
+		return nil, nil
+	}
+	rows, err := s.pool.Query(ctx, `
+		SELECT `+serviceColumns+`
+		FROM services s LEFT JOIN agents a ON a.id = s.agent_id
+		WHERE s.removed_at IS NOT NULL AND s.kind = $1 AND NOT s.self AND s.id <> $2
+		ORDER BY (s.compose_project, s.compose_service, s.swarm_service) = ($3, $4, $5) DESC,
+		         s.removed_at DESC, s.id DESC
+		LIMIT 50`,
+		string(to.Kind), toID, to.ComposeProject, to.ComposeService, to.SwarmService)
+	if err != nil {
+		return nil, fmt.Errorf("merge sources: %w", err)
+	}
+	defer rows.Close()
+	var out []model.Service
+	for rows.Next() {
+		m, err := scanService(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan service: %w", err)
+		}
+		out = append(out, m)
+	}
+	return out, rows.Err()
+}
+
+// mergeRow is what MergeService needs to know about one of the two rows.
+type mergeRow struct {
+	kind    string
+	agentID *int64
+	removed bool
+	self    bool
+}
+
+// MergeService moves the live service toID onto the removed service fromID:
+// the old row keeps its id, first_seen_at, update mode and desired ref, takes
+// the location and inventory of the live row (and its available suggestion),
+// becomes live, and the live row is deleted. The jobs of the live row move to
+// the old one. This is the only place a services row is deleted.
+//
+// It returns *model.MergeRefusedError when the pair cannot be merged,
+// model.ErrJobActive when either row has a queued or running job,
+// model.ErrRevoked when the live row's agent is revoked, model.ErrNotFound
+// when a row does not exist, and model.ErrConflict when the live row moved to
+// another agent while the merge started.
+//
+// Locks are taken in the order inventory writes use: the agent row first,
+// then the service rows by id. The live row is deleted before the old one is
+// updated because location_key is unique.
+func (s *Store) MergeService(ctx context.Context, fromID, toID int64) error {
+	if fromID == toID {
+		return &model.MergeRefusedError{Reason: model.ReasonMergeSame}
+	}
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return fmt.Errorf("begin merge: %w", err)
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+
+	var toAgent *int64
+	err = tx.QueryRow(ctx, `SELECT agent_id FROM services WHERE id = $1`, toID).Scan(&toAgent)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return model.ErrNotFound
+	}
+	if err != nil {
+		return fmt.Errorf("merge: find new service: %w", err)
+	}
+	if toAgent != nil {
+		if err := lockActiveAgent(ctx, tx, *toAgent); err != nil {
+			return err
+		}
+	}
+
+	rows, err := tx.Query(ctx, `
+		SELECT id, kind, agent_id, removed_at IS NOT NULL, self FROM services
+		WHERE id IN ($1, $2) ORDER BY id FOR UPDATE`, fromID, toID)
+	if err != nil {
+		return fmt.Errorf("merge: lock services: %w", err)
+	}
+	locked := map[int64]mergeRow{}
+	for rows.Next() {
+		var id int64
+		var r mergeRow
+		if err := rows.Scan(&id, &r.kind, &r.agentID, &r.removed, &r.self); err != nil {
+			rows.Close()
+			return fmt.Errorf("merge: scan service: %w", err)
+		}
+		locked[id] = r
+	}
+	rows.Close()
+	if err := rows.Err(); err != nil {
+		return fmt.Errorf("merge: lock services: %w", err)
+	}
+	from, okFrom := locked[fromID]
+	to, okTo := locked[toID]
+	if !okFrom || !okTo {
+		return model.ErrNotFound
+	}
+	if (to.agentID == nil) != (toAgent == nil) || (toAgent != nil && *to.agentID != *toAgent) {
+		return model.ErrConflict
+	}
+	if reason := mergeRefusal(from, to); reason != "" {
+		return &model.MergeRefusedError{Reason: reason}
+	}
+
+	var active bool
+	if err := tx.QueryRow(ctx, `
+		SELECT EXISTS (SELECT 1 FROM update_jobs
+		               WHERE service_id IN ($1, $2) AND status IN ('queued', 'running'))`,
+		fromID, toID).Scan(&active); err != nil {
+		return fmt.Errorf("merge: check active jobs: %w", err)
+	}
+	if active {
+		return model.ErrJobActive
+	}
+
+	if _, err := tx.Exec(ctx, `UPDATE update_jobs SET service_id = $1 WHERE service_id = $2`, fromID, toID); err != nil {
+		return fmt.Errorf("merge: move jobs: %w", err)
+	}
+	var t model.Service
+	var lastTagDigest string
+	err = tx.QueryRow(ctx, `
+		SELECT agent_id, location_key, compose_project, compose_service, working_dir, config_files, built,
+		       swarm_cluster_id, swarm_service, container_name, image_repo, actual_ref, actual_digest, status, self,
+		       updated_at, available_ref, available_digest, available_source, available_at, last_poll_error,
+		       last_tag_digest
+		FROM services WHERE id = $1`, toID).Scan(
+		&t.AgentID, &t.LocationKey, &t.ComposeProject, &t.ComposeService, &t.WorkingDir, &t.ConfigFiles, &t.Built,
+		&t.SwarmClusterID, &t.SwarmService, &t.ContainerName, &t.ImageRepo, &t.ActualRef, &t.ActualDigest, &t.Status, &t.Self,
+		&t.UpdatedAt, &t.AvailableRef, &t.AvailableDigest, &t.AvailableSource, &t.AvailableAt, &t.LastPollError,
+		&lastTagDigest)
+	if err != nil {
+		return fmt.Errorf("merge: read new service: %w", err)
+	}
+	if _, err := tx.Exec(ctx, `DELETE FROM services WHERE id = $1`, toID); err != nil {
+		return fmt.Errorf("merge: delete new service: %w", err)
+	}
+	if _, err := tx.Exec(ctx, `
+		UPDATE services SET
+			agent_id = $2, location_key = $3, compose_project = $4, compose_service = $5, working_dir = $6,
+			config_files = $7, built = $8, swarm_cluster_id = $9, swarm_service = $10, container_name = $11,
+			image_repo = $12, actual_ref = $13, actual_digest = $14, status = $15, self = $16, updated_at = $17,
+			available_ref = $18, available_digest = $19, available_source = $20, available_at = $21,
+			last_poll_error = $22,
+			last_tag_digest = CASE WHEN last_tag_digest <> '' THEN last_tag_digest ELSE $23 END,
+			removed_at = NULL, awaiting_inventory = false
+		WHERE id = $1`,
+		fromID, t.AgentID, t.LocationKey, t.ComposeProject, t.ComposeService, t.WorkingDir,
+		t.ConfigFiles, t.Built, t.SwarmClusterID, t.SwarmService, t.ContainerName,
+		t.ImageRepo, t.ActualRef, t.ActualDigest, t.Status, t.Self, t.UpdatedAt,
+		t.AvailableRef, t.AvailableDigest, t.AvailableSource, t.AvailableAt,
+		t.LastPollError, lastTagDigest); err != nil {
+		return fmt.Errorf("merge: update old service: %w", err)
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return fmt.Errorf("commit merge: %w", err)
+	}
+	return nil
+}
+
+// mergeRefusal returns why from (the old row) cannot be merged with to (the
+// live one), or "".
+func mergeRefusal(from, to mergeRow) string {
+	switch {
+	case !from.removed:
+		return model.ReasonMergeLiveOld
+	case to.removed:
+		return model.ReasonMergeGoneNew
+	case from.kind != to.kind:
+		return model.ReasonMergeKind
+	case from.kind == string(protocol.KindContainer):
+		return model.ReasonMergeContainer
+	case from.self || to.self:
+		return model.ReasonSelf
+	}
+	return ""
+}

internal/master/store/migrations/0002_jobs.sql

@@ -0,0 +1,28 @@
+ALTER TABLE services ADD COLUMN config_files text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN desired_ref  text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN built        boolean NOT NULL DEFAULT false;
+
+CREATE TABLE update_jobs (
+    id               bigserial PRIMARY KEY,
+    service_id       bigint NOT NULL REFERENCES services (id),
+    agent_id         bigint REFERENCES agents (id),
+    from_ref         text NOT NULL DEFAULT '',
+    to_ref           text NOT NULL,
+    trigger          text NOT NULL CHECK (trigger IN ('manual', 'webhook', 'poll')),
+    requested_by     text NOT NULL DEFAULT '',
+    status           text NOT NULL CHECK (status IN ('queued', 'running', 'succeeded', 'failed')),
+    output           text NOT NULL DEFAULT '',
+    output_truncated boolean NOT NULL DEFAULT false,
+    error            text NOT NULL DEFAULT '',
+    created_at       timestamptz NOT NULL,
+    started_at       timestamptz,
+    finished_at      timestamptz
+);
+
+-- One queued or running job per service. CreateJob relies on this index to
+-- turn a second concurrent click into ErrJobActive.
+CREATE UNIQUE INDEX update_jobs_one_active ON update_jobs (service_id)
+    WHERE status IN ('queued', 'running');
+CREATE INDEX update_jobs_by_service ON update_jobs (service_id, id DESC);
+CREATE INDEX update_jobs_active_by_agent ON update_jobs (agent_id)
+    WHERE status IN ('queued', 'running');

internal/master/store/migrations/0003_webhook.sql

@@ -0,0 +1,6 @@
+ALTER TABLE services ADD COLUMN available_ref    text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN available_source text NOT NULL DEFAULT ''
+    CHECK (available_source IN ('', 'webhook'));
+ALTER TABLE services ADD COLUMN available_at     timestamptz;
+
+CREATE INDEX services_by_image_repo ON services (image_repo) WHERE removed_at IS NULL;

internal/master/store/migrations/0004_polling.sql

@@ -0,0 +1,20 @@
+ALTER TABLE services DROP CONSTRAINT services_available_source_check;
+ALTER TABLE services ADD CONSTRAINT services_available_source_check
+    CHECK (available_source IN ('', 'webhook', 'poll'));
+ALTER TABLE services ADD COLUMN available_digest text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN last_poll_error  text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN last_tag_digest text NOT NULL DEFAULT '';
+
+-- Every version the master has seen. '' means "no tag" / "digest unknown";
+-- never NULL, so the unique constraint deduplicates.
+CREATE TABLE image_versions (
+    id            bigserial   PRIMARY KEY,
+    repo          text        NOT NULL,
+    tag           text        NOT NULL DEFAULT '',
+    digest        text        NOT NULL DEFAULT '',
+    source        text        NOT NULL CHECK (source IN ('webhook', 'poll', 'running')),
+    first_seen_at timestamptz NOT NULL,
+    CONSTRAINT image_versions_tag_or_digest CHECK (tag <> '' OR digest <> ''),
+    CONSTRAINT image_versions_unique UNIQUE (repo, tag, digest)
+);
+CREATE INDEX image_versions_recent ON image_versions (repo, first_seen_at DESC, id DESC);

internal/master/store/migrations/0005_drift_merge.sql

@@ -0,0 +1,7 @@
+-- True from a successful job until the next inventory write for the row. The
+-- agent's upsert arrives a debounce interval after job.result, and until then
+-- desired_ref is newer than actual_ref without the service having drifted.
+ALTER TABLE services ADD COLUMN awaiting_inventory boolean NOT NULL DEFAULT false;
+
+-- Move suggestions look up Compose rows by project and service across agents.
+CREATE INDEX services_by_compose_name ON services (compose_project, compose_service) WHERE kind = 'compose';

internal/master/store/services.go

@@ -39,35 +39,62 @@ func lockActiveAgent(ctx context.Context, tx pgx.Tx, agentID int64) error {
 
 const upsertServiceSQL = `
 INSERT INTO services (
-	kind, location_key, agent_id, compose_project, compose_service, working_dir,
-	swarm_cluster_id, swarm_service, container_name, image_repo, actual_ref, actual_digest,
+	kind, location_key, agent_id, compose_project, compose_service, working_dir, config_files,
+	built, swarm_cluster_id, swarm_service, container_name, image_repo, actual_ref, actual_digest,
 	status, self, first_seen_at, updated_at)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $15)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $17)
 ON CONFLICT (location_key) DO UPDATE SET
 	agent_id      = EXCLUDED.agent_id,
 	working_dir   = EXCLUDED.working_dir,
+	config_files  = EXCLUDED.config_files,
+	built         = EXCLUDED.built,
 	image_repo    = EXCLUDED.image_repo,
 	actual_ref    = EXCLUDED.actual_ref,
 	actual_digest = EXCLUDED.actual_digest,
 	status        = EXCLUDED.status,
 	self          = EXCLUDED.self,
 	updated_at    = EXCLUDED.updated_at,
-	removed_at    = NULL`
+	removed_at    = NULL,
+	awaiting_inventory = false`
 
 func upsertService(ctx context.Context, db execer, agentID int64, clusterID string, svc protocol.Service, at time.Time) error {
+	image := imageref.Parse(svc.Image)
 	clusterCol := ""
 	if svc.Kind == protocol.KindSwarm {
 		clusterCol = clusterID
 	}
 	_, err := db.Exec(ctx, upsertServiceSQL,
 		string(svc.Kind), model.LocationKey(agentID, clusterID, svc.Key()), agentID,
-		svc.ComposeProject, svc.ComposeService, svc.WorkingDir,
+		svc.ComposeProject, svc.ComposeService, svc.WorkingDir, svc.ConfigFiles, svc.Built,
 		clusterCol, svc.SwarmService, svc.ContainerName,
-		imageref.Parse(svc.Image).Repo, svc.Image, svc.Digest,
+		image.Repo, svc.Image, svc.Digest,
 		svc.Status, svc.Self, at)
 	if err != nil {
 		return fmt.Errorf("upsert service %s: %w", svc.Key(), err)
 	}
+	if imageref.IsDigest(svc.Digest) {
+		// A bare image runs "latest", as jobs.FollowedRef sees it, so it
+		// shares its row with the polled one. repo@digest has no tag.
+		if image.Tag == "" && image.Digest == "" {
+			image.Tag = "latest"
+		}
+		if err := insertVersion(ctx, db, image.Repo, image.Tag, svc.Digest, model.SourceRunning, at); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// insertVersion records a version in image_versions unless the same
+// (repo, tag, digest) is already there; the first sighting keeps its source
+// and first_seen_at.
+func insertVersion(ctx context.Context, db execer, repo, tag, digest, source string, at time.Time) error {
+	if _, err := db.Exec(ctx, `
+		INSERT INTO image_versions (repo, tag, digest, source, first_seen_at)
+		VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`,
+		repo, tag, digest, source, at); err != nil {
+		return fmt.Errorf("record version %s: %w", imageref.Ref{Repo: repo, Tag: tag, Digest: digest}, err)
+	}
 	return nil
 }
 
@@ -144,33 +171,203 @@ func (s *Store) RemoveService(ctx context.Context, agentID int64, clusterID, age
 	return nil
 }
 
+const serviceColumns = `s.id, s.kind, s.location_key, s.agent_id, COALESCE(a.name, ''),
+	s.compose_project, s.compose_service, s.working_dir, s.config_files, s.built, s.swarm_cluster_id,
+	s.swarm_service, s.container_name, s.image_repo, s.desired_ref, s.actual_ref, s.actual_digest,
+	s.status, s.self, s.update_mode, s.available_ref, s.available_digest, s.available_source, s.available_at,
+	s.last_poll_error, s.last_tag_digest, s.awaiting_inventory, s.first_seen_at, s.updated_at, s.removed_at`
+
+func scanService(row pgx.Row) (model.Service, error) {
+	var m model.Service
+	var kind string
+	err := row.Scan(&m.ID, &kind, &m.LocationKey, &m.AgentID, &m.AgentName,
+		&m.ComposeProject, &m.ComposeService, &m.WorkingDir, &m.ConfigFiles, &m.Built, &m.SwarmClusterID,
+		&m.SwarmService, &m.ContainerName, &m.ImageRepo, &m.DesiredRef, &m.ActualRef, &m.ActualDigest,
+		&m.Status, &m.Self, &m.UpdateMode, &m.AvailableRef, &m.AvailableDigest, &m.AvailableSource, &m.AvailableAt,
+		&m.LastPollError, &m.LastTagDigest, &m.AwaitingInventory, &m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt)
+	m.Kind = protocol.Kind(kind)
+	return m, err
+}
+
 // ListServices returns all services not marked removed: machine-bound
 // services grouped by agent name first, Swarm services last.
 func (s *Store) ListServices(ctx context.Context) ([]model.Service, error) {
+	return s.queryServices(ctx, "list services", "", nil)
+}
+
+// ServicesByRepo returns the services not marked removed whose image repo
+// equals repo (case-sensitive), in the same order as ListServices.
+func (s *Store) ServicesByRepo(ctx context.Context, repo string) ([]model.Service, error) {
+	return s.queryServices(ctx, "services by repo", "AND s.image_repo = $1", []any{repo})
+}
+
+func (s *Store) queryServices(ctx context.Context, what, extraWhere string, args []any) ([]model.Service, error) {
 	rows, err := s.pool.Query(ctx, `
-		SELECT s.id, s.kind, s.location_key, s.agent_id, COALESCE(a.name, ''),
-		       s.compose_project, s.compose_service, s.working_dir, s.swarm_cluster_id,
-		       s.swarm_service, s.container_name, s.image_repo, s.actual_ref, s.actual_digest,
-		       s.status, s.self, s.update_mode, s.first_seen_at, s.updated_at
+		SELECT `+serviceColumns+`
 		FROM services s LEFT JOIN agents a ON a.id = s.agent_id
-		WHERE s.removed_at IS NULL
-		ORDER BY s.kind = 'swarm', COALESCE(a.name, ''), s.location_key`)
+		WHERE s.removed_at IS NULL `+extraWhere+`
+		ORDER BY s.kind = 'swarm', COALESCE(a.name, ''), s.location_key`, args...)
 	if err != nil {
-		return nil, fmt.Errorf("list services: %w", err)
+		return nil, fmt.Errorf("%s: %w", what, err)
 	}
 	defer rows.Close()
 	var out []model.Service
 	for rows.Next() {
-		var m model.Service
-		var kind string
-		if err := rows.Scan(&m.ID, &kind, &m.LocationKey, &m.AgentID, &m.AgentName,
-			&m.ComposeProject, &m.ComposeService, &m.WorkingDir, &m.SwarmClusterID,
-			&m.SwarmService, &m.ContainerName, &m.ImageRepo, &m.ActualRef, &m.ActualDigest,
-			&m.Status, &m.Self, &m.UpdateMode, &m.FirstSeenAt, &m.UpdatedAt); err != nil {
+		m, err := scanService(rows)
+		if err != nil {
 			return nil, fmt.Errorf("scan service: %w", err)
 		}
-		m.Kind = protocol.Kind(kind)
 		out = append(out, m)
 	}
 	return out, rows.Err()
 }
+
+// SetAvailable records a newer ref for the service, replacing any earlier
+// suggestion, and records the version in image_versions in the same
+// transaction (the tag is parsed from a.Ref).
+func (s *Store) SetAvailable(ctx context.Context, id int64, a model.Available) error {
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return fmt.Errorf("set available: begin: %w", err)
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+	if _, err := tx.Exec(ctx, `
+		UPDATE services SET available_ref = $2, available_digest = $3, available_source = $4, available_at = $5
+		WHERE id = $1`, id, a.Ref, a.Digest, a.Source, a.At); err != nil {
+		return fmt.Errorf("set available: %w", err)
+	}
+	ref := imageref.Parse(a.Ref)
+	if err := insertVersion(ctx, tx, ref.Repo, ref.Tag, a.Digest, a.Source, a.At); err != nil {
+		return err
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return fmt.Errorf("set available: commit: %w", err)
+	}
+	return nil
+}
+
+// SetPollError sets last_poll_error on the given services; msg "" clears it.
+func (s *Store) SetPollError(ctx context.Context, ids []int64, msg string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	if _, err := s.pool.Exec(ctx,
+		`UPDATE services SET last_poll_error = $2 WHERE id = ANY($1)`, ids, msg); err != nil {
+		return fmt.Errorf("set poll error: %w", err)
+	}
+	return nil
+}
+
+// SetLastTagDigest sets last_tag_digest; see model.Service.LastTagDigest. The
+// poller and jobs.RestoreDesired call it, and FinishJob sets the same column
+// after a tag job. Inventory writes never change it.
+func (s *Store) SetLastTagDigest(ctx context.Context, id int64, digest string) error {
+	if _, err := s.pool.Exec(ctx,
+		`UPDATE services SET last_tag_digest = $2 WHERE id = $1`, id, digest); err != nil {
+		return fmt.Errorf("set last tag digest: %w", err)
+	}
+	return nil
+}
+
+// RecentVersions returns up to limit versions of repo, most recently first
+// seen first. A version without a digest is left out when a version with the
+// same tag and a digest was first seen at or after it.
+func (s *Store) RecentVersions(ctx context.Context, repo string, limit int) ([]model.ImageVersion, error) {
+	rows, err := s.pool.Query(ctx, `
+		SELECT id, repo, tag, digest, source, first_seen_at FROM image_versions v
+		WHERE v.repo = $1
+		  AND NOT (v.digest = '' AND EXISTS (
+		        SELECT 1 FROM image_versions d
+		        WHERE d.repo = v.repo AND d.tag = v.tag AND d.digest <> ''
+		          AND d.first_seen_at >= v.first_seen_at))
+		ORDER BY v.first_seen_at DESC, v.id DESC
+		LIMIT $2`, repo, limit)
+	if err != nil {
+		return nil, fmt.Errorf("recent versions: %w", err)
+	}
+	defer rows.Close()
+	var out []model.ImageVersion
+	for rows.Next() {
+		var v model.ImageVersion
+		if err := rows.Scan(&v.ID, &v.Repo, &v.Tag, &v.Digest, &v.Source, &v.FirstSeenAt); err != nil {
+			return nil, fmt.Errorf("scan version: %w", err)
+		}
+		out = append(out, v)
+	}
+	return out, rows.Err()
+}
+
+// GetImageVersion returns one version. Returns model.ErrNotFound if the ID
+// does not exist.
+func (s *Store) GetImageVersion(ctx context.Context, id int64) (model.ImageVersion, error) {
+	var v model.ImageVersion
+	err := s.pool.QueryRow(ctx, `
+		SELECT id, repo, tag, digest, source, first_seen_at FROM image_versions WHERE id = $1`, id,
+	).Scan(&v.ID, &v.Repo, &v.Tag, &v.Digest, &v.Source, &v.FirstSeenAt)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return model.ImageVersion{}, model.ErrNotFound
+	}
+	if err != nil {
+		return model.ImageVersion{}, fmt.Errorf("get image version: %w", err)
+	}
+	return v, nil
+}
+
+// SetUpdateMode sets the service's update mode (model.ModeManual or
+// model.ModeAuto). Returns model.ErrNotFound if the service does not exist
+// or is marked removed.
+func (s *Store) SetUpdateMode(ctx context.Context, id int64, mode string) error {
+	tag, err := s.pool.Exec(ctx,
+		`UPDATE services SET update_mode = $2 WHERE id = $1 AND removed_at IS NULL`, id, mode)
+	if err != nil {
+		return fmt.Errorf("set update mode: %w", err)
+	}
+	if tag.RowsAffected() == 0 {
+		return model.ErrNotFound
+	}
+	return nil
+}
+
+// AcceptRunning makes the running version the desired one: desired_ref =
+// desired, provided the service is live, has been updated before and still
+// reports actualRef and actualDigest (what the caller saw when it decided).
+// It starts no job and leaves last_tag_digest, the available suggestion and
+// the update mode alone. Returns model.ErrNotFound for an unknown or removed
+// service and model.ErrConflict when the service was never updated or the
+// running version changed meanwhile.
+func (s *Store) AcceptRunning(ctx context.Context, id int64, actualRef, actualDigest, desired string) error {
+	tag, err := s.pool.Exec(ctx, `
+		UPDATE services SET desired_ref = $4, awaiting_inventory = false
+		WHERE id = $1 AND removed_at IS NULL AND desired_ref <> ''
+		  AND actual_ref = $2 AND actual_digest = $3`, id, actualRef, actualDigest, desired)
+	if err != nil {
+		return fmt.Errorf("accept running: %w", err)
+	}
+	if tag.RowsAffected() == 1 {
+		return nil
+	}
+	svc, err := s.GetService(ctx, id)
+	if err != nil {
+		return err
+	}
+	if svc.RemovedAt != nil {
+		return model.ErrNotFound
+	}
+	return model.ErrConflict
+}
+
+// GetService returns one service, including a removed one (RemovedAt set).
+// Returns model.ErrNotFound if the ID does not exist.
+func (s *Store) GetService(ctx context.Context, id int64) (model.Service, error) {
+	m, err := scanService(s.pool.QueryRow(ctx, `
+		SELECT `+serviceColumns+`
+		FROM services s LEFT JOIN agents a ON a.id = s.agent_id
+		WHERE s.id = $1`, id))
+	if errors.Is(err, pgx.ErrNoRows) {
+		return model.Service{}, model.ErrNotFound
+	}
+	if err != nil {
+		return model.Service{}, fmt.Errorf("get service: %w", err)
+	}
+	return m, nil
+}