From 7e55ec3b331390251ee0ddf78da5763a1dd89483 Mon Sep 17 00:00:00 2001 From: Samuel Enocsson Date: Mon, 12 Oct 2026 17:13:15 +0200 Subject: [PATCH 01/29] docs: implementation plan for master/agent phase 2 (jobs) (#7) --- .../2026-10-12-master-agent-phase-2-jobs.md | 1313 +++++++++++++++++ 1 file changed, 1313 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md diff --git a/docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md b/docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md new file mode 100644 index 0000000..6c1c708 --- /dev/null +++ b/docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md @@ -0,0 +1,1313 @@ +# Master/Agent Phase 2 — Jobs and Manual Update Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. Code in "Frozen contracts" is final; code inside tasks is given only where it is subtle — elsewhere the task gives the contract and the test cases, and the implementer writes the code. + +**Goal:** A person clicks *Update* on a service in the master UI; the owning agent re-pulls the service's current tag (resolved to `repo:tag@digest`) and redeploys it, streaming its log live to the browser. Job history is kept in Postgres. `cmd/server` and the old single-instance updater code are removed. + +**Architecture:** The master creates an `update_jobs` row and sends `job.start` over the agent's existing WebSocket. The agent queues it in a single-worker FIFO runner, resolves the tag's digest through the Docker daemon, and deploys: Compose by writing an agent-owned `compose.override.yaml` and running `docker compose pull` + `up -d`; Swarm by `ServiceUpdate` with the pinned image. Output flows back as `job.progress`, the outcome as `job.result`, which the agent keeps until the master answers `job.ack`. On every hello the agent lists the job IDs it holds; the master marks its own queued/running jobs for that agent that are not listed as failed ("lost"). The browser re-fetches a job's full log from Postgres on each SSE signal, so dropped signals are harmless. + +**Tech Stack:** unchanged from phase 1, plus `gopkg.in/yaml.v3` and `github.com/distribution/reference` promoted from indirect to direct. `github.com/prometheus/*` disappears with `internal/metrics`. + +**Spec:** `docs/superpowers/specs/2026-10-12-master-agent-design.md`, phase 2. Phase 1 plan: `docs/superpowers/plans/2026-10-12-master-agent-phase-1-inventory.md` (its conventions still apply). + +## Global Constraints + +- Go `1.26.3` in `go.mod`; Dockerfile stays on `golang:1.26-alpine`. +- `go test ./...` passes with no Docker daemon and no Postgres. Postgres tests carry `//go:build integration` and need `DATABASE_URL`. Run the race detector on every package that gets goroutines in this phase: `go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/`. +- Interfaces are declared in the consuming package. Hand-written fakes, `stretchr/testify`, table-driven where there is more than one case. +- Every exported identifier gets a GoDoc comment. Conventional Commits; every commit message ends with ` (#7)`. +- UI text is English. Every state-changing UI request is a POST carrying the CSRF token. +- WebSocket read limit stays `1 << 20` on both sides. Every websocket write goes through one writer goroutine per connection (both sides) — no other goroutine calls `wsjson.Write`. +- Never edit operator files: the agent writes exactly one file, `/compose.override.yaml`, and only if it is absent or starts with the header line. +- Agent tokens and `AGENT_TOKEN` values never appear in logs (unchanged). + +## Deliberate deviations from the spec in this phase + +- **Only "re-pull the current tag" exists.** The job's target is always the service's current `repo:tag`; the agent resolves the digest. No version picker (phase 4), no webhook (phase 3). Services without a tag (`repo@digest`, "follows no tag") cannot be updated; the UI says why. +- **`cmd/server`, `internal/api`, `internal/config`, `internal/discovery`, `internal/selfupdate` and `internal/metrics` are deleted.** There is no `POST /update` anywhere until phase 3 (see PR notes). `internal/selfupdate` does not compile without `discovery.Job`, and the problem it solves (flushing an HTTP response before recreating yourself) does not exist in the agent model; git history keeps it as the starting point the spec mentions. `internal/metrics` was only used by the old queue and server. +- **`internal/updater` moves to `internal/agent/updater`** (`git mv`, history kept). Its blocking `Queue.Submit` (built for the synchronous webhook) is replaced by `Runner` (async, outbox, live output). The executors are rewritten against `protocol.JobStart`. `discovery.IsInsideRoot` moves into the same package; image matching is dropped (`imageref.Parse(x).Repo` is equivalent to `NormaliseImage`). +- **Override file: one name, and refuse when any other override exists.** Verified empirically against Docker Compose v5.1.2: Compose auto-loads `compose.override.{yml,yaml}` / `docker-compose.override.{yml,yaml}` next to *any* default base file, but loads **only one** (first of `compose.override.yml`, `compose.override.yaml`, `docker-compose.override.yml`, `docker-compose.override.yaml`) and warns about the rest. So the agent writes `compose.override.yaml` and refuses the job when any of the other three exists — writing ours next to an operator's `docker-compose.override.yml` would silently disable theirs on the next manual `docker compose up`. Stacks started with explicit `-f` files do not load the override on a manual `up`; the agent's own jobs always pass it explicitly. +- **The override file is never pruned.** If a service is deleted from the stack file, its `image:` entry stays in `compose.override.yaml` and a manual `docker compose up -d` would recreate it. Documented in the README; pruning needs the base file parsed and is deferred. +- **`job.start` carries no paths.** The agent reads `working_dir` and `config_files` from its own Docker daemon (container labels) at execution time. The inventory still reports `ConfigFiles` (shown on the service page), but a stale or forged path from the master never reaches `docker compose`. +- **`protocol.Service.ConfigFiles` is the raw comma-joined label string**, not a slice, so `Service` stays comparable with `==` (`inventory.Diff` depends on it). +- **Swarm "succeeded" means the daemon accepted the new spec**, not that tasks converged. The inventory shows `degraded` until they do. +- **Job timestamps:** `created_at` is the master's clock; `started_at`/`finished_at` come from the agent's `JobResult` (the master sets `started_at` from its own clock on the first `job.progress` only if the result never arrives). +- **Output cap 128 KiB (tail)**, not 256 KiB: a `JobResult` carrying the whole output must fit the 1 MiB read limit even with worst-case JSON escaping (6×). +- **SSE topic for one job is `job-`** (not `job:`), so it is unambiguous inside `hx-trigger="sse:job-42"`. +- **Revoking an agent fails its queued/running jobs** ("agent revoked"). Not in the spec, but without it a job on a machine that never comes back would block that service forever (one active job per service). +- **A job whose agent never reconnects stays `running`** until the agent is revoked. The master has no job timeout of its own; the agent's `UPDATE_TIMEOUT` applies on the agent. +- **Registry auth supports inline `auths` entries only** (`auth` or `identitytoken` in `config.json`). `credsStore`/`credHelpers` are not available inside the agent container; the agent logs a warning at startup if the file names them. +- **Self-protection is enforced on the master only.** The agent does not re-check whether a job targets itself. +- **Unknown message types are logged and ignored on both sides** (phase 1 closed the connection). Unparseable messages still close it. + +## Review Focus + +| Failure case | Test that covers it | +|---|---| +| A job finishes while the agent is disconnected: the result is resent after reconnect and recorded exactly once; a result resent after a lost `job.ack` is acked again but not re-applied. | e2e `TestJobFinishedWhileDisconnectedIsRecordedOnce` (W5-1); store `TestFinishJob_OnlyOnceAndOnlyByOwner` (W3-1) | +| Master restart mid-job: the agent still has the job → it is **not** marked lost and its result lands. Agent restart mid-job: the job is marked "lost". | e2e `TestMasterRestartKeepsJobTheAgentKnows`, `TestAgentRestartMarksJobLost` (W5-1); hub `TestHub_ReconcilesBeforeRoutable` (W3-2) | +| A job is never sent over a connection that replaced the one routed to (agent reconnected in between); the job fails "agent offline" instead of running after being marked lost. | hub `TestHub_StartJobOnReplacedConnectionFails` (W3-2) | +| An agent cannot write progress to or finish another agent's job (leaked token). | store `TestJobProgress_OwnerOnly`, `TestFinishJob_OnlyOnceAndOnlyByOwner` (W3-1) | +| Two clicks on Update at once create one job; the second gets the first back. | store `TestCreateJob_OneActivePerService` (concurrent, W3-1); jobs `TestTrigger` "active job" case (W3-3) | +| The master refuses jobs for itself and the agents (same image repo as `AGENT_IMAGE`) and for rows flagged `self`. | jobs `TestUpdatable` (W3-3); web `TestServicePage_DisabledReason` (W4-1) | +| The agent never overwrites an operator's override file, and never writes next to another auto-loaded override name. | updater `TestWriteOverride` (W2-4) | +| Every config file and the override path are inside `STACKS_ROOT`, also after resolving symlinks. | updater `TestComposeFiles` (W2-4) | +| Concurrent writers on one websocket (job progress, acks, inventory) are serialised. | `go test -race` on conn and hub tests that start a job while inventory flows (W2-6, W3-2) | +| A phase-1 agent (protocol 0) never receives `job.start`; the job fails "agent too old for updates". | e2e `TestPhaseOneAgentGetsNoJobs` (W5-1); hub `TestRoute` (W3-2) | +| A Swarm job routed to an agent whose daemon is in a different cluster is refused before any update. | updater `TestSwarmExecutor` "cluster mismatch" (W2-5) | +| A job running past `UPDATE_TIMEOUT` is cancelled and fails with a timeout error. | updater `TestRunner_Timeout` (W2-3) | +| A 128 KiB output of escape characters still arrives in one `job.result`. | hub `TestHub_LargeResultArrives` (W3-2) | +| A message type the peer does not know is ignored without dropping the connection. | hub `TestHub_IgnoresUnknownType` (W3-2); conn `TestAgent_IgnoresUnknownType` (W2-6) | + +## PR notes (put this in the PR description) + +> **Production risk — read before merging or tagging a release.** This PR removes `cmd/server` and changes the image's default `ENTRYPOINT` to `updater-agent`. The currently running single-instance updater (`docker-compose.example.yml` style: `image: …/package-updater:latest`, default entrypoint, and its own `se.shcizo.auto-update=true` label) will, the next time anything pulls `:latest` for it — its own `/update` webhook, Watchtower or Shepherd — restart as an agent without `MASTER_URL` and exit in a crash loop. Its `/update` endpoint, which the Gitea action calls, disappears with it. +> Before the next `v*` tag after this merges, the operator must either pin the old updater's image to the last pre-phase-2 tag (and remove its opt-in label), or not release until phase 3 has moved `POST /update` to the master. Agents and the master must be on this release together: a phase-1 agent connects fine but every job sent to it fails "agent too old for updates". + +## Verified facts this plan relies on + +- **Compose override loading** (Docker Compose v5.1.2, tested in a temp dir with `docker compose config` and a throwaway `docker compose create`): see the deviation above. The `com.docker.compose.project.config_files` label lists every file used, as absolute paths joined by `,`, **including an auto-loaded override**. The agent therefore removes `/compose.override.yaml` from the label's list and appends it once, last. +- **`ServiceUpdate` by name:** the daemon's `getService` (`daemon/cluster/helpers.go`) resolves full ID, then exact name, then ID prefix. The executor still inspects by name, checks `Spec.Name`, and updates by the returned `ID` with the returned `Version`, so it updates exactly the object it inspected. +- **Digest resolution:** `client.DistributionInspect(ctx, "repo:tag", encodedAuth)` (API ≥ 1.30) returns `Descriptor.Digest`; the daemon contacts the registry, so it works for any registry the daemon can pull from. +- **Registry auth encoding:** `registry.EncodeAuthConfig(registry.AuthConfig{...})` (`api/types/registry`). + +## Workstreams and file ownership + +W1 lands first (one small commit set). W2, W3, W4 then run in parallel. W5 runs after W2–W4 are merged into the branch. + +| Workstream | Owner | Owns (only this workstream edits these) | +|---|---|---| +| W1 contracts | first implementer | `internal/protocol/protocol.go`, `protocol_test.go`; `internal/master/model/model.go`, new `model/job.go`, `model_test.go`; `internal/master/events/broker.go`, `broker_test.go`; new `internal/master/store/migrations/0002_jobs.sql` | +| W2 agent | implementer-agent | `internal/agent/**` (incl. new `internal/agent/updater/`), `cmd/agent/`, deletions of `cmd/server`, `internal/api`, `internal/config`, `internal/discovery`, `internal/updater`, `internal/selfupdate`, `internal/metrics`; `Dockerfile`; `docker-compose.example.yml`, `docker-compose.local.yml`, `env.sample` (deleted); new `deploy/agent.compose.yml`; `go.mod`, `go.sum` | +| W3 master | implementer-master | `internal/master/store/**` (except the migration file W1 adds), `internal/master/hub/**`, new `internal/master/jobs/`, `internal/master/config/`, `cmd/master/` | +| W4 web | implementer-web | `internal/master/web/**` (Go, templates, `static/app.css`, tests), `internal/master/webauth/auth.go` + `auth_test.go` (one helper only) | +| W5 integration | implementer-master | `internal/e2e/**`, `CLAUDE.md`, `README.md`, `deploy/master.stack.yml` | + +Shared touch points, resolved by contract rather than co-editing: +- `cmd/master/main.go` (W3) calls `web.New` with W4's new signature — frozen below. W3 makes that call compile; if W4 is not merged yet, W3's branch fails to build at that one call, which is expected until both land. +- `go.mod`/`go.sum`: only W2 runs `go mod tidy`. W3/W4 add no dependencies. +- `internal/master/model` is W1's. If W3 or W4 needs a model change, they ask the W1 owner (lead) rather than editing it. + +--- + +## Frozen contracts + +These are final. W2–W4 code against them; W1 commits them verbatim. + +### C1. `internal/protocol/protocol.go` (complete file) + +```go +// Package protocol defines the messages agents and the master exchange over +// their WebSocket connection. Every message is an Envelope whose Payload is +// one of the payload types below, selected by Type. +// +// Both sides close the connection on a message they cannot parse (bad JSON, +// or a known Type whose payload does not decode). A message with an unknown +// Type is logged and ignored, so a newer peer can add message types without +// breaking an older one. +package protocol + +import ( + "encoding/json" + "fmt" + "time" +) + +// Version is the protocol version this build speaks. Agents send it in +// Hello. Phase-1 agents do not send the field, which reads as 0. +const Version = 2 + +// MinJobVersion is the lowest agent protocol version the master sends jobs to. +const MinJobVersion = 2 + +// MaxJobOutput caps the job output an agent keeps and the master stores, in +// bytes. Only the tail is kept. With JSON escaping (worst case 6 bytes per +// byte) a JobResult carrying this much output still fits the 1 MiB read limit. +const MaxJobOutput = 128 << 10 + +// Message types sent from agent to master. +const ( + TypeHello = "hello" + TypeSnapshot = "snapshot" + TypeUpsert = "upsert" + TypeRemove = "remove" + TypeJobProgress = "job.progress" + TypeJobResult = "job.result" +) + +// Message types sent from master to agent. +const ( + TypeJobStart = "job.start" + TypeJobAck = "job.ack" +) + +// Envelope wraps every message on the wire. ID is the job ID (decimal) on +// job.* messages and empty on inventory messages. +type Envelope struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` + Payload json.RawMessage `json:"payload,omitempty"` +} + +// Hello is the first message an agent sends on every connection. +type Hello struct { + AgentVersion string `json:"agent_version"` + ProtocolVersion int `json:"protocol_version,omitempty"` + Hostname string `json:"hostname"` + SwarmManager bool `json:"swarm_manager"` + SwarmClusterID string `json:"swarm_cluster_id,omitempty"` + // KnownJobs lists every job ID the agent holds: queued, running, or + // finished but not yet acknowledged. The master fails its queued and + // running jobs for this agent that are not listed ("lost"). + KnownJobs []string `json:"known_jobs,omitempty"` +} + +// Kind says how a service is deployed. +type Kind string + +// Service kinds. +const ( + KindCompose Kind = "compose" + KindSwarm Kind = "swarm" + KindContainer Kind = "container" +) + +// Service is one deployable unit as an agent sees it. Which location fields +// are set depends on Kind. +type Service struct { + Kind Kind `json:"kind"` + ComposeProject string `json:"compose_project,omitempty"` + ComposeService string `json:"compose_service,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + // ConfigFiles is the raw com.docker.compose.project.config_files label: + // absolute host paths joined by ",". Kept as a string so Service stays + // comparable with ==. + ConfigFiles string `json:"config_files,omitempty"` + SwarmService string `json:"swarm_service,omitempty"` + ContainerName string `json:"container_name,omitempty"` + Image string `json:"image"` + Digest string `json:"digest,omitempty"` + Status string `json:"status"` + Self bool `json:"self,omitempty"` +} + +// Key identifies the service within one agent's report. It is stable across +// restarts and is what Remove refers to. +func (s Service) Key() string { + switch s.Kind { + case KindCompose: + return "compose|" + s.ComposeProject + "|" + s.ComposeService + case KindSwarm: + return "swarm|" + s.SwarmService + default: + return "container|" + s.ContainerName + } +} + +// Snapshot is the agent's complete inventory. Sent after every Hello. +type Snapshot struct { + Services []Service `json:"services"` +} + +// Upsert reports a service that appeared or changed. +type Upsert struct { + Service Service `json:"service"` +} + +// Remove reports that the service with Key is gone. +type Remove struct { + Key string `json:"key"` +} + +// JobStart asks the agent to deploy Ref to one service. Ref is "repo:tag"; +// the agent resolves the tag's current digest and deploys "repo:tag@digest". +// The agent reads the Compose working directory and config files from its +// own Docker daemon, not from this message. +type JobStart struct { + Kind Kind `json:"kind"` + ComposeProject string `json:"compose_project,omitempty"` + ComposeService string `json:"compose_service,omitempty"` + SwarmClusterID string `json:"swarm_cluster_id,omitempty"` + SwarmService string `json:"swarm_service,omitempty"` + Ref string `json:"ref"` +} + +// JobProgress carries output produced since the previous JobProgress for the +// same job. The first JobProgress of a job (Output may be empty) tells the +// master the job is running. +type JobProgress struct { + Output string `json:"output"` +} + +// Job result statuses. +const ( + ResultSucceeded = "succeeded" + ResultFailed = "failed" +) + +// JobResult is a job's outcome. The agent keeps it until the master sends +// JobAck for the same ID, and resends it after every reconnect until then. +type JobResult struct { + Status string `json:"status"` // ResultSucceeded or ResultFailed + Ref string `json:"ref,omitempty"` // deployed "repo:tag@digest", set on success + Error string `json:"error,omitempty"` + // Output is the job's complete output, tail-capped at MaxJobOutput. It + // replaces whatever the master assembled from JobProgress messages. + Output string `json:"output"` + OutputTruncated bool `json:"output_truncated,omitempty"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` +} + +// JobAck tells the agent the master has stored the JobResult with the same +// ID. It has no fields. +type JobAck struct{} + +// Encode wraps payload in an Envelope of the given type. +func Encode(typ string, payload any) (Envelope, error) { + raw, err := json.Marshal(payload) + if err != nil { + return Envelope{}, fmt.Errorf("encode %s: %w", typ, err) + } + return Envelope{Type: typ, Payload: raw}, nil +} + +// EncodeJob wraps a job message's payload in an Envelope carrying the job ID. +func EncodeJob(typ, jobID string, payload any) (Envelope, error) { + env, err := Encode(typ, payload) + if err != nil { + return Envelope{}, err + } + env.ID = jobID + return env, nil +} + +// Decode unpacks env's payload into T. +func Decode[T any](env Envelope) (T, error) { + var v T + if err := json.Unmarshal(env.Payload, &v); err != nil { + return v, fmt.Errorf("decode %s: %w", env.Type, err) + } + return v, nil +} +``` + +Wire rules both sides follow: + +| Message | Direction | `ID` | Receiver on bad payload | Receiver on unknown/foreign job ID | +|---|---|---|---|---| +| `job.start` | M→A | job ID | close | (n/a) duplicate ID ignored | +| `job.ack` | M→A | job ID | close | ignore | +| `job.progress` | A→M | job ID | close | ignore (non-integer ID: log + ignore) | +| `job.result` | A→M | job ID | close | **ack anyway** (non-integer ID too), so the agent drops it; log `job_result_ignored` | +| anything else unknown | both | — | — | log at warn, ignore, keep the connection | + +Agent send order on each connection: `hello` (with `ProtocolVersion: protocol.Version`, `KnownJobs`) → `snapshot` → every unacknowledged `job.result` → then live traffic. A store error while the master handles a job message closes the connection (as in phase 1), so the agent reconnects and resends. + +### C2. `internal/master/model` additions + +`model.go` — add three fields to `Service` (W1): + +```go + WorkingDir string + ConfigFiles string // raw config_files label, paths joined by "," + SwarmClusterID string + ... + DesiredRef string // set by the last successful job; "" if never updated + ActualRef string + ... + UpdatedAt time.Time + RemovedAt *time.Time +``` + +New file `job.go` (complete): + +```go +package model + +import ( + "errors" + "time" +) + +// Sentinel errors for jobs. +var ( + // ErrJobActive means the service already has a queued or running job. + // CreateJob and jobs.Service.Trigger return it together with that job. + ErrJobActive = errors.New("an update is already in progress for this service") + // ErrAgentOffline means no connected agent can run the job. + ErrAgentOffline = errors.New("agent offline") + // ErrAgentTooOld means the only agent that could run the job speaks a + // protocol version without jobs. + ErrAgentTooOld = errors.New("agent too old for updates") +) + +// NotUpdatableError says why a service cannot be updated. Reason is shown +// in the UI as is. +type NotUpdatableError struct { + Reason string +} + +func (e *NotUpdatableError) Error() string { return "not updatable: " + e.Reason } + +// Reasons a service cannot be updated. +const ( + ReasonSelf = "updated manually outside the system" + ReasonContainer = "plain containers cannot be updated" + ReasonNoTag = "follows no tag" + ReasonRemoved = "service no longer exists" +) + +// JobStatus is the state of an update job. +type JobStatus string + +// Job statuses. A job moves queued -> running -> succeeded|failed, or +// straight from queued to failed. +const ( + JobQueued JobStatus = "queued" + JobRunning JobStatus = "running" + JobSucceeded JobStatus = "succeeded" + JobFailed JobStatus = "failed" +) + +// JobTrigger says what created a job. Only TriggerManual is used in phase 2. +type JobTrigger string + +// Job triggers. +const ( + TriggerManual JobTrigger = "manual" + TriggerWebhook JobTrigger = "webhook" + TriggerPoll JobTrigger = "poll" +) + +// Job is one update of one service. +type Job struct { + ID int64 + ServiceID int64 + ServiceName string // Service.DisplayName() of the job's service + AgentID *int64 // agent the job was sent to; nil if it never was + AgentName string + FromRef string + ToRef string // "repo:tag" when created; "repo:tag@digest" after success + Trigger JobTrigger + RequestedBy string // OIDC subject for manual jobs + Status JobStatus + Output string // empty in ListJobs results + OutputTruncated bool + Error string + CreatedAt time.Time + StartedAt *time.Time + FinishedAt *time.Time +} + +// Active reports whether the job is queued or running. +func (j Job) Active() bool { return j.Status == JobQueued || j.Status == JobRunning } + +// NewJob is what CreateJob needs to insert a queued job. +type NewJob struct { + ServiceID int64 + AgentID *int64 + FromRef string + ToRef string + Trigger JobTrigger + RequestedBy string + CreatedAt time.Time +} + +// JobFinish is a job's outcome as reported by the agent. +type JobFinish struct { + Status JobStatus // JobSucceeded or JobFailed + Ref string // deployed "repo:tag@digest"; set on success + Error string + Output string + OutputTruncated bool + StartedAt time.Time // agent clock + FinishedAt time.Time // agent clock +} + +// JobFilter selects jobs for ListJobs. ServiceID 0 means all services. +// Limit 0 means 100. +type JobFilter struct { + ServiceID int64 + Limit int +} + +// Route is the hub's choice of agent connection for a job. ConnID identifies +// the connection, so a job is never sent over a connection that replaced the +// one chosen. +type Route struct { + AgentID int64 + ConnID uint64 +} +``` + +### C3. `internal/master/events/broker.go` additions + +```go +import ( + "strconv" + "sync" +) + +// Topics. +const ( + Services Topic = "services" + Agents Topic = "agents" + Jobs Topic = "jobs" // a job was created or changed status +) + +// JobTopic is signalled when job id gets new output or changes status. The +// name avoids ":" so it is safe inside hx-trigger="sse:job-42". +func JobTopic(id int64) Topic { return Topic("job-" + strconv.FormatInt(id, 10)) } +``` + +Who publishes what (W3 implements, W4 listens): + +| Event | Topics | +|---|---| +| job created (any outcome) | `Jobs`, `JobTopic(id)` | +| `job.progress` applied | `Jobs`, `JobTopic(id)` | +| job finished / failed / lost / revoked | `Jobs`, `JobTopic(id)`; plus `Services` on success (desired ref changed) | + +`web.events` needs no change: it already writes `event: `. + +### C4. Migration `internal/master/store/migrations/0002_jobs.sql` (complete) + +```sql +ALTER TABLE services ADD COLUMN config_files text NOT NULL DEFAULT ''; +ALTER TABLE services ADD COLUMN desired_ref text NOT NULL DEFAULT ''; + +CREATE TABLE update_jobs ( + id bigserial PRIMARY KEY, + service_id bigint NOT NULL REFERENCES services (id), + agent_id bigint REFERENCES agents (id), + from_ref text NOT NULL DEFAULT '', + to_ref text NOT NULL, + trigger text NOT NULL CHECK (trigger IN ('manual', 'webhook', 'poll')), + requested_by text NOT NULL DEFAULT '', + status text NOT NULL CHECK (status IN ('queued', 'running', 'succeeded', 'failed')), + output text NOT NULL DEFAULT '', + output_truncated boolean NOT NULL DEFAULT false, + error text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL, + started_at timestamptz, + finished_at timestamptz +); + +-- One queued or running job per service. CreateJob relies on this index to +-- turn a second concurrent click into ErrJobActive. +CREATE UNIQUE INDEX update_jobs_one_active ON update_jobs (service_id) + WHERE status IN ('queued', 'running'); +CREATE INDEX update_jobs_by_service ON update_jobs (service_id, id DESC); +CREATE INDEX update_jobs_active_by_agent ON update_jobs (agent_id) + WHERE status IN ('queued', 'running'); +``` + +### C5. Store methods (W3 implements on `*store.Store`; W3/W4 consumers declare the subsets they use) + +```go +// GetService returns one service, including a removed one (RemovedAt set). +// model.ErrNotFound if the ID does not exist. +GetService(ctx context.Context, id int64) (model.Service, error) + +// CreateJob inserts a queued job. If the service already has a queued or +// running job, it returns that job and model.ErrJobActive (detected via the +// unique index, so it holds under concurrent calls). +CreateJob(ctx context.Context, j model.NewJob) (model.Job, error) + +// FailJob sets a queued or running job to failed with msg and finished_at = +// at. Returns false (no error) if the job is no longer active. +FailJob(ctx context.Context, id int64, msg string, at time.Time) (bool, error) + +// GetJob returns one job with its output. model.ErrNotFound if missing. +GetJob(ctx context.Context, id int64) (model.Job, error) + +// ListJobs returns jobs newest first, without Output. +ListJobs(ctx context.Context, f model.JobFilter) ([]model.Job, error) + +// FailLostJobs fails every queued or running job sent to agentID whose ID is +// not in known, with error "lost: the agent did not know this job when it +// reconnected". Returns the IDs it failed. +FailLostJobs(ctx context.Context, agentID int64, known []int64, at time.Time) ([]int64, error) + +// JobProgress appends chunk to the output of a queued or running job sent to +// agentID, keeping the last protocol.MaxJobOutput characters, and moves a +// queued job to running (started_at = at if unset). Returns false if no +// such active job belongs to agentID. +JobProgress(ctx context.Context, agentID, jobID int64, chunk string, at time.Time) (bool, error) + +// FinishJob records the agent's result for a queued or running job sent to +// agentID: status, error, output (replacing what progress assembled), +// started_at (kept if already set), finished_at; to_ref = f.Ref when set. +// On success it also sets the service's desired_ref = f.Ref, in the same +// transaction. Returns false if no such active job belongs to agentID. +FinishJob(ctx context.Context, agentID, jobID int64, f model.JobFinish) (bool, error) +``` + +Changed existing methods (W3): `upsertService`/`ApplySnapshot`/`UpsertService` also write `config_files`; `ListServices` also returns `ConfigFiles` and `DesiredRef`; `RevokeAgent` also fails the agent's queued/running jobs with "agent revoked" in its transaction. + +### C6. `internal/master/jobs` (W3) — consumed by web (W4) + +```go +// Package jobs decides whether a service may be updated and starts update +// jobs on the right agent. +package jobs + +// Store is the storage jobs needs. *store.Store satisfies it. +type Store interface { + GetService(ctx context.Context, id int64) (model.Service, error) + CreateJob(ctx context.Context, j model.NewJob) (model.Job, error) + FailJob(ctx context.Context, id int64, msg string, at time.Time) (bool, error) + GetJob(ctx context.Context, id int64) (model.Job, error) +} + +// Hub routes jobs to agent connections. *hub.Hub satisfies it. +type Hub interface { + // Route picks the connection for svc: model.ErrAgentOffline or + // model.ErrAgentTooOld when there is none. + Route(svc model.Service) (model.Route, error) + // StartJob sends job.start over exactly the routed connection. + // model.ErrAgentOffline if that connection is gone or was replaced. + StartJob(r model.Route, jobID int64, start protocol.JobStart) error +} + +// Publisher receives change signals. *events.Broker satisfies it. +type Publisher interface{ Publish(t events.Topic) } + +// New returns a Service. agentImage is the master's AGENT_IMAGE; services +// whose image repo equals its repo are the updater itself and are refused. +func New(store Store, hub Hub, pub Publisher, agentImage string, log *slog.Logger) *Service + +// Updatable returns nil or a *model.NotUpdatableError. It looks only at the +// service row, so the UI can call it for every page render. +func (s *Service) Updatable(svc model.Service) error + +// Trigger starts a manual update of the service to its current tag. It +// returns the job, which may already be failed ("agent offline", "agent too +// old for updates"); a *model.NotUpdatableError when the service cannot be +// updated (no job created); the active job and model.ErrJobActive when one +// is queued or running; model.ErrNotFound for an unknown service. +func (s *Service) Trigger(ctx context.Context, serviceID int64, requestedBy string) (model.Job, error) +``` + +### C7. `internal/master/web` constructor (W4) — called by `cmd/master` (W3) + +```go +// Jobs starts updates and explains why a service cannot be updated. +// *jobs.Service satisfies it. +type Jobs interface { + Updatable(svc model.Service) error + Trigger(ctx context.Context, serviceID int64, requestedBy string) (model.Job, error) +} + +// Store gains (in addition to phase 1's methods): + GetService(ctx context.Context, id int64) (model.Service, error) + GetJob(ctx context.Context, id int64) (model.Job, error) + ListJobs(ctx context.Context, f model.JobFilter) ([]model.Job, error) + +func New(store Store, hub Hub, jobs Jobs, broker Broker, cfg Config, log *slog.Logger) (*Server, error) +``` + +`cmd/master` wiring (W3): + +```go + agentHub := hub.New(st, broker, logger) + jobService := jobs.New(st, agentHub, broker, cfg.AgentImage, logger) + ui, err := web.New(st, agentHub, jobService, broker, web.Config{...}, logger) +``` + +### C8. Agent configuration and public agent APIs (W2; used by W4 snippet and W5 e2e) + +Environment (added to `internal/agent/config.Config`): + +| Variable | Field | Default | Validation | +|---|---|---|---| +| `STACKS_ROOT` | `StacksRoot string` | `""` (Compose jobs then fail "refused: STACKS_ROOT is not set on this agent") | if set: absolute and an existing directory, else fail fast | +| `UPDATE_TIMEOUT` | `UpdateTimeout time.Duration` | `5m` | `time.ParseDuration`, must be > 0 | +| `DOCKER_CONFIG` | `DockerConfig string` | `""` (anonymous registry access) | if set: `$DOCKER_CONFIG/config.json` must exist and parse, else fail fast. Same variable `docker compose` reads, so `pull` uses the same credentials. | + +`STACKS_ROOT` must be mounted read-write **at the same path as on the host**, because the labels contain host paths. + +```go +// package conn +type Jobs interface { + Submit(id string, start protocol.JobStart) + Ack(id string) + Known() []string + TakeProgress() []updater.Progress + Pending() []updater.Finished + Changed() <-chan struct{} +} +// Config gains ProgressInterval time.Duration // default 500ms +func New(cfg Config, src Source, jobs Jobs, log *slog.Logger) *Agent + +// package updater +type Executor interface { + Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) +} +type Progress struct{ ID, Output string } +type Finished struct { + ID string + Result protocol.JobResult +} +func NewRunner(exec Executor, timeout time.Duration) *Runner // *Runner satisfies conn.Jobs +func (r *Runner) Run(ctx context.Context) +``` + +--- + +## W1 — Contracts + +### Task W1-1: Protocol, model, events, migration + +**Files:** +- Modify: `internal/protocol/protocol.go`, `internal/protocol/protocol_test.go` +- Modify: `internal/master/model/model.go`, `internal/master/model/model_test.go`; Create: `internal/master/model/job.go` +- Modify: `internal/master/events/broker.go`, `internal/master/events/broker_test.go` +- Create: `internal/master/store/migrations/0002_jobs.sql` + +- [ ] **Step 1: Write the failing tests** + - `protocol_test.go`: `TestEncodeJob_CarriesID` — `EncodeJob(TypeJobStart, "42", JobStart{Kind: KindSwarm, SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"})` round-trips through `json.Marshal`/`Unmarshal` + `Decode[JobStart]` with `ID == "42"`; `TestHello_PhaseOneDecodesAsVersionZero` — `{"agent_version":"x","hostname":"h","swarm_manager":false}` decodes to `ProtocolVersion 0`, `KnownJobs nil`; `TestEnvelope_InventoryHasNoID` — `Encode(TypeSnapshot, …)` marshals without an `"id"` key. + - `model_test.go`: `TestJob_Active` table over the four statuses. + - `broker_test.go`: `TestJobTopic` — `JobTopic(42) == "job-42"`. +- [ ] **Step 2:** Run `go test ./internal/protocol ./internal/master/model ./internal/master/events` — FAIL (undefined identifiers). +- [ ] **Step 3:** Apply C1, C2, C3 exactly; add C4's file. +- [ ] **Step 4:** Run `go build ./... && go test ./...` — PASS. (The rest of the tree must still compile: `ConfigFiles` is a string, so `inventory.Diff`'s `!=` still works.) +- [ ] **Step 5: Commit** — `feat(protocol): job messages, protocol version and job model (#7)` + +--- + +## W2 — Agent + +### Task W2-1: Remove the old updater and move what survives + +**Files:** +- `git mv internal/updater internal/agent/updater`; then delete `internal/agent/updater/{queue.go,queue_test.go,worker.go,worker_test.go,executor.go}`. Keep `compose_executor.go`, `swarm_executor.go`, `swarm_executor_test.go` for W2-4/W2-5 to rewrite (history follows the rename). +- `git mv internal/discovery/pathcheck.go internal/agent/updater/pathcheck.go` and `pathcheck_test.go` likewise; change `package discovery` → `package updater`. +- Delete: `cmd/server/`, `internal/api/`, `internal/config/`, the rest of `internal/discovery/`, `internal/selfupdate/`, `internal/metrics/`. + +- [ ] **Step 1:** Do the moves and deletions. The moved executors no longer compile (they reference `discovery.Job`) until W2-4/W2-5 rewrite them, so W2-1, W2-3, W2-4 and W2-5 land as **one commit** once `go build ./...` is green. Never commit a tree that does not build. +- [ ] **Step 2:** `go mod tidy` after everything compiles; confirm `github.com/prometheus/*` left `go.mod` and `gopkg.in/yaml.v3`, `github.com/distribution/reference` are direct. +- [ ] **Step 3:** `grep -rn "internal/\(api\|config\|discovery\|selfupdate\|metrics\|updater\)\"" --include='*.go' .` returns nothing. +- [ ] **Step 4: Commit** (with W2-3..W2-5) — `refactor(agent)!: move updater under the agent and remove cmd/server (#7)` with a `BREAKING CHANGE:` footer naming the removed `/update` endpoint and the new default entrypoint. + +### Task W2-2: Agent config and inventory `ConfigFiles` + +**Files:** `internal/agent/config/config.go`, `config_test.go`; `internal/agent/inventory/collector.go`, `collector_test.go` + +- [ ] **Step 1: Failing tests** + - `TestLoad_JobSettings` (table): defaults (`StacksRoot ""`, `UpdateTimeout 5m`, `DockerConfig ""`); `STACKS_ROOT=relative/path` → error; `STACKS_ROOT=/missing` → error; `STACKS_ROOT=` → ok; `UPDATE_TIMEOUT=90s` → 90s; `UPDATE_TIMEOUT=0`/`abc` → error; `DOCKER_CONFIG=` → error; with a valid file → ok. + - Collector: a Compose container with label `com.docker.compose.project.config_files=/s/a/compose.yaml,/s/a/compose.override.yaml` yields `ConfigFiles` equal to the raw label value. +- [ ] **Step 2:** Run `go test ./internal/agent/config ./internal/agent/inventory` — FAIL. +- [ ] **Step 3:** Implement. `config.Load` only validates `DOCKER_CONFIG`'s file exists and is JSON; parsing into credentials is W2-5's `LoadRegistryAuth`. +- [ ] **Step 4:** PASS. **Commit:** `feat(agent): STACKS_ROOT, UPDATE_TIMEOUT and DOCKER_CONFIG settings (#7)` + +### Task W2-3: Job runner (queue, outbox, timeout) + +**Files:** Create `internal/agent/updater/runner.go`, `runner_test.go` + +**Contract:** C8 `Executor`, `Progress`, `Finished`, `NewRunner`, plus `Submit`, `Changed`, `Known`, `TakeProgress`, `Pending`, `Ack`, `Run`. + +- [ ] **Step 1: Failing tests** (fake executor that writes `"pulling \n"` to `out` and blocks on a channel; fails for refs starting with `bad`): + - `TestRunner_FIFOSingleWorker`: submit 1, 2; executor records max concurrency = 1; results in `Pending()` in finishing order 1, 2; 1 succeeded with `Ref "a:1@sha256:aa"` and `Output "pulling a:1\n"`; 2 failed with the executor's error. + - `TestRunner_DuplicateSubmitIgnored`: second `Submit("1", …)` with a different ref does not run. + - `TestRunner_KnownAndAck`: `Known()` lists queued, running and finished-unacked IDs (sorted); after `Ack("1")` it is gone; `Ack` of a running job is ignored. + - `TestRunner_TakeProgress`: a started job appears once with `""` output even before writing; output written between two calls appears once; a finished job no longer appears. + - `TestRunner_Timeout`: timeout 20ms, executor blocks on ctx → result failed, `Error == "timed out after 20ms"`. + - `TestRunner_OutputCapped`: executor writes `protocol.MaxJobOutput + 10` bytes → `Result.Output` is the last `MaxJobOutput` bytes, `OutputTruncated` true. + - Run all with `-race`. +- [ ] **Step 2:** `go test -race ./internal/agent/updater -run Runner` — FAIL. +- [ ] **Step 3: Implement.** The state handling is subtle; use this shape (prototyped and passing the tests above under `-race`): + +```go +type jobState struct { + start protocol.JobStart + running bool + result *protocol.JobResult + output tailBuffer // everything, capped at protocol.MaxJobOutput + unsent tailBuffer // since the last TakeProgress, capped at maxProgress (32 KiB) + reported bool // a Progress has been taken since the job started + finished time.Time // orders Pending +} + +// Runner: mu guards jobs and queue; wake (cap 1) wakes the worker; changed +// (cap 1) tells the connection there is something to send. signal() does a +// non-blocking send so neither side ever blocks on the other. +type Runner struct { + exec Executor + timeout time.Duration + now func() time.Time + mu sync.Mutex + jobs map[string]*jobState + queue []string + wake chan struct{} + changed chan struct{} +} + +func (r *Runner) runOne(ctx context.Context, id string, start protocol.JobStart) { + started := r.now() + jobCtx, cancel := context.WithTimeout(ctx, r.timeout) + ref, err := r.exec.Execute(jobCtx, start, &jobWriter{r: r, id: id}) + timedOut := errors.Is(jobCtx.Err(), context.DeadlineExceeded) + cancel() + res := protocol.JobResult{Status: protocol.ResultSucceeded, Ref: ref, StartedAt: started, FinishedAt: r.now()} + switch { + case timedOut: + res = protocol.JobResult{Status: protocol.ResultFailed, Error: fmt.Sprintf("timed out after %s", r.timeout), StartedAt: started, FinishedAt: res.FinishedAt} + case err != nil: + res = protocol.JobResult{Status: protocol.ResultFailed, Error: err.Error(), StartedAt: started, FinishedAt: res.FinishedAt} + } + r.mu.Lock() + j := r.jobs[id] + res.Output, res.OutputTruncated = j.output.String(), j.output.Truncated() + j.running, j.result, j.finished = false, &res, res.FinishedAt + j.unsent.Reset() + r.mu.Unlock() + signal(r.changed) +} +``` + +`jobWriter.Write` appends to both buffers under `mu`, then `signal(r.changed)`. `tailBuffer` keeps the last `max` bytes and a `truncated` flag. `TakeProgress` skips jobs that are not running, and running jobs that were already reported and have no new output. `Run` returns on ctx end; queued jobs are dropped (the master marks them lost on the next hello). + +- [ ] **Step 4:** PASS with `-race`. Commit with W2-1. + +### Task W2-4: Compose executor and override file + +**Files:** Create `internal/agent/updater/override.go`, `override_test.go`, `resolve.go`, `resolve_test.go`; rewrite `compose_executor.go` (→ `compose.go` via `git mv`), add `compose_test.go`. + +**Contracts:** + +```go +// Resolver turns "repo:tag" into "repo:tag@digest" via the daemon. +type DistributionClient interface { + DistributionInspect(ctx context.Context, imageRef, encodedRegistryAuth string) (registry.DistributionInspect, error) +} +func NewResolver(cli DistributionClient, auth *RegistryAuth) *Resolver +// Resolve fails for a ref without a tag or with a digest already. +func (r *Resolver) Resolve(ctx context.Context, ref string) (string, error) + +// CommandRunner runs "docker " in dir with stdout and stderr to out. +type CommandRunner func(ctx context.Context, dir string, out io.Writer, args ...string) error +// ExecDocker is the real CommandRunner (exec.CommandContext, WaitDelay 10s). +func ExecDocker(ctx context.Context, dir string, out io.Writer, args ...string) error + +type ContainerLister interface { + ContainerList(ctx context.Context, opts container.ListOptions) ([]container.Summary, error) +} +func NewComposeExecutor(cli ContainerLister, resolve *Resolver, stacksRoot string, run CommandRunner) *ComposeExecutor +func (e *ComposeExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) +``` + +`ComposeExecutor.Execute` steps, in this order (each failure stops the job before the next step): +1. List containers (`All: true`) filtered by labels `com.docker.compose.project=

` and `com.docker.compose.service=`; none → error `no container for compose service p/s on this machine`. Take `working_dir` and `config_files` labels from the first. +2. `composeFiles(stacksRoot, configFiles)` → `-f` list ending with the override, and the stack dir (code below). +3. `Resolve(job.Ref)` → `pinned`. Write `resolved \n` to `out`. +4. `WriteOverride(dir, job.ComposeService, pinned)`. +5. `run(ctx, workingDir, out, "compose", "-p", p, "-f", f1, …, "-f", override, "pull", s)`, then the same with `"up", "-d", s`. Return `pinned`. + +Override writer and path check (prototyped; tests below pass): + +```go +// OverrideName is the override file the agent owns. docker compose loads it +// automatically next to compose.yaml or docker-compose.yml. +const OverrideName = "compose.override.yaml" + +// overrideHeader marks a file the agent may rewrite. A file without it +// belongs to the operator and is never touched. +const overrideHeader = "# managed by package-updater — do not edit\n" + +// overrideNames are the override files docker compose auto-loads. It loads +// only the first one present (in this order), so any other one next to ours +// would either hide ours or be hidden by it. +var overrideNames = []string{ + "compose.override.yml", "compose.override.yaml", + "docker-compose.override.yml", "docker-compose.override.yaml", +} + +// WriteOverride sets services..image to ref in dir/OverrideName, +// keeping every other entry in the file, and returns the file's path. It +// refuses when the file exists without the header, or when another +// auto-loaded override file exists in dir. The write is atomic. +func WriteOverride(dir, service, ref string) (string, error) { + for _, name := range overrideNames { + if name == OverrideName { + continue + } + if _, err := os.Lstat(filepath.Join(dir, name)); err == nil { + return "", fmt.Errorf("refused: %s exists; the agent only manages %s and never edits operator files", name, OverrideName) + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("check %s: %w", name, err) + } + } + path := filepath.Join(dir, OverrideName) + doc := map[string]any{} + raw, err := os.ReadFile(path) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + return "", fmt.Errorf("read override: %w", err) + case !bytes.HasPrefix(raw, []byte(overrideHeader)): + return "", fmt.Errorf("refused: %s exists and was not written by package-updater", path) + default: + if err := yaml.Unmarshal(raw, &doc); err != nil { + return "", fmt.Errorf("parse override: %w", err) + } + if doc == nil { + doc = map[string]any{} + } + } + services, _ := doc["services"].(map[string]any) + if services == nil { + services = map[string]any{} + } + entry, _ := services[service].(map[string]any) + if entry == nil { + entry = map[string]any{} + } + entry["image"] = ref + services[service] = entry + doc["services"] = services + body, err := yaml.Marshal(doc) + if err != nil { + return "", fmt.Errorf("encode override: %w", err) + } + // Temp file in the same directory + rename: a crash never leaves a + // half-written override that compose would then load. + tmp, err := os.CreateTemp(dir, "."+OverrideName+".*") + if err != nil { + return "", fmt.Errorf("write override: %w", err) + } + defer os.Remove(tmp.Name()) // no-op after a successful rename + if _, err := tmp.WriteString(overrideHeader + string(body)); err != nil { + tmp.Close() + return "", fmt.Errorf("write override: %w", err) + } + if err := tmp.Chmod(0o644); err != nil { + tmp.Close() + return "", fmt.Errorf("write override: %w", err) + } + if err := tmp.Close(); err != nil { + return "", fmt.Errorf("write override: %w", err) + } + if err := os.Rename(tmp.Name(), path); err != nil { + return "", fmt.Errorf("write override: %w", err) + } + return path, nil +} + +// composeFiles returns the -f arguments for a job: the stack's config files +// from the container label with any copy of the override removed, then the +// override last so it wins. Every path, and the real location of the +// stack directory, must be inside root. +func composeFiles(root, configFilesLabel string) (files []string, dir string, err error) { + if root == "" { + return nil, "", errors.New("refused: STACKS_ROOT is not set on this agent") + } + for _, f := range strings.Split(configFilesLabel, ",") { + if f = strings.TrimSpace(f); f != "" { + files = append(files, f) + } + } + if len(files) == 0 { + return nil, "", errors.New("container has no com.docker.compose.project.config_files label") + } + dir = filepath.Dir(files[0]) + override := filepath.Join(dir, OverrideName) + out := files[:0] + for _, f := range files { + if !filepath.IsAbs(f) || !IsInsideRoot(root, f) { + return nil, "", fmt.Errorf("refused: %s is outside STACKS_ROOT", f) + } + if filepath.Clean(f) != override { + out = append(out, f) + } + } + real, err := filepath.EvalSymlinks(dir) + if err != nil { + return nil, "", fmt.Errorf("resolve %s: %w", dir, err) + } + realRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return nil, "", fmt.Errorf("resolve STACKS_ROOT: %w", err) + } + if !IsInsideRoot(realRoot, real) { + return nil, "", fmt.Errorf("refused: %s resolves to %s, outside STACKS_ROOT", dir, real) + } + return append(out, override), dir, nil +} +``` + +- [ ] **Step 1: Failing tests** + - `TestWriteOverride` (table, each case on its own `t.TempDir()`): new file → starts with header, contains `services.web.image`; existing ours with `db` entry → `db` kept, `web` set; existing ours with `web` → replaced; `compose.override.yaml` without header → refused, file unchanged; each of `compose.override.yml`, `docker-compose.override.yml`, `docker-compose.override.yaml` present → refused, ours not created; after success the directory holds no temp files. + - `TestComposeFiles` (table): label `a/compose.yaml` → `[a/compose.yaml, a/compose.override.yaml]`; label already containing the override → listed once, last; empty label → error; `STACKS_ROOT ""` → refused; a file outside root → refused; root/`evil` symlinked to a dir outside root → refused; `/foo-evil` vs root `/foo` → refused (lexical guard still applies). + - `TestResolver` (fake `DistributionClient` recording args): `"gitea.shcizo.se/shcizo/slusk:1.4"` → called with that ref and the auth string for `gitea.shcizo.se`; returns `"gitea.shcizo.se/shcizo/slusk:1.4@sha256:d1"`; `"repo@sha256:x"` and `"repo"` → error without calling the daemon; daemon error is wrapped. + - `TestComposeExecutor` (fake lister, fake resolver client, recording `CommandRunner`, temp `STACKS_ROOT`): happy path runs exactly `compose -p slusk -f /slusk/compose.yaml -f /slusk/compose.override.yaml pull web` then `… up -d web` in `working_dir`, override written before the first command, returns pinned ref; no matching container → error, no commands; resolve fails → no override written, no commands; pull fails → `up` not run, error contains the command output; override refused → no commands. +- [ ] **Step 2:** FAIL. **Step 3:** Implement. **Step 4:** `go test -race ./internal/agent/updater` PASS. + +### Task W2-5: Swarm executor, registry auth, dispatch + +**Files:** `git mv internal/agent/updater/swarm_executor.go swarm.go` (+ test), create `registryauth.go`, `registryauth_test.go`, `dispatch.go`, `dispatch_test.go`. + +**Contracts:** + +```go +// RegistryAuth holds inline credentials from a docker config.json. A nil +// *RegistryAuth means anonymous access. +func LoadRegistryAuth(dockerConfigDir string, log *slog.Logger) (*RegistryAuth, error) // "" → nil, nil +// For returns the X-Registry-Auth value for repo's registry, or "". +func (a *RegistryAuth) For(repo string) string + +type SwarmClient interface { + Info(ctx context.Context) (system.Info, error) + ServiceInspectWithRaw(ctx context.Context, serviceID string, opts swarm.ServiceInspectOptions) (swarm.Service, []byte, error) + ServiceUpdate(ctx context.Context, serviceID string, version swarm.Version, service swarm.ServiceSpec, opts swarm.ServiceUpdateOptions) (swarm.ServiceUpdateResponse, error) +} +func NewSwarmExecutor(cli SwarmClient, resolve *Resolver, auth *RegistryAuth) *SwarmExecutor + +// Dispatch picks the executor by job.Kind; KindContainer and unknown kinds fail. +func NewDispatch(compose, swarm Executor) Executor +``` + +Registry host mapping: `reference.ParseNormalizedNamed(repo)` → `reference.Domain(named)`; `docker.io` matches config keys `https://index.docker.io/v1/`, `index.docker.io`, `docker.io`, `registry-1.docker.io`; any other host matches its key with or without `https://`/`http://` and trailing path. Entries with `auth` (base64 `user:pass`) or `identitytoken` become `registry.EncodeAuthConfig(registry.AuthConfig{Username, Password, IdentityToken, ServerAddress})`. If the file has `credsStore` or `credHelpers`, log `registry_cred_helpers_unsupported` once. + +`SwarmExecutor.Execute` steps: +1. `Info`: not `Swarm.ControlAvailable` → `refused: this agent's daemon is not a swarm manager`; `Swarm.Cluster == nil || Cluster.ID != job.SwarmClusterID` → `refused: job is for swarm cluster X, this daemon is in Y`. No inspect, no update. +2. `ServiceInspectWithRaw(job.SwarmService)`; check `svc.Spec.Name == job.SwarmService`. +3. `Resolve(job.Ref)` → pinned; write `resolved \n`. +4. `spec := svc.Spec; spec.TaskTemplate.ContainerSpec.Image = pinned`; `ServiceUpdate(svc.ID, svc.Version, spec, opts)` with `QueryRegistry: true` and `EncodedRegistryAuth: auth.For(repo)` if non-empty, else `RegistryAuthFrom: swarm.RegistryAuthFromSpec`. Write any `resp.Warnings` to `out`. Return pinned. + +- [ ] **Step 1: Failing tests** + - `TestSwarmExecutor` (table, fake client): happy path updates by ID `"svc-id"` with inspected version and image `repo:tag@sha256:…`; auth known → `EncodedRegistryAuth` set; unknown → `RegistryAuthFrom "spec"`; cluster mismatch → refused, zero `ServiceInspectWithRaw`/`ServiceUpdate` calls; worker node → refused; inspect returns a different name → error, no update; nil `ContainerSpec` → error. + - `TestRegistryAuth`: hub keys variants; `gitea.shcizo.se` with `https://gitea.shcizo.se` key; unknown host → `""`; empty dir → nil and `For` on nil returns `""`; malformed JSON → error. + - `TestDispatch`: routes by kind; container → error. +- [ ] **Step 2–4:** FAIL → implement → `go test -race ./internal/agent/updater` PASS. Commit W2-1..W2-5 together (see W2-1 message). + +### Task W2-6: Connection — writer goroutine, job messages, resend + +**Files:** `internal/agent/conn/conn.go`, `conn_test.go` + +**Changes:** +- `New(cfg, src, jobs, log)` (C8). Hello: `ProtocolVersion = protocol.Version`, `KnownJobs = jobs.Known()`. +- One writer goroutine per session reading from `out chan protocol.Envelope` (cap 64); every send (hello, snapshot, upserts, removes, progress, results) goes through it. Each write uses a 10s timeout; a write error cancels the session. +- Reader goroutine: decodes envelopes. `job.start` with empty `ID` or undecodable payload → close session. Valid → `jobs.Submit(id, start)`. `job.ack` → `jobs.Ack(id)` and forget it from the session's `sent` set. Unknown type → `log.Warn("unknown_message_type", "type", …)`, continue. +- After the snapshot: `flush()` once, then the main loop also selects on `jobs.Changed()`. `flush()` sends `job.progress` for each `TakeProgress()` entry, then `job.result` for each `Pending()` entry not yet in this session's `sent` set. Throttle: at most one `flush()` per `ProgressInterval`; a signal arriving earlier arms a timer for the remainder instead of being dropped. + +- [ ] **Step 1: Failing tests** (extend `fakeMaster` so the test can also *send* envelopes; fake `Jobs` records calls and returns scripted `Known`/`Pending`/`TakeProgress`): + - `TestAgent_HelloCarriesVersionAndKnownJobs`. + - `TestAgent_JobStartSubmitsAndAckAcks`. + - `TestAgent_SendsPendingResultsAfterSnapshot`: order hello → snapshot → result(s). + - `TestAgent_ResendsUnackedResultAfterReconnect`: result sent, master drops connection without ack, result sent again on the next session; after ack it is not resent on a third session. + - `TestAgent_ProgressIsThrottled`: 20 changed signals in 50ms with `ProgressInterval 100ms` → at most 2 progress batches in 150ms, and the last chunk is delivered. + - `TestAgent_IgnoresUnknownType`: master sends `{"type":"future.thing"}` → connection stays, a later `job.start` still works. + - `TestAgent_ClosesOnMalformedJobStart`: `job.start` without ID → agent reconnects (new connection observed). + - Run with `-race`; one test sends inventory changes and job progress concurrently. +- [ ] **Step 2–4:** FAIL → implement → `go test -race ./internal/agent/...` PASS. +- [ ] **Step 5: Commit** — `feat(agent): run jobs from the master and resend results until acked (#7)` + +### Task W2-7: Agent binary, image, examples + +**Files:** `cmd/agent/main.go`, `Dockerfile`, delete `docker-compose.example.yml`, `docker-compose.local.yml`, `env.sample`; create `deploy/agent.compose.yml`. + +- [ ] **Step 1:** `cmd/agent` wiring: + +```go + auth, err := updater.LoadRegistryAuth(cfg.DockerConfig, logger) + resolver := updater.NewResolver(dockerCli, auth) + exec := updater.NewDispatch( + updater.NewComposeExecutor(dockerCli, resolver, cfg.StacksRoot, updater.ExecDocker), + updater.NewSwarmExecutor(dockerCli, resolver, auth), + ) + runner := updater.NewRunner(exec, cfg.UpdateTimeout) + go runner.Run(ctx) + agent := conn.New(conn.Config{...}, collector, runner, logger) +``` + +Log `agent_starting` with `stacks_root`, `update_timeout`, `registry_auth` (bool) — never the token. +- [ ] **Step 2: Dockerfile:** build only `master:updater-master agent:updater-agent`; `EXPOSE 8080 8090`; `ENTRYPOINT ["/usr/local/bin/updater-agent"]`. Why the agent: it is the image's many-copies deployment (one per machine, pasted from the UI), the master stack already sets its own entrypoint, and an old container that accidentally picks up the new image fails fast on missing `MASTER_URL` instead of serving anything. +- [ ] **Step 3:** `deploy/agent.compose.yml` = the UI snippet (W4-4) with placeholders and comments. +- [ ] **Step 4:** `go build ./cmd/agent && docker build -t package-updater:dev .` then `docker run --rm package-updater:dev` exits with `fatal: MASTER_URL is required`. Remove the built `./agent` binary. +- [ ] **Step 5: Commit** — `build: agent is the image's default entrypoint; drop the old server examples (#7)` + +--- + +## W3 — Master + +### Task W3-1: Store — jobs, config files, desired ref + +**Files:** Create `internal/master/store/jobs.go`, `jobs_integration_test.go`; modify `services.go`, `agents.go`, `services_integration_test.go`, `agents_integration_test.go`. + +- [ ] **Step 1: Failing integration tests** (`//go:build integration`): + - `TestCreateJob_OneActivePerService`: two goroutines `CreateJob` for the same service concurrently → exactly one succeeds; the other gets `model.ErrJobActive` and the same job ID. After `FailJob`, a new `CreateJob` succeeds. + - `TestFailLostJobs`: agent A has jobs 1 (queued), 2 (running), 3 (succeeded); agent B has 4 (running). `FailLostJobs(A, [2])` → returns `[1]`; 1 failed with the "lost" message; 2, 3, 4 unchanged. + - `TestJobProgress_OwnerOnly`: progress from A moves queued → running and sets `started_at`; progress for the same job from B returns false and changes nothing; progress after finish returns false. + - `TestJobProgress_KeepsTail`: appending beyond `MaxJobOutput` keeps the last `MaxJobOutput` characters and sets `output_truncated`. + - `TestFinishJob_OnlyOnceAndOnlyByOwner`: B → false; A succeeded with `Ref "r:1@sha256:d"` → true, job `to_ref` and service `desired_ref` both `r:1@sha256:d`, output replaced; second call → false, nothing changes; a failed finish leaves `desired_ref` untouched. + - `TestGetJobAndListJobs`: `ServiceName` filled from the service row; newest first; `ServiceID` filter; `Limit`; list has empty `Output`. + - `TestGetService_IncludesRemoved`; snapshot round-trips `ConfigFiles`. + - `TestRevokeAgent_FailsActiveJobs`: queued/running jobs of the agent → failed "agent revoked"; other agents' jobs untouched. +- [ ] **Step 2:** `DATABASE_URL=… go test -tags integration ./internal/master/store/` — FAIL. +- [ ] **Step 3: Implement.** The two statements with subtle semantics: + +```sql +-- JobProgress ($1 job, $2 agent, $3 chunk, $4 at, $5 = protocol.MaxJobOutput) +UPDATE update_jobs SET + status = 'running', + started_at = COALESCE(started_at, $4), + output_truncated = output_truncated OR length(output) + length($3) > $5, + output = right(output || $3, $5) +WHERE id = $1 AND agent_id = $2 AND status IN ('queued', 'running') + +-- FinishJob, inside one transaction +UPDATE update_jobs SET + status = $3, error = $4, output = $5, output_truncated = $6, + to_ref = CASE WHEN $7 <> '' THEN $7 ELSE to_ref END, + started_at = COALESCE(started_at, $8), finished_at = $9 +WHERE id = $1 AND agent_id = $2 AND status IN ('queued', 'running') +RETURNING service_id +-- then, only if a row came back and $3 = 'succeeded': +UPDATE services SET desired_ref = $7 WHERE id = $service_id +``` + +`CreateJob`: plain `INSERT … RETURNING`; on a unique violation of `update_jobs_one_active` (`pgconn.PgError` code `23505`, constraint name) select the active job for the service and return it with `model.ErrJobActive`. +- [ ] **Step 4:** PASS. `go vet ./... && go test ./...` (non-integration) still PASS. +- [ ] **Step 5: Commit** — `feat(master): store update jobs (#7)` + +### Task W3-2: Hub — per-connection writer, routing, reconcile, job messages + +**Files:** `internal/master/hub/hub.go` (split into `hub.go`, `route.go`, `jobs.go` if it helps), `hub_test.go`, new `route_test.go`. + +**Changes:** +- `conns map[int64]*agentConn` where `agentConn{ws, id uint64 (from an atomic counter), out chan protocol.Envelope (cap 64), ctx, cancel, hello *protocol.Hello}`. A writer goroutine per connection does every write (10s timeout per write; on error `CloseNow` + cancel). `Online` and `Disconnect` keep their meaning. +- `Store` interface gains `FailLostJobs`, `JobProgress`, `FinishJob` (C5 signatures). +- Hello handling, in this order: `RecordHello` → parse `KnownJobs` (skip non-integers with a warning) → `FailLostJobs` → publish `Jobs` and each lost `JobTopic` → under `h.mu`, set `conn.hello` (from now on the connection is routable) → publish `Agents`. A connection is never routable before its reconcile ran. +- `Route(svc)`: copy the routable connections under `h.mu`, then: + +```go +// route picks the connection a job for svc goes to. Compose services and +// containers can only run on the agent that reported them. Swarm services +// run on the reporting agent if it qualifies, else on any other connected +// manager of the same cluster (lowest agent ID first). Too-old wins over +// offline only when a connection would otherwise have qualified. +func route(svc model.Service, conns []candidate) (model.Route, error) { + sort.Slice(conns, func(a, b int) bool { return conns[a].agentID < conns[b].agentID }) + tooOld := false + try := func(c candidate) bool { + if svc.Kind == protocol.KindSwarm { + if !c.hello.SwarmManager || c.hello.SwarmClusterID == "" || c.hello.SwarmClusterID != svc.SwarmClusterID { + return false + } + } + if c.hello.ProtocolVersion < protocol.MinJobVersion { + tooOld = true + return false + } + return true + } + if svc.AgentID != nil { + for _, c := range conns { + if c.agentID == *svc.AgentID && try(c) { + return model.Route{AgentID: c.agentID, ConnID: c.connID}, nil + } + } + } + if svc.Kind == protocol.KindSwarm { + for _, c := range conns { + if try(c) { + return model.Route{AgentID: c.agentID, ConnID: c.connID}, nil + } + } + } + if tooOld { + return model.Route{}, model.ErrAgentTooOld + } + return model.Route{}, model.ErrAgentOffline +} +``` + +- `StartJob(r, jobID, start)`: under `h.mu` look up `conns[r.AgentID]`; missing, `id != r.ConnID` or `hello == nil` → `model.ErrAgentOffline`. Otherwise enqueue `EncodeJob(TypeJobStart, strconv.FormatInt(jobID,10), start)` (blocking on `out` or the connection's ctx; ctx done → `ErrAgentOffline`). An enqueued job whose write later fails stays queued and is settled by the next hello's reconcile. +- `job.progress`: non-integer ID → warn, ignore. `JobProgress(agentID, id, payload.Output, now)`; applied → publish `Jobs`, `JobTopic(id)`. +- `job.result`: non-integer ID → warn, ack, return. Status other than `succeeded`/`failed` → store as failed with error `agent sent invalid status ""`. `FinishJob(agentID, id, …)`: store error → return it (closes the connection; agent resends after reconnect). Not applied → warn `job_result_ignored`. Always enqueue `job.ack` afterwards. Applied → publish `Jobs`, `JobTopic(id)`, and `Services` if succeeded. +- Unknown type (after hello) → warn `unknown_message_type`, continue. Before hello any non-hello still closes (phase 1 rule). + +- [ ] **Step 1: Failing tests** (extend `fakeStore` with the three methods, recording calls and returning scripted results): + - `TestRoute` (table on the pure `route` function): compose on own agent v2 → ok; own agent v0 → `ErrAgentTooOld`; own agent missing → `ErrAgentOffline`; another agent online for a compose service → still offline; swarm: reporting agent qualifies → it; reporting agent offline, other manager same cluster v2 → the other; other manager different cluster → offline; only a v0 manager of the cluster → too old; worker of the cluster → offline; manager with empty cluster ID → offline. + - `TestHub_ReconcilesBeforeRoutable`: fake store blocks `FailLostJobs` until released; while blocked, `Route` for the agent's service returns `ErrAgentOffline`; after release it routes. `FailLostJobs` got `known = [7, 9]` from `KnownJobs ["7","9","x"]`. + - `TestHub_StartJobSendsEnvelope`: client receives `job.start` with `ID "42"` and the payload. + - `TestHub_StartJobOnReplacedConnectionFails`: route, then the agent reconnects (new connection), `StartJob` with the old route → `ErrAgentOffline`, nothing written to the new connection. + - `TestHub_ProgressAndResult`: progress → `JobProgress(agent, 42, "out", …)` and `job-42` published; result → `FinishJob`, ack received with `ID "42"`; not-applied result → still acked; result with ID `"abc"` → acked, no store call; result with status `"weird"` → stored as failed. + - `TestHub_ResultStoreErrorClosesConnection`. + - `TestHub_IgnoresUnknownType`: `{"type":"future.thing"}` after hello → connection stays open, a following upsert is stored. + - `TestHub_LargeResultArrives`: a result whose `Output` is `protocol.MaxJobOutput` bytes of `"\x1b"` reaches `FinishJob` intact. + - `TestHub_ConcurrentWritesAreSerialised` (`-race`): 50 `StartJob` calls from goroutines while the client sends 50 results (acks written concurrently) — all 50 job.starts and 50 acks arrive, each a valid JSON envelope. + - Existing phase-1 hub tests keep passing. +- [ ] **Step 2–4:** FAIL → implement → `go test -race ./internal/master/hub` PASS. +- [ ] **Step 5: Commit** — `feat(master): route jobs to agents and reconcile them on reconnect (#7)` + +### Task W3-3: Jobs service + +**Files:** Create `internal/master/jobs/jobs.go`, `jobs_test.go`. + +**Contract:** C6. Rules, in this order: + +```go +// Updatable returns nil or a *model.NotUpdatableError. +func (s *Service) Updatable(svc model.Service) error { + switch { + case svc.RemovedAt != nil: + return &model.NotUpdatableError{Reason: model.ReasonRemoved} + case svc.Self || imageref.Parse(svc.ActualRef).Repo == s.selfRepo: + return &model.NotUpdatableError{Reason: model.ReasonSelf} + case svc.Kind == protocol.KindContainer: + return &model.NotUpdatableError{Reason: model.ReasonContainer} + case imageref.Parse(svc.ActualRef).Tag == "": + return &model.NotUpdatableError{Reason: model.ReasonNoTag} + } + return nil +} +``` + +`selfRepo = imageref.Parse(agentImage).Repo` (master and agent are the same image; `AGENT_IMAGE` defaults to `gitea.shcizo.se/shcizo/package-updater:latest`). + +`Trigger`: `GetService` → `Updatable` → `route, routeErr := hub.Route(svc)` → `CreateJob(NewJob{ServiceID, AgentID: &route.AgentID or nil, FromRef: svc.ActualRef, ToRef: repo:tag, Trigger: manual, RequestedBy, CreatedAt: now})` (on `ErrJobActive` return it as is) → if `routeErr != nil`: `FailJob(job.ID, routeErr.Error())` → else `StartJob(route, job.ID, start)`; on error `FailJob(job.ID, model.ErrAgentOffline.Error())` → publish `Jobs` + `JobTopic` → return `GetJob(job.ID)` (so the caller sees the final status). `JobStart` for compose: `Kind, ComposeProject, ComposeService, Ref`; for swarm: `Kind, SwarmClusterID, SwarmService, Ref`. + +- [ ] **Step 1: Failing tests** + - `TestUpdatable` (table): removed; `Self`; same repo as `AGENT_IMAGE` with a different tag (`…/package-updater:1.2.0@sha256:…`); container; `repo@sha256:…` (no tag); `postgres:17.10` → nil; `localhost:5000/foo:v1` → nil. + - `TestTrigger` (table, fake store + fake hub + fake publisher): happy compose → job queued, hub got exact `JobStart` with `Ref "gitea.shcizo.se/shcizo/slusk:1.4.2"` from `ActualRef "gitea.shcizo.se/shcizo/slusk:1.4.2@sha256:d1"`, `FromRef` = full actual ref, `RequestedBy` passed, `Jobs` published; swarm → `SwarmClusterID`/`SwarmService` set; not updatable → `*NotUpdatableError`, no job created; active job → that job + `ErrJobActive`, no hub call; route offline → job created and failed "agent offline", no `StartJob`; route too old → failed "agent too old for updates"; `StartJob` fails → failed "agent offline"; unknown service → `ErrNotFound`. +- [ ] **Step 2–4:** FAIL → implement → PASS. +- [ ] **Step 5: Commit** — `feat(master): start manual update jobs with self-protection (#7)` + +### Task W3-4: Master wiring + +**Files:** `cmd/master/main.go`. + +- [ ] **Step 1:** Wire per C7. No new env vars (`AGENT_IMAGE` exists). +- [ ] **Step 2:** `go build ./cmd/master` (requires W4 merged for the `web.New` signature). Remove the built binary. +- [ ] **Step 3: Commit** — `feat(master): wire jobs into the master (#7)` + +--- + +## W4 — Web + +W4 works against C5/C6/C7 with fakes; it does not need W3's code to compile its own package. Integration point: `cmd/master` (W3-4) passes `*jobs.Service` and `*store.Store`. + +### Task W4-1: Service page and Update action + +**Files:** `internal/master/web/server.go`, `views.go`, new `templates/service.html`, `templates/overview.html`, `templates/layout.html`, `static/app.css`, `server_test.go`; `internal/master/webauth/auth.go` + `auth_test.go`. + +**Routes:** + +| Route | Does | +|---|---| +| `GET /services/{id}` | details (name, kind, agent, image `ActualRef`, digest, status, desired ref, config files split on `,`), Update button or the disabled reason from `jobs.Updatable`, job history (`ListJobs{ServiceID, Limit: 20}`) | +| `GET /fragments/services/{id}/jobs` | the history table only; `hx-trigger="sse:jobs"` | +| `POST /services/{id}/update` | CSRF hidden field (as revoke); `jobs.Trigger(ctx, id, sess.Subject)`; success or `ErrJobActive` → `303` to `/jobs/{job.ID}`; `*NotUpdatableError` → `409` service page with the reason; `ErrNotFound` → `404`; other → `500` | + +`webauth`: add `// WithSession returns ctx carrying sess, as Require does.` `func WithSession(ctx context.Context, sess model.Session) context.Context`, and make `Require` use it. Tests use it in their `protect` wrapper to inject a session with `Subject "alice"`. + +Overview: the service name links to `/services/{id}`; nothing else on the overview changes. Nav: `Services · Jobs · Agents`. + +- [ ] **Step 1: Failing tests** (fake store gains `GetService`, `GetJob`, `ListJobs`; fake `Jobs` returns scripted `Updatable` and `Trigger` results and records `requestedBy`): + - `TestServicePage_ShowsUpdateButton` (updatable) and `TestServicePage_DisabledReason` (table over the four reasons: page shows the reason text, no enabled Update button). + - `TestServicePage_NotFound` → 404; bad ID → 400. + - `TestUpdate_RedirectsToJob` (`requestedBy == "alice"`), `TestUpdate_ActiveJobRedirectsToExisting`, `TestUpdate_NotUpdatableShows409`. + - `TestServiceJobsFragment_IsNotAFullPage`. + - `TestOverview_LinksToServicePage`. +- [ ] **Step 2–4:** FAIL → implement → PASS. +- [ ] **Step 5: Commit** — `feat(web): service page with manual update (#7)` + +### Task W4-2: Jobs list and live job log + +**Files:** `server.go`, new `templates/jobs.html`, `templates/job.html`, `server_test.go`. + +| Route | Does | +|---|---| +| `GET /jobs` | `ListJobs{Limit: 100}`: id, service (link), status, from → to, requested by, created, duration | +| `GET /fragments/jobs` | table only; `hx-trigger="sse:jobs"` | +| `GET /jobs/{id}` | job details, error, `

` with `Output` (plus "earlier output truncated" when `OutputTruncated`) |
+| `GET /fragments/jobs/{id}` | details + log; `hx-trigger="sse:job-{{.Job.ID}}"`. Always re-reads the job from the store: the DB is the source of truth, so a dropped signal or SSE reconnect just means the next fetch catches up. |
+
+- [ ] **Step 1: Failing tests:** list renders statuses and links; job page shows output and error; truncated note; fragment is not a full page and contains `sse:job-42`; unknown job → 404.
+- [ ] **Step 2–4:** FAIL → implement → PASS.
+- [ ] **Step 5: Commit** — `feat(web): jobs list and live job log (#7)`
+
+### Task W4-3: Revoke publishes jobs
+
+**Files:** `server.go`, `server_test.go`.
+
+- [ ] `revokeAgent` also publishes `events.Jobs` (the store failed the agent's active jobs). Test: `TestRevokeAgent_RevokesAndDisconnects` asserts both topics. **Commit** with W4-4.
+
+### Task W4-4: Agent snippet
+
+**Files:** `server.go` (`agentSnippet`), `server_test.go`.
+
+```yaml
+services:
+  updater-agent:
+    image: 
+    entrypoint: ["/usr/local/bin/updater-agent"]
+    restart: unless-stopped
+    environment:
+      MASTER_URL: 
+      MASTER_KEY: 
+      AGENT_TOKEN: 
+      # Compose stacks this agent may update. Mount it read-write at the
+      # same path as on the host: Compose labels hold host paths.
+      STACKS_ROOT: /home/shcizo/self-hosted
+      UPDATE_TIMEOUT: 5m
+      # Optional registry credentials (a config.json written by docker login):
+      # DOCKER_CONFIG: /docker-config
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /home/shcizo/self-hosted:/home/shcizo/self-hosted
+      # - /root/.docker/config.json:/docker-config/config.json:ro
+```
+
+- [ ] Test: snippet contains `STACKS_ROOT`, the same-path rw mount line (no `:ro`), `UPDATE_TIMEOUT`, the commented `DOCKER_CONFIG`. **Commit** — `feat(web): agent snippet mounts stacks read-write for updates (#7)`
+
+---
+
+## W5 — Integration (after W2–W4 are on the branch)
+
+### Task W5-1: End-to-end jobs
+
+**Files:** `internal/e2e/agent_master_test.go` (split into `jobs_test.go` if long).
+
+- [ ] **Step 1:** Extend `memStore` with in-memory `GetService`, `CreateJob` (one active per service), `FailJob`, `GetJob`, `FailLostJobs`, `JobProgress`, `FinishJob` (same ownership and "only active" rules as C5). `startMaster` returns a `restart()` that builds a new `hub.New` over the **same** `memStore`, swaps it into the mux through an `atomic.Pointer[hub.Hub]`, and calls `srv.CloseClientConnections()` — the hub holds no job state, so this is a faithful master restart. Agents use a real `updater.Runner` with a fake `Executor` that writes output and blocks until the test releases it.
+- [ ] **Step 2: Tests**
+  - `TestManualJobFlowsToAgentAndBack`: compose service in the store, `jobs.Service.Trigger` → job succeeded, `to_ref`/`desired_ref = "@sha256:e2e"`, output contains the executor's lines, runner `Known()` empty (acked).
+  - `TestJobFinishedWhileDisconnectedIsRecordedOnce`: start job, `CloseClientConnections` while the executor blocks, release, agent reconnects → `FinishJob` applied exactly once (count calls that returned true), runner empty afterwards.
+  - `TestMasterRestartKeepsJobTheAgentKnows`: start job, `restart()` while it runs, release → job succeeded, never failed "lost".
+  - `TestAgentRestartMarksJobLost`: start job, cancel the agent's `Run` and drop its runner, start a new agent (new runner) with the same token → job failed with the "lost" message.
+  - `TestPhaseOneAgentGetsNoJobs`: a raw websocket client sends a phase-1 hello (no `protocol_version`) + snapshot; `Trigger` → job failed "agent too old for updates"; the client receives no `job.start` within 200ms.
+- [ ] **Step 3:** `go test -race ./internal/e2e/` PASS.
+- [ ] **Step 4: Commit** — `test: jobs end to end over pinned TLS (#7)`
+
+### Task W5-2: Documentation
+
+**Files:** `CLAUDE.md`, `README.md`, `deploy/master.stack.yml`.
+
+- [ ] `CLAUDE.md`:
+  - Intro: master/agent fleet updater (no longer "webhook-driven, single binary").
+  - Commands: drop `go build ./cmd/server` and the local smoke test (`docker-compose.local.yml`, `env.sample` are gone); keep `go build ./cmd/master ./cmd/agent`, integration tests, docker build.
+  - Architecture: remove `cmd/server`, `internal/api`, `internal/config`, `internal/discovery`, `internal/updater`, `internal/selfupdate`, `internal/metrics`; add `internal/agent/updater` (runner + Compose/Swarm executors + override file + STACKS_ROOT check + registry auth) and `internal/master/jobs`.
+  - Design intents: remove "Defense in depth: token AND opt-in label AND STACKS_ROOT", "Auth on /update only", "metrics may be nil", "Stateless", "MODE"; replace with: single FIFO worker **per agent** (`internal/agent/updater/runner.go`); one active job per service (unique index); the agent writes only `compose.override.yaml` with the header and refuses when any other override name exists; `STACKS_ROOT` check on every config file and the override path, after symlink resolution; self-protection by `AGENT_IMAGE` repo + `self` flag; unknown message types ignored, malformed messages close.
+  - Gotchas: remove the selfupdate, `MODE`, `STACKS_ROOT` prod-default, "stateless/label gate describe cmd/server only" entries; update the `IsInsideRoot` path; add: Compose loads only one override file; stacks started with explicit `-f` don't load the override on a manual `up`; override entries are never pruned; `STACKS_ROOT` must be mounted rw at the host path; `job.start` carries no paths; default entrypoint is the agent.
+- [ ] `README.md`: replace the webhook-era sections (trigger flow, opt-in label, Swarm `MODE`, quick start with `UPDATER_API_KEY`) with: what master and agents do, deploying the master (`deploy/master.stack.yml`), adding agents (UI snippet / `deploy/agent.compose.yml`), the agent env table (C8), how Compose updates work and their limits (override file, explicit `-f`, operator overrides refused, no pruning), Swarm updates, and "the webhook returns in phase 3; until then the Gitea action has nothing to call".
+- [ ] `deploy/master.stack.yml`: header comment "phase 2: inventory and manual updates".
+- [ ] **Commit** — `docs: master/agent phase 2 — jobs and manual update (#7)`
+
+---
+
+## Final verification
+
+Run from the worktree root:
+
+```bash
+go vet ./... && go test ./... && go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/ \
+  && go build ./cmd/master ./cmd/agent && docker build -t package-updater:dev .
+rm -f master agent
+DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/   # when a Postgres is available
+docker run --rm package-updater:dev   # expect: fatal: MASTER_URL is required
+```
+
+Manual smoke test (one machine with a throwaway Compose stack under `STACKS_ROOT`): deploy the master, add an agent, open the stack's service page, click *Update*, watch the log stream, confirm `compose.override.yaml` appeared with the header and `repo:tag@digest`, run `docker compose up -d` by hand in the stack dir and confirm the container keeps the pinned digest.

From eb41967cd10bd9117b4e578add62b2deb44e042a Mon Sep 17 00:00:00 2001
From: Samuel Enocsson 
Date: Mon, 12 Oct 2026 18:37:28 +0200
Subject: [PATCH 02/29] docs: plan revisions after review (#7)

---
 .../2026-10-12-master-agent-phase-2-jobs.md   | 154 +++++++++++++-----
 1 file changed, 115 insertions(+), 39 deletions(-)

diff --git a/docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md b/docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md
index 6c1c708..b0ee316 100644
--- a/docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md
+++ b/docs/superpowers/plans/2026-10-12-master-agent-phase-2-jobs.md
@@ -20,6 +20,8 @@
 - WebSocket read limit stays `1 << 20` on both sides. Every websocket write goes through one writer goroutine per connection (both sides) — no other goroutine calls `wsjson.Write`.
 - Never edit operator files: the agent writes exactly one file, `/compose.override.yaml`, and only if it is absent or starts with the header line.
 - Agent tokens and `AGENT_TOKEN` values never appear in logs (unchanged).
+- **One shared worktree, one Go module, implementers in parallel.** While W2–W4 run, each implementer runs `go vet`/`go test` only on the packages they own (listed per workstream below), because another workstream's package may be mid-edit. Stage only your own paths with `git add ` — never `git add -A` or `git add .`. If `git` reports `index.lock` exists, wait a few seconds and retry; never delete the lock file. Full `go vet ./... && go test ./...` runs in W5 and in the final verification.
+- The agent may only touch Compose stacks in directories that are bind-mounted into its own container read-write **at the same path as on the host**. There is no `STACKS_ROOT` setting.
 
 ## Deliberate deviations from the spec in this phase
 
@@ -28,6 +30,7 @@
 - **`internal/updater` moves to `internal/agent/updater`** (`git mv`, history kept). Its blocking `Queue.Submit` (built for the synchronous webhook) is replaced by `Runner` (async, outbox, live output). The executors are rewritten against `protocol.JobStart`. `discovery.IsInsideRoot` moves into the same package; image matching is dropped (`imageref.Parse(x).Repo` is equivalent to `NormaliseImage`).
 - **Override file: one name, and refuse when any other override exists.** Verified empirically against Docker Compose v5.1.2: Compose auto-loads `compose.override.{yml,yaml}` / `docker-compose.override.{yml,yaml}` next to *any* default base file, but loads **only one** (first of `compose.override.yml`, `compose.override.yaml`, `docker-compose.override.yml`, `docker-compose.override.yaml`) and warns about the rest. So the agent writes `compose.override.yaml` and refuses the job when any of the other three exists — writing ours next to an operator's `docker-compose.override.yml` would silently disable theirs on the next manual `docker compose up`. Stacks started with explicit `-f` files do not load the override on a manual `up`; the agent's own jobs always pass it explicitly.
 - **The override file is never pruned.** If a service is deleted from the stack file, its `image:` entry stays in `compose.override.yaml` and a manual `docker compose up -d` would recreate it. Documented in the README; pruning needs the base file parsed and is deferred.
+- **`STACKS_ROOT` is replaced by the agent's own mounts.** The spec keeps "the `STACKS_ROOT` path check on the agent"; this phase keeps the check but changes where the allowed directories come from. The agent inspects its own container (`SELF_ID`, default the hostname) and treats every bind mount that is read-write and has `Source == Destination` as an allowed root. Every config file and the override path must be inside one of them (`filepath.Rel` check, also after resolving symlinks). One less setting to keep in sync with the volume list, and a read-only or path-translated mount — which could not work anyway, since labels hold host paths — is refused with a clear message. Mounts are read on every Compose job (one `ContainerInspect`), not cached: it costs one local API call per job and never acts on stale data. An agent not running in a container, or whose `SELF_ID` does not resolve, has no roots, and Compose jobs fail; the agent logs a warning at startup. On Docker Desktop/OrbStack, bind sources are translated host paths and may not equal the destination; that is a development-only limitation.
 - **`job.start` carries no paths.** The agent reads `working_dir` and `config_files` from its own Docker daemon (container labels) at execution time. The inventory still reports `ConfigFiles` (shown on the service page), but a stale or forged path from the master never reaches `docker compose`.
 - **`protocol.Service.ConfigFiles` is the raw comma-joined label string**, not a slice, so `Service` stays comparable with `==` (`inventory.Diff` depends on it).
 - **Swarm "succeeded" means the daemon accepted the new spec**, not that tasks converged. The inventory shows `degraded` until they do.
@@ -51,7 +54,7 @@
 | Two clicks on Update at once create one job; the second gets the first back. | store `TestCreateJob_OneActivePerService` (concurrent, W3-1); jobs `TestTrigger` "active job" case (W3-3) |
 | The master refuses jobs for itself and the agents (same image repo as `AGENT_IMAGE`) and for rows flagged `self`. | jobs `TestUpdatable` (W3-3); web `TestServicePage_DisabledReason` (W4-1) |
 | The agent never overwrites an operator's override file, and never writes next to another auto-loaded override name. | updater `TestWriteOverride` (W2-4) |
-| Every config file and the override path are inside `STACKS_ROOT`, also after resolving symlinks. | updater `TestComposeFiles` (W2-4) |
+| Every config file and the override path are inside a read-write, same-path bind mount of the agent's own container, also after resolving symlinks; read-only and path-translated mounts are refused. | updater `TestMountRoots`, `TestComposeFiles` (W2-4) |
 | Concurrent writers on one websocket (job progress, acks, inventory) are serialised. | `go test -race` on conn and hub tests that start a job while inventory flows (W2-6, W3-2) |
 | A phase-1 agent (protocol 0) never receives `job.start`; the job fails "agent too old for updates". | e2e `TestPhaseOneAgentGetsNoJobs` (W5-1); hub `TestRoute` (W3-2) |
 | A Swarm job routed to an agent whose daemon is in a different cluster is refused before any update. | updater `TestSwarmExecutor` "cluster mismatch" (W2-5) |
@@ -73,7 +76,7 @@
 
 ## Workstreams and file ownership
 
-W1 lands first (one small commit set). W2, W3, W4 then run in parallel. W5 runs after W2–W4 are merged into the branch.
+W1 lands first (one small commit set). W2, W3, W4 then run in parallel in the same worktree. W5 runs after W2–W4 have committed everything.
 
 | Workstream | Owner | Owns (only this workstream edits these) |
 |---|---|---|
@@ -84,7 +87,7 @@ W1 lands first (one small commit set). W2, W3, W4 then run in parallel. W5 runs
 | W5 integration | implementer-master | `internal/e2e/**`, `CLAUDE.md`, `README.md`, `deploy/master.stack.yml` |
 
 Shared touch points, resolved by contract rather than co-editing:
-- `cmd/master/main.go` (W3) calls `web.New` with W4's new signature — frozen below. W3 makes that call compile; if W4 is not merged yet, W3's branch fails to build at that one call, which is expected until both land.
+- `cmd/master/main.go` (W3) calls `web.New` with W4's new signature (C7). W4's **first** commit (Task W4-0) is that signature change plus the matching one-line change to the call in `cmd/master` (the only line W4 touches there), so the module builds at every commit. W3 does W3-4 only after W4-0 is committed.
 - `go.mod`/`go.sum`: only W2 runs `go mod tidy`. W3/W4 add no dependencies.
 - `internal/master/model` is W1's. If W3 or W4 needs a model change, they ask the W1 owner (lead) rather than editing it.
 
@@ -622,11 +625,10 @@ Environment (added to `internal/agent/config.Config`):
 
 | Variable | Field | Default | Validation |
 |---|---|---|---|
-| `STACKS_ROOT` | `StacksRoot string` | `""` (Compose jobs then fail "refused: STACKS_ROOT is not set on this agent") | if set: absolute and an existing directory, else fail fast |
 | `UPDATE_TIMEOUT` | `UpdateTimeout time.Duration` | `5m` | `time.ParseDuration`, must be > 0 |
 | `DOCKER_CONFIG` | `DockerConfig string` | `""` (anonymous registry access) | if set: `$DOCKER_CONFIG/config.json` must exist and parse, else fail fast. Same variable `docker compose` reads, so `pull` uses the same credentials. |
 
-`STACKS_ROOT` must be mounted read-write **at the same path as on the host**, because the labels contain host paths.
+There is no `STACKS_ROOT`. The allowed stack directories are the agent container's bind mounts that are read-write with `Source == Destination` (read on every Compose job via `ContainerInspect(SELF_ID)`). Each stacks directory must be mounted as `/path:/path` (no `:ro`), because Compose labels hold host paths.
 
 ```go
 // package conn
@@ -687,20 +689,22 @@ func (r *Runner) Run(ctx context.Context)
 - Delete: `cmd/server/`, `internal/api/`, `internal/config/`, the rest of `internal/discovery/`, `internal/selfupdate/`, `internal/metrics/`.
 
 - [ ] **Step 1:** Do the moves and deletions. The moved executors no longer compile (they reference `discovery.Job`) until W2-4/W2-5 rewrite them, so W2-1, W2-3, W2-4 and W2-5 land as **one commit** once `go build ./...` is green. Never commit a tree that does not build.
-- [ ] **Step 2:** `go mod tidy` after everything compiles; confirm `github.com/prometheus/*` left `go.mod` and `gopkg.in/yaml.v3`, `github.com/distribution/reference` are direct.
+- [ ] **Step 2:** Promote the two dependencies with `go get gopkg.in/yaml.v3 github.com/distribution/reference`. Run `go mod tidy` only when `go build ./...` is green for the whole module (other workstreams may be mid-edit); if it is not, leave tidy to W5. Confirm `github.com/prometheus/*` left `go.mod` and `gopkg.in/yaml.v3`, `github.com/distribution/reference` are direct.
 - [ ] **Step 3:** `grep -rn "internal/\(api\|config\|discovery\|selfupdate\|metrics\|updater\)\"" --include='*.go' .` returns nothing.
 - [ ] **Step 4: Commit** (with W2-3..W2-5) — `refactor(agent)!: move updater under the agent and remove cmd/server (#7)` with a `BREAKING CHANGE:` footer naming the removed `/update` endpoint and the new default entrypoint.
 
+Packages W2 tests while W2–W4 run in parallel: `go vet ./internal/agent/... ./cmd/agent/ && go test -race ./internal/agent/...`.
+
 ### Task W2-2: Agent config and inventory `ConfigFiles`
 
 **Files:** `internal/agent/config/config.go`, `config_test.go`; `internal/agent/inventory/collector.go`, `collector_test.go`
 
 - [ ] **Step 1: Failing tests**
-  - `TestLoad_JobSettings` (table): defaults (`StacksRoot ""`, `UpdateTimeout 5m`, `DockerConfig ""`); `STACKS_ROOT=relative/path` → error; `STACKS_ROOT=/missing` → error; `STACKS_ROOT=` → ok; `UPDATE_TIMEOUT=90s` → 90s; `UPDATE_TIMEOUT=0`/`abc` → error; `DOCKER_CONFIG=` → error; with a valid file → ok.
+  - `TestLoad_JobSettings` (table): defaults (`UpdateTimeout 5m`, `DockerConfig ""`); `UPDATE_TIMEOUT=90s` → 90s; `UPDATE_TIMEOUT=0`/`abc` → error; `DOCKER_CONFIG=` → error; with a valid file → ok.
   - Collector: a Compose container with label `com.docker.compose.project.config_files=/s/a/compose.yaml,/s/a/compose.override.yaml` yields `ConfigFiles` equal to the raw label value.
 - [ ] **Step 2:** Run `go test ./internal/agent/config ./internal/agent/inventory` — FAIL.
 - [ ] **Step 3:** Implement. `config.Load` only validates `DOCKER_CONFIG`'s file exists and is JSON; parsing into credentials is W2-5's `LoadRegistryAuth`.
-- [ ] **Step 4:** PASS. **Commit:** `feat(agent): STACKS_ROOT, UPDATE_TIMEOUT and DOCKER_CONFIG settings (#7)`
+- [ ] **Step 4:** PASS. **Commit:** `feat(agent): UPDATE_TIMEOUT and DOCKER_CONFIG settings (#7)`
 
 ### Task W2-3: Job runner (queue, outbox, timeout)
 
@@ -791,16 +795,25 @@ type CommandRunner func(ctx context.Context, dir string, out io.Writer, args ...
 // ExecDocker is the real CommandRunner (exec.CommandContext, WaitDelay 10s).
 func ExecDocker(ctx context.Context, dir string, out io.Writer, args ...string) error
 
-type ContainerLister interface {
+// SelfInspector is the subset of the Docker SDK MountRoots needs.
+type SelfInspector interface {
+	ContainerInspect(ctx context.Context, containerID string) (container.InspectResponse, error)
+}
+// MountRoots returns the agent container's read-write bind mounts whose
+// host path equals the path inside the container.
+func MountRoots(ctx context.Context, cli SelfInspector, selfID string) ([]string, error)
+
+type ComposeDocker interface {
+	SelfInspector
 	ContainerList(ctx context.Context, opts container.ListOptions) ([]container.Summary, error)
 }
-func NewComposeExecutor(cli ContainerLister, resolve *Resolver, stacksRoot string, run CommandRunner) *ComposeExecutor
+func NewComposeExecutor(cli ComposeDocker, resolve *Resolver, selfID string, run CommandRunner) *ComposeExecutor
 func (e *ComposeExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error)
 ```
 
 `ComposeExecutor.Execute` steps, in this order (each failure stops the job before the next step):
 1. List containers (`All: true`) filtered by labels `com.docker.compose.project=
` and `com.docker.compose.service=`; none → error `no container for compose service p/s on this machine`. Take `working_dir` and `config_files` labels from the first.
-2. `composeFiles(stacksRoot, configFiles)` → `-f` list ending with the override, and the stack dir (code below).
+2. `MountRoots(ctx, cli, selfID)` (inspect error → job fails with it), then `composeFiles(roots, configFiles)` → `-f` list ending with the override, and the stack dir (code below).
 3. `Resolve(job.Ref)` → `pinned`. Write `resolved \n` to `out`.
 4. `WriteOverride(dir, job.ComposeService, pinned)`.
 5. `run(ctx, workingDir, out, "compose", "-p", p, "-f", f1, …, "-f", override, "pull", s)`, then the same with `"up", "-d", s`. Return `pinned`.
@@ -895,13 +908,38 @@ func WriteOverride(dir, service, ref string) (string, error) {
 	return path, nil
 }
 
+// errNoRoots is returned when the agent has no directory it may write stacks in.
+var errNoRoots = errors.New("refused: no stack directory is mounted into the agent at the same path (read-write); " +
+	"mount each stacks directory as /path:/path and check that SELF_ID (default: the hostname) names the agent's container")
+
+// MountRoots returns the directories the agent may update Compose stacks in:
+// its own container's read-write bind mounts whose host path equals the
+// path inside the container. Compose labels hold host paths, so only those
+// mounts make a label path usable as is.
+func MountRoots(ctx context.Context, cli SelfInspector, selfID string) ([]string, error) {
+	if selfID == "" {
+		return nil, errNoRoots
+	}
+	insp, err := cli.ContainerInspect(ctx, selfID)
+	if err != nil {
+		return nil, fmt.Errorf("inspect own container %s: %w", selfID, err)
+	}
+	var roots []string
+	for _, m := range insp.Mounts {
+		if m.Type == mount.TypeBind && m.RW && filepath.Clean(m.Source) == filepath.Clean(m.Destination) {
+			roots = append(roots, filepath.Clean(m.Destination))
+		}
+	}
+	return roots, nil
+}
+
 // composeFiles returns the -f arguments for a job: the stack's config files
 // from the container label with any copy of the override removed, then the
-// override last so it wins. Every path, and the real location of the
-// stack directory, must be inside root.
-func composeFiles(root, configFilesLabel string) (files []string, dir string, err error) {
-	if root == "" {
-		return nil, "", errors.New("refused: STACKS_ROOT is not set on this agent")
+// override last so it wins. Every file, and the real location of the stack
+// directory after resolving symlinks, must be inside one of roots.
+func composeFiles(roots []string, configFilesLabel string) (files []string, dir string, err error) {
+	if len(roots) == 0 {
+		return nil, "", errNoRoots
 	}
 	for _, f := range strings.Split(configFilesLabel, ",") {
 		if f = strings.TrimSpace(f); f != "" {
@@ -915,8 +953,8 @@ func composeFiles(root, configFilesLabel string) (files []string, dir string, er
 	override := filepath.Join(dir, OverrideName)
 	out := files[:0]
 	for _, f := range files {
-		if !filepath.IsAbs(f) || !IsInsideRoot(root, f) {
-			return nil, "", fmt.Errorf("refused: %s is outside STACKS_ROOT", f)
+		if !filepath.IsAbs(f) || !insideAny(roots, f) {
+			return nil, "", notMounted(filepath.Dir(f))
 		}
 		if filepath.Clean(f) != override {
 			out = append(out, f)
@@ -926,22 +964,42 @@ func composeFiles(root, configFilesLabel string) (files []string, dir string, er
 	if err != nil {
 		return nil, "", fmt.Errorf("resolve %s: %w", dir, err)
 	}
-	realRoot, err := filepath.EvalSymlinks(root)
-	if err != nil {
-		return nil, "", fmt.Errorf("resolve STACKS_ROOT: %w", err)
+	// Only directories count once resolved: this drops a same-path mount of
+	// a single file such as /var/run/docker.sock.
+	var realRoots []string
+	for _, r := range roots {
+		if rr, err := filepath.EvalSymlinks(r); err == nil {
+			if fi, err := os.Stat(rr); err == nil && fi.IsDir() {
+				realRoots = append(realRoots, rr)
+			}
+		}
 	}
-	if !IsInsideRoot(realRoot, real) {
-		return nil, "", fmt.Errorf("refused: %s resolves to %s, outside STACKS_ROOT", dir, real)
+	if !insideAny(realRoots, real) {
+		return nil, "", notMounted(dir)
 	}
 	return append(out, override), dir, nil
 }
+
+func insideAny(roots []string, path string) bool {
+	for _, r := range roots {
+		if IsInsideRoot(r, path) {
+			return true
+		}
+	}
+	return false
+}
+
+func notMounted(dir string) error {
+	return fmt.Errorf("stack directory %s is not mounted into the agent at the same path (read-write)", dir)
+}
 ```
 
 - [ ] **Step 1: Failing tests**
   - `TestWriteOverride` (table, each case on its own `t.TempDir()`): new file → starts with header, contains `services.web.image`; existing ours with `db` entry → `db` kept, `web` set; existing ours with `web` → replaced; `compose.override.yaml` without header → refused, file unchanged; each of `compose.override.yml`, `docker-compose.override.yml`, `docker-compose.override.yaml` present → refused, ours not created; after success the directory holds no temp files.
-  - `TestComposeFiles` (table): label `a/compose.yaml` → `[a/compose.yaml, a/compose.override.yaml]`; label already containing the override → listed once, last; empty label → error; `STACKS_ROOT ""` → refused; a file outside root → refused; root/`evil` symlinked to a dir outside root → refused; `/foo-evil` vs root `/foo` → refused (lexical guard still applies).
+  - `TestMountRoots` (table, fake inspector): rw bind `/srv/stacks:/srv/stacks` → allowed; ro same-path bind → excluded; rw bind `/home/x/stacks:/stacks` (different path) → excluded; named volume → excluded; inspect error → error; `selfID ""` → the no-roots error.
+  - `TestComposeFiles` (table, roots are `t.TempDir()` paths): label `/a/compose.yaml` → `[…/compose.yaml, …/compose.override.yaml]`; label already containing the override → listed once, last; nested path `/apps/x/y/compose.yaml` → allowed; second root also allowed; empty label → error; no roots → error mentioning "no stack directory is mounted"; a file outside every root → exactly `stack directory 
 is not mounted into the agent at the same path (read-write)`; `/evil` symlinked to a dir outside → refused with the same message; `/foo-evil` vs root `/foo` → refused (lexical guard still applies); a root that is a file (socket stand-in) does not allow paths under it.
   - `TestResolver` (fake `DistributionClient` recording args): `"gitea.shcizo.se/shcizo/slusk:1.4"` → called with that ref and the auth string for `gitea.shcizo.se`; returns `"gitea.shcizo.se/shcizo/slusk:1.4@sha256:d1"`; `"repo@sha256:x"` and `"repo"` → error without calling the daemon; daemon error is wrapped.
-  - `TestComposeExecutor` (fake lister, fake resolver client, recording `CommandRunner`, temp `STACKS_ROOT`): happy path runs exactly `compose -p slusk -f /slusk/compose.yaml -f /slusk/compose.override.yaml pull web` then `… up -d web` in `working_dir`, override written before the first command, returns pinned ref; no matching container → error, no commands; resolve fails → no override written, no commands; pull fails → `up` not run, error contains the command output; override refused → no commands.
+  - `TestComposeExecutor` (fake Docker client returning containers and the agent's own mounts, fake resolver client, recording `CommandRunner`, temp dir as the mounted root): happy path runs exactly `compose -p slusk -f /slusk/compose.yaml -f /slusk/compose.override.yaml pull web` then `… up -d web` in `working_dir`, override written before the first command, returns pinned ref; no matching container → error, no commands; resolve fails → no override written, no commands; pull fails → `up` not run, error contains the command output; override refused → no commands; agent container mounts the root read-only → `not mounted` error, no override written, no commands.
 - [ ] **Step 2:** FAIL. **Step 3:** Implement. **Step 4:** `go test -race ./internal/agent/updater` PASS.
 
 ### Task W2-5: Swarm executor, registry auth, dispatch
@@ -1014,7 +1072,7 @@ Registry host mapping: `reference.ParseNormalizedNamed(repo)` → `reference.Dom
 	auth, err := updater.LoadRegistryAuth(cfg.DockerConfig, logger)
 	resolver := updater.NewResolver(dockerCli, auth)
 	exec := updater.NewDispatch(
-		updater.NewComposeExecutor(dockerCli, resolver, cfg.StacksRoot, updater.ExecDocker),
+		updater.NewComposeExecutor(dockerCli, resolver, cfg.SelfID, updater.ExecDocker),
 		updater.NewSwarmExecutor(dockerCli, resolver, auth),
 	)
 	runner := updater.NewRunner(exec, cfg.UpdateTimeout)
@@ -1022,7 +1080,7 @@ Registry host mapping: `reference.ParseNormalizedNamed(repo)` → `reference.Dom
 	agent := conn.New(conn.Config{...}, collector, runner, logger)
 ```
 
-Log `agent_starting` with `stacks_root`, `update_timeout`, `registry_auth` (bool) — never the token.
+Log `agent_starting` with `update_timeout`, `registry_auth` (bool) — never the token. At startup call `updater.MountRoots(ctx, dockerCli, cfg.SelfID)` once, only to log: on error or zero roots log `no_stack_mounts` at warn ("Compose updates will fail; mount each stacks directory at the same path, read-write"), else log `stack_mounts` with the roots. Jobs read the mounts again themselves.
 - [ ] **Step 2: Dockerfile:** build only `master:updater-master agent:updater-agent`; `EXPOSE 8080 8090`; `ENTRYPOINT ["/usr/local/bin/updater-agent"]`. Why the agent: it is the image's many-copies deployment (one per machine, pasted from the UI), the master stack already sets its own entrypoint, and an old container that accidentally picks up the new image fails fast on missing `MASTER_URL` instead of serving anything.
 - [ ] **Step 3:** `deploy/agent.compose.yml` = the UI snippet (W4-4) with placeholders and comments.
 - [ ] **Step 4:** `go build ./cmd/agent && docker build -t package-updater:dev .` then `docker run --rm package-updater:dev` exits with `fatal: MASTER_URL is required`. Remove the built `./agent` binary.
@@ -1032,6 +1090,8 @@ Log `agent_starting` with `stacks_root`, `update_timeout`, `registry_auth` (bool
 
 ## W3 — Master
 
+Packages W3 tests while W2–W4 run in parallel: `go vet ./internal/master/store/ ./internal/master/hub/ ./internal/master/jobs/ ./cmd/master/ && go test -race ./internal/master/store/ ./internal/master/hub/ ./internal/master/jobs/` (plus the `integration` tag for the store when `DATABASE_URL` is set).
+
 ### Task W3-1: Store — jobs, config files, desired ref
 
 **Files:** Create `internal/master/store/jobs.go`, `jobs_integration_test.go`; modify `services.go`, `agents.go`, `services_integration_test.go`, `agents_integration_test.go`.
@@ -1180,8 +1240,8 @@ func (s *Service) Updatable(svc model.Service) error {
 
 **Files:** `cmd/master/main.go`.
 
-- [ ] **Step 1:** Wire per C7. No new env vars (`AGENT_IMAGE` exists).
-- [ ] **Step 2:** `go build ./cmd/master` (requires W4 merged for the `web.New` signature). Remove the built binary.
+- [ ] **Step 1:** Wire per C7, replacing W4-0's temporary `nil` with `jobService`. No new env vars (`AGENT_IMAGE` exists).
+- [ ] **Step 2:** `go build ./cmd/master` (W4-0 is committed before this task starts, so `web.New` already has the C7 signature). Remove the built binary.
 - [ ] **Step 3: Commit** — `feat(master): wire jobs into the master (#7)`
 
 ---
@@ -1190,6 +1250,17 @@ func (s *Service) Updatable(svc model.Service) error {
 
 W4 works against C5/C6/C7 with fakes; it does not need W3's code to compile its own package. Integration point: `cmd/master` (W3-4) passes `*jobs.Service` and `*store.Store`.
 
+Packages W4 tests while W2–W4 run in parallel: `go vet ./internal/master/web/ ./internal/master/webauth/ && go test ./internal/master/web/ ./internal/master/webauth/`.
+
+### Task W4-0: Constructor signature first
+
+**Files:** `internal/master/web/server.go`, `server_test.go`, `cmd/master/main.go` (this one call only — the single exception to W3 owning `cmd/master`; tell the W3 owner when it is committed).
+
+- [ ] **Step 1:** Add the `Jobs` interface and the three `Store` methods from C7, change `New` to `New(store, hub, jobs, broker, cfg, log)` and store `jobs` on `Server`. No routes use it yet. Extend `fakeStore` with the three methods and add a `fakeJobs` in `server_test.go`.
+- [ ] **Step 2:** In `cmd/master/main.go`, change only the `web.New` call: pass `nil` as `jobs`, with the comment `// wired in W3-4`. This is the one temporary value in the plan, and it is there only so the shared module builds at every commit: the real `*jobs.Service` cannot be built until W3-1..W3-3 exist (it needs `*store.Store` and `*hub.Hub` to satisfy their interfaces). No handler calls `s.jobs` before W4-1, and W3-4 replaces the `nil` before W5 starts; the final verification greps that the `// wired in W3-4` comment is gone.
+- [ ] **Step 3:** `go build ./cmd/master ./internal/master/web/ && go test ./internal/master/web/` PASS.
+- [ ] **Step 4: Commit** — `refactor(web): web.New takes the jobs service (#7)`
+
 ### Task W4-1: Service page and Update action
 
 **Files:** `internal/master/web/server.go`, `views.go`, new `templates/service.html`, `templates/overview.html`, `templates/layout.html`, `static/app.css`, `server_test.go`; `internal/master/webauth/auth.go` + `auth_test.go`.
@@ -1250,19 +1321,16 @@ services:
       MASTER_URL: 
       MASTER_KEY: 
       AGENT_TOKEN: 
-      # Compose stacks this agent may update. Mount it read-write at the
-      # same path as on the host: Compose labels hold host paths.
-      STACKS_ROOT: /home/shcizo/self-hosted
       UPDATE_TIMEOUT: 5m
       # Optional registry credentials (a config.json written by docker login):
       # DOCKER_CONFIG: /docker-config
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-      - /home/shcizo/self-hosted:/home/shcizo/self-hosted
+      # - /path/to/stacks:/path/to/stacks  # mount each stacks directory at the same path, read-write
       # - /root/.docker/config.json:/docker-config/config.json:ro
 ```
 
-- [ ] Test: snippet contains `STACKS_ROOT`, the same-path rw mount line (no `:ro`), `UPDATE_TIMEOUT`, the commented `DOCKER_CONFIG`. **Commit** — `feat(web): agent snippet mounts stacks read-write for updates (#7)`
+- [ ] Test: snippet does **not** contain `STACKS_ROOT`; contains the commented same-path mount example line exactly as above, `UPDATE_TIMEOUT`, the commented `DOCKER_CONFIG`. **Commit** — `feat(web): agent snippet mounts stacks read-write for updates (#7)`
 
 ---
 
@@ -1289,10 +1357,16 @@ services:
 - [ ] `CLAUDE.md`:
   - Intro: master/agent fleet updater (no longer "webhook-driven, single binary").
   - Commands: drop `go build ./cmd/server` and the local smoke test (`docker-compose.local.yml`, `env.sample` are gone); keep `go build ./cmd/master ./cmd/agent`, integration tests, docker build.
-  - Architecture: remove `cmd/server`, `internal/api`, `internal/config`, `internal/discovery`, `internal/updater`, `internal/selfupdate`, `internal/metrics`; add `internal/agent/updater` (runner + Compose/Swarm executors + override file + STACKS_ROOT check + registry auth) and `internal/master/jobs`.
-  - Design intents: remove "Defense in depth: token AND opt-in label AND STACKS_ROOT", "Auth on /update only", "metrics may be nil", "Stateless", "MODE"; replace with: single FIFO worker **per agent** (`internal/agent/updater/runner.go`); one active job per service (unique index); the agent writes only `compose.override.yaml` with the header and refuses when any other override name exists; `STACKS_ROOT` check on every config file and the override path, after symlink resolution; self-protection by `AGENT_IMAGE` repo + `self` flag; unknown message types ignored, malformed messages close.
-  - Gotchas: remove the selfupdate, `MODE`, `STACKS_ROOT` prod-default, "stateless/label gate describe cmd/server only" entries; update the `IsInsideRoot` path; add: Compose loads only one override file; stacks started with explicit `-f` don't load the override on a manual `up`; override entries are never pruned; `STACKS_ROOT` must be mounted rw at the host path; `job.start` carries no paths; default entrypoint is the agent.
-- [ ] `README.md`: replace the webhook-era sections (trigger flow, opt-in label, Swarm `MODE`, quick start with `UPDATER_API_KEY`) with: what master and agents do, deploying the master (`deploy/master.stack.yml`), adding agents (UI snippet / `deploy/agent.compose.yml`), the agent env table (C8), how Compose updates work and their limits (override file, explicit `-f`, operator overrides refused, no pruning), Swarm updates, and "the webhook returns in phase 3; until then the Gitea action has nothing to call".
+  - Architecture: remove `cmd/server`, `internal/api`, `internal/config`, `internal/discovery`, `internal/updater`, `internal/selfupdate`, `internal/metrics`; add `internal/agent/updater` (runner + Compose/Swarm executors + override file + mount-root check + registry auth) and `internal/master/jobs`.
+  - Design intents: remove "Defense in depth: token AND opt-in label AND STACKS_ROOT", "Auth on /update only", "metrics may be nil", "Stateless", "MODE"; replace with: single FIFO worker **per agent** (`internal/agent/updater/runner.go`); one active job per service (unique index); the agent writes only `compose.override.yaml` with the header and refuses when any other override name exists; every config file and the override path must be inside a read-write, same-path bind mount of the agent's own container (read per job, checked after symlink resolution) — replaces `STACKS_ROOT`; self-protection by `AGENT_IMAGE` repo + `self` flag; unknown message types ignored, malformed messages close.
+  - Gotchas: remove the selfupdate, `MODE`, `STACKS_ROOT` prod-default, "stateless/label gate describe cmd/server only" entries; update the `IsInsideRoot` path; add: Compose loads only one override file; stacks started with explicit `-f` don't load the override on a manual `up`; override entries are never pruned; there is no `STACKS_ROOT`: stacks directories must be mounted rw at the host path, and `SELF_ID` must resolve to the agent's container (on Docker Desktop/OrbStack bind sources are translated, so Compose jobs fail there); `job.start` carries no paths; default entrypoint is the agent.
+- [ ] `README.md`: replace the webhook-era sections (trigger flow, opt-in label, Swarm `MODE`, quick start with `UPDATER_API_KEY`) with: what master and agents do, deploying the master (`deploy/master.stack.yml`), adding agents (UI snippet / `deploy/agent.compose.yml`), the agent env table (C8), mounting stacks directories (`/path:/path`, read-write, one line per directory; no `STACKS_ROOT`), how Compose updates work and their limits (override file, explicit `-f`, operator overrides refused, no pruning), a "check before enabling" snippet listing every stack's config files so the operator can spot existing override files and explicit `-f` stacks:
+
+  ```bash
+  docker ps --format '{{.Label "com.docker.compose.project.config_files"}}' | sort -u
+  ```
+
+  Swarm updates, and "the webhook returns in phase 3; until then the Gitea action has nothing to call".
 - [ ] `deploy/master.stack.yml`: header comment "phase 2: inventory and manual updates".
 - [ ] **Commit** — `docs: master/agent phase 2 — jobs and manual update (#7)`
 
@@ -1303,6 +1377,8 @@ services:
 Run from the worktree root:
 
 ```bash
+go mod tidy && git diff --exit-code go.mod go.sum   # W2 may have left tidy to here; commit if it changed
+! grep -n "wired in W3-4" cmd/master/main.go          # W4-0's temporary nil is gone
 go vet ./... && go test ./... && go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/ \
   && go build ./cmd/master ./cmd/agent && docker build -t package-updater:dev .
 rm -f master agent
@@ -1310,4 +1386,4 @@ DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/
 docker run --rm package-updater:dev   # expect: fatal: MASTER_URL is required
 ```
 
-Manual smoke test (one machine with a throwaway Compose stack under `STACKS_ROOT`): deploy the master, add an agent, open the stack's service page, click *Update*, watch the log stream, confirm `compose.override.yaml` appeared with the header and `repo:tag@digest`, run `docker compose up -d` by hand in the stack dir and confirm the container keeps the pinned digest.
+Manual smoke test (Linux machine, a throwaway Compose stack in a directory mounted into the agent at the same path, read-write): deploy the master, add an agent, open the stack's service page, click *Update*, watch the log stream, confirm `compose.override.yaml` appeared with the header and `repo:tag@digest`, run `docker compose up -d` by hand in the stack dir and confirm the container keeps the pinned digest.

From 9e09de0a16c5debb0dd1e74ebc809a67d19c58cd Mon Sep 17 00:00:00 2001
From: Samuel Enocsson 
Date: Mon, 12 Oct 2026 18:38:39 +0200
Subject: [PATCH 03/29] feat(protocol): job messages, protocol version and job
 model (#7)

---
 internal/master/events/broker.go              |  10 +-
 internal/master/events/broker_test.go         |   4 +
 internal/master/model/job.go                  | 116 +++++++++++++++++
 internal/master/model/model.go                |   3 +
 internal/master/model/model_test.go           |  17 +++
 .../master/store/migrations/0002_jobs.sql     |  27 ++++
 internal/protocol/protocol.go                 | 121 +++++++++++++++---
 internal/protocol/protocol_test.go            |  32 +++++
 8 files changed, 314 insertions(+), 16 deletions(-)
 create mode 100644 internal/master/model/job.go
 create mode 100644 internal/master/store/migrations/0002_jobs.sql

diff --git a/internal/master/events/broker.go b/internal/master/events/broker.go
index 994fe51..7358b5c 100644
--- a/internal/master/events/broker.go
+++ b/internal/master/events/broker.go
@@ -2,7 +2,10 @@
 // open browser streams.
 package events
 
-import "sync"
+import (
+	"strconv"
+	"sync"
+)
 
 // Topic names what changed. Subscribers refetch that part of the UI.
 type Topic string
@@ -11,8 +14,13 @@ type Topic string
 const (
 	Services Topic = "services"
 	Agents   Topic = "agents"
+	Jobs     Topic = "jobs" // a job was created or changed status
 )
 
+// JobTopic is signalled when job id gets new output or changes status. The
+// name avoids ":" so it is safe inside hx-trigger="sse:job-42".
+func JobTopic(id int64) Topic { return Topic("job-" + strconv.FormatInt(id, 10)) }
+
 // Broker is an in-process publish/subscribe hub.
 type Broker struct {
 	mu   sync.Mutex
diff --git a/internal/master/events/broker_test.go b/internal/master/events/broker_test.go
index 7613051..7043fe6 100644
--- a/internal/master/events/broker_test.go
+++ b/internal/master/events/broker_test.go
@@ -46,3 +46,7 @@ func TestBroker_SlowSubscriberDoesNotBlockPublish(t *testing.T) {
 		t.Fatal("Publish blocked on a slow subscriber")
 	}
 }
+
+func TestJobTopic(t *testing.T) {
+	assert.Equal(t, Topic("job-42"), JobTopic(42))
+}
diff --git a/internal/master/model/job.go b/internal/master/model/job.go
new file mode 100644
index 0000000..5907deb
--- /dev/null
+++ b/internal/master/model/job.go
@@ -0,0 +1,116 @@
+package model
+
+import (
+	"errors"
+	"time"
+)
+
+// Sentinel errors for jobs.
+var (
+	// ErrJobActive means the service already has a queued or running job.
+	// CreateJob and jobs.Service.Trigger return it together with that job.
+	ErrJobActive = errors.New("an update is already in progress for this service")
+	// ErrAgentOffline means no connected agent can run the job.
+	ErrAgentOffline = errors.New("agent offline")
+	// ErrAgentTooOld means the only agent that could run the job speaks a
+	// protocol version without jobs.
+	ErrAgentTooOld = errors.New("agent too old for updates")
+)
+
+// NotUpdatableError says why a service cannot be updated. Reason is shown
+// in the UI as is.
+type NotUpdatableError struct {
+	Reason string
+}
+
+func (e *NotUpdatableError) Error() string { return "not updatable: " + e.Reason }
+
+// Reasons a service cannot be updated.
+const (
+	ReasonSelf      = "updated manually outside the system"
+	ReasonContainer = "plain containers cannot be updated"
+	ReasonNoTag     = "follows no tag"
+	ReasonRemoved   = "service no longer exists"
+)
+
+// JobStatus is the state of an update job.
+type JobStatus string
+
+// Job statuses. A job moves queued -> running -> succeeded|failed, or
+// straight from queued to failed.
+const (
+	JobQueued    JobStatus = "queued"
+	JobRunning   JobStatus = "running"
+	JobSucceeded JobStatus = "succeeded"
+	JobFailed    JobStatus = "failed"
+)
+
+// JobTrigger says what created a job. Only TriggerManual is used in phase 2.
+type JobTrigger string
+
+// Job triggers.
+const (
+	TriggerManual  JobTrigger = "manual"
+	TriggerWebhook JobTrigger = "webhook"
+	TriggerPoll    JobTrigger = "poll"
+)
+
+// Job is one update of one service.
+type Job struct {
+	ID              int64
+	ServiceID       int64
+	ServiceName     string // Service.DisplayName() of the job's service
+	AgentID         *int64 // agent the job was sent to; nil if it never was
+	AgentName       string
+	FromRef         string
+	ToRef           string // "repo:tag" when created; "repo:tag@digest" after success
+	Trigger         JobTrigger
+	RequestedBy     string // OIDC subject for manual jobs
+	Status          JobStatus
+	Output          string // empty in ListJobs results
+	OutputTruncated bool
+	Error           string
+	CreatedAt       time.Time
+	StartedAt       *time.Time
+	FinishedAt      *time.Time
+}
+
+// Active reports whether the job is queued or running.
+func (j Job) Active() bool { return j.Status == JobQueued || j.Status == JobRunning }
+
+// NewJob is what CreateJob needs to insert a queued job.
+type NewJob struct {
+	ServiceID   int64
+	AgentID     *int64
+	FromRef     string
+	ToRef       string
+	Trigger     JobTrigger
+	RequestedBy string
+	CreatedAt   time.Time
+}
+
+// JobFinish is a job's outcome as reported by the agent.
+type JobFinish struct {
+	Status          JobStatus // JobSucceeded or JobFailed
+	Ref             string    // deployed "repo:tag@digest"; set on success
+	Error           string
+	Output          string
+	OutputTruncated bool
+	StartedAt       time.Time // agent clock
+	FinishedAt      time.Time // agent clock
+}
+
+// JobFilter selects jobs for ListJobs. ServiceID 0 means all services.
+// Limit 0 means 100.
+type JobFilter struct {
+	ServiceID int64
+	Limit     int
+}
+
+// Route is the hub's choice of agent connection for a job. ConnID identifies
+// the connection, so a job is never sent over a connection that replaced the
+// one chosen.
+type Route struct {
+	AgentID int64
+	ConnID  uint64
+}
diff --git a/internal/master/model/model.go b/internal/master/model/model.go
index e09bc4a..b1a5570 100644
--- a/internal/master/model/model.go
+++ b/internal/master/model/model.go
@@ -41,10 +41,12 @@ type Service struct {
 	ComposeProject string
 	ComposeService string
 	WorkingDir     string
+	ConfigFiles    string // raw config_files label, paths joined by ","
 	SwarmClusterID string
 	SwarmService   string
 	ContainerName  string
 	ImageRepo      string
+	DesiredRef     string // set by the last successful job; "" if never updated
 	ActualRef      string
 	ActualDigest   string
 	Status         string
@@ -52,6 +54,7 @@ type Service struct {
 	UpdateMode     string
 	FirstSeenAt    time.Time
 	UpdatedAt      time.Time
+	RemovedAt      *time.Time
 }
 
 // DisplayName is the human-facing name of the service.
diff --git a/internal/master/model/model_test.go b/internal/master/model/model_test.go
index 0705f69..48c7947 100644
--- a/internal/master/model/model_test.go
+++ b/internal/master/model/model_test.go
@@ -39,3 +39,20 @@ func TestDisplayName(t *testing.T) {
 		})
 	}
 }
+
+func TestJob_Active(t *testing.T) {
+	cases := []struct {
+		status JobStatus
+		want   bool
+	}{
+		{JobQueued, true},
+		{JobRunning, true},
+		{JobSucceeded, false},
+		{JobFailed, false},
+	}
+	for _, c := range cases {
+		t.Run(string(c.status), func(t *testing.T) {
+			assert.Equal(t, c.want, Job{Status: c.status}.Active())
+		})
+	}
+}
diff --git a/internal/master/store/migrations/0002_jobs.sql b/internal/master/store/migrations/0002_jobs.sql
new file mode 100644
index 0000000..eaae8a6
--- /dev/null
+++ b/internal/master/store/migrations/0002_jobs.sql
@@ -0,0 +1,27 @@
+ALTER TABLE services ADD COLUMN config_files text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN desired_ref  text NOT NULL DEFAULT '';
+
+CREATE TABLE update_jobs (
+    id               bigserial PRIMARY KEY,
+    service_id       bigint NOT NULL REFERENCES services (id),
+    agent_id         bigint REFERENCES agents (id),
+    from_ref         text NOT NULL DEFAULT '',
+    to_ref           text NOT NULL,
+    trigger          text NOT NULL CHECK (trigger IN ('manual', 'webhook', 'poll')),
+    requested_by     text NOT NULL DEFAULT '',
+    status           text NOT NULL CHECK (status IN ('queued', 'running', 'succeeded', 'failed')),
+    output           text NOT NULL DEFAULT '',
+    output_truncated boolean NOT NULL DEFAULT false,
+    error            text NOT NULL DEFAULT '',
+    created_at       timestamptz NOT NULL,
+    started_at       timestamptz,
+    finished_at      timestamptz
+);
+
+-- One queued or running job per service. CreateJob relies on this index to
+-- turn a second concurrent click into ErrJobActive.
+CREATE UNIQUE INDEX update_jobs_one_active ON update_jobs (service_id)
+    WHERE status IN ('queued', 'running');
+CREATE INDEX update_jobs_by_service ON update_jobs (service_id, id DESC);
+CREATE INDEX update_jobs_active_by_agent ON update_jobs (agent_id)
+    WHERE status IN ('queued', 'running');
diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go
index a4aa9bb..048587b 100644
--- a/internal/protocol/protocol.go
+++ b/internal/protocol/protocol.go
@@ -1,33 +1,66 @@
 // Package protocol defines the messages agents and the master exchange over
 // their WebSocket connection. Every message is an Envelope whose Payload is
 // one of the payload types below, selected by Type.
+//
+// Both sides close the connection on a message they cannot parse (bad JSON,
+// or a known Type whose payload does not decode). A message with an unknown
+// Type is logged and ignored, so a newer peer can add message types without
+// breaking an older one.
 package protocol
 
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 )
 
+// Version is the protocol version this build speaks. Agents send it in
+// Hello. Phase-1 agents do not send the field, which reads as 0.
+const Version = 2
+
+// MinJobVersion is the lowest agent protocol version the master sends jobs to.
+const MinJobVersion = 2
+
+// MaxJobOutput caps the job output an agent keeps and the master stores, in
+// bytes. Only the tail is kept. With JSON escaping (worst case 6 bytes per
+// byte) a JobResult carrying this much output still fits the 1 MiB read limit.
+const MaxJobOutput = 128 << 10
+
 // Message types sent from agent to master.
 const (
-	TypeHello    = "hello"
-	TypeSnapshot = "snapshot"
-	TypeUpsert   = "upsert"
-	TypeRemove   = "remove"
+	TypeHello       = "hello"
+	TypeSnapshot    = "snapshot"
+	TypeUpsert      = "upsert"
+	TypeRemove      = "remove"
+	TypeJobProgress = "job.progress"
+	TypeJobResult   = "job.result"
 )
 
-// Envelope wraps every message on the wire.
+// Message types sent from master to agent.
+const (
+	TypeJobStart = "job.start"
+	TypeJobAck   = "job.ack"
+)
+
+// Envelope wraps every message on the wire. ID is the job ID (decimal) on
+// job.* messages and empty on inventory messages.
 type Envelope struct {
 	Type    string          `json:"type"`
+	ID      string          `json:"id,omitempty"`
 	Payload json.RawMessage `json:"payload,omitempty"`
 }
 
 // Hello is the first message an agent sends on every connection.
 type Hello struct {
-	AgentVersion   string `json:"agent_version"`
-	Hostname       string `json:"hostname"`
-	SwarmManager   bool   `json:"swarm_manager"`
-	SwarmClusterID string `json:"swarm_cluster_id,omitempty"`
+	AgentVersion    string `json:"agent_version"`
+	ProtocolVersion int    `json:"protocol_version,omitempty"`
+	Hostname        string `json:"hostname"`
+	SwarmManager    bool   `json:"swarm_manager"`
+	SwarmClusterID  string `json:"swarm_cluster_id,omitempty"`
+	// KnownJobs lists every job ID the agent holds: queued, running, or
+	// finished but not yet acknowledged. The master fails its queued and
+	// running jobs for this agent that are not listed ("lost").
+	KnownJobs []string `json:"known_jobs,omitempty"`
 }
 
 // Kind says how a service is deployed.
@@ -47,12 +80,16 @@ type Service struct {
 	ComposeProject string `json:"compose_project,omitempty"`
 	ComposeService string `json:"compose_service,omitempty"`
 	WorkingDir     string `json:"working_dir,omitempty"`
-	SwarmService   string `json:"swarm_service,omitempty"`
-	ContainerName  string `json:"container_name,omitempty"`
-	Image          string `json:"image"`
-	Digest         string `json:"digest,omitempty"`
-	Status         string `json:"status"`
-	Self           bool   `json:"self,omitempty"`
+	// ConfigFiles is the raw com.docker.compose.project.config_files label:
+	// absolute host paths joined by ",". Kept as a string so Service stays
+	// comparable with ==.
+	ConfigFiles   string `json:"config_files,omitempty"`
+	SwarmService  string `json:"swarm_service,omitempty"`
+	ContainerName string `json:"container_name,omitempty"`
+	Image         string `json:"image"`
+	Digest        string `json:"digest,omitempty"`
+	Status        string `json:"status"`
+	Self          bool   `json:"self,omitempty"`
 }
 
 // Key identifies the service within one agent's report. It is stable across
@@ -83,6 +120,50 @@ type Remove struct {
 	Key string `json:"key"`
 }
 
+// JobStart asks the agent to deploy Ref to one service. Ref is "repo:tag";
+// the agent resolves the tag's current digest and deploys "repo:tag@digest".
+// The agent reads the Compose working directory and config files from its
+// own Docker daemon, not from this message.
+type JobStart struct {
+	Kind           Kind   `json:"kind"`
+	ComposeProject string `json:"compose_project,omitempty"`
+	ComposeService string `json:"compose_service,omitempty"`
+	SwarmClusterID string `json:"swarm_cluster_id,omitempty"`
+	SwarmService   string `json:"swarm_service,omitempty"`
+	Ref            string `json:"ref"`
+}
+
+// JobProgress carries output produced since the previous JobProgress for the
+// same job. The first JobProgress of a job (Output may be empty) tells the
+// master the job is running.
+type JobProgress struct {
+	Output string `json:"output"`
+}
+
+// Job result statuses.
+const (
+	ResultSucceeded = "succeeded"
+	ResultFailed    = "failed"
+)
+
+// JobResult is a job's outcome. The agent keeps it until the master sends
+// JobAck for the same ID, and resends it after every reconnect until then.
+type JobResult struct {
+	Status string `json:"status"`        // ResultSucceeded or ResultFailed
+	Ref    string `json:"ref,omitempty"` // deployed "repo:tag@digest", set on success
+	Error  string `json:"error,omitempty"`
+	// Output is the job's complete output, tail-capped at MaxJobOutput. It
+	// replaces whatever the master assembled from JobProgress messages.
+	Output          string    `json:"output"`
+	OutputTruncated bool      `json:"output_truncated,omitempty"`
+	StartedAt       time.Time `json:"started_at"`
+	FinishedAt      time.Time `json:"finished_at"`
+}
+
+// JobAck tells the agent the master has stored the JobResult with the same
+// ID. It has no fields.
+type JobAck struct{}
+
 // Encode wraps payload in an Envelope of the given type.
 func Encode(typ string, payload any) (Envelope, error) {
 	raw, err := json.Marshal(payload)
@@ -92,6 +173,16 @@ func Encode(typ string, payload any) (Envelope, error) {
 	return Envelope{Type: typ, Payload: raw}, nil
 }
 
+// EncodeJob wraps a job message's payload in an Envelope carrying the job ID.
+func EncodeJob(typ, jobID string, payload any) (Envelope, error) {
+	env, err := Encode(typ, payload)
+	if err != nil {
+		return Envelope{}, err
+	}
+	env.ID = jobID
+	return env, nil
+}
+
 // Decode unpacks env's payload into T.
 func Decode[T any](env Envelope) (T, error) {
 	var v T
diff --git a/internal/protocol/protocol_test.go b/internal/protocol/protocol_test.go
index 4c288cb..53cc817 100644
--- a/internal/protocol/protocol_test.go
+++ b/internal/protocol/protocol_test.go
@@ -47,3 +47,35 @@ func TestDecodeRejectsMalformedPayload(t *testing.T) {
 	_, err := Decode[Hello](Envelope{Type: TypeHello, Payload: json.RawMessage(`{"hostname": 5}`)})
 	assert.ErrorContains(t, err, "decode hello")
 }
+
+func TestEncodeJob_CarriesID(t *testing.T) {
+	in := JobStart{Kind: KindSwarm, SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"}
+	env, err := EncodeJob(TypeJobStart, "42", in)
+	require.NoError(t, err)
+
+	raw, err := json.Marshal(env)
+	require.NoError(t, err)
+	var back Envelope
+	require.NoError(t, json.Unmarshal(raw, &back))
+
+	assert.Equal(t, "42", back.ID)
+	assert.Equal(t, TypeJobStart, back.Type)
+	out, err := Decode[JobStart](back)
+	require.NoError(t, err)
+	assert.Equal(t, in, out)
+}
+
+func TestHello_PhaseOneDecodesAsVersionZero(t *testing.T) {
+	var h Hello
+	require.NoError(t, json.Unmarshal([]byte(`{"agent_version":"x","hostname":"h","swarm_manager":false}`), &h))
+	assert.Equal(t, 0, h.ProtocolVersion)
+	assert.Nil(t, h.KnownJobs)
+}
+
+func TestEnvelope_InventoryHasNoID(t *testing.T) {
+	env, err := Encode(TypeSnapshot, Snapshot{})
+	require.NoError(t, err)
+	raw, err := json.Marshal(env)
+	require.NoError(t, err)
+	assert.NotContains(t, string(raw), `"id"`)
+}

From c9c4e0cadb54f91bd83b018624e4e7e3aa1fd405 Mon Sep 17 00:00:00 2001
From: Samuel Enocsson 
Date: Mon, 12 Oct 2026 18:39:57 +0200
Subject: [PATCH 04/29] refactor(web): web.New takes the jobs service (#7)

---
 cmd/master/main.go                 |  2 +-
 internal/master/web/server.go      | 15 +++++++++++++--
 internal/master/web/server_test.go | 18 ++++++++++++++++--
 3 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/cmd/master/main.go b/cmd/master/main.go
index 34e0de7..68d0287 100644
--- a/cmd/master/main.go
+++ b/cmd/master/main.go
@@ -76,7 +76,7 @@ func run() error {
 	}
 	auth := webauth.New(provider, st, cfg.AllowedGroup, strings.HasPrefix(cfg.PublicURL, "https://"), logger)
 
-	ui, err := web.New(st, agentHub, broker, web.Config{
+	ui, err := web.New(st, agentHub, nil, broker, web.Config{ // jobs is nil: wired in W3-4
 		AgentURL: cfg.AgentURL, MasterKey: fingerprint, AgentImage: cfg.AgentImage,
 	}, logger)
 	if err != nil {
diff --git a/internal/master/web/server.go b/internal/master/web/server.go
index bf81f1d..a6433ea 100644
--- a/internal/master/web/server.go
+++ b/internal/master/web/server.go
@@ -28,6 +28,9 @@ var agentNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
 // Store is the storage the UI reads and writes. *store.Store satisfies it.
 type Store interface {
 	ListServices(ctx context.Context) ([]model.Service, error)
+	GetService(ctx context.Context, id int64) (model.Service, error)
+	GetJob(ctx context.Context, id int64) (model.Job, error)
+	ListJobs(ctx context.Context, f model.JobFilter) ([]model.Job, error)
 	ListAgents(ctx context.Context) ([]model.Agent, error)
 	CreateAgent(ctx context.Context, name, tokenHash string) (model.Agent, error)
 	RevokeAgent(ctx context.Context, id int64, at time.Time) error
@@ -39,6 +42,13 @@ type Hub interface {
 	Disconnect(agentID int64)
 }
 
+// Jobs starts updates and explains why a service cannot be updated.
+// *jobs.Service satisfies it.
+type Jobs interface {
+	Updatable(svc model.Service) error
+	Trigger(ctx context.Context, serviceID int64, requestedBy string) (model.Job, error)
+}
+
 // Broker delivers change signals to open streams and accepts signals from
 // UI actions. *events.Broker satisfies it.
 type Broker interface {
@@ -57,6 +67,7 @@ type Config struct {
 type Server struct {
 	store  Store
 	hub    Hub
+	jobs   Jobs
 	broker Broker
 	cfg    Config
 	log    *slog.Logger
@@ -66,7 +77,7 @@ type Server struct {
 
 // New parses the templates. Each page is its own template set so every page
 // can define its own "content" block.
-func New(store Store, hub Hub, broker Broker, cfg Config, log *slog.Logger) (*Server, error) {
+func New(store Store, hub Hub, jobs Jobs, broker Broker, cfg Config, log *slog.Logger) (*Server, error) {
 	funcs := template.FuncMap{"shortRef": shortRef}
 	pages := map[string]*template.Template{}
 	for _, page := range []string{"overview.html", "agents.html", "agent_created.html"} {
@@ -76,7 +87,7 @@ func New(store Store, hub Hub, broker Broker, cfg Config, log *slog.Logger) (*Se
 		}
 		pages[page] = t
 	}
-	return &Server{store: store, hub: hub, broker: broker, cfg: cfg, log: log, now: time.Now, pages: pages}, nil
+	return &Server{store: store, hub: hub, jobs: jobs, broker: broker, cfg: cfg, log: log, now: time.Now, pages: pages}, nil
 }
 
 // Register adds the UI routes to mux. protect wraps every route except
diff --git a/internal/master/web/server_test.go b/internal/master/web/server_test.go
index 8cec09d..d2f5494 100644
--- a/internal/master/web/server_test.go
+++ b/internal/master/web/server_test.go
@@ -34,7 +34,14 @@ type fakeStore struct {
 }
 
 func (f *fakeStore) ListServices(context.Context) ([]model.Service, error) { return f.services, nil }
-func (f *fakeStore) ListAgents(context.Context) ([]model.Agent, error)     { return f.agents, nil }
+func (f *fakeStore) GetService(context.Context, int64) (model.Service, error) {
+	return model.Service{}, model.ErrNotFound
+}
+func (f *fakeStore) GetJob(context.Context, int64) (model.Job, error) {
+	return model.Job{}, model.ErrNotFound
+}
+func (f *fakeStore) ListJobs(context.Context, model.JobFilter) ([]model.Job, error) { return nil, nil }
+func (f *fakeStore) ListAgents(context.Context) ([]model.Agent, error)              { return f.agents, nil }
 
 func (f *fakeStore) CreateAgent(_ context.Context, name, hash string) (model.Agent, error) {
 	f.mu.Lock()
@@ -65,6 +72,13 @@ func (f *fakeStore) revoked() int64 {
 	return f.revokedID
 }
 
+type fakeJobs struct{}
+
+func (fakeJobs) Updatable(model.Service) error { return nil }
+func (fakeJobs) Trigger(context.Context, int64, string) (model.Job, error) {
+	return model.Job{}, model.ErrNotFound
+}
+
 type fakeHub struct {
 	mu           sync.Mutex
 	online       map[int64]bool
@@ -110,7 +124,7 @@ func newTestServer(t *testing.T) (*httptest.Server, *fakeStore, *fakeHub, *event
 	}
 	hub := &fakeHub{online: map[int64]bool{2: true}}
 	broker := events.NewBroker()
-	s, err := New(store, hub, broker, Config{
+	s, err := New(store, hub, fakeJobs{}, broker, Config{
 		AgentURL: "wss://10.0.0.5:8090/agent", MasterKey: "sha256:abc", AgentImage: "gitea.shcizo.se/shcizo/package-updater:latest",
 	}, slog.New(slog.NewTextHandler(io.Discard, nil)))
 	require.NoError(t, err)

From e2b73dfd815ccc290f9f3689612bf084e9ebbe60 Mon Sep 17 00:00:00 2001
From: Samuel Enocsson 
Date: Mon, 12 Oct 2026 18:40:15 +0200
Subject: [PATCH 05/29] feat(agent): UPDATE_TIMEOUT and DOCKER_CONFIG settings
 (#7)

---
 cmd/server/main.go                            | 144 ------------
 internal/agent/config/config.go               |  42 ++++
 internal/agent/config/config_test.go          |  48 +++-
 internal/agent/inventory/collector.go         |  10 +-
 internal/agent/inventory/collector_test.go    |   9 +
 .../{ => agent}/updater/compose_executor.go   |   0
 .../{discovery => agent/updater}/pathcheck.go |   0
 .../updater}/pathcheck_test.go                |   0
 .../{ => agent}/updater/swarm_executor.go     |   0
 .../updater/swarm_executor_test.go            |   0
 internal/api/handlers.go                      | 148 ------------
 internal/api/handlers_test.go                 | 216 ------------------
 internal/api/middleware.go                    | 112 ---------
 internal/api/middleware_test.go               |  92 --------
 internal/api/swarm_result_test.go             |  48 ----
 internal/api/types.go                         |  40 ----
 internal/config/config.go                     |  58 -----
 internal/config/config_test.go                |  82 -------
 internal/discovery/discovery.go               |  96 --------
 internal/discovery/discovery_test.go          | 172 --------------
 internal/discovery/docker_client.go           |  15 --
 internal/discovery/labels.go                  |  71 ------
 internal/discovery/labels_test.go             |  88 -------
 internal/discovery/matching.go                |  33 ---
 internal/discovery/matching_test.go           |  51 -----
 internal/discovery/swarm.go                   |  60 -----
 internal/discovery/swarm_test.go              |  86 -------
 internal/metrics/metrics.go                   |  69 ------
 internal/selfupdate/selfupdate.go             |  56 -----
 internal/selfupdate/selfupdate_test.go        |  87 -------
 internal/updater/executor.go                  |  15 --
 internal/updater/queue.go                     | 144 ------------
 internal/updater/queue_test.go                | 129 -----------
 internal/updater/worker.go                    |   6 -
 internal/updater/worker_test.go               |  38 ---
 35 files changed, 104 insertions(+), 2161 deletions(-)
 delete mode 100644 cmd/server/main.go
 rename internal/{ => agent}/updater/compose_executor.go (100%)
 rename internal/{discovery => agent/updater}/pathcheck.go (100%)
 rename internal/{discovery => agent/updater}/pathcheck_test.go (100%)
 rename internal/{ => agent}/updater/swarm_executor.go (100%)
 rename internal/{ => agent}/updater/swarm_executor_test.go (100%)
 delete mode 100644 internal/api/handlers.go
 delete mode 100644 internal/api/handlers_test.go
 delete mode 100644 internal/api/middleware.go
 delete mode 100644 internal/api/middleware_test.go
 delete mode 100644 internal/api/swarm_result_test.go
 delete mode 100644 internal/api/types.go
 delete mode 100644 internal/config/config.go
 delete mode 100644 internal/config/config_test.go
 delete mode 100644 internal/discovery/discovery.go
 delete mode 100644 internal/discovery/discovery_test.go
 delete mode 100644 internal/discovery/docker_client.go
 delete mode 100644 internal/discovery/labels.go
 delete mode 100644 internal/discovery/labels_test.go
 delete mode 100644 internal/discovery/matching.go
 delete mode 100644 internal/discovery/matching_test.go
 delete mode 100644 internal/discovery/swarm.go
 delete mode 100644 internal/discovery/swarm_test.go
 delete mode 100644 internal/metrics/metrics.go
 delete mode 100644 internal/selfupdate/selfupdate.go
 delete mode 100644 internal/selfupdate/selfupdate_test.go
 delete mode 100644 internal/updater/executor.go
 delete mode 100644 internal/updater/queue.go
 delete mode 100644 internal/updater/queue_test.go
 delete mode 100644 internal/updater/worker.go
 delete mode 100644 internal/updater/worker_test.go

diff --git a/cmd/server/main.go b/cmd/server/main.go
deleted file mode 100644
index 27c1288..0000000
--- a/cmd/server/main.go
+++ /dev/null
@@ -1,144 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-
-	"github.com/docker/docker/client"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/config"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/shcizo/package-updater/internal/metrics"
-	"github.com/shcizo/package-updater/internal/updater"
-)
-
-// Build info is injected at link time via -ldflags="-X main.version=..."
-var (
-	version   = "dev"
-	commit    = "unknown"
-	buildTime = "unknown"
-)
-
-func main() {
-	if err := run(); err != nil {
-		fmt.Fprintln(os.Stderr, "fatal:", err)
-		os.Exit(1)
-	}
-}
-
-func run() error {
-	cfg, err := config.Load()
-	if err != nil {
-		return err
-	}
-
-	logger := logging.New(cfg.LogLevel)
-	logger.Info("starting",
-		"version", version, "commit", commit, "port", cfg.Port,
-		"mode", cfg.Mode, "stacks_root", cfg.StacksRoot, "opt_in_label", cfg.OptInLabel,
-	)
-
-	dockerCli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-	if err != nil {
-		return fmt.Errorf("docker client: %w", err)
-	}
-	defer dockerCli.Close()
-
-	var finder api.Finder
-	var exec updater.Executor
-	switch cfg.Mode {
-	case "swarm":
-		finder = discovery.NewSwarm(dockerCli, cfg.OptInLabel)
-		exec = updater.NewSwarmExecutor(dockerCli)
-	case "compose":
-		finder = discovery.New(dockerCli, cfg.StacksRoot, cfg.OptInLabel)
-		exec = updater.NewComposeExecutor()
-	default:
-		// Unreachable: config.Load already validates Mode, but guard
-		// here too rather than silently falling through to a nil
-		// finder/exec pair.
-		return fmt.Errorf("unknown MODE %q", cfg.Mode)
-	}
-
-	reg := prometheus.NewRegistry()
-	m := metrics.New(reg)
-	m.BuildInfo.WithLabelValues(version, commit).Set(1)
-
-	queue := updater.NewQueue(exec, m)
-	queue.Start(context.Background())
-	defer queue.Stop()
-
-	handlers := api.NewHandlers(finder, &submitterAdapter{queue: queue, timeout: cfg.UpdateTimeout}, dockerCli, version, commit, buildTime, m)
-
-	mux := http.NewServeMux()
-	mux.HandleFunc("POST /update", handlers.Update)
-	mux.HandleFunc("GET /healthz", handlers.Healthz)
-	mux.HandleFunc("GET /version", handlers.Version)
-	mux.Handle("GET /metrics", metrics.Handler(reg))
-
-	authed := api.Auth(cfg.APIKey)
-	handler := api.RequestID(api.RequestLogger(logger, m)(routeAuth(mux, authed)))
-
-	srv := &http.Server{
-		Addr:              ":" + cfg.Port,
-		Handler:           handler,
-		ReadHeaderTimeout: 10 * time.Second,
-	}
-
-	errCh := make(chan error, 1)
-	go func() {
-		logger.Info("listening", "addr", srv.Addr)
-		errCh <- srv.ListenAndServe()
-	}()
-
-	sig := make(chan os.Signal, 1)
-	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
-
-	select {
-	case s := <-sig:
-		logger.Info("shutdown_signal", "signal", s.String())
-	case err := <-errCh:
-		if err != nil && err != http.ErrServerClosed {
-			return err
-		}
-	}
-
-	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	_ = srv.Shutdown(shutdownCtx)
-	return nil
-}
-
-// routeAuth applies the bearer-token middleware to /update only.
-// /healthz, /metrics, /version are unauthenticated by design (internal
-// network only; healthcheck and Prometheus scraper need to reach them
-// without secrets).
-func routeAuth(next *http.ServeMux, mw func(http.Handler) http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/update" {
-			mw(next).ServeHTTP(w, r)
-			return
-		}
-		next.ServeHTTP(w, r)
-	})
-}
-
-// submitterAdapter wraps Queue.Submit to inject the per-request timeout
-// from config, satisfying the api.Submitter interface.
-type submitterAdapter struct {
-	queue   *updater.Queue
-	timeout time.Duration
-}
-
-func (s *submitterAdapter) Submit(ctx context.Context, jobs []discovery.Job) []updater.Result {
-	ctx, cancel := context.WithTimeout(ctx, s.timeout)
-	defer cancel()
-	return s.queue.Submit(ctx, jobs)
-}
diff --git a/internal/agent/config/config.go b/internal/agent/config/config.go
index 1b918d9..2b2620a 100644
--- a/internal/agent/config/config.go
+++ b/internal/agent/config/config.go
@@ -2,10 +2,14 @@
 package config
 
 import (
+	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 )
 
 var masterKeyPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
@@ -17,8 +21,17 @@ type Config struct {
 	Token     string // AGENT_TOKEN, shown once when the agent was added
 	SelfID    string // SELF_ID, defaults to the hostname (= container ID prefix)
 	LogLevel  string // LOG_LEVEL, default info
+
+	// UpdateTimeout bounds one update job (UPDATE_TIMEOUT, default 5m).
+	UpdateTimeout time.Duration
+	// DockerConfig is the directory holding config.json with registry
+	// credentials (DOCKER_CONFIG). Empty means anonymous registry access.
+	DockerConfig string
 }
 
+// defaultUpdateTimeout is used when UPDATE_TIMEOUT is unset.
+const defaultUpdateTimeout = 5 * time.Minute
+
 // Load reads and validates the agent configuration. It fails fast so a
 // misconfigured agent never starts dialling.
 func Load() (*Config, error) {
@@ -28,6 +41,9 @@ func Load() (*Config, error) {
 		Token:     os.Getenv("AGENT_TOKEN"),
 		SelfID:    os.Getenv("SELF_ID"),
 		LogLevel:  os.Getenv("LOG_LEVEL"),
+
+		UpdateTimeout: defaultUpdateTimeout,
+		DockerConfig:  os.Getenv("DOCKER_CONFIG"),
 	}
 	switch {
 	case cfg.MasterURL == "":
@@ -41,6 +57,18 @@ func Load() (*Config, error) {
 	case cfg.Token == "":
 		return nil, errors.New("AGENT_TOKEN is required")
 	}
+	if v := os.Getenv("UPDATE_TIMEOUT"); v != "" {
+		d, err := time.ParseDuration(v)
+		if err != nil || d <= 0 {
+			return nil, fmt.Errorf("UPDATE_TIMEOUT must be a positive duration such as 5m, got %q", v)
+		}
+		cfg.UpdateTimeout = d
+	}
+	if cfg.DockerConfig != "" {
+		if err := checkDockerConfig(cfg.DockerConfig); err != nil {
+			return nil, err
+		}
+	}
 	if cfg.SelfID == "" {
 		cfg.SelfID, _ = os.Hostname()
 	}
@@ -49,3 +77,17 @@ func Load() (*Config, error) {
 	}
 	return cfg, nil
 }
+
+// checkDockerConfig fails when DOCKER_CONFIG is set but its config.json is
+// missing or not JSON, so a typo does not silently turn into anonymous pulls.
+func checkDockerConfig(dir string) error {
+	path := filepath.Join(dir, "config.json")
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("DOCKER_CONFIG: read %s: %w", path, err)
+	}
+	if !json.Valid(raw) {
+		return fmt.Errorf("DOCKER_CONFIG: %s is not valid JSON", path)
+	}
+	return nil
+}
diff --git a/internal/agent/config/config_test.go b/internal/agent/config/config_test.go
index 606018e..6aac5b2 100644
--- a/internal/agent/config/config_test.go
+++ b/internal/agent/config/config_test.go
@@ -2,7 +2,9 @@ package config
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -16,6 +18,8 @@ func setValid(t *testing.T) {
 	t.Setenv("AGENT_TOKEN", "pua_abc")
 	t.Setenv("SELF_ID", "")
 	t.Setenv("LOG_LEVEL", "")
+	t.Setenv("UPDATE_TIMEOUT", "")
+	t.Setenv("DOCKER_CONFIG", "")
 }
 
 func TestLoad_Valid(t *testing.T) {
@@ -25,7 +29,7 @@ func TestLoad_Valid(t *testing.T) {
 	host, _ := os.Hostname()
 	assert.Equal(t, &Config{
 		MasterURL: "wss://10.0.0.5:8090/agent", MasterKey: validKey, Token: "pua_abc",
-		SelfID: host, LogLevel: "info",
+		SelfID: host, LogLevel: "info", UpdateTimeout: 5 * time.Minute,
 	}, cfg)
 }
 
@@ -48,3 +52,45 @@ func TestLoad_Invalid(t *testing.T) {
 		})
 	}
 }
+
+func TestLoad_JobSettings(t *testing.T) {
+	withConfig := func(t *testing.T, body string) string {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(body), 0o600))
+		return dir
+	}
+	cases := []struct {
+		name      string
+		timeout   string
+		dockerCfg func(t *testing.T) string
+		wantTime  time.Duration
+		wantErr   string
+	}{
+		{name: "defaults", wantTime: 5 * time.Minute},
+		{name: "custom timeout", timeout: "90s", wantTime: 90 * time.Second},
+		{name: "zero timeout", timeout: "0", wantErr: "UPDATE_TIMEOUT"},
+		{name: "garbage timeout", timeout: "abc", wantErr: "UPDATE_TIMEOUT"},
+		{name: "docker config without file", dockerCfg: func(t *testing.T) string { return t.TempDir() }, wantErr: "DOCKER_CONFIG"},
+		{name: "docker config not json", dockerCfg: func(t *testing.T) string { return withConfig(t, "nope") }, wantErr: "not valid JSON"},
+		{name: "docker config ok", dockerCfg: func(t *testing.T) string { return withConfig(t, `{"auths":{}}`) }, wantTime: 5 * time.Minute},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			setValid(t)
+			t.Setenv("UPDATE_TIMEOUT", c.timeout)
+			dir := ""
+			if c.dockerCfg != nil {
+				dir = c.dockerCfg(t)
+				t.Setenv("DOCKER_CONFIG", dir)
+			}
+			cfg, err := Load()
+			if c.wantErr != "" {
+				assert.ErrorContains(t, err, c.wantErr)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, c.wantTime, cfg.UpdateTimeout)
+			assert.Equal(t, dir, cfg.DockerConfig)
+		})
+	}
+}
diff --git a/internal/agent/inventory/collector.go b/internal/agent/inventory/collector.go
index 2ee81d1..fbe36bf 100644
--- a/internal/agent/inventory/collector.go
+++ b/internal/agent/inventory/collector.go
@@ -18,10 +18,11 @@ import (
 )
 
 const (
-	labelProject    = "com.docker.compose.project"
-	labelService    = "com.docker.compose.service"
-	labelWorkingDir = "com.docker.compose.project.working_dir"
-	labelSwarmTask  = "com.docker.swarm.service.id"
+	labelProject     = "com.docker.compose.project"
+	labelService     = "com.docker.compose.service"
+	labelWorkingDir  = "com.docker.compose.project.working_dir"
+	labelConfigFiles = "com.docker.compose.project.config_files"
+	labelSwarmTask   = "com.docker.swarm.service.id"
 )
 
 // DockerClient is the subset of the Docker SDK the collector needs.
@@ -148,6 +149,7 @@ func (c *Collector) fromContainer(ct container.Summary, ref string, repoDigests
 		svc.ComposeProject = project
 		svc.ComposeService = service
 		svc.WorkingDir = ct.Labels[labelWorkingDir]
+		svc.ConfigFiles = ct.Labels[labelConfigFiles]
 		return svc
 	}
 	svc.Kind = protocol.KindContainer
diff --git a/internal/agent/inventory/collector_test.go b/internal/agent/inventory/collector_test.go
index b300de5..ff464ff 100644
--- a/internal/agent/inventory/collector_test.go
+++ b/internal/agent/inventory/collector_test.go
@@ -200,3 +200,12 @@ func TestHello(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, protocol.Hello{Hostname: "swarm-mgr", SwarmManager: true, SwarmClusterID: "cluster-1"}, got)
 }
+
+func TestSnapshot_ComposeConfigFilesIsRawLabel(t *testing.T) {
+	ct := composeContainer("abc123", "slusk", "web", "slusk:1", "sha256:img1", "running")
+	ct.Labels[labelConfigFiles] = "/s/a/compose.yaml,/s/a/compose.override.yaml"
+	got, err := New(&fakeDocker{containers: []container.Summary{ct}}, "").Snapshot(context.Background())
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, "/s/a/compose.yaml,/s/a/compose.override.yaml", got[0].ConfigFiles)
+}
diff --git a/internal/updater/compose_executor.go b/internal/agent/updater/compose_executor.go
similarity index 100%
rename from internal/updater/compose_executor.go
rename to internal/agent/updater/compose_executor.go
diff --git a/internal/discovery/pathcheck.go b/internal/agent/updater/pathcheck.go
similarity index 100%
rename from internal/discovery/pathcheck.go
rename to internal/agent/updater/pathcheck.go
diff --git a/internal/discovery/pathcheck_test.go b/internal/agent/updater/pathcheck_test.go
similarity index 100%
rename from internal/discovery/pathcheck_test.go
rename to internal/agent/updater/pathcheck_test.go
diff --git a/internal/updater/swarm_executor.go b/internal/agent/updater/swarm_executor.go
similarity index 100%
rename from internal/updater/swarm_executor.go
rename to internal/agent/updater/swarm_executor.go
diff --git a/internal/updater/swarm_executor_test.go b/internal/agent/updater/swarm_executor_test.go
similarity index 100%
rename from internal/updater/swarm_executor_test.go
rename to internal/agent/updater/swarm_executor_test.go
diff --git a/internal/api/handlers.go b/internal/api/handlers.go
deleted file mode 100644
index 420ff35..0000000
--- a/internal/api/handlers.go
+++ /dev/null
@@ -1,148 +0,0 @@
-package api
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-
-	"github.com/docker/docker/api/types"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/shcizo/package-updater/internal/metrics"
-	"github.com/shcizo/package-updater/internal/updater"
-)
-
-// Finder is the discovery interface used by the update handler.
-type Finder interface {
-	FindJobs(ctx context.Context, image string) ([]discovery.Job, error)
-}
-
-// Submitter is the queue interface used by the update handler.
-type Submitter interface {
-	Submit(ctx context.Context, jobs []discovery.Job) []updater.Result
-}
-
-// Pinger pings the Docker daemon for the healthcheck.
-type Pinger interface {
-	Ping(ctx context.Context) (types.Ping, error)
-}
-
-// Handlers wires the HTTP endpoints to the rest of the service.
-type Handlers struct {
-	finder    Finder
-	submitter Submitter
-	pinger    Pinger
-	version   string
-	commit    string
-	buildTime string
-	metrics   *metrics.Metrics
-}
-
-// NewHandlers constructs a Handlers value with all dependencies injected.
-// m may be nil, in which case metrics recording is silently skipped.
-func NewHandlers(f Finder, s Submitter, p Pinger, version, commit, buildTime string, m *metrics.Metrics) *Handlers {
-	return &Handlers{finder: f, submitter: s, pinger: p, version: version, commit: commit, buildTime: buildTime, metrics: m}
-}
-
-// Update implements POST /update.
-func (h *Handlers) Update(w http.ResponseWriter, r *http.Request) {
-	var req UpdateRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		writeJSONError(w, http.StatusBadRequest, "invalid JSON body")
-		return
-	}
-	if req.Image == "" {
-		writeJSONError(w, http.StatusBadRequest, "image is required")
-		return
-	}
-
-	requestedImage := req.Image
-	if req.Tag != "" {
-		requestedImage = req.Image + ":" + req.Tag
-	}
-
-	jobs, err := h.finder.FindJobs(r.Context(), requestedImage)
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "discovery failed: "+err.Error())
-		return
-	}
-
-	resp := UpdateResponse{
-		RequestID: logging.RequestIDFrom(r.Context()),
-		Image:     req.Image,
-		Tag:       req.Tag,
-		Matched:   len(jobs),
-		Results:   []ResultDTO{},
-	}
-
-	if len(jobs) == 0 {
-		writeJSON(w, http.StatusOK, resp)
-		return
-	}
-
-	results := h.submitter.Submit(r.Context(), jobs)
-	updated, failed := 0, 0
-	for _, res := range results {
-		composeFile := ""
-		if len(res.Job.ConfigFiles) > 0 {
-			composeFile = res.Job.ConfigFiles[0]
-		}
-		resp.Results = append(resp.Results, ResultDTO{
-			Project:     res.Job.Project,
-			Service:     res.Job.Service,
-			ComposeFile: composeFile,
-			ServiceID:   res.Job.ServiceID,
-			Status:      string(res.Status),
-			Error:       res.Error,
-			DurationMs:  res.DurationMs,
-		})
-		if res.Status == updater.StatusUpdated {
-			updated++
-		} else {
-			failed++
-		}
-	}
-
-	switch {
-	case failed == 0:
-		writeJSON(w, http.StatusOK, resp)
-	case updated == 0:
-		writeJSON(w, http.StatusInternalServerError, resp)
-	default:
-		writeJSON(w, http.StatusMultiStatus, resp)
-	}
-}
-
-// Healthz implements GET /healthz.
-func (h *Handlers) Healthz(w http.ResponseWriter, r *http.Request) {
-	if _, err := h.pinger.Ping(r.Context()); err != nil {
-		if h.metrics != nil {
-			h.metrics.DockerPingUp.Set(0)
-		}
-		writeJSON(w, http.StatusServiceUnavailable, HealthResponse{Status: "unhealthy", Docker: "unreachable"})
-		return
-	}
-	if h.metrics != nil {
-		h.metrics.DockerPingUp.Set(1)
-	}
-	writeJSON(w, http.StatusOK, HealthResponse{Status: "ok", Docker: "ok"})
-}
-
-// Version implements GET /version.
-func (h *Handlers) Version(w http.ResponseWriter, _ *http.Request) {
-	writeJSON(w, http.StatusOK, VersionResponse{
-		Version:   h.version,
-		Commit:    h.commit,
-		BuildTime: h.buildTime,
-	})
-}
-
-func writeJSON(w http.ResponseWriter, code int, body any) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(code)
-	_ = json.NewEncoder(w).Encode(body)
-}
-
-func writeJSONError(w http.ResponseWriter, code int, msg string) {
-	writeJSON(w, code, map[string]string{"error": msg})
-}
diff --git a/internal/api/handlers_test.go b/internal/api/handlers_test.go
deleted file mode 100644
index 462540a..0000000
--- a/internal/api/handlers_test.go
+++ /dev/null
@@ -1,216 +0,0 @@
-package api_test
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/docker/docker/api/types"
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/updater"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeFinder struct {
-	jobs []discovery.Job
-	err  error
-
-	// gotImage records the image argument passed to FindJobs, for
-	// assertions on the exact reference the handler built.
-	gotImage string
-	// swarmShaped, when true, makes FindJobs return a Job whose Image
-	// field carries the received image argument, simulating
-	// discovery.SwarmDiscovery.FindJobs.
-	swarmShaped bool
-}
-
-func (f *fakeFinder) FindJobs(_ context.Context, image string) ([]discovery.Job, error) {
-	f.gotImage = image
-	if f.swarmShaped {
-		return []discovery.Job{{Service: "myapp_web", ServiceID: "svc1", Image: image}}, f.err
-	}
-	return f.jobs, f.err
-}
-
-type fakeSubmitter struct {
-	results []updater.Result
-}
-
-func (f *fakeSubmitter) Submit(_ context.Context, jobs []discovery.Job) []updater.Result {
-	if f.results != nil {
-		return f.results
-	}
-	out := make([]updater.Result, len(jobs))
-	for i, j := range jobs {
-		out[i] = updater.Result{Job: j, Status: updater.StatusUpdated}
-	}
-	return out
-}
-
-type fakePinger struct{ err error }
-
-func (f *fakePinger) Ping(_ context.Context) (types.Ping, error) {
-	return types.Ping{}, f.err
-}
-
-func decode[T any](t *testing.T, body io.Reader) T {
-	t.Helper()
-	var v T
-	require.NoError(t, json.NewDecoder(body).Decode(&v))
-	return v
-}
-
-func TestUpdate_ValidationError(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodPost, "/update",
-		strings.NewReader(`{"tag":"v1.2.3"}`))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusBadRequest, w.Code)
-}
-
-func TestUpdate_BadJSON(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`not json`))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusBadRequest, w.Code)
-}
-
-func TestUpdate_DiscoveryFailureReturns500(t *testing.T) {
-	finder := &fakeFinder{err: errors.New("daemon unreachable")}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusInternalServerError, w.Code)
-}
-
-func TestUpdate_ZeroMatchesReturns200(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	resp := decode[api.UpdateResponse](t, w.Body)
-	require.Equal(t, 0, resp.Matched)
-}
-
-func TestUpdate_AllSucceeded200(t *testing.T) {
-	finder := &fakeFinder{jobs: []discovery.Job{
-		{Project: "p", Service: "s", WorkingDir: "/x", ConfigFiles: []string{"/x/c.yml"}},
-	}}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x", Tag: "v1"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	resp := decode[api.UpdateResponse](t, w.Body)
-	require.Equal(t, 1, resp.Matched)
-	require.Equal(t, "updated", resp.Results[0].Status)
-	require.Equal(t, "/x/c.yml", resp.Results[0].ComposeFile)
-}
-
-func TestUpdate_MixedReturns207(t *testing.T) {
-	jobs := []discovery.Job{
-		{Project: "p1", Service: "s", ConfigFiles: []string{"/x/c.yml"}},
-		{Project: "p2", Service: "s", ConfigFiles: []string{"/y/c.yml"}},
-	}
-	finder := &fakeFinder{jobs: jobs}
-	submitter := &fakeSubmitter{results: []updater.Result{
-		{Job: jobs[0], Status: updater.StatusUpdated},
-		{Job: jobs[1], Status: updater.StatusFailed, Error: "boom"},
-	}}
-	h := api.NewHandlers(finder, submitter, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusMultiStatus, w.Code)
-}
-
-func TestUpdate_AllFailedReturns500(t *testing.T) {
-	jobs := []discovery.Job{{Project: "p", Service: "s", ConfigFiles: []string{"/x/c.yml"}}}
-	finder := &fakeFinder{jobs: jobs}
-	submitter := &fakeSubmitter{results: []updater.Result{
-		{Job: jobs[0], Status: updater.StatusFailed, Error: "boom"},
-	}}
-	h := api.NewHandlers(finder, submitter, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "r/x"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusInternalServerError, w.Code)
-}
-
-func TestHealthz_OKWhenDockerUp(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
-	w := httptest.NewRecorder()
-	h.Healthz(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-}
-
-func TestHealthz_503WhenDockerDown(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{err: errors.New("ping fail")}, "v0.0.0", "abc", "now", nil)
-	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
-	w := httptest.NewRecorder()
-	h.Healthz(w, req)
-	require.Equal(t, http.StatusServiceUnavailable, w.Code)
-}
-
-func TestUpdate_FindJobsCalledWithTaggedImage(t *testing.T) {
-	finder := &fakeFinder{}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "registry.example.com/myapp", Tag: "v2"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, "registry.example.com/myapp:v2", finder.gotImage)
-}
-
-func TestUpdate_FindJobsCalledWithBareImageWhenNoTag(t *testing.T) {
-	finder := &fakeFinder{}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "registry.example.com/myapp"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, "registry.example.com/myapp", finder.gotImage)
-}
-
-func TestUpdate_SwarmModeThreadsTagThroughDiscovery(t *testing.T) {
-	finder := &fakeFinder{swarmShaped: true}
-	h := api.NewHandlers(finder, &fakeSubmitter{}, &fakePinger{}, "v0.0.0", "abc", "now", nil)
-	body, _ := json.Marshal(api.UpdateRequest{Image: "registry.example.com/myapp", Tag: "v2"})
-	req := httptest.NewRequest(http.MethodPost, "/update", bytes.NewReader(body))
-	w := httptest.NewRecorder()
-	h.Update(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	// The fake Finder's returned Job.Image (as SwarmDiscovery would build
-	// it) reflects the image argument it received from the handler.
-	// Asserting on the captured argument proves the full request ->
-	// discovery flow carries the tag through to what would reach the
-	// SwarmExecutor.
-	require.Equal(t, "registry.example.com/myapp:v2", finder.gotImage)
-}
-
-func TestVersion(t *testing.T) {
-	h := api.NewHandlers(&fakeFinder{}, &fakeSubmitter{}, &fakePinger{}, "v1.2.3", "abcdef", "2026-05-22T00:00:00Z", nil)
-	req := httptest.NewRequest(http.MethodGet, "/version", nil)
-	w := httptest.NewRecorder()
-	h.Version(w, req)
-	require.Equal(t, http.StatusOK, w.Code)
-	resp := decode[api.VersionResponse](t, w.Body)
-	require.Equal(t, "v1.2.3", resp.Version)
-}
diff --git a/internal/api/middleware.go b/internal/api/middleware.go
deleted file mode 100644
index 8564890..0000000
--- a/internal/api/middleware.go
+++ /dev/null
@@ -1,112 +0,0 @@
-// Package api contains HTTP handlers, middleware, and request/response DTOs.
-package api
-
-import (
-	"crypto/rand"
-	"crypto/subtle"
-	"fmt"
-	"log/slog"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/shcizo/package-updater/internal/metrics"
-)
-
-// Auth returns middleware that requires a matching bearer token.
-// Compares with constant-time to defeat timing attacks.
-func Auth(token string) func(http.Handler) http.Handler {
-	tokenBytes := []byte(token)
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			h := r.Header.Get("Authorization")
-			const prefix = "Bearer "
-			if !strings.HasPrefix(h, prefix) {
-				writeAuthError(w)
-				return
-			}
-			provided := []byte(strings.TrimPrefix(h, prefix))
-			if len(provided) == 0 ||
-				subtle.ConstantTimeCompare(provided, tokenBytes) != 1 {
-				writeAuthError(w)
-				return
-			}
-			next.ServeHTTP(w, r)
-		})
-	}
-}
-
-func writeAuthError(w http.ResponseWriter) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(http.StatusUnauthorized)
-	_, _ = w.Write([]byte(`{"error":"unauthorized"}`))
-}
-
-// newUUID generates a random UUID v4 string using crypto/rand.
-func newUUID() string {
-	var b [16]byte
-	_, _ = rand.Read(b[:])
-	b[6] = (b[6] & 0x0f) | 0x40 // version 4
-	b[8] = (b[8] & 0x3f) | 0x80 // variant bits
-	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
-		b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
-}
-
-// RequestID middleware ensures every request has an X-Request-ID
-// header (generated if absent) and stores it in the request context.
-func RequestID(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		id := r.Header.Get("X-Request-ID")
-		if id == "" {
-			id = newUUID()
-		}
-		w.Header().Set("X-Request-ID", id)
-		ctx := logging.WithRequestID(r.Context(), id)
-		next.ServeHTTP(w, r.WithContext(ctx))
-	})
-}
-
-// RequestLogger emits a structured access-log line per request. m may be nil,
-// in which case metrics recording is silently skipped.
-func RequestLogger(base *slog.Logger, m *metrics.Metrics) func(http.Handler) http.Handler {
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			start := time.Now()
-			sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
-			next.ServeHTTP(sw, r)
-			logger := logging.FromContext(r.Context(), base)
-			logger.Info("http_request",
-				"method", r.Method,
-				"path", r.URL.Path,
-				"status", sw.status,
-				"duration_ms", time.Since(start).Milliseconds(),
-				"client_ip", clientIP(r),
-			)
-			if m != nil {
-				m.HTTPRequests.WithLabelValues(r.URL.Path, strconv.Itoa(sw.status)).Inc()
-			}
-		})
-	}
-}
-
-type statusWriter struct {
-	http.ResponseWriter
-	status int
-}
-
-func (s *statusWriter) WriteHeader(code int) {
-	s.status = code
-	s.ResponseWriter.WriteHeader(code)
-}
-
-func clientIP(r *http.Request) string {
-	if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
-		if comma := strings.Index(xff, ","); comma >= 0 {
-			return strings.TrimSpace(xff[:comma])
-		}
-		return xff
-	}
-	return r.RemoteAddr
-}
diff --git a/internal/api/middleware_test.go b/internal/api/middleware_test.go
deleted file mode 100644
index e973aad..0000000
--- a/internal/api/middleware_test.go
+++ /dev/null
@@ -1,92 +0,0 @@
-package api_test
-
-import (
-	"io"
-	"log/slog"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/logging"
-	"github.com/stretchr/testify/require"
-)
-
-func newTestLogger() *slog.Logger {
-	return slog.New(slog.NewJSONHandler(io.Discard, nil))
-}
-
-func TestAuth_AllowsMatchingToken(t *testing.T) {
-	called := false
-	h := api.Auth("secret")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		called = true
-		w.WriteHeader(http.StatusOK)
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	req.Header.Set("Authorization", "Bearer secret")
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.True(t, called)
-	require.Equal(t, http.StatusOK, w.Code)
-}
-
-func TestAuth_Rejects(t *testing.T) {
-	cases := []struct{ name, header string }{
-		{"missing", ""},
-		{"wrong scheme", "Token secret"},
-		{"wrong value", "Bearer nope"},
-		{"empty bearer", "Bearer "},
-	}
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			h := api.Auth("secret")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-				t.Fatal("handler must not be called")
-			}))
-			req := httptest.NewRequest(http.MethodPost, "/update", nil)
-			if c.header != "" {
-				req.Header.Set("Authorization", c.header)
-			}
-			w := httptest.NewRecorder()
-			h.ServeHTTP(w, req)
-			require.Equal(t, http.StatusUnauthorized, w.Code)
-		})
-	}
-}
-
-func TestRequestID_GeneratesIfMissing(t *testing.T) {
-	var seenID string
-	h := api.RequestID(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
-		seenID = logging.RequestIDFrom(r.Context())
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.NotEmpty(t, seenID)
-	require.Equal(t, seenID, w.Header().Get("X-Request-ID"))
-}
-
-func TestRequestID_UsesIncoming(t *testing.T) {
-	var seenID string
-	h := api.RequestID(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
-		seenID = logging.RequestIDFrom(r.Context())
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	req.Header.Set("X-Request-ID", "given-id")
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.Equal(t, "given-id", seenID)
-}
-
-func TestRequestLogger_LogsAndDelegates(t *testing.T) {
-	logger := newTestLogger()
-	called := false
-	h := api.RequestLogger(logger, nil)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		called = true
-		w.WriteHeader(http.StatusTeapot)
-	}))
-	req := httptest.NewRequest(http.MethodPost, "/update", nil)
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, req)
-	require.True(t, called)
-	require.Equal(t, http.StatusTeapot, w.Code)
-}
diff --git a/internal/api/swarm_result_test.go b/internal/api/swarm_result_test.go
deleted file mode 100644
index e6d978b..0000000
--- a/internal/api/swarm_result_test.go
+++ /dev/null
@@ -1,48 +0,0 @@
-package api_test
-
-import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/docker/docker/api/types"
-	"github.com/shcizo/package-updater/internal/api"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/updater"
-	"github.com/stretchr/testify/require"
-)
-
-type swarmResultFinder struct{ jobs []discovery.Job }
-
-func (f *swarmResultFinder) FindJobs(_ context.Context, _ string) ([]discovery.Job, error) {
-	return f.jobs, nil
-}
-
-type swarmResultSubmitter struct{ results []updater.Result }
-
-func (s *swarmResultSubmitter) Submit(_ context.Context, _ []discovery.Job) []updater.Result {
-	return s.results
-}
-
-type swarmResultPinger struct{}
-
-func (swarmResultPinger) Ping(_ context.Context) (types.Ping, error) { return types.Ping{}, nil }
-
-func TestUpdate_SwarmJobPopulatesServiceID(t *testing.T) {
-	job := discovery.Job{Service: "myapp_web", ServiceID: "svc123", Image: "registry.example.com/myapp:v2"}
-	finder := &swarmResultFinder{jobs: []discovery.Job{job}}
-	submitter := &swarmResultSubmitter{results: []updater.Result{
-		{Job: job, Status: updater.StatusUpdated, DurationMs: 42},
-	}}
-	h := api.NewHandlers(finder, submitter, swarmResultPinger{}, "v", "c", "b", nil)
-
-	req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"image":"registry.example.com/myapp"}`))
-	rec := httptest.NewRecorder()
-	h.Update(rec, req)
-
-	require.Equal(t, http.StatusOK, rec.Code)
-	require.Contains(t, rec.Body.String(), `"service_id":"svc123"`)
-	require.Contains(t, rec.Body.String(), `"service":"myapp_web"`)
-}
diff --git a/internal/api/types.go b/internal/api/types.go
deleted file mode 100644
index 02f1f63..0000000
--- a/internal/api/types.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package api
-
-// UpdateRequest is the body of POST /update.
-type UpdateRequest struct {
-	Image string `json:"image"`
-	Tag   string `json:"tag,omitempty"`
-}
-
-// UpdateResponse is the body of POST /update.
-type UpdateResponse struct {
-	RequestID string      `json:"request_id"`
-	Image     string      `json:"image"`
-	Tag       string      `json:"tag,omitempty"`
-	Matched   int         `json:"matched"`
-	Results   []ResultDTO `json:"results"`
-}
-
-// ResultDTO is one row in UpdateResponse.Results.
-type ResultDTO struct {
-	Project     string `json:"project"`
-	Service     string `json:"service"`
-	ComposeFile string `json:"compose_file"`
-	ServiceID   string `json:"service_id,omitempty"`
-	Status      string `json:"status"`
-	Error       string `json:"error,omitempty"`
-	DurationMs  int64  `json:"duration_ms"`
-}
-
-// HealthResponse is the body of GET /healthz.
-type HealthResponse struct {
-	Status string `json:"status"`
-	Docker string `json:"docker"`
-}
-
-// VersionResponse is the body of GET /version.
-type VersionResponse struct {
-	Version   string `json:"version"`
-	Commit    string `json:"commit"`
-	BuildTime string `json:"build_time"`
-}
diff --git a/internal/config/config.go b/internal/config/config.go
deleted file mode 100644
index b143434..0000000
--- a/internal/config/config.go
+++ /dev/null
@@ -1,58 +0,0 @@
-// Package config loads service configuration from environment variables.
-package config
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"time"
-)
-
-// Config holds all runtime configuration for the service.
-type Config struct {
-	APIKey        string
-	StacksRoot    string
-	Port          string
-	LogLevel      string
-	UpdateTimeout time.Duration
-	OptInLabel    string
-	Mode          string
-}
-
-// Load reads configuration from environment variables, applies defaults,
-// and validates required fields. Returns an error if validation fails so
-// the service can fail-fast at startup.
-func Load() (*Config, error) {
-	cfg := &Config{
-		APIKey:     os.Getenv("UPDATER_API_KEY"),
-		StacksRoot: getenvDefault("STACKS_ROOT", "/home/shcizo/self-hosted"),
-		Port:       getenvDefault("PORT", "8080"),
-		LogLevel:   getenvDefault("LOG_LEVEL", "info"),
-		OptInLabel: getenvDefault("OPT_IN_LABEL", "se.shcizo.auto-update"),
-		Mode:       getenvDefault("MODE", "compose"),
-	}
-
-	if cfg.APIKey == "" {
-		return nil, errors.New("UPDATER_API_KEY is required")
-	}
-
-	if cfg.Mode != "compose" && cfg.Mode != "swarm" {
-		return nil, fmt.Errorf("MODE %q is invalid: must be %q or %q", cfg.Mode, "compose", "swarm")
-	}
-
-	timeoutStr := getenvDefault("UPDATE_TIMEOUT", "5m")
-	d, err := time.ParseDuration(timeoutStr)
-	if err != nil {
-		return nil, fmt.Errorf("UPDATE_TIMEOUT %q is not a valid duration: %w", timeoutStr, err)
-	}
-	cfg.UpdateTimeout = d
-
-	return cfg, nil
-}
-
-func getenvDefault(key, fallback string) string {
-	if v := os.Getenv(key); v != "" {
-		return v
-	}
-	return fallback
-}
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
deleted file mode 100644
index 6e3fe88..0000000
--- a/internal/config/config_test.go
+++ /dev/null
@@ -1,82 +0,0 @@
-package config_test
-
-import (
-	"testing"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/config"
-	"github.com/stretchr/testify/require"
-)
-
-func TestLoad_RequiresAPIKey(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "")
-	_, err := config.Load()
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "UPDATER_API_KEY")
-}
-
-func TestLoad_AppliesDefaults(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("STACKS_ROOT", "")
-	t.Setenv("PORT", "")
-	t.Setenv("LOG_LEVEL", "")
-	t.Setenv("UPDATE_TIMEOUT", "")
-	t.Setenv("OPT_IN_LABEL", "")
-
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "secret", cfg.APIKey)
-	require.Equal(t, "/home/shcizo/self-hosted", cfg.StacksRoot)
-	require.Equal(t, "8080", cfg.Port)
-	require.Equal(t, "info", cfg.LogLevel)
-	require.Equal(t, 5*time.Minute, cfg.UpdateTimeout)
-	require.Equal(t, "se.shcizo.auto-update", cfg.OptInLabel)
-}
-
-func TestLoad_OverridesViaEnv(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("STACKS_ROOT", "/srv/stacks")
-	t.Setenv("PORT", "9090")
-	t.Setenv("LOG_LEVEL", "debug")
-	t.Setenv("UPDATE_TIMEOUT", "30s")
-	t.Setenv("OPT_IN_LABEL", "io.example.update")
-
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "/srv/stacks", cfg.StacksRoot)
-	require.Equal(t, "9090", cfg.Port)
-	require.Equal(t, "debug", cfg.LogLevel)
-	require.Equal(t, 30*time.Second, cfg.UpdateTimeout)
-	require.Equal(t, "io.example.update", cfg.OptInLabel)
-}
-
-func TestLoad_InvalidTimeoutErrors(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("UPDATE_TIMEOUT", "not-a-duration")
-	_, err := config.Load()
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "UPDATE_TIMEOUT")
-}
-
-func TestLoad_DefaultsToComposeMode(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "compose", cfg.Mode)
-}
-
-func TestLoad_AcceptsSwarmMode(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("MODE", "swarm")
-	cfg, err := config.Load()
-	require.NoError(t, err)
-	require.Equal(t, "swarm", cfg.Mode)
-}
-
-func TestLoad_RejectsInvalidMode(t *testing.T) {
-	t.Setenv("UPDATER_API_KEY", "secret")
-	t.Setenv("MODE", "kubernetes")
-	_, err := config.Load()
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "MODE")
-}
diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go
deleted file mode 100644
index 6269169..0000000
--- a/internal/discovery/discovery.go
+++ /dev/null
@@ -1,96 +0,0 @@
-package discovery
-
-import (
-	"context"
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/docker/docker/api/types/container"
-)
-
-// Job describes a single (project, service, config_files) update to execute.
-type Job struct {
-	Project     string
-	Service     string
-	WorkingDir  string
-	ConfigFiles []string
-	// Image is the full image reference the update request asked for.
-	// Compose jobs ignore it (compose.yml already pins the reference to
-	// pull); Swarm jobs need it to know what to set on the service spec.
-	Image string
-	// ServiceID is the Swarm service ID. Empty for Compose jobs.
-	ServiceID string
-	// Refused is true when the WorkingDir falls outside STACKS_ROOT.
-	// The job is returned so the caller can surface a per-job "refused"
-	// result, but it MUST NOT be executed.
-	Refused       bool
-	RefusedReason string
-}
-
-// Discovery orchestrates "given an image, what jobs should we enqueue?".
-type Discovery struct {
-	cli        DockerClient
-	stacksRoot string
-	optInLabel string
-}
-
-// New returns a Discovery bound to the given Docker client and config.
-func New(cli DockerClient, stacksRoot, optInLabel string) *Discovery {
-	return &Discovery{cli: cli, stacksRoot: stacksRoot, optInLabel: optInLabel}
-}
-
-// FindJobs lists running containers, filters by image match + opt-in label,
-// extracts Compose info, applies the path safety check, and deduplicates.
-func (d *Discovery) FindJobs(ctx context.Context, image string) ([]Job, error) {
-	all, err := d.cli.ContainerList(ctx, container.ListOptions{All: true})
-	if err != nil {
-		return nil, fmt.Errorf("docker container list: %w", err)
-	}
-
-	seen := make(map[string]struct{})
-	var jobs []Job
-
-	for _, c := range all {
-		if !ImagesMatch(image, c.Image) {
-			continue
-		}
-		if !HasOptIn(c.Labels, d.optInLabel) {
-			continue
-		}
-		cl, err := ParseComposeLabels(c.Labels)
-		if err != nil {
-			continue
-		}
-
-		refused := !IsInsideRoot(d.stacksRoot, cl.WorkingDir)
-		reason := ""
-		if refused {
-			reason = fmt.Sprintf("working_dir %q outside STACKS_ROOT %q", cl.WorkingDir, d.stacksRoot)
-		}
-
-		key := dedupKey(cl)
-		if _, dup := seen[key]; dup {
-			continue
-		}
-		seen[key] = struct{}{}
-
-		jobs = append(jobs, Job{
-			Project:       cl.Project,
-			Service:       cl.Service,
-			WorkingDir:    cl.WorkingDir,
-			ConfigFiles:   cl.ConfigFiles,
-			Image:         image,
-			Refused:       refused,
-			RefusedReason: reason,
-		})
-	}
-
-	return jobs, nil
-}
-
-func dedupKey(cl ComposeLabels) string {
-	files := append([]string(nil), cl.ConfigFiles...)
-	sort.Strings(files)
-	return cl.Project + "|" + cl.Service + "|" + strings.Join(files, ",")
-}
diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go
deleted file mode 100644
index c1f5b0a..0000000
--- a/internal/discovery/discovery_test.go
+++ /dev/null
@@ -1,172 +0,0 @@
-package discovery_test
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeDockerClient struct {
-	containers []types.Container
-	err        error
-}
-
-func (f *fakeDockerClient) ContainerList(_ context.Context, _ container.ListOptions) ([]types.Container, error) {
-	return f.containers, f.err
-}
-
-func (f *fakeDockerClient) Ping(_ context.Context) (types.Ping, error) {
-	return types.Ping{}, nil
-}
-
-func mkContainer(image string, labels map[string]string) types.Container {
-	return types.Container{Image: image, Labels: labels}
-}
-
-func mkComposeLabels(project, service, workingDir, configFile string, optIn bool) map[string]string {
-	m := map[string]string{
-		"com.docker.compose.project":              project,
-		"com.docker.compose.service":              service,
-		"com.docker.compose.project.working_dir":  workingDir,
-		"com.docker.compose.project.config_files": configFile,
-	}
-	if optIn {
-		m["se.shcizo.auto-update"] = "true"
-	}
-	return m
-}
-
-func TestFindJobs_MatchAndOptIn(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			true,
-		)),
-		mkContainer("registry.example.com/other:v1", mkComposeLabels(
-			"other", "web",
-			"/home/shcizo/self-hosted/other",
-			"/home/shcizo/self-hosted/other/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.Equal(t, "myapp-prod", jobs[0].Project)
-	require.Equal(t, "web", jobs[0].Service)
-	require.Equal(t, "/home/shcizo/self-hosted/myapp-prod", jobs[0].WorkingDir)
-}
-
-func TestFindJobs_SkipsWithoutOptIn(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			false,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}
-
-func TestFindJobs_DedupReplicas(t *testing.T) {
-	labels := mkComposeLabels(
-		"myapp", "web",
-		"/home/shcizo/self-hosted/myapp",
-		"/home/shcizo/self-hosted/myapp/docker-compose.yml",
-		true,
-	)
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", labels),
-		mkContainer("registry.example.com/myapp:v1", labels),
-		mkContainer("registry.example.com/myapp:v1", labels),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-}
-
-func TestFindJobs_OutsideRootProducesRefusedJob(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp", "web",
-			"/opt/elsewhere/myapp",
-			"/opt/elsewhere/myapp/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.True(t, jobs[0].Refused)
-}
-
-func TestFindJobs_MultipleStacksSameImage(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			true,
-		)),
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-staging", "web",
-			"/home/shcizo/self-hosted/myapp-staging",
-			"/home/shcizo/self-hosted/myapp-staging/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 2)
-}
-
-func TestFindJobs_DockerError(t *testing.T) {
-	fake := &fakeDockerClient{err: errors.New("connection refused")}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	_, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.Error(t, err)
-}
-
-func TestFindJobs_NonComposeContainerIsSkipped(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", map[string]string{
-			"se.shcizo.auto-update": "true",
-		}),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}
-
-func TestFindJobs_PopulatesImageField(t *testing.T) {
-	fake := &fakeDockerClient{containers: []types.Container{
-		mkContainer("registry.example.com/myapp:v1", mkComposeLabels(
-			"myapp-prod", "web",
-			"/home/shcizo/self-hosted/myapp-prod",
-			"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-			true,
-		)),
-	}}
-	d := discovery.New(fake, "/home/shcizo/self-hosted", "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.Equal(t, "registry.example.com/myapp", jobs[0].Image)
-	require.Empty(t, jobs[0].ServiceID)
-}
diff --git a/internal/discovery/docker_client.go b/internal/discovery/docker_client.go
deleted file mode 100644
index 1c1aed5..0000000
--- a/internal/discovery/docker_client.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package discovery
-
-import (
-	"context"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-)
-
-// DockerClient is the subset of the Docker SDK we depend on.
-// Defined as an interface so tests can supply a fake.
-type DockerClient interface {
-	ContainerList(ctx context.Context, opts container.ListOptions) ([]types.Container, error)
-	Ping(ctx context.Context) (types.Ping, error)
-}
diff --git a/internal/discovery/labels.go b/internal/discovery/labels.go
deleted file mode 100644
index aecde76..0000000
--- a/internal/discovery/labels.go
+++ /dev/null
@@ -1,71 +0,0 @@
-package discovery
-
-import (
-	"fmt"
-	"strings"
-)
-
-// ComposeLabels captures the four Compose-managed labels we need
-// to drive a `docker compose pull`/`up -d` against the right stack.
-type ComposeLabels struct {
-	Project     string
-	Service     string
-	WorkingDir  string
-	ConfigFiles []string
-}
-
-const (
-	labelProject     = "com.docker.compose.project"
-	labelService     = "com.docker.compose.service"
-	labelWorkingDir  = "com.docker.compose.project.working_dir"
-	labelConfigFiles = "com.docker.compose.project.config_files"
-)
-
-// ParseComposeLabels extracts the Compose labels we need. Returns an
-// error naming the missing label if any required field is absent —
-// in practice this should only happen if the container was not
-// started by Compose.
-func ParseComposeLabels(labels map[string]string) (ComposeLabels, error) {
-	get := func(key string) (string, error) {
-		v, ok := labels[key]
-		if !ok || v == "" {
-			return "", fmt.Errorf("missing required label: %s", key)
-		}
-		return v, nil
-	}
-
-	project, err := get(labelProject)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-	service, err := get(labelService)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-	workingDir, err := get(labelWorkingDir)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-	configFilesRaw, err := get(labelConfigFiles)
-	if err != nil {
-		return ComposeLabels{}, err
-	}
-
-	files := strings.Split(configFilesRaw, ",")
-	for i, f := range files {
-		files[i] = strings.TrimSpace(f)
-	}
-
-	return ComposeLabels{
-		Project:     project,
-		Service:     service,
-		WorkingDir:  workingDir,
-		ConfigFiles: files,
-	}, nil
-}
-
-// HasOptIn reports whether the labels include the opt-in marker with
-// value "true" (exact, case-sensitive — anything else is excluded).
-func HasOptIn(labels map[string]string, key string) bool {
-	return labels[key] == "true"
-}
diff --git a/internal/discovery/labels_test.go b/internal/discovery/labels_test.go
deleted file mode 100644
index 179f5dd..0000000
--- a/internal/discovery/labels_test.go
+++ /dev/null
@@ -1,88 +0,0 @@
-package discovery_test
-
-import (
-	"testing"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-func TestParseComposeLabels_Success(t *testing.T) {
-	labels := map[string]string{
-		"com.docker.compose.project":                  "myapp-prod",
-		"com.docker.compose.service":                  "web",
-		"com.docker.compose.project.working_dir":      "/home/shcizo/self-hosted/myapp-prod",
-		"com.docker.compose.project.config_files":     "/home/shcizo/self-hosted/myapp-prod/docker-compose.yml",
-		"se.shcizo.auto-update":                       "true",
-	}
-	got, err := discovery.ParseComposeLabels(labels)
-	require.NoError(t, err)
-	require.Equal(t, "myapp-prod", got.Project)
-	require.Equal(t, "web", got.Service)
-	require.Equal(t, "/home/shcizo/self-hosted/myapp-prod", got.WorkingDir)
-	require.Equal(t, []string{"/home/shcizo/self-hosted/myapp-prod/docker-compose.yml"}, got.ConfigFiles)
-}
-
-func TestParseComposeLabels_MultipleConfigFiles(t *testing.T) {
-	labels := map[string]string{
-		"com.docker.compose.project":                  "myapp",
-		"com.docker.compose.service":                  "web",
-		"com.docker.compose.project.working_dir":      "/srv/myapp",
-		"com.docker.compose.project.config_files":     "/srv/myapp/docker-compose.yml,/srv/myapp/docker-compose.prod.yml",
-	}
-	got, err := discovery.ParseComposeLabels(labels)
-	require.NoError(t, err)
-	require.Equal(t, []string{
-		"/srv/myapp/docker-compose.yml",
-		"/srv/myapp/docker-compose.prod.yml",
-	}, got.ConfigFiles)
-}
-
-func TestParseComposeLabels_MissingFieldsError(t *testing.T) {
-	cases := []struct {
-		name    string
-		missing string
-		labels  map[string]string
-	}{
-		{"project", "com.docker.compose.project", map[string]string{
-			"com.docker.compose.service":              "web",
-			"com.docker.compose.project.working_dir":  "/x",
-			"com.docker.compose.project.config_files": "/x/y.yml",
-		}},
-		{"service", "com.docker.compose.service", map[string]string{
-			"com.docker.compose.project":              "p",
-			"com.docker.compose.project.working_dir":  "/x",
-			"com.docker.compose.project.config_files": "/x/y.yml",
-		}},
-		{"working_dir", "com.docker.compose.project.working_dir", map[string]string{
-			"com.docker.compose.project":              "p",
-			"com.docker.compose.service":              "web",
-			"com.docker.compose.project.config_files": "/x/y.yml",
-		}},
-		{"config_files", "com.docker.compose.project.config_files", map[string]string{
-			"com.docker.compose.project":             "p",
-			"com.docker.compose.service":             "web",
-			"com.docker.compose.project.working_dir": "/x",
-		}},
-	}
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			_, err := discovery.ParseComposeLabels(c.labels)
-			require.Error(t, err)
-			require.Contains(t, err.Error(), c.missing)
-		})
-	}
-}
-
-func TestHasOptIn(t *testing.T) {
-	require.True(t, discovery.HasOptIn(map[string]string{
-		"se.shcizo.auto-update": "true",
-	}, "se.shcizo.auto-update"))
-	require.False(t, discovery.HasOptIn(map[string]string{
-		"se.shcizo.auto-update": "false",
-	}, "se.shcizo.auto-update"))
-	require.False(t, discovery.HasOptIn(map[string]string{
-		"se.shcizo.auto-update": "TRUE",
-	}, "se.shcizo.auto-update"))
-	require.False(t, discovery.HasOptIn(map[string]string{}, "se.shcizo.auto-update"))
-}
diff --git a/internal/discovery/matching.go b/internal/discovery/matching.go
deleted file mode 100644
index c872009..0000000
--- a/internal/discovery/matching.go
+++ /dev/null
@@ -1,33 +0,0 @@
-package discovery
-
-import "strings"
-
-// NormaliseImage strips the tag and digest from an image reference,
-// returning the bare repository name.
-//
-// Tricky case: "localhost:5000/foo:v1" — the first colon is a port,
-// not a tag. We disambiguate by splitting on "/" first and only
-// treating colons in the last segment as tag separators.
-func NormaliseImage(ref string) string {
-	if at := strings.Index(ref, "@"); at >= 0 {
-		ref = ref[:at]
-	}
-	slash := strings.LastIndex(ref, "/")
-	if slash < 0 {
-		if colon := strings.Index(ref, ":"); colon >= 0 {
-			return ref[:colon]
-		}
-		return ref
-	}
-	prefix, last := ref[:slash], ref[slash+1:]
-	if colon := strings.Index(last, ":"); colon >= 0 {
-		last = last[:colon]
-	}
-	return prefix + "/" + last
-}
-
-// ImagesMatch reports whether two image references resolve to the same
-// repository, ignoring tag and digest. Case-sensitive per spec section 5.2.
-func ImagesMatch(a, b string) bool {
-	return NormaliseImage(a) == NormaliseImage(b)
-}
diff --git a/internal/discovery/matching_test.go b/internal/discovery/matching_test.go
deleted file mode 100644
index 5f20fb1..0000000
--- a/internal/discovery/matching_test.go
+++ /dev/null
@@ -1,51 +0,0 @@
-package discovery_test
-
-import (
-	"testing"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNormaliseImage(t *testing.T) {
-	cases := []struct {
-		in   string
-		want string
-	}{
-		{"registry.example.com/myapp", "registry.example.com/myapp"},
-		{"registry.example.com/myapp:v1.2.3", "registry.example.com/myapp"},
-		{"registry.example.com/myapp:latest", "registry.example.com/myapp"},
-		{"registry.example.com/myapp@sha256:abc123", "registry.example.com/myapp"},
-		{"registry.example.com/myapp:v1.2.3@sha256:abc123", "registry.example.com/myapp"},
-		{"nginx", "nginx"},
-		{"nginx:1.25-alpine", "nginx"},
-		{"library/nginx:latest", "library/nginx"},
-		{"gcr.io/proj/svc:tag", "gcr.io/proj/svc"},
-		{"localhost:5000/myimg:v1", "localhost:5000/myimg"},
-	}
-	for _, c := range cases {
-		t.Run(c.in, func(t *testing.T) {
-			require.Equal(t, c.want, discovery.NormaliseImage(c.in))
-		})
-	}
-}
-
-func TestImagesMatch(t *testing.T) {
-	require.True(t, discovery.ImagesMatch(
-		"registry.example.com/myapp",
-		"registry.example.com/myapp:v1.2.3",
-	))
-	require.True(t, discovery.ImagesMatch(
-		"registry.example.com/myapp:v1.0.0",
-		"registry.example.com/myapp:v9.9.9",
-	))
-	require.False(t, discovery.ImagesMatch(
-		"registry.example.com/myapp",
-		"registry.example.com/otherapp",
-	))
-	// Case-sensitive per spec section 5.2
-	require.False(t, discovery.ImagesMatch(
-		"registry.example.com/MyApp",
-		"registry.example.com/myapp",
-	))
-}
diff --git a/internal/discovery/swarm.go b/internal/discovery/swarm.go
deleted file mode 100644
index 108fad5..0000000
--- a/internal/discovery/swarm.go
+++ /dev/null
@@ -1,60 +0,0 @@
-package discovery
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/docker/docker/api/types/swarm"
-)
-
-// SwarmDockerClient is the subset of the Docker SDK SwarmDiscovery depends
-// on. Defined as an interface so tests can supply a fake.
-type SwarmDockerClient interface {
-	ServiceList(ctx context.Context, opts swarm.ServiceListOptions) ([]swarm.Service, error)
-}
-
-// SwarmDiscovery orchestrates "given an image, which Swarm services should
-// we update?". Unlike Discovery, it has no filesystem path to check — the
-// opt-in label on the service itself is the only gate, since Swarm services
-// have no local compose file to anchor a STACKS_ROOT check against.
-type SwarmDiscovery struct {
-	cli        SwarmDockerClient
-	optInLabel string
-}
-
-// NewSwarm returns a SwarmDiscovery bound to the given Docker client and
-// opt-in label.
-func NewSwarm(cli SwarmDockerClient, optInLabel string) *SwarmDiscovery {
-	return &SwarmDiscovery{cli: cli, optInLabel: optInLabel}
-}
-
-// FindJobs lists Swarm services, filters by image match + opt-in label, and
-// returns one Job per matching service. Signature matches Discovery.FindJobs
-// so both satisfy api.Finder.
-func (d *SwarmDiscovery) FindJobs(ctx context.Context, image string) ([]Job, error) {
-	services, err := d.cli.ServiceList(ctx, swarm.ServiceListOptions{})
-	if err != nil {
-		return nil, fmt.Errorf("docker service list: %w", err)
-	}
-
-	var jobs []Job
-	for _, svc := range services {
-		if svc.Spec.TaskTemplate.ContainerSpec == nil {
-			continue
-		}
-		if !ImagesMatch(image, svc.Spec.TaskTemplate.ContainerSpec.Image) {
-			continue
-		}
-		if !HasOptIn(svc.Spec.Labels, d.optInLabel) {
-			continue
-		}
-
-		jobs = append(jobs, Job{
-			Service:   svc.Spec.Name,
-			ServiceID: svc.ID,
-			Image:     image,
-		})
-	}
-
-	return jobs, nil
-}
diff --git a/internal/discovery/swarm_test.go b/internal/discovery/swarm_test.go
deleted file mode 100644
index 5c3f048..0000000
--- a/internal/discovery/swarm_test.go
+++ /dev/null
@@ -1,86 +0,0 @@
-package discovery_test
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/docker/docker/api/types/swarm"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeSwarmDockerClient struct {
-	services []swarm.Service
-	err      error
-}
-
-func (f *fakeSwarmDockerClient) ServiceList(_ context.Context, _ swarm.ServiceListOptions) ([]swarm.Service, error) {
-	return f.services, f.err
-}
-
-func mkService(id, name, image string, labels map[string]string) swarm.Service {
-	return swarm.Service{
-		ID: id,
-		Spec: swarm.ServiceSpec{
-			Annotations: swarm.Annotations{Name: name, Labels: labels},
-			TaskTemplate: swarm.TaskSpec{
-				ContainerSpec: &swarm.ContainerSpec{Image: image},
-			},
-		},
-	}
-}
-
-func TestSwarmFindJobs_MatchAndOptIn(t *testing.T) {
-	fake := &fakeSwarmDockerClient{services: []swarm.Service{
-		mkService("svc-myapp", "myapp_web", "registry.example.com/myapp:v1", map[string]string{
-			"se.shcizo.auto-update": "true",
-		}),
-		mkService("svc-other", "other_web", "registry.example.com/other:v1", map[string]string{
-			"se.shcizo.auto-update": "true",
-		}),
-	}}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Len(t, jobs, 1)
-	require.Equal(t, "myapp_web", jobs[0].Service)
-	require.Equal(t, "svc-myapp", jobs[0].ServiceID)
-	require.Equal(t, "registry.example.com/myapp", jobs[0].Image)
-	require.False(t, jobs[0].Refused)
-	require.Empty(t, jobs[0].WorkingDir)
-	require.Empty(t, jobs[0].ConfigFiles)
-}
-
-func TestSwarmFindJobs_SkipsWithoutOptIn(t *testing.T) {
-	fake := &fakeSwarmDockerClient{services: []swarm.Service{
-		mkService("svc-myapp", "myapp_web", "registry.example.com/myapp:v1", nil),
-	}}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}
-
-func TestSwarmFindJobs_DockerError(t *testing.T) {
-	fake := &fakeSwarmDockerClient{err: errors.New("connection refused")}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	_, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.Error(t, err)
-}
-
-func TestSwarmFindJobs_NoContainerSpecIsSkipped(t *testing.T) {
-	fake := &fakeSwarmDockerClient{services: []swarm.Service{
-		{
-			ID: "svc-weird",
-			Spec: swarm.ServiceSpec{
-				Annotations:  swarm.Annotations{Name: "weird", Labels: map[string]string{"se.shcizo.auto-update": "true"}},
-				TaskTemplate: swarm.TaskSpec{ContainerSpec: nil},
-			},
-		},
-	}}
-	d := discovery.NewSwarm(fake, "se.shcizo.auto-update")
-	jobs, err := d.FindJobs(context.Background(), "registry.example.com/myapp")
-	require.NoError(t, err)
-	require.Empty(t, jobs)
-}
diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
deleted file mode 100644
index 119d819..0000000
--- a/internal/metrics/metrics.go
+++ /dev/null
@@ -1,69 +0,0 @@
-// Package metrics defines and registers the Prometheus collectors
-// exported on /metrics.
-package metrics
-
-import (
-	"net/http"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-)
-
-// Metrics bundles all collectors. Each field is the public handle the
-// rest of the service uses to record observations.
-type Metrics struct {
-	BuildInfo      *prometheus.GaugeVec
-	HTTPRequests   *prometheus.CounterVec
-	UpdateJobs     *prometheus.CounterVec
-	UpdateDuration *prometheus.HistogramVec
-	QueueDepth     prometheus.Gauge
-	LastUpdateTime *prometheus.GaugeVec
-	DockerPingUp   prometheus.Gauge
-}
-
-// New constructs Metrics and registers them with the given registry.
-// Use prometheus.NewRegistry() in tests so collectors don't leak between
-// runs; production code uses prometheus.DefaultRegisterer.
-func New(reg prometheus.Registerer) *Metrics {
-	m := &Metrics{
-		BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{
-			Name: "package_updater_build_info",
-			Help: "Always 1. Labels carry version/commit for dashboards.",
-		}, []string{"version", "commit"}),
-		HTTPRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
-			Name: "package_updater_http_requests_total",
-			Help: "HTTP requests handled, labelled by endpoint and status.",
-		}, []string{"endpoint", "status_code"}),
-		UpdateJobs: prometheus.NewCounterVec(prometheus.CounterOpts{
-			Name: "package_updater_update_jobs_total",
-			Help: "Update jobs executed, labelled by project/service/status.",
-		}, []string{"project", "service", "status"}),
-		UpdateDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
-			Name:    "package_updater_update_duration_seconds",
-			Help:    "Time taken to pull + up a single service.",
-			Buckets: []float64{0.5, 1, 2, 5, 10, 30, 60, 120, 300},
-		}, []string{"project", "service"}),
-		QueueDepth: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "package_updater_queue_depth",
-			Help: "Current number of submissions waiting in the queue.",
-		}),
-		LastUpdateTime: prometheus.NewGaugeVec(prometheus.GaugeOpts{
-			Name: "package_updater_last_update_timestamp",
-			Help: "Unix timestamp of the most recent successful update per service.",
-		}, []string{"project", "service"}),
-		DockerPingUp: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "package_updater_docker_ping_up",
-			Help: "1 if the Docker socket responded to ping, 0 otherwise.",
-		}),
-	}
-	reg.MustRegister(
-		m.BuildInfo, m.HTTPRequests, m.UpdateJobs, m.UpdateDuration,
-		m.QueueDepth, m.LastUpdateTime, m.DockerPingUp,
-	)
-	return m
-}
-
-// Handler returns the /metrics HTTP handler bound to the given registry.
-func Handler(gatherer prometheus.Gatherer) http.Handler {
-	return promhttp.HandlerFor(gatherer, promhttp.HandlerOpts{})
-}
diff --git a/internal/selfupdate/selfupdate.go b/internal/selfupdate/selfupdate.go
deleted file mode 100644
index e901d03..0000000
--- a/internal/selfupdate/selfupdate.go
+++ /dev/null
@@ -1,56 +0,0 @@
-// Package selfupdate handles the special case where the update target
-// is the running service's own container. We must finish writing the
-// HTTP response (and flush + close the connection) before exec'ing
-// `docker compose up -d` against ourselves, otherwise the response is
-// lost when the container is replaced.
-package selfupdate
-
-import (
-	"context"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-)
-
-// IsSelf reports whether job targets the running service.
-// Matches on Compose project name (which is also typically the
-// service name for single-service stacks).
-func IsSelf(job discovery.Job, selfProject string) bool {
-	return job.Project == selfProject
-}
-
-// innerExec is the executor abstraction we wrap.
-type innerExec interface {
-	Execute(ctx context.Context, job discovery.Job) error
-}
-
-// Wrapped wraps an Executor with self-update-aware deferred execution.
-type Wrapped struct {
-	inner       innerExec
-	selfProject string
-	delay       time.Duration
-}
-
-// Wrap returns a Wrapped that defers exec until after flush() for
-// self-updates. The delay is added after flush before exec, so the
-// kernel TCP buffer has time to drain.
-func Wrap(inner innerExec, selfProject string, delay time.Duration) *Wrapped {
-	return &Wrapped{inner: inner, selfProject: selfProject, delay: delay}
-}
-
-// ExecuteWithFlush runs job, invoking flush() before exec for self-updates
-// and waiting `delay` after flush. For non-self jobs, exec happens
-// normally and flush is not invoked at all (the HTTP layer decides
-// when to flush in that case).
-func (w *Wrapped) ExecuteWithFlush(ctx context.Context, job discovery.Job, flush func()) error {
-	if !IsSelf(job, w.selfProject) {
-		return w.inner.Execute(ctx, job)
-	}
-	flush()
-	select {
-	case <-time.After(w.delay):
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-	return w.inner.Execute(ctx, job)
-}
diff --git a/internal/selfupdate/selfupdate_test.go b/internal/selfupdate/selfupdate_test.go
deleted file mode 100644
index 121181c..0000000
--- a/internal/selfupdate/selfupdate_test.go
+++ /dev/null
@@ -1,87 +0,0 @@
-package selfupdate_test
-
-import (
-	"context"
-	"errors"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/selfupdate"
-	"github.com/stretchr/testify/require"
-)
-
-type recExec struct {
-	mu     sync.Mutex
-	called bool
-	at     time.Time
-}
-
-func (r *recExec) Execute(_ context.Context, _ discovery.Job) error {
-	r.mu.Lock()
-	r.called = true
-	r.at = time.Now()
-	r.mu.Unlock()
-	return nil
-}
-
-func TestIsSelf(t *testing.T) {
-	require.True(t, selfupdate.IsSelf(
-		discovery.Job{Project: "package-updater", Service: "package-updater"},
-		"package-updater",
-	))
-	require.False(t, selfupdate.IsSelf(
-		discovery.Job{Project: "other", Service: "web"},
-		"package-updater",
-	))
-}
-
-func TestWrap_DefersSelf(t *testing.T) {
-	inner := &recExec{}
-	wrapped := selfupdate.Wrap(inner, "package-updater", 30*time.Millisecond)
-
-	flushed := make(chan time.Time, 1)
-	flush := func() { flushed <- time.Now() }
-
-	job := discovery.Job{Project: "package-updater", Service: "package-updater"}
-	require.NoError(t, wrapped.ExecuteWithFlush(context.Background(), job, flush))
-
-	flushAt := <-flushed
-	inner.mu.Lock()
-	require.True(t, inner.called)
-	require.True(t, inner.at.After(flushAt))
-	inner.mu.Unlock()
-}
-
-func TestWrap_NormalJobIsImmediate(t *testing.T) {
-	inner := &recExec{}
-	wrapped := selfupdate.Wrap(inner, "package-updater", 30*time.Millisecond)
-
-	flushed := make(chan time.Time, 1)
-	flush := func() { flushed <- time.Now() }
-
-	job := discovery.Job{Project: "other", Service: "web"}
-	require.NoError(t, wrapped.ExecuteWithFlush(context.Background(), job, flush))
-
-	inner.mu.Lock()
-	require.True(t, inner.called)
-	inner.mu.Unlock()
-	select {
-	case <-flushed:
-		t.Fatal("flush should not be called for non-self jobs")
-	default:
-	}
-}
-
-func TestWrap_PropagatesError(t *testing.T) {
-	wrapped := selfupdate.Wrap(failExec{}, "x", 1*time.Millisecond)
-	err := wrapped.ExecuteWithFlush(context.Background(), discovery.Job{Project: "other"}, func() {})
-	require.Error(t, err)
-}
-
-type failExec struct{}
-
-func (failExec) Execute(_ context.Context, _ discovery.Job) error {
-	return errors.New("boom")
-}
diff --git a/internal/updater/executor.go b/internal/updater/executor.go
deleted file mode 100644
index e767282..0000000
--- a/internal/updater/executor.go
+++ /dev/null
@@ -1,15 +0,0 @@
-// Package updater contains the FIFO job queue, the single worker that
-// drains it, and the abstraction over `docker compose` invocation.
-package updater
-
-import (
-	"context"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-)
-
-// Executor runs `docker compose pull` then `up -d` for a single job.
-// Defined as an interface so the worker can be tested with a fake.
-type Executor interface {
-	Execute(ctx context.Context, job discovery.Job) error
-}
diff --git a/internal/updater/queue.go b/internal/updater/queue.go
deleted file mode 100644
index c7b3a38..0000000
--- a/internal/updater/queue.go
+++ /dev/null
@@ -1,144 +0,0 @@
-package updater
-
-import (
-	"context"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/metrics"
-)
-
-// Status describes the outcome of executing a single Job.
-type Status string
-
-const (
-	StatusUpdated Status = "updated"
-	StatusFailed  Status = "failed"
-	StatusRefused Status = "refused"
-	StatusTimeout Status = "timeout"
-)
-
-// Result captures a per-Job outcome to be surfaced in the HTTP response.
-type Result struct {
-	Job        discovery.Job
-	Status     Status
-	Error      string
-	DurationMs int64
-}
-
-// Queue serialises Job execution through a single worker so we never
-// run two `docker compose` commands against the same stack concurrently.
-// It also satisfies the "single global FIFO worker" design choice in
-// spec section 5.7.
-type Queue struct {
-	exec    Executor
-	metrics *metrics.Metrics
-	ch      chan submission
-	stop    chan struct{}
-	done    chan struct{}
-}
-
-type submission struct {
-	ctx     context.Context
-	jobs    []discovery.Job
-	results chan []Result
-}
-
-// NewQueue constructs a queue bound to the given executor. m may be nil,
-// in which case metrics recording is silently skipped.
-func NewQueue(exec Executor, m *metrics.Metrics) *Queue {
-	return &Queue{
-		exec:    exec,
-		metrics: m,
-		ch:      make(chan submission, 16),
-		stop:    make(chan struct{}),
-		done:    make(chan struct{}),
-	}
-}
-
-// Start launches the single background worker. Call Stop to terminate.
-func (q *Queue) Start(ctx context.Context) {
-	go q.run(ctx)
-}
-
-// Stop signals the worker to exit and waits for it to finish.
-func (q *Queue) Stop() {
-	close(q.stop)
-	<-q.done
-}
-
-// Submit blocks until all jobs in the batch have been processed by the
-// worker, then returns per-job results in the same order. The ctx
-// timeout (if any) applies to each individual Job's execution and is
-// surfaced as StatusTimeout.
-func (q *Queue) Submit(ctx context.Context, jobs []discovery.Job) []Result {
-	resCh := make(chan []Result, 1)
-	q.ch <- submission{ctx: ctx, jobs: jobs, results: resCh}
-	if q.metrics != nil {
-		q.metrics.QueueDepth.Set(float64(len(q.ch)))
-	}
-	return <-resCh
-}
-
-func (q *Queue) run(_ context.Context) {
-	defer close(q.done)
-	for {
-		select {
-		case <-q.stop:
-			return
-		case s := <-q.ch:
-			results := make([]Result, len(s.jobs))
-			for i, j := range s.jobs {
-				results[i] = q.runOne(s.ctx, j)
-			}
-			s.results <- results
-		}
-	}
-}
-
-func (q *Queue) runOne(ctx context.Context, job discovery.Job) Result {
-	start := time.Now()
-	r := Result{Job: job}
-
-	if job.Refused {
-		r.Status = StatusRefused
-		r.Error = job.RefusedReason
-		r.DurationMs = time.Since(start).Milliseconds()
-		if q.metrics != nil {
-			q.metrics.UpdateJobs.WithLabelValues(job.Project, job.Service, string(StatusRefused)).Inc()
-			q.metrics.UpdateDuration.WithLabelValues(job.Project, job.Service).Observe(time.Since(start).Seconds())
-		}
-		return r
-	}
-
-	err := q.exec.Execute(ctx, job)
-	r.DurationMs = time.Since(start).Milliseconds()
-	switch {
-	case err == nil:
-		r.Status = StatusUpdated
-	case errorsIsContextDeadline(err) || ctxDeadlineExceeded(ctx):
-		r.Status = StatusTimeout
-		r.Error = err.Error()
-	default:
-		r.Status = StatusFailed
-		r.Error = err.Error()
-	}
-
-	if q.metrics != nil {
-		elapsed := time.Since(start).Seconds()
-		q.metrics.UpdateJobs.WithLabelValues(job.Project, job.Service, string(r.Status)).Inc()
-		q.metrics.UpdateDuration.WithLabelValues(job.Project, job.Service).Observe(elapsed)
-		if r.Status == StatusUpdated {
-			q.metrics.LastUpdateTime.WithLabelValues(job.Project, job.Service).SetToCurrentTime()
-		}
-	}
-	return r
-}
-
-func errorsIsContextDeadline(err error) bool {
-	return err == context.DeadlineExceeded
-}
-
-func ctxDeadlineExceeded(ctx context.Context) bool {
-	return ctx.Err() == context.DeadlineExceeded
-}
diff --git a/internal/updater/queue_test.go b/internal/updater/queue_test.go
deleted file mode 100644
index c8f99d8..0000000
--- a/internal/updater/queue_test.go
+++ /dev/null
@@ -1,129 +0,0 @@
-package updater_test
-
-import (
-	"context"
-	"errors"
-	"sync"
-	"testing"
-	"time"
-
-	dto "github.com/prometheus/client_model/go"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/metrics"
-	"github.com/shcizo/package-updater/internal/updater"
-	"github.com/stretchr/testify/require"
-)
-
-type fakeExec struct {
-	mu        sync.Mutex
-	calls     []discovery.Job
-	delay     time.Duration
-	errForSvc map[string]error
-}
-
-func (f *fakeExec) Execute(ctx context.Context, job discovery.Job) error {
-	f.mu.Lock()
-	f.calls = append(f.calls, job)
-	f.mu.Unlock()
-	if f.delay > 0 {
-		select {
-		case <-time.After(f.delay):
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-	if err, ok := f.errForSvc[job.Service]; ok {
-		return err
-	}
-	return nil
-}
-
-func (f *fakeExec) callsCopy() []discovery.Job {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	return append([]discovery.Job(nil), f.calls...)
-}
-
-func TestQueue_ProcessesFIFO(t *testing.T) {
-	exec := &fakeExec{delay: 20 * time.Millisecond}
-	q := updater.NewQueue(exec, nil)
-	q.Start(context.Background())
-	defer q.Stop()
-
-	job1 := discovery.Job{Project: "a", Service: "svc"}
-	job2 := discovery.Job{Project: "b", Service: "svc"}
-	job3 := discovery.Job{Project: "c", Service: "svc"}
-
-	results := make(chan []updater.Result, 3)
-	go func() { results <- q.Submit(context.Background(), []discovery.Job{job1}) }()
-	time.Sleep(5 * time.Millisecond)
-	go func() { results <- q.Submit(context.Background(), []discovery.Job{job2}) }()
-	time.Sleep(5 * time.Millisecond)
-	go func() { results <- q.Submit(context.Background(), []discovery.Job{job3}) }()
-
-	for i := 0; i < 3; i++ {
-		<-results
-	}
-
-	calls := exec.callsCopy()
-	require.Equal(t, []string{"a", "b", "c"}, []string{calls[0].Project, calls[1].Project, calls[2].Project})
-}
-
-func TestQueue_ReturnsPerJobResults(t *testing.T) {
-	exec := &fakeExec{errForSvc: map[string]error{"failing": errors.New("boom")}}
-	q := updater.NewQueue(exec, nil)
-	q.Start(context.Background())
-	defer q.Stop()
-
-	jobs := []discovery.Job{
-		{Project: "p1", Service: "ok"},
-		{Project: "p2", Service: "failing"},
-		{Project: "p3", Service: "refused-svc", Refused: true, RefusedReason: "outside root"},
-	}
-	results := q.Submit(context.Background(), jobs)
-
-	require.Len(t, results, 3)
-	require.Equal(t, updater.StatusUpdated, results[0].Status)
-	require.Equal(t, updater.StatusFailed, results[1].Status)
-	require.Contains(t, results[1].Error, "boom")
-	require.Equal(t, updater.StatusRefused, results[2].Status)
-	require.Contains(t, results[2].Error, "outside root")
-}
-
-func TestQueue_Timeout(t *testing.T) {
-	exec := &fakeExec{delay: 200 * time.Millisecond}
-	q := updater.NewQueue(exec, nil)
-	q.Start(context.Background())
-	defer q.Stop()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
-	defer cancel()
-
-	jobs := []discovery.Job{{Project: "slow", Service: "svc"}}
-	results := q.Submit(ctx, jobs)
-
-	require.Len(t, results, 1)
-	require.Equal(t, updater.StatusTimeout, results[0].Status)
-}
-
-func TestQueue_RecordsMetrics(t *testing.T) {
-	reg := prometheus.NewRegistry()
-	m := metrics.New(reg)
-	exec := &fakeExec{}
-	q := updater.NewQueue(exec, m)
-	q.Start(context.Background())
-	defer q.Stop()
-
-	q.Submit(context.Background(), []discovery.Job{{Project: "p", Service: "s"}})
-
-	// Verify UpdateJobs counter was incremented for the successful job.
-	metric := &dto.Metric{}
-	require.NoError(t, m.UpdateJobs.WithLabelValues("p", "s", "updated").Write(metric))
-	require.Equal(t, 1.0, metric.GetCounter().GetValue())
-
-	// Verify LastUpdateTime was set (non-zero).
-	tsMetric := &dto.Metric{}
-	require.NoError(t, m.LastUpdateTime.WithLabelValues("p", "s").Write(tsMetric))
-	require.Greater(t, tsMetric.GetGauge().GetValue(), 0.0)
-}
diff --git a/internal/updater/worker.go b/internal/updater/worker.go
deleted file mode 100644
index 7705bae..0000000
--- a/internal/updater/worker.go
+++ /dev/null
@@ -1,6 +0,0 @@
-package updater
-
-// Worker logic currently lives in queue.go (single in-process worker
-// goroutine). This file is reserved for the eventual per-stack-mutex
-// implementation called out in spec section 15 ("Multi-worker concurrency
-// with per-stack mutex"). Intentionally empty for v1.
diff --git a/internal/updater/worker_test.go b/internal/updater/worker_test.go
deleted file mode 100644
index 9522b8f..0000000
--- a/internal/updater/worker_test.go
+++ /dev/null
@@ -1,38 +0,0 @@
-package updater_test
-
-import (
-	"context"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/shcizo/package-updater/internal/discovery"
-	"github.com/shcizo/package-updater/internal/updater"
-	"github.com/stretchr/testify/require"
-)
-
-type counterExec struct{ n atomic.Int32 }
-
-func (c *counterExec) Execute(_ context.Context, _ discovery.Job) error {
-	c.n.Add(1)
-	time.Sleep(10 * time.Millisecond)
-	return nil
-}
-
-func TestWorker_RunsExactlyOneAtATime(t *testing.T) {
-	exec := &counterExec{}
-	q := updater.NewQueue(exec, nil)
-	q.Start(context.Background())
-	defer q.Stop()
-
-	jobs := make([]discovery.Job, 10)
-	for i := range jobs {
-		jobs[i] = discovery.Job{Project: "p", Service: "svc"}
-	}
-	start := time.Now()
-	q.Submit(context.Background(), jobs)
-	elapsed := time.Since(start)
-
-	require.GreaterOrEqual(t, elapsed, 90*time.Millisecond)
-	require.Equal(t, int32(10), exec.n.Load())
-}

From 9d3a6f808c709d7e991c7b40210ade29faa965dd Mon Sep 17 00:00:00 2001
From: Samuel Enocsson 
Date: Mon, 12 Oct 2026 18:40:48 +0200
Subject: [PATCH 06/29] feat(master): store update jobs (#7)

---
 internal/master/store/agents.go               |   5 +-
 .../master/store/agents_integration_test.go   |  21 ++
 internal/master/store/jobs.go                 | 204 ++++++++++++
 .../master/store/jobs_integration_test.go     | 305 ++++++++++++++++++
 internal/master/store/services.go             |  53 ++-
 .../master/store/services_integration_test.go |  33 ++
 6 files changed, 606 insertions(+), 15 deletions(-)
 create mode 100644 internal/master/store/jobs.go
 create mode 100644 internal/master/store/jobs_integration_test.go

diff --git a/internal/master/store/agents.go b/internal/master/store/agents.go
index c477665..1d363f3 100644
--- a/internal/master/store/agents.go
+++ b/internal/master/store/agents.go
@@ -64,7 +64,7 @@ func (s *Store) AgentByTokenHash(ctx context.Context, hash string) (model.Agent,
 }
 
 // RevokeAgent marks the agent revoked and its Compose and plain-container
-// services removed, in one transaction. Swarm rows are shared with other
+// services removed and its queued and running jobs failed, in one transaction. Swarm rows are shared with other
 // managers and left alone. Its token stops working immediately; closing a
 // live connection is the hub's job.
 func (s *Store) RevokeAgent(ctx context.Context, id int64, at time.Time) error {
@@ -83,6 +83,9 @@ func (s *Store) RevokeAgent(ctx context.Context, id int64, at time.Time) error {
 		 WHERE agent_id = $1 AND kind IN ('compose', 'container') AND removed_at IS NULL`, id, at); err != nil {
 		return fmt.Errorf("revoke agent: remove services: %w", err)
 	}
+	if _, err := tx.Exec(ctx, failActiveJobsSQL, id, "agent revoked", at); err != nil {
+		return fmt.Errorf("revoke agent: fail jobs: %w", err)
+	}
 	if err := tx.Commit(ctx); err != nil {
 		return fmt.Errorf("revoke agent: commit: %w", err)
 	}
diff --git a/internal/master/store/agents_integration_test.go b/internal/master/store/agents_integration_test.go
index 568ffc1..7227d93 100644
--- a/internal/master/store/agents_integration_test.go
+++ b/internal/master/store/agents_integration_test.go
@@ -138,3 +138,24 @@ func TestRevokedAgent_UpsertDoesNotResurrectRemovedRow(t *testing.T) {
 	require.NoError(t, err)
 	assert.Empty(t, got, "the revoked agent's removed row must stay removed")
 }
+
+func TestRevokeAgent_FailsActiveJobs(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svcA := jobFixture(t, s, "a")
+	b, svcB := jobFixture(t, s, "b")
+	ja, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	jb, err := s.CreateJob(ctx, newJob(svcB, b))
+	require.NoError(t, err)
+
+	require.NoError(t, s.RevokeAgent(ctx, a, t0))
+
+	got, err := s.GetJob(ctx, ja.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, got.Status)
+	assert.Equal(t, "agent revoked", got.Error)
+	got, err = s.GetJob(ctx, jb.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobQueued, got.Status)
+}
diff --git a/internal/master/store/jobs.go b/internal/master/store/jobs.go
new file mode 100644
index 0000000..4f708c6
--- /dev/null
+++ b/internal/master/store/jobs.go
@@ -0,0 +1,204 @@
+package store
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+const lostJobMessage = "lost: the agent did not know this job when it reconnected"
+
+// failActiveJobsSQL fails every queued or running job of agent $1 with
+// message $2 at time $3.
+const failActiveJobsSQL = `
+UPDATE update_jobs SET status = 'failed', error = $2, finished_at = $3
+WHERE agent_id = $1 AND status IN ('queued', 'running')`
+
+const jobSelectSQL = `
+SELECT j.id, j.service_id, s.kind, s.compose_project, s.compose_service, s.swarm_service, s.container_name,
+       j.agent_id, COALESCE(a.name, ''), j.from_ref, j.to_ref, j.trigger, j.requested_by, j.status,
+       %s, j.output_truncated, j.error, j.created_at, j.started_at, j.finished_at
+FROM update_jobs j
+JOIN services s ON s.id = j.service_id
+LEFT JOIN agents a ON a.id = j.agent_id`
+
+func scanJob(row pgx.Row) (model.Job, error) {
+	var j model.Job
+	var svc model.Service
+	var kind, trigger, status string
+	err := row.Scan(&j.ID, &j.ServiceID, &kind, &svc.ComposeProject, &svc.ComposeService, &svc.SwarmService, &svc.ContainerName,
+		&j.AgentID, &j.AgentName, &j.FromRef, &j.ToRef, &trigger, &j.RequestedBy, &status,
+		&j.Output, &j.OutputTruncated, &j.Error, &j.CreatedAt, &j.StartedAt, &j.FinishedAt)
+	svc.Kind = protocol.Kind(kind)
+	j.ServiceName = svc.DisplayName()
+	j.Trigger = model.JobTrigger(trigger)
+	j.Status = model.JobStatus(status)
+	return j, err
+}
+
+func (s *Store) getJob(ctx context.Context, id int64) (model.Job, error) {
+	j, err := scanJob(s.pool.QueryRow(ctx, fmt.Sprintf(jobSelectSQL, "j.output")+` WHERE j.id = $1`, id))
+	if errors.Is(err, pgx.ErrNoRows) {
+		return model.Job{}, model.ErrNotFound
+	}
+	if err != nil {
+		return model.Job{}, fmt.Errorf("get job: %w", err)
+	}
+	return j, nil
+}
+
+// CreateJob inserts a queued job. If the service already has a queued or
+// running job, it returns that job and model.ErrJobActive (detected via the
+// unique index, so it holds under concurrent calls).
+func (s *Store) CreateJob(ctx context.Context, j model.NewJob) (model.Job, error) {
+	var id int64
+	err := s.pool.QueryRow(ctx, `
+		INSERT INTO update_jobs (service_id, agent_id, from_ref, to_ref, trigger, requested_by, status, created_at)
+		VALUES ($1, $2, $3, $4, $5, $6, 'queued', $7)
+		RETURNING id`,
+		j.ServiceID, j.AgentID, j.FromRef, j.ToRef, string(j.Trigger), j.RequestedBy, j.CreatedAt).Scan(&id)
+	var pgErr *pgconn.PgError
+	if errors.As(err, &pgErr) && pgErr.Code == "23505" && pgErr.ConstraintName == "update_jobs_one_active" {
+		active, err := scanJob(s.pool.QueryRow(ctx, fmt.Sprintf(jobSelectSQL, "j.output")+
+			` WHERE j.service_id = $1 AND j.status IN ('queued', 'running')`, j.ServiceID))
+		if err != nil {
+			return model.Job{}, fmt.Errorf("find active job: %w", err)
+		}
+		return active, model.ErrJobActive
+	}
+	if err != nil {
+		return model.Job{}, fmt.Errorf("create job: %w", err)
+	}
+	return s.getJob(ctx, id)
+}
+
+// FailJob sets a queued or running job to failed with msg and finished_at =
+// at. Returns false (no error) if the job is no longer active.
+func (s *Store) FailJob(ctx context.Context, id int64, msg string, at time.Time) (bool, error) {
+	tag, err := s.pool.Exec(ctx, `
+		UPDATE update_jobs SET status = 'failed', error = $2, finished_at = $3
+		WHERE id = $1 AND status IN ('queued', 'running')`, id, msg, at)
+	if err != nil {
+		return false, fmt.Errorf("fail job: %w", err)
+	}
+	return tag.RowsAffected() == 1, nil
+}
+
+// GetJob returns one job with its output. Returns model.ErrNotFound if missing.
+func (s *Store) GetJob(ctx context.Context, id int64) (model.Job, error) {
+	return s.getJob(ctx, id)
+}
+
+// ListJobs returns jobs newest first, without Output.
+func (s *Store) ListJobs(ctx context.Context, f model.JobFilter) ([]model.Job, error) {
+	limit := f.Limit
+	if limit <= 0 {
+		limit = 100
+	}
+	rows, err := s.pool.Query(ctx, fmt.Sprintf(jobSelectSQL, "''")+`
+		WHERE ($1 = 0 OR j.service_id = $1)
+		ORDER BY j.id DESC
+		LIMIT $2`, f.ServiceID, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list jobs: %w", err)
+	}
+	defer rows.Close()
+	var out []model.Job
+	for rows.Next() {
+		j, err := scanJob(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan job: %w", err)
+		}
+		out = append(out, j)
+	}
+	return out, rows.Err()
+}
+
+// FailLostJobs fails every queued or running job sent to agentID whose ID is
+// not in known, with a "lost" error. Returns the IDs it failed.
+func (s *Store) FailLostJobs(ctx context.Context, agentID int64, known []int64, at time.Time) ([]int64, error) {
+	if known == nil {
+		known = []int64{}
+	}
+	rows, err := s.pool.Query(ctx, `
+		UPDATE update_jobs SET status = 'failed', error = $3, finished_at = $4
+		WHERE agent_id = $1 AND status IN ('queued', 'running') AND NOT (id = ANY($2))
+		RETURNING id`, agentID, known, lostJobMessage, at)
+	if err != nil {
+		return nil, fmt.Errorf("fail lost jobs: %w", err)
+	}
+	defer rows.Close()
+	var ids []int64
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			return nil, fmt.Errorf("scan lost job: %w", err)
+		}
+		ids = append(ids, id)
+	}
+	return ids, rows.Err()
+}
+
+// JobProgress appends chunk to the output of a queued or running job sent to
+// agentID, keeping the last protocol.MaxJobOutput characters, and moves a
+// queued job to running (started_at = at if unset). Returns false if no such
+// active job belongs to agentID.
+func (s *Store) JobProgress(ctx context.Context, agentID, jobID int64, chunk string, at time.Time) (bool, error) {
+	tag, err := s.pool.Exec(ctx, `
+		UPDATE update_jobs SET
+			status           = 'running',
+			started_at       = COALESCE(started_at, $4),
+			output_truncated = output_truncated OR length(output) + length($3::text) > $5,
+			output           = right(output || $3::text, $5)
+		WHERE id = $1 AND agent_id = $2 AND status IN ('queued', 'running')`,
+		jobID, agentID, chunk, at, protocol.MaxJobOutput)
+	if err != nil {
+		return false, fmt.Errorf("job progress: %w", err)
+	}
+	return tag.RowsAffected() == 1, nil
+}
+
+// FinishJob records the agent's result for a queued or running job sent to
+// agentID: status, error, output (replacing what progress assembled),
+// started_at (kept if already set), finished_at; to_ref = f.Ref when set. On
+// success it also sets the service's desired_ref = f.Ref, in the same
+// transaction. Returns false if no such active job belongs to agentID.
+func (s *Store) FinishJob(ctx context.Context, agentID, jobID int64, f model.JobFinish) (bool, error) {
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return false, fmt.Errorf("finish job: begin: %w", err)
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+
+	var serviceID int64
+	err = tx.QueryRow(ctx, `
+		UPDATE update_jobs SET
+			status = $3, error = $4, output = $5, output_truncated = $6,
+			to_ref = CASE WHEN $7::text <> '' THEN $7::text ELSE to_ref END,
+			started_at = COALESCE(started_at, $8), finished_at = $9
+		WHERE id = $1 AND agent_id = $2 AND status IN ('queued', 'running')
+		RETURNING service_id`,
+		jobID, agentID, string(f.Status), f.Error, f.Output, f.OutputTruncated, f.Ref, f.StartedAt, f.FinishedAt).Scan(&serviceID)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("finish job: %w", err)
+	}
+	if f.Status == model.JobSucceeded {
+		if _, err := tx.Exec(ctx, `UPDATE services SET desired_ref = $2 WHERE id = $1`, serviceID, f.Ref); err != nil {
+			return false, fmt.Errorf("finish job: desired ref: %w", err)
+		}
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return false, fmt.Errorf("finish job: commit: %w", err)
+	}
+	return true, nil
+}
diff --git a/internal/master/store/jobs_integration_test.go b/internal/master/store/jobs_integration_test.go
new file mode 100644
index 0000000..2e7a54d
--- /dev/null
+++ b/internal/master/store/jobs_integration_test.go
@@ -0,0 +1,305 @@
+//go:build integration
+
+package store
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// jobFixture seeds one agent with one Compose service and returns their IDs.
+func jobFixture(t *testing.T, s *Store, agentName string) (agentID, serviceID int64) {
+	t.Helper()
+	ctx := context.Background()
+	agentID = newAgent(t, s, agentName)
+	require.NoError(t, s.ApplySnapshot(ctx, agentID, worker, []protocol.Service{slusk}, t0))
+	svcs, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	for _, svc := range svcs {
+		if svc.AgentID != nil && *svc.AgentID == agentID {
+			return agentID, svc.ID
+		}
+	}
+	t.Fatal("service not found")
+	return
+}
+
+func newJob(serviceID, agentID int64) model.NewJob {
+	return model.NewJob{
+		ServiceID: serviceID, AgentID: &agentID,
+		FromRef: "r:1@sha256:a", ToRef: "r:1", Trigger: model.TriggerManual,
+		RequestedBy: "sub", CreatedAt: t0,
+	}
+}
+
+func TestCreateJob_OneActivePerService(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	agentID, svcID := jobFixture(t, s, "a")
+
+	var wg sync.WaitGroup
+	jobs := make([]model.Job, 2)
+	errs := make([]error, 2)
+	for i := range jobs {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			jobs[i], errs[i] = s.CreateJob(ctx, newJob(svcID, agentID))
+		}()
+	}
+	wg.Wait()
+
+	assert.Equal(t, jobs[0].ID, jobs[1].ID)
+	var failures int
+	for _, err := range errs {
+		if err != nil {
+			assert.ErrorIs(t, err, model.ErrJobActive)
+			failures++
+		}
+	}
+	assert.Equal(t, 1, failures)
+
+	ok, err := s.FailJob(ctx, jobs[0].ID, "x", t0)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	ok, err = s.FailJob(ctx, jobs[0].ID, "x", t0)
+	require.NoError(t, err)
+	assert.False(t, ok)
+
+	next, err := s.CreateJob(ctx, newJob(svcID, agentID))
+	require.NoError(t, err)
+	assert.NotEqual(t, jobs[0].ID, next.ID)
+}
+
+func TestFailLostJobs(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svcA := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	require.NoError(t, s.ApplySnapshot(ctx, b, worker, []protocol.Service{adhoc}, t0))
+	all, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	var svcB int64
+	for _, svc := range all {
+		if svc.ID != svcA {
+			svcB = svc.ID
+		}
+	}
+
+	// Only one active job per service, so A's jobs go through the lifecycle.
+	j3, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	ok, err := s.FinishJob(ctx, a, j3.ID, model.JobFinish{Status: model.JobSucceeded, Ref: "r:1@sha256:d", StartedAt: t0, FinishedAt: t0})
+	require.NoError(t, err)
+	require.True(t, ok)
+	j1, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	j4, err := s.CreateJob(ctx, newJob(svcB, b))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, b, j4.ID, "x", t0)
+	require.NoError(t, err)
+
+	ids, err := s.FailLostJobs(ctx, a, []int64{j3.ID}, t0)
+	require.NoError(t, err)
+	assert.Equal(t, []int64{j1.ID}, ids)
+
+	got, err := s.GetJob(ctx, j1.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, got.Status)
+	assert.Contains(t, got.Error, "lost")
+	got, err = s.GetJob(ctx, j3.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobSucceeded, got.Status)
+	got, err = s.GetJob(ctx, j4.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobRunning, got.Status)
+
+	// A running job the agent still knows is kept.
+	j5, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j5.ID, "", t0)
+	require.NoError(t, err)
+	ids, err = s.FailLostJobs(ctx, a, []int64{j5.ID}, t0)
+	require.NoError(t, err)
+	assert.Empty(t, ids)
+}
+
+func TestJobProgress_OwnerOnly(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+
+	started := t0.Add(time.Second)
+	ok, err := s.JobProgress(ctx, a, j.ID, "hello ", started)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobRunning, got.Status)
+	require.NotNil(t, got.StartedAt)
+	assert.True(t, started.Equal(*got.StartedAt))
+
+	ok, err = s.JobProgress(ctx, b, j.ID, "evil", t0.Add(time.Minute))
+	require.NoError(t, err)
+	assert.False(t, ok)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "hello ", got.Output)
+
+	ok, err = s.JobProgress(ctx, a, j.ID, "world", t0.Add(time.Minute))
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "hello world", got.Output)
+	assert.True(t, started.Equal(*got.StartedAt))
+
+	_, err = s.FailJob(ctx, j.ID, "x", t0)
+	require.NoError(t, err)
+	ok, err = s.JobProgress(ctx, a, j.ID, "late", t0)
+	require.NoError(t, err)
+	assert.False(t, ok)
+}
+
+func TestJobProgress_KeepsTail(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+
+	_, err = s.JobProgress(ctx, a, j.ID, strings.Repeat("a", protocol.MaxJobOutput), t0)
+	require.NoError(t, err)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.False(t, got.OutputTruncated)
+
+	_, err = s.JobProgress(ctx, a, j.ID, "TAIL", t0)
+	require.NoError(t, err)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.True(t, got.OutputTruncated)
+	assert.Len(t, got.Output, protocol.MaxJobOutput)
+	assert.True(t, strings.HasSuffix(got.Output, "aaTAIL"))
+}
+
+func TestFinishJob_OnlyOnceAndOnlyByOwner(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svc := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	j, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j.ID, "partial", t0)
+	require.NoError(t, err)
+
+	fin := model.JobFinish{Status: model.JobSucceeded, Ref: "r:1@sha256:d", Output: "full", StartedAt: t0.Add(time.Hour), FinishedAt: t0.Add(2 * time.Hour)}
+	ok, err := s.FinishJob(ctx, b, j.ID, fin)
+	require.NoError(t, err)
+	assert.False(t, ok)
+
+	ok, err = s.FinishJob(ctx, a, j.ID, fin)
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err := s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobSucceeded, got.Status)
+	assert.Equal(t, "r:1@sha256:d", got.ToRef)
+	assert.Equal(t, "full", got.Output)
+	assert.True(t, t0.Equal(*got.StartedAt), "started_at set by the first progress is kept")
+	assert.True(t, fin.FinishedAt.Equal(*got.FinishedAt))
+	service, err := s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, "r:1@sha256:d", service.DesiredRef)
+
+	again := fin
+	again.Ref = "r:2@sha256:e"
+	again.Output = "other"
+	ok, err = s.FinishJob(ctx, a, j.ID, again)
+	require.NoError(t, err)
+	assert.False(t, ok)
+	got, err = s.GetJob(ctx, j.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "full", got.Output)
+	service, err = s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, "r:1@sha256:d", service.DesiredRef)
+
+	j2, err := s.CreateJob(ctx, newJob(svc, a))
+	require.NoError(t, err)
+	ok, err = s.FinishJob(ctx, a, j2.ID, model.JobFinish{Status: model.JobFailed, Error: "boom", Output: "o", StartedAt: t0, FinishedAt: t0})
+	require.NoError(t, err)
+	assert.True(t, ok)
+	got, err = s.GetJob(ctx, j2.ID)
+	require.NoError(t, err)
+	assert.Equal(t, model.JobFailed, got.Status)
+	assert.Equal(t, "boom", got.Error)
+	assert.Equal(t, "r:1", got.ToRef)
+	service, err = s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, "r:1@sha256:d", service.DesiredRef)
+}
+
+func TestGetJobAndListJobs(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	a, svcA := jobFixture(t, s, "a")
+	b := newAgent(t, s, "b")
+	require.NoError(t, s.ApplySnapshot(ctx, b, worker, []protocol.Service{adhoc}, t0))
+	all, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	var svcB int64
+	for _, svc := range all {
+		if svc.ID != svcA {
+			svcB = svc.ID
+		}
+	}
+
+	j1, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	_, err = s.JobProgress(ctx, a, j1.ID, "out", t0)
+	require.NoError(t, err)
+	_, err = s.FailJob(ctx, j1.ID, "x", t0)
+	require.NoError(t, err)
+	j2, err := s.CreateJob(ctx, newJob(svcA, a))
+	require.NoError(t, err)
+	j3, err := s.CreateJob(ctx, newJob(svcB, b))
+	require.NoError(t, err)
+
+	got, err := s.GetJob(ctx, j1.ID)
+	require.NoError(t, err)
+	assert.Equal(t, "slusk/web", got.ServiceName)
+	assert.Equal(t, "a", got.AgentName)
+	assert.Equal(t, "out", got.Output)
+
+	_, err = s.GetJob(ctx, 99999)
+	assert.ErrorIs(t, err, model.ErrNotFound)
+
+	list, err := s.ListJobs(ctx, model.JobFilter{})
+	require.NoError(t, err)
+	require.Len(t, list, 3)
+	assert.Equal(t, []int64{j3.ID, j2.ID, j1.ID}, []int64{list[0].ID, list[1].ID, list[2].ID})
+	for _, j := range list {
+		assert.Empty(t, j.Output)
+	}
+
+	list, err = s.ListJobs(ctx, model.JobFilter{ServiceID: svcA})
+	require.NoError(t, err)
+	assert.Len(t, list, 2)
+
+	list, err = s.ListJobs(ctx, model.JobFilter{Limit: 1})
+	require.NoError(t, err)
+	assert.Len(t, list, 1)
+}
diff --git a/internal/master/store/services.go b/internal/master/store/services.go
index 6f858eb..a36ceee 100644
--- a/internal/master/store/services.go
+++ b/internal/master/store/services.go
@@ -39,13 +39,14 @@ func lockActiveAgent(ctx context.Context, tx pgx.Tx, agentID int64) error {
 
 const upsertServiceSQL = `
 INSERT INTO services (
-	kind, location_key, agent_id, compose_project, compose_service, working_dir,
+	kind, location_key, agent_id, compose_project, compose_service, working_dir, config_files,
 	swarm_cluster_id, swarm_service, container_name, image_repo, actual_ref, actual_digest,
 	status, self, first_seen_at, updated_at)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $15)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $16)
 ON CONFLICT (location_key) DO UPDATE SET
 	agent_id      = EXCLUDED.agent_id,
 	working_dir   = EXCLUDED.working_dir,
+	config_files  = EXCLUDED.config_files,
 	image_repo    = EXCLUDED.image_repo,
 	actual_ref    = EXCLUDED.actual_ref,
 	actual_digest = EXCLUDED.actual_digest,
@@ -61,7 +62,7 @@ func upsertService(ctx context.Context, db execer, agentID int64, clusterID stri
 	}
 	_, err := db.Exec(ctx, upsertServiceSQL,
 		string(svc.Kind), model.LocationKey(agentID, clusterID, svc.Key()), agentID,
-		svc.ComposeProject, svc.ComposeService, svc.WorkingDir,
+		svc.ComposeProject, svc.ComposeService, svc.WorkingDir, svc.ConfigFiles,
 		clusterCol, svc.SwarmService, svc.ContainerName,
 		imageref.Parse(svc.Image).Repo, svc.Image, svc.Digest,
 		svc.Status, svc.Self, at)
@@ -144,14 +145,27 @@ func (s *Store) RemoveService(ctx context.Context, agentID int64, clusterID, age
 	return nil
 }
 
+const serviceColumns = `s.id, s.kind, s.location_key, s.agent_id, COALESCE(a.name, ''),
+	s.compose_project, s.compose_service, s.working_dir, s.config_files, s.swarm_cluster_id,
+	s.swarm_service, s.container_name, s.image_repo, s.desired_ref, s.actual_ref, s.actual_digest,
+	s.status, s.self, s.update_mode, s.first_seen_at, s.updated_at, s.removed_at`
+
+func scanService(row pgx.Row) (model.Service, error) {
+	var m model.Service
+	var kind string
+	err := row.Scan(&m.ID, &kind, &m.LocationKey, &m.AgentID, &m.AgentName,
+		&m.ComposeProject, &m.ComposeService, &m.WorkingDir, &m.ConfigFiles, &m.SwarmClusterID,
+		&m.SwarmService, &m.ContainerName, &m.ImageRepo, &m.DesiredRef, &m.ActualRef, &m.ActualDigest,
+		&m.Status, &m.Self, &m.UpdateMode, &m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt)
+	m.Kind = protocol.Kind(kind)
+	return m, err
+}
+
 // ListServices returns all services not marked removed: machine-bound
 // services grouped by agent name first, Swarm services last.
 func (s *Store) ListServices(ctx context.Context) ([]model.Service, error) {
 	rows, err := s.pool.Query(ctx, `
-		SELECT s.id, s.kind, s.location_key, s.agent_id, COALESCE(a.name, ''),
-		       s.compose_project, s.compose_service, s.working_dir, s.swarm_cluster_id,
-		       s.swarm_service, s.container_name, s.image_repo, s.actual_ref, s.actual_digest,
-		       s.status, s.self, s.update_mode, s.first_seen_at, s.updated_at
+		SELECT `+serviceColumns+`
 		FROM services s LEFT JOIN agents a ON a.id = s.agent_id
 		WHERE s.removed_at IS NULL
 		ORDER BY s.kind = 'swarm', COALESCE(a.name, ''), s.location_key`)
@@ -161,16 +175,27 @@ func (s *Store) ListServices(ctx context.Context) ([]model.Service, error) {
 	defer rows.Close()
 	var out []model.Service
 	for rows.Next() {
-		var m model.Service
-		var kind string
-		if err := rows.Scan(&m.ID, &kind, &m.LocationKey, &m.AgentID, &m.AgentName,
-			&m.ComposeProject, &m.ComposeService, &m.WorkingDir, &m.SwarmClusterID,
-			&m.SwarmService, &m.ContainerName, &m.ImageRepo, &m.ActualRef, &m.ActualDigest,
-			&m.Status, &m.Self, &m.UpdateMode, &m.FirstSeenAt, &m.UpdatedAt); err != nil {
+		m, err := scanService(rows)
+		if err != nil {
 			return nil, fmt.Errorf("scan service: %w", err)
 		}
-		m.Kind = protocol.Kind(kind)
 		out = append(out, m)
 	}
 	return out, rows.Err()
 }
+
+// GetService returns one service, including a removed one (RemovedAt set).
+// Returns model.ErrNotFound if the ID does not exist.
+func (s *Store) GetService(ctx context.Context, id int64) (model.Service, error) {
+	m, err := scanService(s.pool.QueryRow(ctx, `
+		SELECT `+serviceColumns+`
+		FROM services s LEFT JOIN agents a ON a.id = s.agent_id
+		WHERE s.id = $1`, id))
+	if errors.Is(err, pgx.ErrNoRows) {
+		return model.Service{}, model.ErrNotFound
+	}
+	if err != nil {
+		return model.Service{}, fmt.Errorf("get service: %w", err)
+	}
+	return m, nil
+}
diff --git a/internal/master/store/services_integration_test.go b/internal/master/store/services_integration_test.go
index 87d8013..728c8bc 100644
--- a/internal/master/store/services_integration_test.go
+++ b/internal/master/store/services_integration_test.go
@@ -160,3 +160,36 @@ func TestListServices_SwarmLast(t *testing.T) {
 }
 
 func itoa(n int64) string { return strconv.FormatInt(n, 10) }
+
+func TestGetService_IncludesRemoved(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	id := newAgent(t, s, "node-2")
+	require.NoError(t, s.ApplySnapshot(ctx, id, worker, []protocol.Service{slusk}, t0))
+	svcs, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	require.Len(t, svcs, 1)
+
+	require.NoError(t, s.RemoveService(ctx, id, "", slusk.Key(), t0))
+	got, err := s.GetService(ctx, svcs[0].ID)
+	require.NoError(t, err)
+	assert.NotNil(t, got.RemovedAt)
+	assert.Equal(t, "node-2", got.AgentName)
+
+	_, err = s.GetService(ctx, 99999)
+	assert.ErrorIs(t, err, model.ErrNotFound)
+}
+
+func TestApplySnapshot_StoresConfigFiles(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	id := newAgent(t, s, "node-2")
+	withFiles := slusk
+	withFiles.ConfigFiles = "/srv/slusk/compose.yaml,/srv/slusk/compose.override.yaml"
+	require.NoError(t, s.ApplySnapshot(ctx, id, worker, []protocol.Service{withFiles}, t0))
+
+	got, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, withFiles.ConfigFiles, got[0].ConfigFiles)
+}

From 574f425267682f3595fb3f5f7e3f96923126ff40 Mon Sep 17 00:00:00 2001
From: Samuel Enocsson 
Date: Mon, 12 Oct 2026 18:41:46 +0200
Subject: [PATCH 07/29] feat(web): service page, manual update, jobs list and
 live job log (#7)

Also publishes the jobs topic on agent revoke and updates the agent
snippet to mount stacks read-write at the same path.
---
 internal/master/web/server.go               | 168 ++++++++++++-
 internal/master/web/server_test.go          | 250 ++++++++++++++++++--
 internal/master/web/static/app.css          |   7 +
 internal/master/web/templates/job.html      |  21 ++
 internal/master/web/templates/jobs.html     |  27 +++
 internal/master/web/templates/layout.html   |   1 +
 internal/master/web/templates/overview.html |   2 +-
 internal/master/web/templates/service.html  |  46 ++++
 internal/master/web/views.go                |  13 +
 internal/master/webauth/auth.go             |   7 +-
 internal/master/webauth/auth_test.go        |   7 +
 11 files changed, 526 insertions(+), 23 deletions(-)
 create mode 100644 internal/master/web/templates/job.html
 create mode 100644 internal/master/web/templates/jobs.html
 create mode 100644 internal/master/web/templates/service.html

diff --git a/internal/master/web/server.go b/internal/master/web/server.go
index a6433ea..761d684 100644
--- a/internal/master/web/server.go
+++ b/internal/master/web/server.go
@@ -78,9 +78,9 @@ type Server struct {
 // New parses the templates. Each page is its own template set so every page
 // can define its own "content" block.
 func New(store Store, hub Hub, jobs Jobs, broker Broker, cfg Config, log *slog.Logger) (*Server, error) {
-	funcs := template.FuncMap{"shortRef": shortRef}
+	funcs := template.FuncMap{"shortRef": shortRef, "duration": jobDuration}
 	pages := map[string]*template.Template{}
-	for _, page := range []string{"overview.html", "agents.html", "agent_created.html"} {
+	for _, page := range []string{"overview.html", "agents.html", "agent_created.html", "service.html", "jobs.html", "job.html"} {
 		t, err := template.New(page).Funcs(funcs).ParseFS(assets, "templates/layout.html", "templates/"+page)
 		if err != nil {
 			return nil, fmt.Errorf("parse %s: %w", page, err)
@@ -97,6 +97,13 @@ func (s *Server) Register(mux *http.ServeMux, protect func(http.Handler) http.Ha
 	route := func(pattern string, h http.HandlerFunc) { mux.Handle(pattern, protect(h)) }
 	route("GET /{$}", s.overview)
 	route("GET /fragments/services", s.servicesFragment)
+	route("GET /services/{id}", s.servicePage)
+	route("GET /fragments/services/{id}/jobs", s.serviceJobsFragment)
+	route("POST /services/{id}/update", s.updateService)
+	route("GET /jobs", s.jobsPage)
+	route("GET /fragments/jobs", s.jobsFragment)
+	route("GET /jobs/{id}", s.jobPage)
+	route("GET /fragments/jobs/{id}", s.jobFragment)
 	route("GET /agents", s.agents)
 	route("GET /fragments/agents", s.agentsFragment)
 	route("POST /agents", s.createAgent)
@@ -112,6 +119,12 @@ type pageData struct {
 	Error   string
 	Created model.Agent
 	Snippet string
+
+	Service     model.Service
+	ConfigFiles []string
+	Reason      string // why Service cannot be updated; "" when it can
+	Jobs        []model.Job
+	Job         model.Job
 }
 
 func (s *Server) base(r *http.Request, title string) pageData {
@@ -155,6 +168,151 @@ func (s *Server) servicesFragment(w http.ResponseWriter, r *http.Request) {
 	s.render(w, http.StatusOK, "overview.html", "services", data)
 }
 
+// pathID parses the {id} path value. On failure it has already answered 400.
+func pathID(w http.ResponseWriter, r *http.Request) (int64, bool) {
+	id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
+	if err != nil {
+		http.Error(w, "invalid id", http.StatusBadRequest)
+		return 0, false
+	}
+	return id, true
+}
+
+// failLookup answers 404 for model.ErrNotFound and 500 otherwise.
+func (s *Server) failLookup(w http.ResponseWriter, err error) {
+	if errors.Is(err, model.ErrNotFound) {
+		http.Error(w, "not found", http.StatusNotFound)
+		return
+	}
+	s.fail(w, err)
+}
+
+func (s *Server) serviceDetail(r *http.Request, id int64) (pageData, error) {
+	svc, err := s.store.GetService(r.Context(), id)
+	if err != nil {
+		return pageData{}, err
+	}
+	data := s.base(r, svc.DisplayName())
+	data.Service = svc
+	if svc.ConfigFiles != "" {
+		data.ConfigFiles = strings.Split(svc.ConfigFiles, ",")
+	}
+	var nu *model.NotUpdatableError
+	if err := s.jobs.Updatable(svc); errors.As(err, &nu) {
+		data.Reason = nu.Reason
+	} else if err != nil {
+		return data, err
+	}
+	data.Jobs, err = s.store.ListJobs(r.Context(), model.JobFilter{ServiceID: id, Limit: 20})
+	return data, err
+}
+
+func (s *Server) servicePage(w http.ResponseWriter, r *http.Request) {
+	id, ok := pathID(w, r)
+	if !ok {
+		return
+	}
+	data, err := s.serviceDetail(r, id)
+	if err != nil {
+		s.failLookup(w, err)
+		return
+	}
+	s.render(w, http.StatusOK, "service.html", "layout", data)
+}
+
+func (s *Server) serviceJobsFragment(w http.ResponseWriter, r *http.Request) {
+	id, ok := pathID(w, r)
+	if !ok {
+		return
+	}
+	jobs, err := s.store.ListJobs(r.Context(), model.JobFilter{ServiceID: id, Limit: 20})
+	if err != nil {
+		s.fail(w, err)
+		return
+	}
+	data := s.base(r, "Service")
+	data.Jobs = jobs
+	s.render(w, http.StatusOK, "service.html", "history", data)
+}
+
+func (s *Server) updateService(w http.ResponseWriter, r *http.Request) {
+	id, ok := pathID(w, r)
+	if !ok {
+		return
+	}
+	sess, _ := webauth.SessionFrom(r.Context())
+	job, err := s.jobs.Trigger(r.Context(), id, sess.Subject)
+	var nu *model.NotUpdatableError
+	switch {
+	case err == nil || errors.Is(err, model.ErrJobActive):
+		http.Redirect(w, r, "/jobs/"+strconv.FormatInt(job.ID, 10), http.StatusSeeOther)
+	case errors.As(err, &nu):
+		data, derr := s.serviceDetail(r, id)
+		if derr != nil {
+			s.failLookup(w, derr)
+			return
+		}
+		data.Error = "Cannot update: " + nu.Reason + "."
+		s.render(w, http.StatusConflict, "service.html", "layout", data)
+	default:
+		s.failLookup(w, err)
+	}
+}
+
+func (s *Server) jobsData(r *http.Request) (pageData, error) {
+	data := s.base(r, "Jobs")
+	var err error
+	data.Jobs, err = s.store.ListJobs(r.Context(), model.JobFilter{Limit: 100})
+	return data, err
+}
+
+func (s *Server) jobsPage(w http.ResponseWriter, r *http.Request) {
+	data, err := s.jobsData(r)
+	if err != nil {
+		s.fail(w, err)
+		return
+	}
+	s.render(w, http.StatusOK, "jobs.html", "layout", data)
+}
+
+func (s *Server) jobsFragment(w http.ResponseWriter, r *http.Request) {
+	data, err := s.jobsData(r)
+	if err != nil {
+		s.fail(w, err)
+		return
+	}
+	s.render(w, http.StatusOK, "jobs.html", "jobs", data)
+}
+
+// jobData always reads the job from the store, so a fragment fetched after
+// a dropped signal still shows the full current output.
+func (s *Server) jobData(w http.ResponseWriter, r *http.Request) (pageData, bool) {
+	id, ok := pathID(w, r)
+	if !ok {
+		return pageData{}, false
+	}
+	job, err := s.store.GetJob(r.Context(), id)
+	if err != nil {
+		s.failLookup(w, err)
+		return pageData{}, false
+	}
+	data := s.base(r, fmt.Sprintf("Job #%d", job.ID))
+	data.Job = job
+	return data, true
+}
+
+func (s *Server) jobPage(w http.ResponseWriter, r *http.Request) {
+	if data, ok := s.jobData(w, r); ok {
+		s.render(w, http.StatusOK, "job.html", "layout", data)
+	}
+}
+
+func (s *Server) jobFragment(w http.ResponseWriter, r *http.Request) {
+	if data, ok := s.jobData(w, r); ok {
+		s.render(w, http.StatusOK, "job.html", "job", data)
+	}
+}
+
 func (s *Server) agentData(r *http.Request) (pageData, error) {
 	data := s.base(r, "Agents")
 	agents, err := s.store.ListAgents(r.Context())
@@ -234,6 +392,7 @@ func (s *Server) revokeAgent(w http.ResponseWriter, r *http.Request) {
 	}
 	s.hub.Disconnect(id)
 	s.broker.Publish(events.Services) // the agent's services were marked removed
+	s.broker.Publish(events.Jobs)     // the agent's active jobs were failed
 	http.Redirect(w, r, "/agents", http.StatusSeeOther)
 }
 
@@ -293,7 +452,12 @@ func agentSnippet(cfg Config, token string) string {
       MASTER_URL: %s
       MASTER_KEY: %s
       AGENT_TOKEN: %s
+      UPDATE_TIMEOUT: 5m
+      # Optional registry credentials (a config.json written by docker login):
+      # DOCKER_CONFIG: /docker-config
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      # - /path/to/stacks:/path/to/stacks  # mount each stacks directory at the same path, read-write
+      # - /root/.docker/config.json:/docker-config/config.json:ro
 `, cfg.AgentImage, cfg.AgentURL, cfg.MasterKey, token)
 }
diff --git a/internal/master/web/server_test.go b/internal/master/web/server_test.go
index d2f5494..9cb0c26 100644
--- a/internal/master/web/server_test.go
+++ b/internal/master/web/server_test.go
@@ -20,6 +20,7 @@ import (
 	"github.com/shcizo/package-updater/internal/agenttoken"
 	"github.com/shcizo/package-updater/internal/master/events"
 	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/master/webauth"
 	"github.com/shcizo/package-updater/internal/protocol"
 )
 
@@ -28,20 +29,42 @@ import (
 type fakeStore struct {
 	mu        sync.Mutex
 	services  []model.Service
+	jobs      []model.Job
 	agents    []model.Agent
 	created   map[string]string // name → token hash
 	revokedID int64
 }
 
 func (f *fakeStore) ListServices(context.Context) ([]model.Service, error) { return f.services, nil }
-func (f *fakeStore) GetService(context.Context, int64) (model.Service, error) {
+func (f *fakeStore) GetService(_ context.Context, id int64) (model.Service, error) {
+	for _, s := range f.services {
+		if s.ID == id {
+			return s, nil
+		}
+	}
 	return model.Service{}, model.ErrNotFound
 }
-func (f *fakeStore) GetJob(context.Context, int64) (model.Job, error) {
+
+func (f *fakeStore) GetJob(_ context.Context, id int64) (model.Job, error) {
+	for _, j := range f.jobs {
+		if j.ID == id {
+			return j, nil
+		}
+	}
 	return model.Job{}, model.ErrNotFound
 }
-func (f *fakeStore) ListJobs(context.Context, model.JobFilter) ([]model.Job, error) { return nil, nil }
-func (f *fakeStore) ListAgents(context.Context) ([]model.Agent, error)              { return f.agents, nil }
+
+func (f *fakeStore) ListJobs(_ context.Context, flt model.JobFilter) ([]model.Job, error) {
+	var out []model.Job
+	for _, j := range f.jobs {
+		if flt.ServiceID == 0 || j.ServiceID == flt.ServiceID {
+			out = append(out, j)
+		}
+	}
+	return out, nil
+}
+
+func (f *fakeStore) ListAgents(context.Context) ([]model.Agent, error) { return f.agents, nil }
 
 func (f *fakeStore) CreateAgent(_ context.Context, name, hash string) (model.Agent, error) {
 	f.mu.Lock()
@@ -72,11 +95,29 @@ func (f *fakeStore) revoked() int64 {
 	return f.revokedID
 }
 
-type fakeJobs struct{}
+// fakeJobs returns scripted results and records who asked for the update.
+type fakeJobs struct {
+	updatable error
+	job       model.Job
+	err       error
 
-func (fakeJobs) Updatable(model.Service) error { return nil }
-func (fakeJobs) Trigger(context.Context, int64, string) (model.Job, error) {
-	return model.Job{}, model.ErrNotFound
+	mu          sync.Mutex
+	requestedBy string
+}
+
+func (f *fakeJobs) Updatable(model.Service) error { return f.updatable }
+
+func (f *fakeJobs) Trigger(_ context.Context, _ int64, by string) (model.Job, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.requestedBy = by
+	return f.job, f.err
+}
+
+func (f *fakeJobs) triggeredBy() string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.requestedBy
 }
 
 type fakeHub struct {
@@ -104,16 +145,22 @@ func (h *fakeHub) lastDisconnected() int64 {
 }
 
 func newTestServer(t *testing.T) (*httptest.Server, *fakeStore, *fakeHub, *events.Broker) {
+	t.Helper()
+	srv, store, hub, broker, _ := newTestEnv(t)
+	return srv, store, hub, broker
+}
+
+func newTestEnv(t *testing.T) (*httptest.Server, *fakeStore, *fakeHub, *events.Broker, *fakeJobs) {
 	t.Helper()
 	agentID := int64(2)
 	store := &fakeStore{
 		created: map[string]string{"node-2": "x"},
 		services: []model.Service{
-			{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", AgentID: &agentID, AgentName: "node-2",
+			{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", AgentID: &agentID, AgentName: "node-2",
 				ActualRef: "gitea.shcizo.se/shcizo/slusk:1.4.2", Status: "running", UpdateMode: "manual"},
-			{Kind: protocol.KindSwarm, SwarmService: "jellyfin_jellyfin",
+			{ID: 2, Kind: protocol.KindSwarm, SwarmService: "jellyfin_jellyfin",
 				ActualRef: "lscr.io/linuxserver/jellyfin@sha256:0f42497a69fa0441bfd5f9d6bba8694f2a656ec984e571d0ac04c6dd91250039", Status: "degraded", UpdateMode: "manual"},
-			{Kind: protocol.KindContainer, ContainerName: "updater-agent", AgentID: &agentID, AgentName: "node-2",
+			{ID: 3, Kind: protocol.KindContainer, ContainerName: "updater-agent", AgentID: &agentID, AgentName: "node-2",
 				ActualRef: "pu:1", Status: "running", UpdateMode: "manual", Self: true},
 		},
 		agents: []model.Agent{
@@ -124,15 +171,21 @@ func newTestServer(t *testing.T) (*httptest.Server, *fakeStore, *fakeHub, *event
 	}
 	hub := &fakeHub{online: map[int64]bool{2: true}}
 	broker := events.NewBroker()
-	s, err := New(store, hub, fakeJobs{}, broker, Config{
+	jobs := &fakeJobs{}
+	s, err := New(store, hub, jobs, broker, Config{
 		AgentURL: "wss://10.0.0.5:8090/agent", MasterKey: "sha256:abc", AgentImage: "gitea.shcizo.se/shcizo/package-updater:latest",
 	}, slog.New(slog.NewTextHandler(io.Discard, nil)))
 	require.NoError(t, err)
 	mux := http.NewServeMux()
-	s.Register(mux, func(h http.Handler) http.Handler { return h })
+	s.Register(mux, func(h http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx := webauth.WithSession(r.Context(), model.Session{Subject: "alice", CSRFToken: "csrf"})
+			h.ServeHTTP(w, r.WithContext(ctx))
+		})
+	})
 	srv := httptest.NewServer(mux)
 	t.Cleanup(srv.Close)
-	return srv, store, hub, broker
+	return srv, store, hub, broker, jobs
 }
 
 func ptr[T any](v T) *T { return &v }
@@ -233,12 +286,17 @@ func TestRevokeAgent_RevokesAndDisconnects(t *testing.T) {
 	assert.Equal(t, "/agents", resp.Header.Get("Location"))
 	assert.Equal(t, int64(2), store.revoked())
 	assert.Equal(t, int64(2), hub.lastDisconnected(), "a revoked agent must be cut off immediately")
-	select {
-	case topic := <-ch:
-		assert.Equal(t, events.Services, topic, "the overview must drop the revoked agent's services")
-	default:
-		t.Fatal("revoke published no services event")
+	var got []events.Topic
+	for len(got) < 2 {
+		select {
+		case topic := <-ch:
+			got = append(got, topic)
+		default:
+			t.Fatalf("revoke published %v, want services and jobs", got)
+		}
 	}
+	assert.ElementsMatch(t, []events.Topic{events.Services, events.Jobs}, got,
+		"the overview drops the agent's services and job lists show its failed jobs")
 }
 
 func TestRevokeAgent_BadID(t *testing.T) {
@@ -277,3 +335,157 @@ func TestEvents_StreamsTopics(t *testing.T) {
 	}
 	assert.Equal(t, "event: services\n", line)
 }
+
+func TestAgentSnippet(t *testing.T) {
+	snippet := agentSnippet(Config{AgentURL: "wss://m/agent", MasterKey: "sha256:abc", AgentImage: "img:1"}, "tok")
+	assert.NotContains(t, snippet, "STACKS_ROOT")
+	assert.Contains(t, snippet, "UPDATE_TIMEOUT: 5m")
+	assert.Contains(t, snippet, "      # DOCKER_CONFIG: /docker-config\n")
+	assert.Contains(t, snippet, "      # - /path/to/stacks:/path/to/stacks  # mount each stacks directory at the same path, read-write\n")
+	assert.Contains(t, snippet, "AGENT_TOKEN: tok")
+}
+
+func TestOverview_LinksToServicePage(t *testing.T) {
+	srv, _, _, _ := newTestServer(t)
+	_, body := get(t, srv, "/")
+	assert.Contains(t, body, `href="/services/1"`)
+	assert.Contains(t, body, `href="/jobs"`)
+}
+
+func TestServicePage_ShowsUpdateButton(t *testing.T) {
+	srv, store, _, _, _ := newTestEnv(t)
+	store.services[0].ConfigFiles = "/srv/slusk/compose.yaml,/srv/slusk/compose.override.yaml"
+	store.jobs = []model.Job{{ID: 7, ServiceID: 1, Status: model.JobSucceeded, FromRef: "a:1", ToRef: "a:1", RequestedBy: "alice", CreatedAt: time.Now()}}
+	code, body := get(t, srv, "/services/1")
+	assert.Equal(t, http.StatusOK, code)
+	assert.Contains(t, body, `action="/services/1/update"`)
+	assert.Contains(t, body, `name="csrf_token" value="csrf"`)
+	assert.Contains(t, body, "/srv/slusk/compose.override.yaml")
+	assert.Contains(t, body, `href="/jobs/7"`)
+	assert.Contains(t, body, "sse:jobs")
+}
+
+func TestServicePage_DisabledReason(t *testing.T) {
+	for _, reason := range []string{model.ReasonSelf, model.ReasonContainer, model.ReasonNoTag, model.ReasonRemoved} {
+		t.Run(reason, func(t *testing.T) {
+			srv, _, _, _, jobs := newTestEnv(t)
+			jobs.updatable = &model.NotUpdatableError{Reason: reason}
+			code, body := get(t, srv, "/services/1")
+			assert.Equal(t, http.StatusOK, code)
+			assert.Contains(t, body, reason)
+			assert.NotContains(t, body, `action="/services/1/update"`)
+		})
+	}
+}
+
+func TestServicePage_NotFoundAndBadID(t *testing.T) {
+	srv, _, _, _ := newTestServer(t)
+	code, _ := get(t, srv, "/services/99")
+	assert.Equal(t, http.StatusNotFound, code)
+	code, _ = get(t, srv, "/services/abc")
+	assert.Equal(t, http.StatusBadRequest, code)
+}
+
+func postUpdate(t *testing.T, srv *httptest.Server) (*http.Response, string) {
+	t.Helper()
+	resp, err := noRedirectClient().PostForm(srv.URL+"/services/1/update", url.Values{"csrf_token": {"csrf"}})
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(resp.Body)
+	return resp, string(body)
+}
+
+func TestUpdate_RedirectsToJob(t *testing.T) {
+	srv, _, _, _, jobs := newTestEnv(t)
+	jobs.job = model.Job{ID: 42}
+	resp, _ := postUpdate(t, srv)
+	assert.Equal(t, http.StatusSeeOther, resp.StatusCode)
+	assert.Equal(t, "/jobs/42", resp.Header.Get("Location"))
+	assert.Equal(t, "alice", jobs.triggeredBy())
+}
+
+func TestUpdate_ActiveJobRedirectsToExisting(t *testing.T) {
+	srv, _, _, _, jobs := newTestEnv(t)
+	jobs.job, jobs.err = model.Job{ID: 41}, model.ErrJobActive
+	resp, _ := postUpdate(t, srv)
+	assert.Equal(t, http.StatusSeeOther, resp.StatusCode)
+	assert.Equal(t, "/jobs/41", resp.Header.Get("Location"))
+}
+
+func TestUpdate_NotUpdatableShows409(t *testing.T) {
+	srv, _, _, _, jobs := newTestEnv(t)
+	jobs.err = &model.NotUpdatableError{Reason: model.ReasonNoTag}
+	jobs.updatable = jobs.err
+	resp, body := postUpdate(t, srv)
+	assert.Equal(t, http.StatusConflict, resp.StatusCode)
+	assert.Contains(t, body, model.ReasonNoTag)
+}
+
+func TestUpdate_UnknownService404(t *testing.T) {
+	srv, _, _, _, jobs := newTestEnv(t)
+	jobs.err = model.ErrNotFound
+	resp, _ := postUpdate(t, srv)
+	assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+}
+
+func TestServiceJobsFragment_IsNotAFullPage(t *testing.T) {
+	srv, store, _, _, _ := newTestEnv(t)
+	store.jobs = []model.Job{{ID: 7, ServiceID: 1, Status: model.JobFailed, CreatedAt: time.Now()}}
+	code, body := get(t, srv, "/fragments/services/1/jobs")
+	assert.Equal(t, http.StatusOK, code)
+	assert.NotContains(t, body, "boom", Error: "pull failed", OutputTruncated: true, CreatedAt: time.Now()}}
+	code, body := get(t, srv, "/jobs/42")
+	assert.Equal(t, http.StatusOK, code)
+	assert.Contains(t, body, "Pulling web")
+	assert.Contains(t, body, "<b>boom", "output is escaped")
+	assert.Contains(t, body, "pull failed")
+	assert.Contains(t, body, "Earlier output truncated")
+	assert.Contains(t, body, `hx-trigger="sse:job-42"`)
+}
+
+func TestJobFragment_RereadsFromStore(t *testing.T) {
+	srv, store, _, _, _ := newTestEnv(t)
+	store.jobs = []model.Job{{ID: 42, ServiceID: 1, Status: model.JobRunning, Output: "step 1", CreatedAt: time.Now()}}
+	code, body := get(t, srv, "/fragments/jobs/42")
+	assert.Equal(t, http.StatusOK, code)
+	assert.NotContains(t, body, "Job #{{.Job.ID}}
+
{{.Job.ServiceName}}

+

+  {{template "job" .}}
+

+{{end}}
+
+{{define "job"}}
+
+  
+  
+  
+  
+  
+  
+
Status{{.Job.Status}}
From{{shortRef .Job.FromRef}}
To{{shortRef .Job.ToRef}}
Requested by{{.Job.RequestedBy}}
Created{{.Job.CreatedAt.Format "2006-01-02 15:04:05"}}
Duration{{duration .Job}}

+{{if .Job.Error}}
{{.Job.Error}}
{{end}}
+{{if .Job.OutputTruncated}}
Earlier output truncated.
{{end}}
+
{{.Job.Output}}

+{{end}}
diff --git a/internal/master/web/templates/jobs.html b/internal/master/web/templates/jobs.html
new file mode 100644
index 0000000..a226ee5
--- /dev/null
+++ b/internal/master/web/templates/jobs.html
@@ -0,0 +1,27 @@
+{{define "content"}}
+
Jobs

+

+  {{template "jobs" .}}
+

+{{end}}
+
+{{define "jobs"}}
+
+  
+  
+  {{range .Jobs}}
+    
+      
+      
+      
+      
+      
+      
+      
+    
+  {{else}}
+    
+  {{end}}
+  
+
JobServiceStatusFrom → toRequested byCreatedDuration
#{{.ID}}{{.ServiceName}}{{.Status}}{{shortRef .FromRef}} → {{shortRef .ToRef}}{{.RequestedBy}}{{.CreatedAt.Format "2006-01-02 15:04"}}{{duration .}}
No jobs yet.

+{{end}}
diff --git a/internal/master/web/templates/layout.html b/internal/master/web/templates/layout.html
index d8eaad9..1be320d 100644
--- a/internal/master/web/templates/layout.html
+++ b/internal/master/web/templates/layout.html
@@ -11,6 +11,7 @@