feat: self-update of master and agents (#15) (#32)

internal/agent/updater/compose.go

@@ -6,13 +6,12 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"slices"
 	"strings"
 	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
-
-	"github.com/shcizo/package-updater/internal/protocol"
 )
 
 const (
@@ -71,6 +70,7 @@ func dockerEnv() []string {
 // ComposeDocker is the subset of the Docker SDK ComposeExecutor needs.
 type ComposeDocker interface {
 	SelfInspector
+	HelperDocker
 	ContainerList(ctx context.Context, opts container.ListOptions) ([]container.Summary, error)
 }
 
@@ -95,7 +95,10 @@ func NewComposeExecutor(cli ComposeDocker, resolve *Resolver, selfID string, run
 // is), writes it to the stack's override file and recreates the service.
 // Working directory and config files come from the service's container
 // labels on this machine, never from the job. It returns the pinned ref.
-func (e *ComposeExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+//
+// When the service is the agent's own container, the recreate runs in a
+// helper container instead (see selfUpdate), because it stops this process.
+func (e *ComposeExecutor) Execute(ctx context.Context, job Job, out io.Writer) (string, error) {
 	project, service := job.ComposeProject, job.ComposeService
 	cts, err := e.cli.ContainerList(ctx, container.ListOptions{
 		All: true,
@@ -110,14 +113,24 @@ func (e *ComposeExecutor) Execute(ctx context.Context, job protocol.JobStart, ou
 	if len(cts) == 0 {
 		return "", fmt.Errorf("no container for compose service %s/%s on this machine", project, service)
 	}
+	self, err := inspectSelf(ctx, e.cli, e.selfID)
+	if err != nil {
+		return "", err
+	}
+	isSelf := self.ContainerJSONBase != nil && self.ID != "" &&
+		slices.ContainsFunc(cts, func(ct container.Summary) bool { return ct.ID == self.ID })
+	// What an update of the agent's own service has printed so far goes into
+	// the helper's labels.
+	var soFar *tailBuffer
+	if isSelf {
+		soFar = &tailBuffer{max: selfUpdateOutputMax}
+		out = io.MultiWriter(out, tailWriter{soFar})
+	}
 	workingDir := cts[0].Labels[labelWorkingDir]
 	configFiles := cts[0].Labels[labelConfigFiles]
 	envFileLabel := cts[0].Labels[labelEnvFile]
 
-	roots, err := MountRoots(ctx, e.cli, e.selfID)
-	if err != nil {
-		return "", err
-	}
+	roots := rootsOf(self)
 	files, dir, err := composeFiles(roots, configFiles)
 	if err != nil {
 		return "", err
@@ -163,13 +176,29 @@ func (e *ComposeExecutor) Execute(ctx context.Context, job protocol.JobStart, ou
 	}
 
 	base := composeBase(project, workingDir, files, envs)
-	for _, step := range [][]string{{"pull", service}, {"up", "-d", service}} {
-		args := append(append([]string(nil), base...), step...)
+	run := func(args []string) error {
 		fmt.Fprintf(out, "$ docker %s\n", strings.Join(args, " "))
 		tail := &tailBuffer{max: errorOutputTail}
 		if err := e.run(ctx, workingDir, io.MultiWriter(out, tailWriter{tail}), args...); err != nil {
-			return "", fmt.Errorf("docker compose %s: %w (output: %s)", step[0], err, strings.TrimSpace(tail.String()))
+			return composeError(args[len(base)], err, tail)
 		}
+		return nil
+	}
+	if err := run(append(append([]string(nil), base...), "pull", service)); err != nil {
+		return "", err
+	}
+	up := append(append([]string(nil), base...), "up", "-d", service)
+	if isSelf {
+		return e.selfUpdate(ctx, job, self, workingDir, up, pinned, out, soFar)
+	}
+	if err := run(up); err != nil {
+		return "", err
 	}
 	return pinned, nil
 }
+
+// composeError describes a failed `docker compose <step>` call with the tail
+// of its output.
+func composeError(step string, err error, tail *tailBuffer) error {
+	return fmt.Errorf("docker compose %s: %w (output: %s)", step, err, strings.TrimSpace(tail.String()))
+}

internal/agent/updater/compose_test.go

@@ -1,6 +1,7 @@
 package updater
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"io"
@@ -8,9 +9,14 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
+	"unicode/utf8"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/pkg/stdcopy"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -26,6 +32,123 @@ func (f fakeCompose) ContainerList(context.Context, container.ListOptions) ([]co
 	return f.containers, nil
 }
 
+// fakeCompose is not meant to start helper containers; these fail the test
+// run if an executor tries.
+func (fakeCompose) ContainerCreate(context.Context, *container.Config, *container.HostConfig, *network.NetworkingConfig, *ocispec.Platform, string) (container.CreateResponse, error) {
+	panic("unexpected ContainerCreate")
+}
+func (fakeCompose) ContainerStart(context.Context, string, container.StartOptions) error {
+	panic("unexpected ContainerStart")
+}
+func (fakeCompose) ContainerWait(context.Context, string, container.WaitCondition) (<-chan container.WaitResponse, <-chan error) {
+	panic("unexpected ContainerWait")
+}
+func (fakeCompose) ContainerLogs(context.Context, string, container.LogsOptions) (io.ReadCloser, error) {
+	panic("unexpected ContainerLogs")
+}
+func (fakeCompose) ContainerRemove(context.Context, string, container.RemoveOptions) error {
+	panic("unexpected ContainerRemove")
+}
+
+// createdHelper is what a fakeHelper was asked to create.
+type createdHelper struct {
+	cfg  *container.Config
+	host *container.HostConfig
+}
+
+// fakeHelper is a helper container the test controls. Its logs are framed
+// like Docker's multiplexed stream.
+type fakeHelper struct {
+	createErr error
+	exit      int64
+	hold      bool   // ContainerWait never reports an exit
+	logs      string // what the helper printed
+	onStart   func()
+	state     container.ContainerState // what an inspect reports; exited when empty
+
+	created  []createdHelper
+	started  []string
+	removed  []string
+	waitCond container.WaitCondition
+}
+
+func (h *fakeHelper) ContainerCreate(_ context.Context, cfg *container.Config, host *container.HostConfig, _ *network.NetworkingConfig, _ *ocispec.Platform, _ string) (container.CreateResponse, error) {
+	if h.createErr != nil {
+		return container.CreateResponse{}, h.createErr
+	}
+	h.created = append(h.created, createdHelper{cfg, host})
+	return container.CreateResponse{ID: "helper1"}, nil
+}
+
+func (h *fakeHelper) ContainerInspect(context.Context, string) (container.InspectResponse, error) {
+	state := h.state
+	if state == "" {
+		state = container.StateExited
+	}
+	return container.InspectResponse{ContainerJSONBase: &container.ContainerJSONBase{State: &container.State{Status: state}}}, nil
+}
+
+func (h *fakeHelper) ContainerStart(_ context.Context, id string, _ container.StartOptions) error {
+	h.started = append(h.started, id)
+	if h.onStart != nil {
+		h.onStart()
+	}
+	return nil
+}
+
+func (h *fakeHelper) ContainerWait(_ context.Context, _ string, cond container.WaitCondition) (<-chan container.WaitResponse, <-chan error) {
+	h.waitCond = cond
+	res, errs := make(chan container.WaitResponse, 1), make(chan error, 1)
+	if !h.hold {
+		res <- container.WaitResponse{StatusCode: h.exit}
+	}
+	return res, errs
+}
+
+func (h *fakeHelper) ContainerLogs(context.Context, string, container.LogsOptions) (io.ReadCloser, error) {
+	var buf bytes.Buffer
+	stdcopy.NewStdWriter(&buf, stdcopy.Stdout).Write([]byte(h.logs))
+	return io.NopCloser(&buf), nil
+}
+
+func (h *fakeHelper) ContainerRemove(_ context.Context, id string, opts container.RemoveOptions) error {
+	if !opts.Force {
+		return errors.New("remove without force")
+	}
+	h.removed = append(h.removed, id)
+	return nil
+}
+
+// fakeHelperCompose is a fakeCompose whose helper containers are h.
+type fakeHelperCompose struct {
+	fakeCompose
+	h *fakeHelper
+}
+
+// ContainerInspect answers for the agent's own container ("me") like fakeCompose
+// and for any other ID like the helper.
+func (f fakeHelperCompose) ContainerInspect(ctx context.Context, id string) (container.InspectResponse, error) {
+	if id == "me" {
+		return f.fakeCompose.ContainerInspect(ctx, id)
+	}
+	return f.h.ContainerInspect(ctx, id)
+}
+func (f fakeHelperCompose) ContainerCreate(ctx context.Context, cfg *container.Config, host *container.HostConfig, n *network.NetworkingConfig, p *ocispec.Platform, name string) (container.CreateResponse, error) {
+	return f.h.ContainerCreate(ctx, cfg, host, n, p, name)
+}
+func (f fakeHelperCompose) ContainerStart(ctx context.Context, id string, o container.StartOptions) error {
+	return f.h.ContainerStart(ctx, id, o)
+}
+func (f fakeHelperCompose) ContainerWait(ctx context.Context, id string, c container.WaitCondition) (<-chan container.WaitResponse, <-chan error) {
+	return f.h.ContainerWait(ctx, id, c)
+}
+func (f fakeHelperCompose) ContainerLogs(ctx context.Context, id string, o container.LogsOptions) (io.ReadCloser, error) {
+	return f.h.ContainerLogs(ctx, id, o)
+}
+func (f fakeHelperCompose) ContainerRemove(ctx context.Context, id string, o container.RemoveOptions) error {
+	return f.h.ContainerRemove(ctx, id, o)
+}
+
 // outRunner is a fake OutputRunner for `docker compose config`.
 type outRunner struct {
 	json string
@@ -70,7 +193,7 @@ func TestComposeExecutor(t *testing.T) {
 	stack := filepath.Join(root, "slusk")
 	require.NoError(t, os.MkdirAll(stack, 0o755))
 	composeFile := stackFile(t, stack)
-	job := protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "gitea.shcizo.se/shcizo/slusk:1.4"}
+	job := Job{ID: "7", Started: time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC), JobStart: protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "gitea.shcizo.se/shcizo/slusk:1.4"}}
 	pinned := job.Ref + "@" + d
 
 	ct := container.Summary{Labels: map[string]string{
@@ -319,3 +442,139 @@ func TestExecDocker_UsesMinimalEnv(t *testing.T) {
 	assert.NotContains(t, string(raw), "AGENT_TOKEN")
 	assert.Contains(t, string(raw), "DOCKER_CONFIG=/docker-config")
 }
+
+func TestComposeSelfUpdate(t *testing.T) {
+	const d = "sha256:d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1"
+	root, _ := filepath.EvalSymlinks(t.TempDir())
+	stack := filepath.Join(root, "updater")
+	require.NoError(t, os.MkdirAll(stack, 0o755))
+	stackFile(t, stack)
+	job := Job{ID: "7", Started: time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC), JobStart: protocol.JobStart{
+		Kind: protocol.KindCompose, ComposeProject: "updater", ComposeService: "agent", Ref: "gitea.shcizo.se/shcizo/package-updater:1.4", Self: true}}
+	pinned := job.Ref + "@" + d
+	overridePath := filepath.Join(stack, OverrideName)
+	t.Cleanup(func() { os.Remove(overridePath) })
+
+	selfCt := container.Summary{ID: "selfid", Labels: map[string]string{
+		labelWorkingDir:  stack,
+		labelConfigFiles: filepath.Join(stack, "compose.yaml"),
+	}}
+	mounts := []container.MountPoint{
+		{Type: mount.TypeBind, Source: root, Destination: root, RW: true},
+		{Type: mount.TypeBind, Source: "/var/run/docker.sock", Destination: "/var/run/docker.sock", RW: true},
+		{Type: mount.TypeBind, Source: "/root/.docker/config.json", Destination: "/docker-config/config.json", RW: false},
+		{Type: mount.TypeVolume, Name: "agent-data", Source: "/var/lib/docker/volumes/agent-data/_data", Destination: "/data", RW: true},
+		{Type: mount.TypeTmpfs, Destination: "/tmp", RW: true},
+	}
+	self := fakeSelf{id: "selfid", image: "sha256:agentimage", mounts: mounts}
+	wantMounts := []mount.Mount{
+		{Type: mount.TypeBind, Source: root, Target: root},
+		{Type: mount.TypeBind, Source: "/var/run/docker.sock", Target: "/var/run/docker.sock"},
+		{Type: mount.TypeBind, Source: "/root/.docker/config.json", Target: "/docker-config/config.json", ReadOnly: true},
+		{Type: mount.TypeVolume, Source: "agent-data", Target: "/data"},
+	}
+	base := []string{"compose", "-p", "updater", "--project-directory", stack,
+		"-f", filepath.Join(stack, "compose.yaml"), "-f", overridePath}
+	withStep := func(step ...string) []string { return append(append([]string(nil), base...), step...) }
+
+	t.Setenv("PATH", "/usr/bin")
+	t.Setenv("DOCKER_CONFIG", "/docker-config")
+	t.Setenv("AGENT_TOKEN", "secret")
+
+	cases := []struct {
+		name        string
+		helper      *fakeHelper
+		cancelled   bool // ctx is cancelled when the helper starts
+		wantErr     string
+		wantCancel  bool
+		wantRemoved []string
+		wantOut     []string
+	}{
+		{name: "helper succeeds", helper: &fakeHelper{logs: "Container updater-agent-1  Recreated\n"},
+			wantRemoved: []string{"helper1"}, wantOut: []string{"Container updater-agent-1  Recreated"}},
+		{name: "helper fails", helper: &fakeHelper{exit: 1, logs: "network not found\n"},
+			wantErr: "docker compose up: exit status 1 (output: network not found)", wantRemoved: []string{"helper1"},
+			wantOut: []string{"network not found"}},
+		{name: "ctx ends while waiting", helper: &fakeHelper{hold: true}, cancelled: true, wantCancel: true},
+		{name: "create fails", helper: &fakeHelper{createErr: errors.New("no such image")}, wantErr: "create update helper: no such image"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			t.Cleanup(func() { os.Remove(overridePath) })
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			if c.cancelled {
+				c.helper.onStart = cancel
+			}
+			rec := &recorder{}
+			cli := fakeHelperCompose{fakeCompose{self, []container.Summary{selfCt}}, c.helper}
+			e := NewComposeExecutor(cli, NewResolver(&fakeDist{digest: d}, nil), "me", rec.run, (&outRunner{json: `{"services":{"agent":{"image":"x"}}}`}).run)
+
+			var out strings.Builder
+			got, err := e.Execute(ctx, job, &out)
+
+			require.Len(t, rec.cmds, 1, "pull ran, up did not")
+			assert.Equal(t, ranCmd{stack, withStep("pull", "agent")}, rec.cmds[0])
+			assert.Equal(t, c.wantRemoved, c.helper.removed)
+			for _, s := range c.wantOut {
+				assert.Contains(t, out.String(), s)
+			}
+			switch {
+			case c.wantCancel:
+				assert.ErrorIs(t, err, context.Canceled)
+			case c.wantErr != "":
+				assert.EqualError(t, err, c.wantErr)
+				assert.Empty(t, got)
+			default:
+				require.NoError(t, err)
+				assert.Equal(t, pinned, got)
+			}
+			if len(c.helper.created) == 0 {
+				return
+			}
+			cfg, host := c.helper.created[0].cfg, c.helper.created[0].host
+			assert.Equal(t, "sha256:agentimage", cfg.Image)
+			assert.Equal(t, []string{"docker"}, []string(cfg.Entrypoint))
+			assert.Equal(t, withStep("up", "-d", "agent"), []string(cfg.Cmd))
+			assert.Equal(t, stack, cfg.WorkingDir)
+			assert.Equal(t, dockerEnv(), cfg.Env)
+			assert.Equal(t, wantMounts, host.Mounts)
+			assert.False(t, host.AutoRemove)
+			assert.Equal(t, container.RestartPolicy{}, host.RestartPolicy)
+			assert.Equal(t, []string{"helper1"}, c.helper.started)
+			assert.Equal(t, container.WaitConditionNextExit, c.helper.waitCond)
+			labels := cfg.Labels
+			assert.Len(t, labels, 6, "no compose labels")
+			assert.Equal(t, "7", labels[labelSelfUpdateJob])
+			assert.Equal(t, pinned, labels[labelSelfUpdateRef])
+			assert.Equal(t, "2026-10-13T09:00:00Z", labels[labelSelfUpdateStarted])
+			assert.Equal(t, "updater", labels[labelSelfUpdateProject])
+			assert.Equal(t, "agent", labels[labelSelfUpdateService])
+			assert.Contains(t, labels[labelSelfUpdateOutput], "resolved "+pinned)
+			assert.Contains(t, labels[labelSelfUpdateOutput], "$ docker "+strings.Join(withStep("pull", "agent"), " "))
+		})
+	}
+
+	t.Run("a service that is not the agent's own never uses a helper", func(t *testing.T) {
+		t.Cleanup(func() { os.Remove(overridePath) })
+		other := self
+		other.id = "otherid" // fakeCompose panics on ContainerCreate
+		rec := &recorder{}
+		e := NewComposeExecutor(fakeCompose{other, []container.Summary{selfCt}}, NewResolver(&fakeDist{digest: d}, nil), "me", rec.run,
+			(&outRunner{json: `{"services":{"agent":{"image":"x"}}}`}).run)
+		got, err := e.Execute(context.Background(), job, io.Discard)
+		require.NoError(t, err)
+		assert.Equal(t, pinned, got)
+		require.Len(t, rec.cmds, 2)
+		assert.Equal(t, withStep("up", "-d", "agent"), rec.cmds[1].args)
+	})
+
+	t.Run("output label keeps the tail and says it was cut", func(t *testing.T) {
+		b := &tailBuffer{max: 4}
+		b.Write([]byte("abcdef"))
+		assert.Equal(t, truncatedMarker+"cdef", outputLabel(b))
+		b2 := &tailBuffer{max: 3}
+		b2.Write([]byte("aéé")) // the cut falls inside a character
+		assert.True(t, utf8.ValidString(outputLabel(b2)))
+	})
+}

internal/agent/updater/dispatch.go

@@ -16,7 +16,7 @@ func NewDispatch(compose, swarm Executor) Executor {
 	return dispatch{compose: compose, swarm: swarm}
 }
 
-func (d dispatch) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+func (d dispatch) Execute(ctx context.Context, job Job, out io.Writer) (string, error) {
 	switch job.Kind {
 	case protocol.KindCompose:
 		return d.compose.Execute(ctx, job, out)

internal/agent/updater/dispatch_test.go

@@ -12,20 +12,20 @@ import (
 )
 
 func TestDispatch(t *testing.T) {
-	compose := execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "compose", nil })
-	swarm := execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "swarm", nil })
+	compose := execFunc(func(context.Context, Job, io.Writer) (string, error) { return "compose", nil })
+	swarm := execFunc(func(context.Context, Job, io.Writer) (string, error) { return "swarm", nil })
 	d := NewDispatch(compose, swarm)
 
-	got, err := d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindCompose}, io.Discard)
+	got, err := d.Execute(context.Background(), Job{JobStart: protocol.JobStart{Kind: protocol.KindCompose}}, io.Discard)
 	require.NoError(t, err)
 	assert.Equal(t, "compose", got)
 
-	got, err = d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindSwarm}, io.Discard)
+	got, err = d.Execute(context.Background(), Job{JobStart: protocol.JobStart{Kind: protocol.KindSwarm}}, io.Discard)
 	require.NoError(t, err)
 	assert.Equal(t, "swarm", got)
 
-	_, err = d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindContainer}, io.Discard)
+	_, err = d.Execute(context.Background(), Job{JobStart: protocol.JobStart{Kind: protocol.KindContainer}}, io.Discard)
 	assert.ErrorContains(t, err, "cannot update")
-	_, err = d.Execute(context.Background(), protocol.JobStart{Kind: "weird"}, io.Discard)
+	_, err = d.Execute(context.Background(), Job{JobStart: protocol.JobStart{Kind: "weird"}}, io.Discard)
 	assert.ErrorContains(t, err, "cannot update")
 }

internal/agent/updater/override.go

@@ -177,20 +177,35 @@ var errNoRoots = errors.New("refused: no stack directory is mounted into the age
 // path inside the container. Compose labels hold host paths, so only those
 // mounts make a label path usable as is.
 func MountRoots(ctx context.Context, cli SelfInspector, selfID string) ([]string, error) {
+	insp, err := inspectSelf(ctx, cli, selfID)
+	if err != nil {
+		return nil, err
+	}
+	return rootsOf(insp), nil
+}
+
+// inspectSelf inspects the agent's own container.
+func inspectSelf(ctx context.Context, cli SelfInspector, selfID string) (container.InspectResponse, error) {
 	if selfID == "" {
-		return nil, errNoRoots
+		return container.InspectResponse{}, errNoRoots
 	}
 	insp, err := cli.ContainerInspect(ctx, selfID)
 	if err != nil {
-		return nil, fmt.Errorf("inspect own container %s: %w", selfID, err)
+		return container.InspectResponse{}, fmt.Errorf("inspect own container %s: %w", selfID, err)
 	}
+	return insp, nil
+}
+
+// rootsOf returns the directories MountRoots describes, from an inspected
+// container.
+func rootsOf(insp container.InspectResponse) []string {
 	var roots []string
 	for _, m := range insp.Mounts {
 		if m.Type == mount.TypeBind && m.RW && filepath.Clean(m.Source) == filepath.Clean(m.Destination) {
 			roots = append(roots, filepath.Clean(m.Destination))
 		}
 	}
-	return roots, nil
+	return roots
 }
 
 // composeFiles returns the -f arguments for a job: the stack's config files

internal/agent/updater/override_test.go

@@ -175,10 +175,20 @@ func TestWriteOverride(t *testing.T) {
 type fakeSelf struct {
 	mounts []container.MountPoint
 	err    error
+	id     string            // the container's ID, when set
+	image  string            // the container's image ID, when set
+	labels map[string]string // the container's labels, when set
 }
 
 func (f fakeSelf) ContainerInspect(context.Context, string) (container.InspectResponse, error) {
-	return container.InspectResponse{Mounts: f.mounts}, f.err
+	insp := container.InspectResponse{Mounts: f.mounts}
+	if f.id != "" || f.image != "" {
+		insp.ContainerJSONBase = &container.ContainerJSONBase{ID: f.id, Image: f.image}
+	}
+	if f.labels != nil {
+		insp.Config = &container.Config{Labels: f.labels}
+	}
+	return insp, f.err
 }
 
 func TestMountRoots(t *testing.T) {

internal/agent/updater/runner.go

@@ -16,12 +16,25 @@ import (
 // chatty job cannot grow a single job.progress message without bound.
 const maxProgress = 32 << 10
 
+// Job is a job as an Executor sees it: what the master asked for, plus the
+// ID and start time the runner gave it. A self-update records both in the
+// helper container, so the new agent can finish the job under the same ID.
+type Job struct {
+	protocol.JobStart
+	ID      string
+	Started time.Time
+}
+
 // Executor deploys one job and returns the "repo:tag@digest" it deployed.
 // Everything it writes to out is streamed to the master as job output.
 type Executor interface {
-	Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error)
+	Execute(ctx context.Context, job Job, out io.Writer) (string, error)
 }
 
+// ResumeFunc finishes a job that an earlier agent process began. It returns
+// the "repo:tag@digest" that was deployed, like Executor.Execute.
+type ResumeFunc func(ctx context.Context, out io.Writer) (string, error)
+
 // Progress is output a running job produced since the previous TakeProgress.
 type Progress struct{ ID, Output string }
 
@@ -52,6 +65,8 @@ func (b *tailBuffer) Reset()          { b.buf, b.truncated = nil, false }
 
 type jobState struct {
 	start    protocol.JobStart
+	resume   ResumeFunc // set for a job resumed after a restart; start is then empty
+	started  time.Time  // set with resume
 	running  bool
 	result   *protocol.JobResult
 	output   tailBuffer // everything, capped at protocol.MaxJobOutput
@@ -117,6 +132,28 @@ func (r *Runner) Submit(id string, start protocol.JobStart) {
 	signal(r.wake)
 }
 
+// Resume registers a job that an earlier agent process began and runs fn for
+// it, before every job that is already queued. The job is in Known at once,
+// so the agent's next hello does not report it lost. The result carries
+// started as its start time. A job whose ID the runner already holds is
+// ignored.
+func (r *Runner) Resume(id string, started time.Time, fn ResumeFunc) {
+	r.mu.Lock()
+	if _, ok := r.jobs[id]; ok {
+		r.mu.Unlock()
+		return
+	}
+	r.jobs[id] = &jobState{
+		resume:  fn,
+		started: started,
+		output:  tailBuffer{max: protocol.MaxJobOutput},
+		unsent:  tailBuffer{max: maxProgress},
+	}
+	r.queue = append([]string{id}, r.queue...)
+	r.mu.Unlock()
+	signal(r.wake)
+}
+
 // Ack forgets a finished job. Acks for unknown, queued or running jobs are
 // ignored.
 func (r *Runner) Ack(id string) {
@@ -196,10 +233,12 @@ func (r *Runner) Pending() []Finished {
 func (r *Runner) Changed() <-chan struct{} { return r.changed }
 
 // Run works through the queue until ctx ends. Jobs still queued then are
-// dropped; the master marks them lost on the agent's next hello.
+// dropped; the master marks them lost on the agent's next hello. A job that
+// ctx interrupts is dropped too, without a result: a self-updating agent is
+// stopped that way, and its successor reports the outcome.
 func (r *Runner) Run(ctx context.Context) {
 	for {
-		id, start, ok := r.next()
+		id, ok := r.next()
 		if !ok {
 			select {
 			case <-ctx.Done():
@@ -211,32 +250,62 @@ func (r *Runner) Run(ctx context.Context) {
 		if ctx.Err() != nil {
 			return
 		}
-		r.runOne(ctx, id, start)
+		r.runOne(ctx, id)
 	}
 }
 
-// next pops the next queued job and marks it running.
-func (r *Runner) next() (string, protocol.JobStart, bool) {
+// next pops the next job to run and marks it running. Jobs for the agent's
+// own service run last: a master and an agent on the same host share an
+// image, and the master's job must finish before the agent replaces itself.
+func (r *Runner) next() (string, bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if len(r.queue) == 0 {
-		return "", protocol.JobStart{}, false
+		return "", false
 	}
-	id := r.queue[0]
-	r.queue = r.queue[1:]
-	j := r.jobs[id]
-	j.running = true
+	pick := 0
+	for i, id := range r.queue {
+		if !r.jobs[id].start.Self {
+			pick = i
+			break
+		}
+	}
+	id := r.queue[pick]
+	r.queue = append(r.queue[:pick:pick], r.queue[pick+1:]...)
+	r.jobs[id].running = true
 	signal(r.changed)
-	return id, j.start, true
+	return id, true
 }
 
-func (r *Runner) runOne(ctx context.Context, id string, start protocol.JobStart) {
-	started := r.now()
+func (r *Runner) runOne(ctx context.Context, id string) {
+	r.mu.Lock()
+	j := r.jobs[id]
+	start, resume, started := j.start, j.resume, j.started
+	r.mu.Unlock()
+	if resume == nil {
+		started = r.now()
+	}
+
 	jobCtx, cancel := context.WithTimeout(ctx, r.timeout)
-	ref, err := r.exec.Execute(jobCtx, start, &jobWriter{r: r, id: id})
+	var ref string
+	var err error
+	if resume != nil {
+		ref, err = resume(jobCtx, &jobWriter{r: r, id: id})
+	} else {
+		ref, err = r.exec.Execute(jobCtx, Job{JobStart: start, ID: id, Started: started}, &jobWriter{r: r, id: id})
+	}
 	timedOut := errors.Is(jobCtx.Err(), context.DeadlineExceeded)
 	cancel()
 
+	if err != nil && ctx.Err() != nil && !timedOut {
+		// The agent is stopping, not the job failing. Nothing is recorded, so
+		// no result can pre-empt the one a successor agent may report.
+		r.mu.Lock()
+		delete(r.jobs, id)
+		r.mu.Unlock()
+		return
+	}
+
 	res := protocol.JobResult{Status: protocol.ResultSucceeded, Ref: ref, StartedAt: started, FinishedAt: r.now()}
 	switch {
 	case timedOut:
@@ -246,7 +315,6 @@ func (r *Runner) runOne(ctx context.Context, id string, start protocol.JobStart)
 	}
 
 	r.mu.Lock()
-	j := r.jobs[id]
 	res.Output, res.OutputTruncated = j.output.String(), j.output.Truncated()
 	j.running, j.result, j.finished = false, &res, res.FinishedAt
 	j.unsent.Reset()

internal/agent/updater/runner_test.go

@@ -29,7 +29,7 @@ func newFakeExec() *fakeExec {
 	return &fakeExec{gate: make(chan struct{}), started: make(chan string, 10)}
 }
 
-func (f *fakeExec) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+func (f *fakeExec) Execute(ctx context.Context, job Job, out io.Writer) (string, error) {
 	n := f.cur.Add(1)
 	defer f.cur.Add(-1)
 	for {
@@ -148,7 +148,7 @@ func TestRunner_TakeProgress(t *testing.T) {
 func TestRunner_TakeProgressReportsStartedJobOnce(t *testing.T) {
 	gate := make(chan struct{})
 	started := make(chan struct{})
-	r := startRunner(t, execFunc(func(ctx context.Context, _ protocol.JobStart, _ io.Writer) (string, error) {
+	r := startRunner(t, execFunc(func(ctx context.Context, _ Job, _ io.Writer) (string, error) {
 		close(started)
 		<-gate
 		return "x", nil
@@ -160,14 +160,14 @@ func TestRunner_TakeProgressReportsStartedJobOnce(t *testing.T) {
 	close(gate)
 }
 
-type execFunc func(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error)
+type execFunc func(ctx context.Context, job Job, out io.Writer) (string, error)
 
-func (f execFunc) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+func (f execFunc) Execute(ctx context.Context, job Job, out io.Writer) (string, error) {
 	return f(ctx, job, out)
 }
 
 func TestRunner_Timeout(t *testing.T) {
-	r := startRunner(t, execFunc(func(ctx context.Context, _ protocol.JobStart, _ io.Writer) (string, error) {
+	r := startRunner(t, execFunc(func(ctx context.Context, _ Job, _ io.Writer) (string, error) {
 		<-ctx.Done()
 		return "", ctx.Err()
 	}), 20*time.Millisecond)
@@ -178,7 +178,7 @@ func TestRunner_Timeout(t *testing.T) {
 }
 
 func TestRunner_OutputCapped(t *testing.T) {
-	r := startRunner(t, execFunc(func(_ context.Context, _ protocol.JobStart, out io.Writer) (string, error) {
+	r := startRunner(t, execFunc(func(_ context.Context, _ Job, out io.Writer) (string, error) {
 		io.WriteString(out, strings.Repeat("a", 10))
 		io.WriteString(out, strings.Repeat("b", protocol.MaxJobOutput))
 		return "x", nil
@@ -190,7 +190,7 @@ func TestRunner_OutputCapped(t *testing.T) {
 }
 
 func TestRunner_ChangedSignals(t *testing.T) {
-	r := startRunner(t, execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "x", nil }), time.Minute)
+	r := startRunner(t, execFunc(func(context.Context, Job, io.Writer) (string, error) { return "x", nil }), time.Minute)
 	r.Submit("1", protocol.JobStart{Ref: "a:1"})
 	select {
 	case <-r.Changed():
@@ -198,3 +198,105 @@ func TestRunner_ChangedSignals(t *testing.T) {
 		t.Fatal("no change signal")
 	}
 }
+
+func TestRunner_ResumeJumpsTheQueue(t *testing.T) {
+	fe := newFakeExec()
+	r := NewRunner(fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	started := time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC)
+	r.Resume("9", started, func(_ context.Context, out io.Writer) (string, error) {
+		io.WriteString(out, "from the helper\n")
+		return "resumed@sha256:bb", nil
+	})
+	assert.Equal(t, []string{"1", "9"}, r.Known(), "known before the worker starts")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() { defer wg.Done(); r.Run(ctx) }()
+	t.Cleanup(func() { cancel(); wg.Wait() })
+
+	got := waitPending(t, r, 1)
+	require.Len(t, got, 1, "the resumed job runs first; job 1 waits on the gate")
+	assert.Equal(t, "9", got[0].ID)
+	assert.Equal(t, protocol.ResultSucceeded, got[0].Result.Status)
+	assert.Equal(t, "resumed@sha256:bb", got[0].Result.Ref)
+	assert.Equal(t, "from the helper\n", got[0].Result.Output)
+	assert.True(t, started.Equal(got[0].Result.StartedAt))
+
+	r.Resume("9", started, nil) // already held: ignored
+	assert.Equal(t, "a:1", <-fe.started)
+	fe.gate <- struct{}{}
+	waitPending(t, r, 2)
+}
+
+func TestRunner_ResumedJobFailure(t *testing.T) {
+	r := startRunner(t, newFakeExec(), time.Minute)
+	r.Resume("9", time.Now(), func(context.Context, io.Writer) (string, error) { return "", errors.New("helper failed") })
+	got := waitPending(t, r, 1)
+	assert.Equal(t, protocol.ResultFailed, got[0].Result.Status)
+	assert.Equal(t, "helper failed", got[0].Result.Error)
+}
+
+func TestRunner_SelfJobsRunLast(t *testing.T) {
+	fe := newFakeExec()
+	r := NewRunner(fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "self:1", Self: true})
+	r.Submit("2", protocol.JobStart{Ref: "a:2"})
+	r.Submit("3", protocol.JobStart{Ref: "self:3", Self: true})
+	r.Submit("4", protocol.JobStart{Ref: "a:4"})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() { defer wg.Done(); r.Run(ctx) }()
+	t.Cleanup(func() { cancel(); wg.Wait() })
+
+	var order []string
+	for i := 0; i < 4; i++ {
+		order = append(order, <-fe.started)
+		fe.gate <- struct{}{}
+	}
+	assert.Equal(t, []string{"a:2", "a:4", "self:1", "self:3"}, order)
+}
+
+func TestRunner_SelfJobRunsWhenAlone(t *testing.T) {
+	fe := newFakeExec()
+	r := startRunner(t, fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "self:1", Self: true})
+	assert.Equal(t, "self:1", <-fe.started)
+	fe.gate <- struct{}{}
+	waitPending(t, r, 1)
+}
+
+func TestRunner_StoppingTheAgentRecordsNoResult(t *testing.T) {
+	r := NewRunner(execFunc(func(ctx context.Context, _ Job, _ io.Writer) (string, error) {
+		<-ctx.Done()
+		return "", ctx.Err()
+	}), time.Minute)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { r.Run(ctx); close(done) }()
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	require.Eventually(t, func() bool { return len(r.TakeProgress()) > 0 }, time.Second, 5*time.Millisecond)
+
+	cancel()
+	<-done
+	assert.Empty(t, r.Pending(), "no \"context canceled\" result")
+	assert.Empty(t, r.Known(), "the job is dropped, so the master marks it lost unless a successor resumes it")
+}
+
+func TestRunner_ExecutorGetsIDAndStart(t *testing.T) {
+	var got Job
+	r := startRunner(t, execFunc(func(_ context.Context, job Job, _ io.Writer) (string, error) {
+		got = job
+		return "x", nil
+	}), time.Minute)
+	now := time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC)
+	r.now = func() time.Time { return now }
+	r.Submit("5", protocol.JobStart{Ref: "a:1"})
+	waitPending(t, r, 1)
+	assert.Equal(t, "5", got.ID)
+	assert.Equal(t, "a:1", got.Ref)
+	assert.True(t, now.Equal(got.Started))
+}

internal/agent/updater/selfupdate.go

@@ -0,0 +1,279 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/pkg/stdcopy"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Labels on a self-update helper container. The new agent finds the helper
+// by them, and the inventory skips containers that carry the job label.
+const (
+	labelSelfUpdateJob     = "se.shcizo.package-updater.self-update.job"
+	labelSelfUpdateRef     = "se.shcizo.package-updater.self-update.ref"
+	labelSelfUpdateStarted = "se.shcizo.package-updater.self-update.started"
+	labelSelfUpdateProject = "se.shcizo.package-updater.self-update.project"
+	labelSelfUpdateService = "se.shcizo.package-updater.self-update.service"
+	labelSelfUpdateOutput  = "se.shcizo.package-updater.self-update.output"
+)
+
+// selfUpdateOutputMax caps the job output stored in the helper container's
+// output label. The old agent's output so far is all the new agent has of it.
+const selfUpdateOutputMax = 32 << 10
+
+// truncatedMarker starts the stored output when its head was cut off.
+const truncatedMarker = "…truncated\n"
+
+// resumeSeparator follows the old agent's output in a resumed job's output.
+const resumeSeparator = "--- agent restarted; output of the update helper ---\n"
+
+// HelperDocker is the subset of the Docker SDK the self-update helper
+// container needs. *client.Client satisfies it.
+type HelperDocker interface {
+	ContainerInspect(ctx context.Context, containerID string) (container.InspectResponse, error)
+	ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *ocispec.Platform, containerName string) (container.CreateResponse, error)
+	ContainerStart(ctx context.Context, containerID string, options container.StartOptions) error
+	ContainerWait(ctx context.Context, containerID string, condition container.WaitCondition) (<-chan container.WaitResponse, <-chan error)
+	ContainerLogs(ctx context.Context, containerID string, options container.LogsOptions) (io.ReadCloser, error)
+	ContainerRemove(ctx context.Context, containerID string, options container.RemoveOptions) error
+}
+
+// selfUpdate runs `up -d` for the agent's own service in a short-lived helper
+// container. Recreating the service stops this process, so the helper, made
+// from the agent's own image with the agent's own mounts, does it and
+// outlives the agent. args are the complete `docker` arguments of the call.
+//
+// If the helper exits while this agent still runs, selfUpdate copies its logs
+// into the job output, removes it and returns the outcome. If ctx ends first
+// (the agent is being replaced, or the job timed out) it returns ctx's error
+// and leaves the helper to the new agent, see ResumeSelfUpdates.
+func (e *ComposeExecutor) selfUpdate(ctx context.Context, job Job, self container.InspectResponse, workingDir string, args []string, pinned string, out io.Writer, soFar *tailBuffer) (string, error) {
+	fmt.Fprintf(out, "$ docker %s (in an update helper container)\n", strings.Join(args, " "))
+	cfg := &container.Config{
+		Image:      self.Image,
+		Entrypoint: []string{"docker"},
+		Cmd:        args,
+		WorkingDir: workingDir,
+		Env:        dockerEnv(),
+		Labels: map[string]string{
+			labelSelfUpdateJob:     job.ID,
+			labelSelfUpdateRef:     pinned,
+			labelSelfUpdateStarted: job.Started.UTC().Format(time.RFC3339Nano),
+			labelSelfUpdateProject: job.ComposeProject,
+			labelSelfUpdateService: job.ComposeService,
+			labelSelfUpdateOutput:  outputLabel(soFar),
+		},
+	}
+	created, err := e.cli.ContainerCreate(ctx, cfg, &container.HostConfig{Mounts: helperMounts(self)}, nil, nil, "")
+	if err != nil {
+		return "", fmt.Errorf("create update helper: %w", err)
+	}
+	// Wait is registered before the start so a fast exit is not missed.
+	waitCh, errCh := e.cli.ContainerWait(ctx, created.ID, container.WaitConditionNextExit)
+	if err := e.cli.ContainerStart(ctx, created.ID, container.StartOptions{}); err != nil {
+		if rmErr := e.cli.ContainerRemove(context.WithoutCancel(ctx), created.ID, container.RemoveOptions{Force: true}); rmErr != nil {
+			fmt.Fprintf(out, "warning: remove the update helper: %v\n", rmErr)
+		}
+		return "", fmt.Errorf("start update helper: %w", err)
+	}
+	code, err := awaitExit(ctx, waitCh, errCh)
+	if err != nil {
+		return "", err
+	}
+
+	tail := &tailBuffer{max: errorOutputTail}
+	if err := finishHelper(ctx, e.cli, created.ID, io.MultiWriter(out, tailWriter{tail})); err != nil {
+		fmt.Fprintf(out, "warning: %v\n", err)
+	}
+	if code != 0 {
+		return "", exitError(code, tail)
+	}
+	return pinned, nil
+}
+
+// outputLabel returns the job output so far as a label value.
+func outputLabel(b *tailBuffer) string {
+	s := strings.ToValidUTF8(b.String(), "")
+	if b.Truncated() {
+		s = truncatedMarker + s
+	}
+	return s
+}
+
+// helperMounts copies the agent's own bind and volume mounts: the Docker
+// socket, the stack directories and the registry credentials the compose call
+// needs are all among them.
+func helperMounts(self container.InspectResponse) []mount.Mount {
+	var mounts []mount.Mount
+	for _, m := range self.Mounts {
+		switch m.Type {
+		case mount.TypeBind:
+			mounts = append(mounts, mount.Mount{Type: mount.TypeBind, Source: m.Source, Target: m.Destination, ReadOnly: !m.RW})
+		case mount.TypeVolume:
+			mounts = append(mounts, mount.Mount{Type: mount.TypeVolume, Source: m.Name, Target: m.Destination, ReadOnly: !m.RW})
+		}
+	}
+	return mounts
+}
+
+// awaitExit waits for a ContainerWait result and returns the exit code. When
+// ctx ends it returns ctx's error.
+func awaitExit(ctx context.Context, waitCh <-chan container.WaitResponse, errCh <-chan error) (int64, error) {
+	select {
+	case r := <-waitCh:
+		if r.Error != nil {
+			return 0, fmt.Errorf("wait for update helper: %s", r.Error.Message)
+		}
+		return r.StatusCode, nil
+	case err := <-errCh:
+		if ctx.Err() != nil {
+			return 0, ctx.Err()
+		}
+		return 0, fmt.Errorf("wait for update helper: %w", err)
+	case <-ctx.Done():
+		return 0, ctx.Err()
+	}
+}
+
+// finishHelper copies the exited helper's logs to w and removes the helper.
+// Both are attempted; the first failure is returned.
+func finishHelper(ctx context.Context, cli HelperDocker, id string, w io.Writer) error {
+	var first error
+	rc, err := cli.ContainerLogs(ctx, id, container.LogsOptions{ShowStdout: true, ShowStderr: true})
+	if err == nil {
+		_, err = stdcopy.StdCopy(w, w, rc)
+		rc.Close()
+	}
+	if err != nil {
+		first = fmt.Errorf("read the update helper's logs: %w", err)
+	}
+	if err := cli.ContainerRemove(ctx, id, container.RemoveOptions{Force: true}); err != nil && first == nil {
+		first = fmt.Errorf("remove the update helper: %w", err)
+	}
+	return first
+}
+
+// exitError describes a failed helper like a failed `docker compose` call.
+func exitError(code int64, tail *tailBuffer) error {
+	return composeError("up", fmt.Errorf("exit status %d", code), tail)
+}
+
+// ResumeDocker is the subset of the Docker SDK ResumeSelfUpdates needs.
+// *client.Client satisfies it.
+type ResumeDocker interface {
+	SelfInspector
+	HelperDocker
+	ContainerList(ctx context.Context, opts container.ListOptions) ([]container.Summary, error)
+}
+
+// ResumeSelfUpdates finishes the jobs a previous agent process started with
+// selfUpdate. It finds the helper containers made for this agent's own
+// Compose service and registers each with runner.Resume, which reports the
+// job under its old ID once the helper has exited. Call it before the runner
+// and the connection start, so the first hello already lists these jobs. A
+// helper with unusable labels is resumed too; its job fails with an error
+// that says so.
+func ResumeSelfUpdates(ctx context.Context, cli ResumeDocker, selfID string, runner *Runner, log *slog.Logger) error {
+	self, err := inspectSelf(ctx, cli, selfID)
+	if err != nil {
+		return err
+	}
+	if self.Config == nil {
+		return nil
+	}
+	project, service := self.Config.Labels[labelComposeProject], self.Config.Labels[labelComposeService]
+	if project == "" || service == "" {
+		return nil
+	}
+	cts, err := cli.ContainerList(ctx, container.ListOptions{
+		All:     true,
+		Filters: filters.NewArgs(filters.Arg("label", labelSelfUpdateJob)),
+	})
+	if err != nil {
+		return fmt.Errorf("list update helpers: %w", err)
+	}
+	for _, ct := range cts {
+		if ct.Labels[labelSelfUpdateProject] != project || ct.Labels[labelSelfUpdateService] != service {
+			continue
+		}
+		jobID := ct.Labels[labelSelfUpdateJob]
+		if jobID == "" {
+			log.Warn("self_update_helper_without_job", "helper", ct.ID)
+			continue
+		}
+		ref, started, labelErr := parseHelperLabels(ct.Labels)
+		if started.IsZero() {
+			started = time.Now()
+		}
+		log.Info("self_update_resumed", "job_id", jobID, "helper", ct.ID)
+		runner.Resume(jobID, started, resumeFunc(cli, ct.ID, ct.Labels[labelSelfUpdateOutput], ref, labelErr))
+	}
+	return nil
+}
+
+// parseHelperLabels reads the pinned ref and start time from a helper's
+// labels. started is zero when it is missing or unreadable, and err then says
+// what is wrong.
+func parseHelperLabels(labels map[string]string) (ref string, started time.Time, err error) {
+	ref = labels[labelSelfUpdateRef]
+	if ref == "" {
+		err = errors.New("update helper has no ref label")
+	}
+	raw := labels[labelSelfUpdateStarted]
+	if t, perr := time.Parse(time.RFC3339Nano, raw); perr == nil {
+		started = t
+	} else if err == nil {
+		err = fmt.Errorf("update helper has no usable started label %q", raw)
+	}
+	return ref, started, err
+}
+
+// resumeFunc waits for the helper to stop, then reports the old agent's
+// output, a separator line and the helper's own logs as the job output,
+// removes the helper and returns the pinned ref, or an error when the helper
+// failed or its labels were unusable.
+func resumeFunc(cli HelperDocker, helperID, earlier, ref string, labelErr error) ResumeFunc {
+	return func(ctx context.Context, out io.Writer) (string, error) {
+		// A helper that was created but never started would read as exited
+		// with code 0, so it must not be mistaken for a finished update.
+		if insp, err := cli.ContainerInspect(ctx, helperID); err == nil && insp.ContainerJSONBase != nil &&
+			insp.State != nil && insp.State.Status == container.StateCreated {
+			if err := cli.ContainerRemove(context.WithoutCancel(ctx), helperID, container.RemoveOptions{Force: true}); err != nil {
+				fmt.Fprintf(out, "warning: remove the update helper: %v\n", err)
+			}
+			return "", errors.New("update helper was never started")
+		}
+		waitCh, errCh := cli.ContainerWait(ctx, helperID, container.WaitConditionNotRunning)
+		code, err := awaitExit(ctx, waitCh, errCh)
+		if err != nil {
+			return "", err
+		}
+		io.WriteString(out, earlier)
+		if earlier != "" && !strings.HasSuffix(earlier, "\n") {
+			io.WriteString(out, "\n")
+		}
+		io.WriteString(out, resumeSeparator)
+		tail := &tailBuffer{max: errorOutputTail}
+		if err := finishHelper(ctx, cli, helperID, io.MultiWriter(out, tailWriter{tail})); err != nil {
+			fmt.Fprintf(out, "warning: %v\n", err)
+		}
+		switch {
+		case labelErr != nil:
+			return "", labelErr
+		case code != 0:
+			return "", exitError(code, tail)
+		}
+		return ref, nil
+	}
+}

internal/agent/updater/selfupdate_test.go

@@ -0,0 +1,116 @@
+package updater
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+func TestResumeSelfUpdates(t *testing.T) {
+	const pinned = "gitea.shcizo.se/shcizo/package-updater:1.4@sha256:aa"
+	started := time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC)
+	self := fakeSelf{id: "selfid", labels: map[string]string{labelComposeProject: "updater", labelComposeService: "agent"}}
+
+	helper := func(id, project, service string, extra map[string]string) container.Summary {
+		labels := map[string]string{
+			labelSelfUpdateJob:     "j-" + id,
+			labelSelfUpdateRef:     pinned,
+			labelSelfUpdateStarted: started.Format(time.RFC3339Nano),
+			labelSelfUpdateProject: project,
+			labelSelfUpdateService: service,
+			labelSelfUpdateOutput:  "resolved " + pinned + "\n$ docker compose pull agent\n",
+		}
+		for k, v := range extra {
+			if v == "" {
+				delete(labels, k)
+			} else {
+				labels[k] = v
+			}
+		}
+		return container.Summary{ID: id, Labels: labels}
+	}
+	separator := "--- agent restarted; output of the update helper ---\n"
+
+	cases := []struct {
+		name       string
+		helper     container.Summary
+		exit       int64
+		state      container.ContainerState
+		noStarted  bool // the old agent's start time is lost with the label
+		wantStatus string
+		wantErr    string
+		wantOutput string
+	}{
+		{name: "helper succeeded", helper: helper("h1", "updater", "agent", nil), wantStatus: protocol.ResultSucceeded,
+			wantOutput: "resolved " + pinned + "\n$ docker compose pull agent\n" + separator + "Recreated\n"},
+		{name: "helper failed", helper: helper("h1", "updater", "agent", nil), exit: 1, wantStatus: protocol.ResultFailed,
+			wantErr: "docker compose up: exit status 1 (output: Recreated)"},
+		{name: "no ref label", helper: helper("h1", "updater", "agent", map[string]string{labelSelfUpdateRef: ""}),
+			wantStatus: protocol.ResultFailed, wantErr: "update helper has no ref label"},
+		{name: "no started label", helper: helper("h1", "updater", "agent", map[string]string{labelSelfUpdateStarted: ""}),
+			noStarted:  true,
+			wantStatus: protocol.ResultFailed, wantErr: `update helper has no usable started label ""`},
+		{name: "helper never started", helper: helper("h1", "updater", "agent", nil), state: container.StateCreated,
+			wantStatus: protocol.ResultFailed, wantErr: "update helper was never started"},
+		{name: "no output label", helper: helper("h1", "updater", "agent", map[string]string{labelSelfUpdateOutput: ""}),
+			wantStatus: protocol.ResultSucceeded, wantOutput: separator + "Recreated\n"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			h := &fakeHelper{exit: c.exit, state: c.state, logs: "Recreated\n"}
+			unrelated := []container.Summary{
+				helper("other-project", "other", "agent", nil),
+				helper("other-service", "updater", "web", nil),
+			}
+			cli := fakeHelperCompose{fakeCompose{self, append(unrelated, c.helper)}, h}
+			r := NewRunner(execFunc(func(context.Context, Job, io.Writer) (string, error) {
+				t.Error("a resumed job must not reach the executor")
+				return "", nil
+			}), time.Minute)
+
+			require.NoError(t, ResumeSelfUpdates(context.Background(), cli, "me", r, slog.New(slog.NewTextHandler(io.Discard, nil))))
+			assert.Equal(t, []string{"j-h1"}, r.Known(), "known before the runner starts, so the first hello lists it")
+
+			ctx, cancel := context.WithCancel(context.Background())
+			var wg sync.WaitGroup
+			wg.Add(1)
+			go func() { defer wg.Done(); r.Run(ctx) }()
+			t.Cleanup(func() { cancel(); wg.Wait() })
+
+			got := waitPending(t, r, 1)
+			require.Len(t, got, 1)
+			assert.Equal(t, "j-h1", got[0].ID)
+			res := got[0].Result
+			assert.Equal(t, c.wantStatus, res.Status)
+			if c.state != container.StateCreated {
+				assert.Equal(t, container.WaitConditionNotRunning, h.waitCond)
+			}
+			assert.Equal(t, []string{"h1"}, h.removed, "only the matching helper is touched")
+			if c.wantStatus == protocol.ResultSucceeded {
+				assert.Equal(t, pinned, res.Ref)
+				assert.Equal(t, c.wantOutput, res.Output)
+			} else {
+				assert.Equal(t, c.wantErr, res.Error)
+			}
+			if !c.noStarted {
+				assert.True(t, started.Equal(res.StartedAt), "the old agent's start time is kept")
+			}
+		})
+	}
+
+	t.Run("an agent outside Compose resumes nothing", func(t *testing.T) {
+		cli := fakeHelperCompose{fakeCompose{fakeSelf{id: "selfid"}, []container.Summary{helper("h1", "updater", "agent", nil)}}, &fakeHelper{}}
+		r := NewRunner(nil, time.Minute)
+		require.NoError(t, ResumeSelfUpdates(context.Background(), cli, "me", r, slog.New(slog.NewTextHandler(io.Discard, nil))))
+		assert.Empty(t, r.Known())
+	})
+}

internal/agent/updater/swarm.go

@@ -9,7 +9,6 @@ import (
 	"github.com/docker/docker/api/types/system"
 
 	"github.com/shcizo/package-updater/internal/imageref"
-	"github.com/shcizo/package-updater/internal/protocol"
 )
 
 // SwarmClient is the subset of the Docker SDK SwarmExecutor depends on.
@@ -40,7 +39,7 @@ func NewSwarmExecutor(cli SwarmClient, resolve *Resolver, auth *RegistryAuth) *S
 // the service that was inspected, so it changes exactly that object. Success
 // means the daemon accepted the new spec, not that the tasks have converged.
 // It returns the pinned ref.
-func (e *SwarmExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+func (e *SwarmExecutor) Execute(ctx context.Context, job Job, out io.Writer) (string, error) {
 	info, err := e.cli.Info(ctx)
 	if err != nil {
 		return "", fmt.Errorf("docker info: %w", err)

internal/agent/updater/swarm_test.go

@@ -79,7 +79,7 @@ func swarmSvc(name string) swarm.Service {
 
 func TestSwarmUsesPinnedRef(t *testing.T) {
 	const ref = "gitea/gitea:1.26@sha256:abababababababababababababababababababababababababababababababab"
-	job := protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: ref}
+	job := Job{JobStart: protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: ref}}
 	fake := &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_gitea")}
 	dist := &fakeDist{digest: "sha256:other"}
 	var out bytes.Buffer
@@ -94,7 +94,7 @@ func TestSwarmUsesPinnedRef(t *testing.T) {
 }
 
 func TestSwarmExecutor(t *testing.T) {
-	job := protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"}
+	job := Job{JobStart: protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"}}
 	workerInfo := managerOf("c1")
 	workerInfo.Swarm.ControlAvailable = false
 	noContainer := swarmSvc("gitea_gitea")