feat: self-update of master and agents (#15)

internal/agent/inventory/collector.go

@@ -131,6 +131,9 @@ func (c *Collector) Snapshot(ctx context.Context) ([]protocol.Service, error) {
 		if _, isTask := ct.Labels[labelSwarmTask]; isTask {
 			continue
 		}
+		if _, isHelper := ct.Labels[protocol.LabelSelfUpdateJob]; isHelper {
+			continue
+		}
 		ref, err := c.imageRef(ctx, ct)
 		if err != nil {
 			return nil, err

internal/agent/inventory/collector_test.go

@@ -135,6 +135,19 @@ func TestSnapshot_PlainContainerAndSwarmTaskSkipped(t *testing.T) {
 	}}, got)
 }
 
+func TestSnapshot_SelfUpdateHelperSkipped(t *testing.T) {
+	cli := &fakeDocker{containers: []container.Summary{
+		{ID: "h1", Names: []string{"/quirky_helper"}, Image: "sha256:agentimage", State: "running",
+			Labels: map[string]string{protocol.LabelSelfUpdateJob: "7"}},
+		{ID: "p1", Names: []string{"/adhoc"}, Image: "redis:alpine", State: "exited"},
+	}}
+	got, err := New(cli, "").Snapshot(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, []protocol.Service{{
+		Kind: protocol.KindContainer, ContainerName: "adhoc", Image: "redis:alpine", Status: "stopped",
+	}}, got)
+}
+
 func TestSnapshot_SelfFlag(t *testing.T) {
 	cli := &fakeDocker{containers: []container.Summary{
 		composeContainer("0123456789abcdef", "updater-agent", "agent", "pu:1", "sha256:i", "running"),

internal/agent/updater/compose.go

@@ -11,8 +11,6 @@ import (
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
-
-	"github.com/shcizo/package-updater/internal/protocol"
 )
 
 const (
@@ -71,6 +69,7 @@ func dockerEnv() []string {
 // ComposeDocker is the subset of the Docker SDK ComposeExecutor needs.
 type ComposeDocker interface {
 	SelfInspector
+	HelperDocker
 	ContainerList(ctx context.Context, opts container.ListOptions) ([]container.Summary, error)
 }
 
@@ -95,7 +94,14 @@ func NewComposeExecutor(cli ComposeDocker, resolve *Resolver, selfID string, run
 // is), writes it to the stack's override file and recreates the service.
 // Working directory and config files come from the service's container
 // labels on this machine, never from the job. It returns the pinned ref.
-func (e *ComposeExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+//
+// When the service is the agent's own container, the recreate runs in a
+// helper container instead (see selfUpdate), because it stops this process.
+func (e *ComposeExecutor) Execute(ctx context.Context, job Job, out io.Writer) (string, error) {
+	// What the job has printed so far goes into the helper's labels.
+	soFar := &tailBuffer{max: selfUpdateOutputMax}
+	out = io.MultiWriter(out, tailWriter{soFar})
+
 	project, service := job.ComposeProject, job.ComposeService
 	cts, err := e.cli.ContainerList(ctx, container.ListOptions{
 		All: true,
@@ -114,10 +120,11 @@ func (e *ComposeExecutor) Execute(ctx context.Context, job protocol.JobStart, ou
 	configFiles := cts[0].Labels[labelConfigFiles]
 	envFileLabel := cts[0].Labels[labelEnvFile]
 
-	roots, err := MountRoots(ctx, e.cli, e.selfID)
+	self, err := inspectSelf(ctx, e.cli, e.selfID)
 	if err != nil {
 		return "", err
 	}
+	roots := rootsOf(self)
 	files, dir, err := composeFiles(roots, configFiles)
 	if err != nil {
 		return "", err
@@ -163,8 +170,17 @@ func (e *ComposeExecutor) Execute(ctx context.Context, job protocol.JobStart, ou
 	}
 
 	base := composeBase(project, workingDir, files, envs)
+	isSelf := false
+	if self.ContainerJSONBase != nil && self.ID != "" {
+		for _, ct := range cts {
+			isSelf = isSelf || ct.ID == self.ID
+		}
+	}
 	for _, step := range [][]string{{"pull", service}, {"up", "-d", service}} {
 		args := append(append([]string(nil), base...), step...)
+		if isSelf && step[0] == "up" {
+			return e.selfUpdate(ctx, job, self, workingDir, args, pinned, out, soFar)
+		}
 		fmt.Fprintf(out, "$ docker %s\n", strings.Join(args, " "))
 		tail := &tailBuffer{max: errorOutputTail}
 		if err := e.run(ctx, workingDir, io.MultiWriter(out, tailWriter{tail}), args...); err != nil {

internal/agent/updater/compose_test.go

@@ -1,6 +1,7 @@
 package updater
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"io"
@@ -8,9 +9,14 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
+	"unicode/utf8"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/pkg/stdcopy"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -26,6 +32,106 @@ func (f fakeCompose) ContainerList(context.Context, container.ListOptions) ([]co
 	return f.containers, nil
 }
 
+// fakeCompose is not meant to start helper containers; these fail the test
+// run if an executor tries.
+func (fakeCompose) ContainerCreate(context.Context, *container.Config, *container.HostConfig, *network.NetworkingConfig, *ocispec.Platform, string) (container.CreateResponse, error) {
+	panic("unexpected ContainerCreate")
+}
+func (fakeCompose) ContainerStart(context.Context, string, container.StartOptions) error {
+	panic("unexpected ContainerStart")
+}
+func (fakeCompose) ContainerWait(context.Context, string, container.WaitCondition) (<-chan container.WaitResponse, <-chan error) {
+	panic("unexpected ContainerWait")
+}
+func (fakeCompose) ContainerLogs(context.Context, string, container.LogsOptions) (io.ReadCloser, error) {
+	panic("unexpected ContainerLogs")
+}
+func (fakeCompose) ContainerRemove(context.Context, string, container.RemoveOptions) error {
+	panic("unexpected ContainerRemove")
+}
+
+// createdHelper is what a fakeHelper was asked to create.
+type createdHelper struct {
+	cfg  *container.Config
+	host *container.HostConfig
+}
+
+// fakeHelper is a helper container the test controls. Its logs are framed
+// like Docker's multiplexed stream.
+type fakeHelper struct {
+	createErr error
+	exit      int64
+	hold      bool   // ContainerWait never reports an exit
+	logs      string // what the helper printed
+	onStart   func()
+
+	created  []createdHelper
+	started  []string
+	removed  []string
+	waitCond container.WaitCondition
+}
+
+func (h *fakeHelper) ContainerCreate(_ context.Context, cfg *container.Config, host *container.HostConfig, _ *network.NetworkingConfig, _ *ocispec.Platform, _ string) (container.CreateResponse, error) {
+	if h.createErr != nil {
+		return container.CreateResponse{}, h.createErr
+	}
+	h.created = append(h.created, createdHelper{cfg, host})
+	return container.CreateResponse{ID: "helper1"}, nil
+}
+
+func (h *fakeHelper) ContainerStart(_ context.Context, id string, _ container.StartOptions) error {
+	h.started = append(h.started, id)
+	if h.onStart != nil {
+		h.onStart()
+	}
+	return nil
+}
+
+func (h *fakeHelper) ContainerWait(_ context.Context, _ string, cond container.WaitCondition) (<-chan container.WaitResponse, <-chan error) {
+	h.waitCond = cond
+	res, errs := make(chan container.WaitResponse, 1), make(chan error, 1)
+	if !h.hold {
+		res <- container.WaitResponse{StatusCode: h.exit}
+	}
+	return res, errs
+}
+
+func (h *fakeHelper) ContainerLogs(context.Context, string, container.LogsOptions) (io.ReadCloser, error) {
+	var buf bytes.Buffer
+	stdcopy.NewStdWriter(&buf, stdcopy.Stdout).Write([]byte(h.logs))
+	return io.NopCloser(&buf), nil
+}
+
+func (h *fakeHelper) ContainerRemove(_ context.Context, id string, opts container.RemoveOptions) error {
+	if !opts.Force {
+		return errors.New("remove without force")
+	}
+	h.removed = append(h.removed, id)
+	return nil
+}
+
+// fakeHelperCompose is a fakeCompose whose helper containers are h.
+type fakeHelperCompose struct {
+	fakeCompose
+	h *fakeHelper
+}
+
+func (f fakeHelperCompose) ContainerCreate(ctx context.Context, cfg *container.Config, host *container.HostConfig, n *network.NetworkingConfig, p *ocispec.Platform, name string) (container.CreateResponse, error) {
+	return f.h.ContainerCreate(ctx, cfg, host, n, p, name)
+}
+func (f fakeHelperCompose) ContainerStart(ctx context.Context, id string, o container.StartOptions) error {
+	return f.h.ContainerStart(ctx, id, o)
+}
+func (f fakeHelperCompose) ContainerWait(ctx context.Context, id string, c container.WaitCondition) (<-chan container.WaitResponse, <-chan error) {
+	return f.h.ContainerWait(ctx, id, c)
+}
+func (f fakeHelperCompose) ContainerLogs(ctx context.Context, id string, o container.LogsOptions) (io.ReadCloser, error) {
+	return f.h.ContainerLogs(ctx, id, o)
+}
+func (f fakeHelperCompose) ContainerRemove(ctx context.Context, id string, o container.RemoveOptions) error {
+	return f.h.ContainerRemove(ctx, id, o)
+}
+
 // outRunner is a fake OutputRunner for `docker compose config`.
 type outRunner struct {
 	json string
@@ -70,7 +176,7 @@ func TestComposeExecutor(t *testing.T) {
 	stack := filepath.Join(root, "slusk")
 	require.NoError(t, os.MkdirAll(stack, 0o755))
 	composeFile := stackFile(t, stack)
-	job := protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "gitea.shcizo.se/shcizo/slusk:1.4"}
+	job := Job{ID: "7", Started: time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC), JobStart: protocol.JobStart{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", Ref: "gitea.shcizo.se/shcizo/slusk:1.4"}}
 	pinned := job.Ref + "@" + d
 
 	ct := container.Summary{Labels: map[string]string{
@@ -319,3 +425,139 @@ func TestExecDocker_UsesMinimalEnv(t *testing.T) {
 	assert.NotContains(t, string(raw), "AGENT_TOKEN")
 	assert.Contains(t, string(raw), "DOCKER_CONFIG=/docker-config")
 }
+
+func TestComposeSelfUpdate(t *testing.T) {
+	const d = "sha256:d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1"
+	root, _ := filepath.EvalSymlinks(t.TempDir())
+	stack := filepath.Join(root, "updater")
+	require.NoError(t, os.MkdirAll(stack, 0o755))
+	stackFile(t, stack)
+	job := Job{ID: "7", Started: time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC), JobStart: protocol.JobStart{
+		Kind: protocol.KindCompose, ComposeProject: "updater", ComposeService: "agent", Ref: "gitea.shcizo.se/shcizo/package-updater:1.4", Self: true}}
+	pinned := job.Ref + "@" + d
+	overridePath := filepath.Join(stack, OverrideName)
+	t.Cleanup(func() { os.Remove(overridePath) })
+
+	selfCt := container.Summary{ID: "selfid", Labels: map[string]string{
+		labelWorkingDir:  stack,
+		labelConfigFiles: filepath.Join(stack, "compose.yaml"),
+	}}
+	mounts := []container.MountPoint{
+		{Type: mount.TypeBind, Source: root, Destination: root, RW: true},
+		{Type: mount.TypeBind, Source: "/var/run/docker.sock", Destination: "/var/run/docker.sock", RW: true},
+		{Type: mount.TypeBind, Source: "/root/.docker/config.json", Destination: "/docker-config/config.json", RW: false},
+		{Type: mount.TypeVolume, Name: "agent-data", Source: "/var/lib/docker/volumes/agent-data/_data", Destination: "/data", RW: true},
+		{Type: mount.TypeTmpfs, Destination: "/tmp", RW: true},
+	}
+	self := fakeSelf{id: "selfid", image: "sha256:agentimage", mounts: mounts}
+	wantMounts := []mount.Mount{
+		{Type: mount.TypeBind, Source: root, Target: root},
+		{Type: mount.TypeBind, Source: "/var/run/docker.sock", Target: "/var/run/docker.sock"},
+		{Type: mount.TypeBind, Source: "/root/.docker/config.json", Target: "/docker-config/config.json", ReadOnly: true},
+		{Type: mount.TypeVolume, Source: "agent-data", Target: "/data"},
+	}
+	base := []string{"compose", "-p", "updater", "--project-directory", stack,
+		"-f", filepath.Join(stack, "compose.yaml"), "-f", overridePath}
+	withStep := func(step ...string) []string { return append(append([]string(nil), base...), step...) }
+
+	t.Setenv("PATH", "/usr/bin")
+	t.Setenv("DOCKER_CONFIG", "/docker-config")
+	t.Setenv("AGENT_TOKEN", "secret")
+
+	cases := []struct {
+		name        string
+		helper      *fakeHelper
+		cancelled   bool // ctx is cancelled when the helper starts
+		wantErr     string
+		wantCancel  bool
+		wantRemoved []string
+		wantOut     []string
+	}{
+		{name: "helper succeeds", helper: &fakeHelper{logs: "Container updater-agent-1  Recreated\n"},
+			wantRemoved: []string{"helper1"}, wantOut: []string{"Container updater-agent-1  Recreated"}},
+		{name: "helper fails", helper: &fakeHelper{exit: 1, logs: "network not found\n"},
+			wantErr: "docker compose up: exit status 1 (output: network not found)", wantRemoved: []string{"helper1"},
+			wantOut: []string{"network not found"}},
+		{name: "ctx ends while waiting", helper: &fakeHelper{hold: true}, cancelled: true, wantCancel: true},
+		{name: "create fails", helper: &fakeHelper{createErr: errors.New("no such image")}, wantErr: "create update helper: no such image"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			t.Cleanup(func() { os.Remove(overridePath) })
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			if c.cancelled {
+				c.helper.onStart = cancel
+			}
+			rec := &recorder{}
+			cli := fakeHelperCompose{fakeCompose{self, []container.Summary{selfCt}}, c.helper}
+			e := NewComposeExecutor(cli, NewResolver(&fakeDist{digest: d}, nil), "me", rec.run, (&outRunner{json: `{"services":{"agent":{"image":"x"}}}`}).run)
+
+			var out strings.Builder
+			got, err := e.Execute(ctx, job, &out)
+
+			require.Len(t, rec.cmds, 1, "pull ran, up did not")
+			assert.Equal(t, ranCmd{stack, withStep("pull", "agent")}, rec.cmds[0])
+			assert.Equal(t, c.wantRemoved, c.helper.removed)
+			for _, s := range c.wantOut {
+				assert.Contains(t, out.String(), s)
+			}
+			switch {
+			case c.wantCancel:
+				assert.ErrorIs(t, err, context.Canceled)
+			case c.wantErr != "":
+				assert.EqualError(t, err, c.wantErr)
+				assert.Empty(t, got)
+			default:
+				require.NoError(t, err)
+				assert.Equal(t, pinned, got)
+			}
+			if len(c.helper.created) == 0 {
+				return
+			}
+			cfg, host := c.helper.created[0].cfg, c.helper.created[0].host
+			assert.Equal(t, "sha256:agentimage", cfg.Image)
+			assert.Equal(t, []string{"docker"}, []string(cfg.Entrypoint))
+			assert.Equal(t, withStep("up", "-d", "agent"), []string(cfg.Cmd))
+			assert.Equal(t, stack, cfg.WorkingDir)
+			assert.Equal(t, dockerEnv(), cfg.Env)
+			assert.Equal(t, wantMounts, host.Mounts)
+			assert.False(t, host.AutoRemove)
+			assert.Equal(t, container.RestartPolicy{}, host.RestartPolicy)
+			assert.Equal(t, []string{"helper1"}, c.helper.started)
+			assert.Equal(t, container.WaitConditionNextExit, c.helper.waitCond)
+			labels := cfg.Labels
+			assert.Len(t, labels, 6, "no compose labels")
+			assert.Equal(t, "7", labels[protocol.LabelSelfUpdateJob])
+			assert.Equal(t, pinned, labels[protocol.LabelSelfUpdateRef])
+			assert.Equal(t, "2026-10-13T09:00:00Z", labels[protocol.LabelSelfUpdateStarted])
+			assert.Equal(t, "updater", labels[protocol.LabelSelfUpdateProject])
+			assert.Equal(t, "agent", labels[protocol.LabelSelfUpdateService])
+			assert.Contains(t, labels[protocol.LabelSelfUpdateOutput], "resolved "+pinned)
+			assert.Contains(t, labels[protocol.LabelSelfUpdateOutput], "$ docker "+strings.Join(withStep("pull", "agent"), " "))
+		})
+	}
+
+	t.Run("a service that is not the agent's own never uses a helper", func(t *testing.T) {
+		t.Cleanup(func() { os.Remove(overridePath) })
+		other := self
+		other.id = "otherid" // fakeCompose panics on ContainerCreate
+		rec := &recorder{}
+		e := NewComposeExecutor(fakeCompose{other, []container.Summary{selfCt}}, NewResolver(&fakeDist{digest: d}, nil), "me", rec.run,
+			(&outRunner{json: `{"services":{"agent":{"image":"x"}}}`}).run)
+		got, err := e.Execute(context.Background(), job, io.Discard)
+		require.NoError(t, err)
+		assert.Equal(t, pinned, got)
+		require.Len(t, rec.cmds, 2)
+		assert.Equal(t, withStep("up", "-d", "agent"), rec.cmds[1].args)
+	})
+
+	t.Run("output label keeps the tail and says it was cut", func(t *testing.T) {
+		b := &tailBuffer{max: 4}
+		b.Write([]byte("abcdef"))
+		assert.Equal(t, truncatedMarker+"cdef", outputLabel(b))
+		b2 := &tailBuffer{max: 3}
+		b2.Write([]byte("aéé")) // the cut falls inside a character
+		assert.True(t, utf8.ValidString(outputLabel(b2)))
+	})
+}

internal/agent/updater/dispatch.go

@@ -16,7 +16,7 @@ func NewDispatch(compose, swarm Executor) Executor {
 	return dispatch{compose: compose, swarm: swarm}
 }
 
-func (d dispatch) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+func (d dispatch) Execute(ctx context.Context, job Job, out io.Writer) (string, error) {
 	switch job.Kind {
 	case protocol.KindCompose:
 		return d.compose.Execute(ctx, job, out)

internal/agent/updater/dispatch_test.go

@@ -12,20 +12,20 @@ import (
 )
 
 func TestDispatch(t *testing.T) {
-	compose := execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "compose", nil })
-	swarm := execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "swarm", nil })
+	compose := execFunc(func(context.Context, Job, io.Writer) (string, error) { return "compose", nil })
+	swarm := execFunc(func(context.Context, Job, io.Writer) (string, error) { return "swarm", nil })
 	d := NewDispatch(compose, swarm)
 
-	got, err := d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindCompose}, io.Discard)
+	got, err := d.Execute(context.Background(), Job{JobStart: protocol.JobStart{Kind: protocol.KindCompose}}, io.Discard)
 	require.NoError(t, err)
 	assert.Equal(t, "compose", got)
 
-	got, err = d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindSwarm}, io.Discard)
+	got, err = d.Execute(context.Background(), Job{JobStart: protocol.JobStart{Kind: protocol.KindSwarm}}, io.Discard)
 	require.NoError(t, err)
 	assert.Equal(t, "swarm", got)
 
-	_, err = d.Execute(context.Background(), protocol.JobStart{Kind: protocol.KindContainer}, io.Discard)
+	_, err = d.Execute(context.Background(), Job{JobStart: protocol.JobStart{Kind: protocol.KindContainer}}, io.Discard)
 	assert.ErrorContains(t, err, "cannot update")
-	_, err = d.Execute(context.Background(), protocol.JobStart{Kind: "weird"}, io.Discard)
+	_, err = d.Execute(context.Background(), Job{JobStart: protocol.JobStart{Kind: "weird"}}, io.Discard)
 	assert.ErrorContains(t, err, "cannot update")
 }

internal/agent/updater/override.go

@@ -177,20 +177,35 @@ var errNoRoots = errors.New("refused: no stack directory is mounted into the age
 // path inside the container. Compose labels hold host paths, so only those
 // mounts make a label path usable as is.
 func MountRoots(ctx context.Context, cli SelfInspector, selfID string) ([]string, error) {
+	insp, err := inspectSelf(ctx, cli, selfID)
+	if err != nil {
+		return nil, err
+	}
+	return rootsOf(insp), nil
+}
+
+// inspectSelf inspects the agent's own container.
+func inspectSelf(ctx context.Context, cli SelfInspector, selfID string) (container.InspectResponse, error) {
 	if selfID == "" {
-		return nil, errNoRoots
+		return container.InspectResponse{}, errNoRoots
 	}
 	insp, err := cli.ContainerInspect(ctx, selfID)
 	if err != nil {
-		return nil, fmt.Errorf("inspect own container %s: %w", selfID, err)
+		return container.InspectResponse{}, fmt.Errorf("inspect own container %s: %w", selfID, err)
 	}
+	return insp, nil
+}
+
+// rootsOf returns the directories MountRoots describes, from an inspected
+// container.
+func rootsOf(insp container.InspectResponse) []string {
 	var roots []string
 	for _, m := range insp.Mounts {
 		if m.Type == mount.TypeBind && m.RW && filepath.Clean(m.Source) == filepath.Clean(m.Destination) {
 			roots = append(roots, filepath.Clean(m.Destination))
 		}
 	}
-	return roots, nil
+	return roots
 }
 
 // composeFiles returns the -f arguments for a job: the stack's config files

internal/agent/updater/override_test.go

@@ -175,10 +175,20 @@ func TestWriteOverride(t *testing.T) {
 type fakeSelf struct {
 	mounts []container.MountPoint
 	err    error
+	id     string            // the container's ID, when set
+	image  string            // the container's image ID, when set
+	labels map[string]string // the container's labels, when set
 }
 
 func (f fakeSelf) ContainerInspect(context.Context, string) (container.InspectResponse, error) {
-	return container.InspectResponse{Mounts: f.mounts}, f.err
+	insp := container.InspectResponse{Mounts: f.mounts}
+	if f.id != "" || f.image != "" {
+		insp.ContainerJSONBase = &container.ContainerJSONBase{ID: f.id, Image: f.image}
+	}
+	if f.labels != nil {
+		insp.Config = &container.Config{Labels: f.labels}
+	}
+	return insp, f.err
 }
 
 func TestMountRoots(t *testing.T) {

internal/agent/updater/runner.go

@@ -16,12 +16,25 @@ import (
 // chatty job cannot grow a single job.progress message without bound.
 const maxProgress = 32 << 10
 
+// Job is a job as an Executor sees it: what the master asked for, plus the
+// ID and start time the runner gave it. A self-update records both in the
+// helper container, so the new agent can finish the job under the same ID.
+type Job struct {
+	protocol.JobStart
+	ID      string
+	Started time.Time
+}
+
 // Executor deploys one job and returns the "repo:tag@digest" it deployed.
 // Everything it writes to out is streamed to the master as job output.
 type Executor interface {
-	Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error)
+	Execute(ctx context.Context, job Job, out io.Writer) (string, error)
 }
 
+// ResumeFunc finishes a job that an earlier agent process began. It returns
+// the "repo:tag@digest" that was deployed, like Executor.Execute.
+type ResumeFunc func(ctx context.Context, out io.Writer) (string, error)
+
 // Progress is output a running job produced since the previous TakeProgress.
 type Progress struct{ ID, Output string }
 
@@ -52,6 +65,8 @@ func (b *tailBuffer) Reset()          { b.buf, b.truncated = nil, false }
 
 type jobState struct {
 	start    protocol.JobStart
+	resume   ResumeFunc // set for a job resumed after a restart; start is then empty
+	started  time.Time  // set with resume
 	running  bool
 	result   *protocol.JobResult
 	output   tailBuffer // everything, capped at protocol.MaxJobOutput
@@ -117,6 +132,28 @@ func (r *Runner) Submit(id string, start protocol.JobStart) {
 	signal(r.wake)
 }
 
+// Resume registers a job that an earlier agent process began and runs fn for
+// it, before every job that is already queued. The job is in Known at once,
+// so the agent's next hello does not report it lost. The result carries
+// started as its start time. A job whose ID the runner already holds is
+// ignored.
+func (r *Runner) Resume(id string, started time.Time, fn ResumeFunc) {
+	r.mu.Lock()
+	if _, ok := r.jobs[id]; ok {
+		r.mu.Unlock()
+		return
+	}
+	r.jobs[id] = &jobState{
+		resume:  fn,
+		started: started,
+		output:  tailBuffer{max: protocol.MaxJobOutput},
+		unsent:  tailBuffer{max: maxProgress},
+	}
+	r.queue = append([]string{id}, r.queue...)
+	r.mu.Unlock()
+	signal(r.wake)
+}
+
 // Ack forgets a finished job. Acks for unknown, queued or running jobs are
 // ignored.
 func (r *Runner) Ack(id string) {
@@ -196,10 +233,12 @@ func (r *Runner) Pending() []Finished {
 func (r *Runner) Changed() <-chan struct{} { return r.changed }
 
 // Run works through the queue until ctx ends. Jobs still queued then are
-// dropped; the master marks them lost on the agent's next hello.
+// dropped; the master marks them lost on the agent's next hello. A job that
+// ctx interrupts is dropped too, without a result: a self-updating agent is
+// stopped that way, and its successor reports the outcome.
 func (r *Runner) Run(ctx context.Context) {
 	for {
-		id, start, ok := r.next()
+		id, ok := r.next()
 		if !ok {
 			select {
 			case <-ctx.Done():
@@ -211,32 +250,62 @@ func (r *Runner) Run(ctx context.Context) {
 		if ctx.Err() != nil {
 			return
 		}
-		r.runOne(ctx, id, start)
+		r.runOne(ctx, id)
 	}
 }
 
-// next pops the next queued job and marks it running.
-func (r *Runner) next() (string, protocol.JobStart, bool) {
+// next pops the next job to run and marks it running. Jobs for the agent's
+// own service run last: a master and an agent on the same host share an
+// image, and the master's job must finish before the agent replaces itself.
+func (r *Runner) next() (string, bool) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 	if len(r.queue) == 0 {
-		return "", protocol.JobStart{}, false
+		return "", false
 	}
-	id := r.queue[0]
-	r.queue = r.queue[1:]
-	j := r.jobs[id]
-	j.running = true
+	pick := 0
+	for i, id := range r.queue {
+		if !r.jobs[id].start.Self {
+			pick = i
+			break
+		}
+	}
+	id := r.queue[pick]
+	r.queue = append(r.queue[:pick:pick], r.queue[pick+1:]...)
+	r.jobs[id].running = true
 	signal(r.changed)
-	return id, j.start, true
+	return id, true
 }
 
-func (r *Runner) runOne(ctx context.Context, id string, start protocol.JobStart) {
-	started := r.now()
+func (r *Runner) runOne(ctx context.Context, id string) {
+	r.mu.Lock()
+	j := r.jobs[id]
+	start, resume, started := j.start, j.resume, j.started
+	r.mu.Unlock()
+	if resume == nil {
+		started = r.now()
+	}
+
 	jobCtx, cancel := context.WithTimeout(ctx, r.timeout)
-	ref, err := r.exec.Execute(jobCtx, start, &jobWriter{r: r, id: id})
+	var ref string
+	var err error
+	if resume != nil {
+		ref, err = resume(jobCtx, &jobWriter{r: r, id: id})
+	} else {
+		ref, err = r.exec.Execute(jobCtx, Job{JobStart: start, ID: id, Started: started}, &jobWriter{r: r, id: id})
+	}
 	timedOut := errors.Is(jobCtx.Err(), context.DeadlineExceeded)
 	cancel()
 
+	if err != nil && ctx.Err() != nil && !timedOut {
+		// The agent is stopping, not the job failing. Nothing is recorded, so
+		// no result can pre-empt the one a successor agent may report.
+		r.mu.Lock()
+		delete(r.jobs, id)
+		r.mu.Unlock()
+		return
+	}
+
 	res := protocol.JobResult{Status: protocol.ResultSucceeded, Ref: ref, StartedAt: started, FinishedAt: r.now()}
 	switch {
 	case timedOut:
@@ -246,7 +315,6 @@ func (r *Runner) runOne(ctx context.Context, id string, start protocol.JobStart)
 	}
 
 	r.mu.Lock()
-	j := r.jobs[id]
 	res.Output, res.OutputTruncated = j.output.String(), j.output.Truncated()
 	j.running, j.result, j.finished = false, &res, res.FinishedAt
 	j.unsent.Reset()

internal/agent/updater/runner_test.go

@@ -29,7 +29,7 @@ func newFakeExec() *fakeExec {
 	return &fakeExec{gate: make(chan struct{}), started: make(chan string, 10)}
 }
 
-func (f *fakeExec) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+func (f *fakeExec) Execute(ctx context.Context, job Job, out io.Writer) (string, error) {
 	n := f.cur.Add(1)
 	defer f.cur.Add(-1)
 	for {
@@ -148,7 +148,7 @@ func TestRunner_TakeProgress(t *testing.T) {
 func TestRunner_TakeProgressReportsStartedJobOnce(t *testing.T) {
 	gate := make(chan struct{})
 	started := make(chan struct{})
-	r := startRunner(t, execFunc(func(ctx context.Context, _ protocol.JobStart, _ io.Writer) (string, error) {
+	r := startRunner(t, execFunc(func(ctx context.Context, _ Job, _ io.Writer) (string, error) {
 		close(started)
 		<-gate
 		return "x", nil
@@ -160,14 +160,14 @@ func TestRunner_TakeProgressReportsStartedJobOnce(t *testing.T) {
 	close(gate)
 }
 
-type execFunc func(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error)
+type execFunc func(ctx context.Context, job Job, out io.Writer) (string, error)
 
-func (f execFunc) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+func (f execFunc) Execute(ctx context.Context, job Job, out io.Writer) (string, error) {
 	return f(ctx, job, out)
 }
 
 func TestRunner_Timeout(t *testing.T) {
-	r := startRunner(t, execFunc(func(ctx context.Context, _ protocol.JobStart, _ io.Writer) (string, error) {
+	r := startRunner(t, execFunc(func(ctx context.Context, _ Job, _ io.Writer) (string, error) {
 		<-ctx.Done()
 		return "", ctx.Err()
 	}), 20*time.Millisecond)
@@ -178,7 +178,7 @@ func TestRunner_Timeout(t *testing.T) {
 }
 
 func TestRunner_OutputCapped(t *testing.T) {
-	r := startRunner(t, execFunc(func(_ context.Context, _ protocol.JobStart, out io.Writer) (string, error) {
+	r := startRunner(t, execFunc(func(_ context.Context, _ Job, out io.Writer) (string, error) {
 		io.WriteString(out, strings.Repeat("a", 10))
 		io.WriteString(out, strings.Repeat("b", protocol.MaxJobOutput))
 		return "x", nil
@@ -190,7 +190,7 @@ func TestRunner_OutputCapped(t *testing.T) {
 }
 
 func TestRunner_ChangedSignals(t *testing.T) {
-	r := startRunner(t, execFunc(func(context.Context, protocol.JobStart, io.Writer) (string, error) { return "x", nil }), time.Minute)
+	r := startRunner(t, execFunc(func(context.Context, Job, io.Writer) (string, error) { return "x", nil }), time.Minute)
 	r.Submit("1", protocol.JobStart{Ref: "a:1"})
 	select {
 	case <-r.Changed():
@@ -198,3 +198,105 @@ func TestRunner_ChangedSignals(t *testing.T) {
 		t.Fatal("no change signal")
 	}
 }
+
+func TestRunner_ResumeJumpsTheQueue(t *testing.T) {
+	fe := newFakeExec()
+	r := NewRunner(fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	started := time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC)
+	r.Resume("9", started, func(_ context.Context, out io.Writer) (string, error) {
+		io.WriteString(out, "from the helper\n")
+		return "resumed@sha256:bb", nil
+	})
+	assert.Equal(t, []string{"1", "9"}, r.Known(), "known before the worker starts")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() { defer wg.Done(); r.Run(ctx) }()
+	t.Cleanup(func() { cancel(); wg.Wait() })
+
+	got := waitPending(t, r, 1)
+	require.Len(t, got, 1, "the resumed job runs first; job 1 waits on the gate")
+	assert.Equal(t, "9", got[0].ID)
+	assert.Equal(t, protocol.ResultSucceeded, got[0].Result.Status)
+	assert.Equal(t, "resumed@sha256:bb", got[0].Result.Ref)
+	assert.Equal(t, "from the helper\n", got[0].Result.Output)
+	assert.True(t, started.Equal(got[0].Result.StartedAt))
+
+	r.Resume("9", started, nil) // already held: ignored
+	assert.Equal(t, "a:1", <-fe.started)
+	fe.gate <- struct{}{}
+	waitPending(t, r, 2)
+}
+
+func TestRunner_ResumedJobFailure(t *testing.T) {
+	r := startRunner(t, newFakeExec(), time.Minute)
+	r.Resume("9", time.Now(), func(context.Context, io.Writer) (string, error) { return "", errors.New("helper failed") })
+	got := waitPending(t, r, 1)
+	assert.Equal(t, protocol.ResultFailed, got[0].Result.Status)
+	assert.Equal(t, "helper failed", got[0].Result.Error)
+}
+
+func TestRunner_SelfJobsRunLast(t *testing.T) {
+	fe := newFakeExec()
+	r := NewRunner(fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "self:1", Self: true})
+	r.Submit("2", protocol.JobStart{Ref: "a:2"})
+	r.Submit("3", protocol.JobStart{Ref: "self:3", Self: true})
+	r.Submit("4", protocol.JobStart{Ref: "a:4"})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() { defer wg.Done(); r.Run(ctx) }()
+	t.Cleanup(func() { cancel(); wg.Wait() })
+
+	var order []string
+	for i := 0; i < 4; i++ {
+		order = append(order, <-fe.started)
+		fe.gate <- struct{}{}
+	}
+	assert.Equal(t, []string{"a:2", "a:4", "self:1", "self:3"}, order)
+}
+
+func TestRunner_SelfJobRunsWhenAloneAndNonSelfSubmittedLater(t *testing.T) {
+	fe := newFakeExec()
+	r := startRunner(t, fe, time.Minute)
+	r.Submit("1", protocol.JobStart{Ref: "self:1", Self: true})
+	assert.Equal(t, "self:1", <-fe.started)
+	fe.gate <- struct{}{}
+	waitPending(t, r, 1)
+}
+
+func TestRunner_StoppingTheAgentRecordsNoResult(t *testing.T) {
+	r := NewRunner(execFunc(func(ctx context.Context, _ Job, _ io.Writer) (string, error) {
+		<-ctx.Done()
+		return "", ctx.Err()
+	}), time.Minute)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { r.Run(ctx); close(done) }()
+	r.Submit("1", protocol.JobStart{Ref: "a:1"})
+	require.Eventually(t, func() bool { return len(r.TakeProgress()) > 0 }, time.Second, 5*time.Millisecond)
+
+	cancel()
+	<-done
+	assert.Empty(t, r.Pending(), "no \"context canceled\" result")
+	assert.Empty(t, r.Known(), "the job is dropped, so the master marks it lost unless a successor resumes it")
+}
+
+func TestRunner_ExecutorGetsIDAndStart(t *testing.T) {
+	var got Job
+	r := startRunner(t, execFunc(func(_ context.Context, job Job, _ io.Writer) (string, error) {
+		got = job
+		return "x", nil
+	}), time.Minute)
+	now := time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC)
+	r.now = func() time.Time { return now }
+	r.Submit("5", protocol.JobStart{Ref: "a:1"})
+	waitPending(t, r, 1)
+	assert.Equal(t, "5", got.ID)
+	assert.Equal(t, "a:1", got.Ref)
+	assert.True(t, now.Equal(got.Started))
+}

internal/agent/updater/selfupdate.go

@@ -0,0 +1,258 @@
+package updater
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/pkg/stdcopy"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+// selfUpdateOutputMax caps the job output stored in the helper container's
+// output label. The old agent's output so far is all the new agent has of it.
+const selfUpdateOutputMax = 32 << 10
+
+// truncatedMarker starts the stored output when its head was cut off.
+const truncatedMarker = "…truncated\n"
+
+// resumeSeparator follows the old agent's output in a resumed job's output.
+const resumeSeparator = "--- agent restarted; output of the update helper ---\n"
+
+// HelperDocker is the subset of the Docker SDK the self-update helper
+// container needs. *client.Client satisfies it.
+type HelperDocker interface {
+	ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *ocispec.Platform, containerName string) (container.CreateResponse, error)
+	ContainerStart(ctx context.Context, containerID string, options container.StartOptions) error
+	ContainerWait(ctx context.Context, containerID string, condition container.WaitCondition) (<-chan container.WaitResponse, <-chan error)
+	ContainerLogs(ctx context.Context, containerID string, options container.LogsOptions) (io.ReadCloser, error)
+	ContainerRemove(ctx context.Context, containerID string, options container.RemoveOptions) error
+}
+
+// selfUpdate runs `up -d` for the agent's own service in a short-lived helper
+// container. Recreating the service stops this process, so the helper, made
+// from the agent's own image with the agent's own mounts, does it and
+// outlives the agent. args are the complete `docker` arguments of the call.
+//
+// If the helper exits while this agent still runs, selfUpdate copies its logs
+// into the job output, removes it and returns the outcome. If ctx ends first
+// (the agent is being replaced, or the job timed out) it returns ctx's error
+// and leaves the helper to the new agent, see ResumeSelfUpdates.
+func (e *ComposeExecutor) selfUpdate(ctx context.Context, job Job, self container.InspectResponse, workingDir string, args []string, pinned string, out io.Writer, soFar *tailBuffer) (string, error) {
+	fmt.Fprintf(out, "$ docker %s (in an update helper container)\n", strings.Join(args, " "))
+	cfg := &container.Config{
+		Image:      self.Image,
+		Entrypoint: []string{"docker"},
+		Cmd:        args,
+		WorkingDir: workingDir,
+		Env:        dockerEnv(),
+		Labels: map[string]string{
+			protocol.LabelSelfUpdateJob:     job.ID,
+			protocol.LabelSelfUpdateRef:     pinned,
+			protocol.LabelSelfUpdateStarted: job.Started.UTC().Format(time.RFC3339Nano),
+			protocol.LabelSelfUpdateProject: job.ComposeProject,
+			protocol.LabelSelfUpdateService: job.ComposeService,
+			protocol.LabelSelfUpdateOutput:  outputLabel(soFar),
+		},
+	}
+	created, err := e.cli.ContainerCreate(ctx, cfg, &container.HostConfig{Mounts: helperMounts(self)}, nil, nil, "")
+	if err != nil {
+		return "", fmt.Errorf("create update helper: %w", err)
+	}
+	// Wait is registered before the start so a fast exit is not missed.
+	waitCh, errCh := e.cli.ContainerWait(ctx, created.ID, container.WaitConditionNextExit)
+	if err := e.cli.ContainerStart(ctx, created.ID, container.StartOptions{}); err != nil {
+		e.cli.ContainerRemove(context.WithoutCancel(ctx), created.ID, container.RemoveOptions{Force: true})
+		return "", fmt.Errorf("start update helper: %w", err)
+	}
+	code, err := awaitExit(ctx, waitCh, errCh)
+	if err != nil {
+		return "", err
+	}
+
+	tail := &tailBuffer{max: errorOutputTail}
+	if err := finishHelper(ctx, e.cli, created.ID, io.MultiWriter(out, tailWriter{tail})); err != nil {
+		fmt.Fprintf(out, "warning: %v\n", err)
+	}
+	if code != 0 {
+		return "", exitError(code, tail)
+	}
+	return pinned, nil
+}
+
+// outputLabel returns the job output so far as a label value.
+func outputLabel(b *tailBuffer) string {
+	s := strings.ToValidUTF8(b.String(), "")
+	if b.Truncated() {
+		s = truncatedMarker + s
+	}
+	return s
+}
+
+// helperMounts copies the agent's own bind and volume mounts: the Docker
+// socket, the stack directories and the registry credentials the compose call
+// needs are all among them.
+func helperMounts(self container.InspectResponse) []mount.Mount {
+	var mounts []mount.Mount
+	for _, m := range self.Mounts {
+		switch m.Type {
+		case mount.TypeBind:
+			mounts = append(mounts, mount.Mount{Type: mount.TypeBind, Source: m.Source, Target: m.Destination, ReadOnly: !m.RW})
+		case mount.TypeVolume:
+			mounts = append(mounts, mount.Mount{Type: mount.TypeVolume, Source: m.Name, Target: m.Destination, ReadOnly: !m.RW})
+		}
+	}
+	return mounts
+}
+
+// awaitExit waits for a ContainerWait result and returns the exit code. When
+// ctx ends it returns ctx's error.
+func awaitExit(ctx context.Context, waitCh <-chan container.WaitResponse, errCh <-chan error) (int64, error) {
+	select {
+	case r := <-waitCh:
+		if r.Error != nil {
+			return 0, fmt.Errorf("wait for update helper: %s", r.Error.Message)
+		}
+		return r.StatusCode, nil
+	case err := <-errCh:
+		if ctx.Err() != nil {
+			return 0, ctx.Err()
+		}
+		return 0, fmt.Errorf("wait for update helper: %w", err)
+	case <-ctx.Done():
+		return 0, ctx.Err()
+	}
+}
+
+// finishHelper copies the exited helper's logs to w and removes the helper.
+// Both are attempted; the first failure is returned.
+func finishHelper(ctx context.Context, cli HelperDocker, id string, w io.Writer) error {
+	var first error
+	rc, err := cli.ContainerLogs(ctx, id, container.LogsOptions{ShowStdout: true, ShowStderr: true})
+	if err == nil {
+		_, err = stdcopy.StdCopy(w, w, rc)
+		rc.Close()
+	}
+	if err != nil {
+		first = fmt.Errorf("read the update helper's logs: %w", err)
+	}
+	if err := cli.ContainerRemove(ctx, id, container.RemoveOptions{Force: true}); err != nil && first == nil {
+		first = fmt.Errorf("remove the update helper: %w", err)
+	}
+	return first
+}
+
+// exitError describes a failed helper like a failed `docker compose` call.
+func exitError(code int64, tail *tailBuffer) error {
+	return fmt.Errorf("docker compose up: exit status %d (output: %s)", code, strings.TrimSpace(tail.String()))
+}
+
+// ResumeDocker is the subset of the Docker SDK ResumeSelfUpdates needs.
+// *client.Client satisfies it.
+type ResumeDocker interface {
+	SelfInspector
+	HelperDocker
+	ContainerList(ctx context.Context, opts container.ListOptions) ([]container.Summary, error)
+}
+
+// ResumeSelfUpdates finishes the jobs a previous agent process started with
+// selfUpdate. It finds the helper containers made for this agent's own
+// Compose service and registers each with runner.Resume, which reports the
+// job under its old ID once the helper has exited. Call it before the runner
+// and the connection start, so the first hello already lists these jobs. A
+// helper with unusable labels is resumed too; its job fails with an error
+// that says so.
+func ResumeSelfUpdates(ctx context.Context, cli ResumeDocker, selfID string, runner *Runner, log *slog.Logger) error {
+	self, err := inspectSelf(ctx, cli, selfID)
+	if err != nil {
+		return err
+	}
+	if self.Config == nil {
+		return nil
+	}
+	project, service := self.Config.Labels[labelComposeProject], self.Config.Labels[labelComposeService]
+	if project == "" || service == "" {
+		return nil
+	}
+	cts, err := cli.ContainerList(ctx, container.ListOptions{
+		All:     true,
+		Filters: filters.NewArgs(filters.Arg("label", protocol.LabelSelfUpdateJob)),
+	})
+	if err != nil {
+		return fmt.Errorf("list update helpers: %w", err)
+	}
+	for _, ct := range cts {
+		if ct.Labels[protocol.LabelSelfUpdateProject] != project || ct.Labels[protocol.LabelSelfUpdateService] != service {
+			continue
+		}
+		jobID := ct.Labels[protocol.LabelSelfUpdateJob]
+		if jobID == "" {
+			log.Warn("self_update_helper_without_job", "helper", ct.ID)
+			continue
+		}
+		ref, started, labelErr := parseHelperLabels(ct.Labels)
+		if started.IsZero() {
+			started = time.Now()
+		}
+		log.Info("self_update_resumed", "job_id", jobID, "helper", ct.ID)
+		runner.Resume(jobID, started, resumeFunc(cli, ct.ID, ct.Labels[protocol.LabelSelfUpdateOutput], ref, labelErr))
+	}
+	return nil
+}
+
+// parseHelperLabels reads the pinned ref and start time from a helper's
+// labels. started is zero when it is missing or unreadable, and err then says
+// what is wrong.
+func parseHelperLabels(labels map[string]string) (ref string, started time.Time, err error) {
+	ref = labels[protocol.LabelSelfUpdateRef]
+	if ref == "" {
+		err = errors.New("update helper has no ref label")
+	}
+	raw := labels[protocol.LabelSelfUpdateStarted]
+	if t, perr := time.Parse(time.RFC3339Nano, raw); perr == nil {
+		started = t
+	} else if err == nil {
+		err = fmt.Errorf("update helper has no usable started label %q", raw)
+	}
+	return ref, started, err
+}
+
+// resumeFunc waits for the helper to stop, then reports the old agent's
+// output, a separator line and the helper's own logs as the job output,
+// removes the helper and returns the pinned ref, or an error when the helper
+// failed or its labels were unusable.
+func resumeFunc(cli HelperDocker, helperID, earlier, ref string, labelErr error) ResumeFunc {
+	return func(ctx context.Context, out io.Writer) (string, error) {
+		waitCh, errCh := cli.ContainerWait(ctx, helperID, container.WaitConditionNotRunning)
+		code, err := awaitExit(ctx, waitCh, errCh)
+		if err != nil {
+			return "", err
+		}
+		io.WriteString(out, earlier)
+		if earlier != "" && !strings.HasSuffix(earlier, "\n") {
+			io.WriteString(out, "\n")
+		}
+		io.WriteString(out, resumeSeparator)
+		tail := &tailBuffer{max: errorOutputTail}
+		if err := finishHelper(ctx, cli, helperID, io.MultiWriter(out, tailWriter{tail})); err != nil {
+			fmt.Fprintf(out, "warning: %v\n", err)
+		}
+		switch {
+		case labelErr != nil:
+			return "", labelErr
+		case code != 0:
+			return "", exitError(code, tail)
+		}
+		return ref, nil
+	}
+}

internal/agent/updater/selfupdate_test.go

@@ -0,0 +1,109 @@
+package updater
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+func TestResumeSelfUpdates(t *testing.T) {
+	const pinned = "gitea.shcizo.se/shcizo/package-updater:1.4@sha256:aa"
+	started := time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC)
+	self := fakeSelf{id: "selfid", labels: map[string]string{labelComposeProject: "updater", labelComposeService: "agent"}}
+
+	helper := func(id, project, service string, extra map[string]string) container.Summary {
+		labels := map[string]string{
+			protocol.LabelSelfUpdateJob:     "j-" + id,
+			protocol.LabelSelfUpdateRef:     pinned,
+			protocol.LabelSelfUpdateStarted: started.Format(time.RFC3339Nano),
+			protocol.LabelSelfUpdateProject: project,
+			protocol.LabelSelfUpdateService: service,
+			protocol.LabelSelfUpdateOutput:  "resolved " + pinned + "\n$ docker compose pull agent\n",
+		}
+		for k, v := range extra {
+			if v == "" {
+				delete(labels, k)
+			} else {
+				labels[k] = v
+			}
+		}
+		return container.Summary{ID: id, Labels: labels}
+	}
+	separator := "--- agent restarted; output of the update helper ---\n"
+
+	cases := []struct {
+		name       string
+		helper     container.Summary
+		exit       int64
+		wantStatus string
+		wantErr    string
+		wantOutput string
+	}{
+		{name: "helper succeeded", helper: helper("h1", "updater", "agent", nil), wantStatus: protocol.ResultSucceeded,
+			wantOutput: "resolved " + pinned + "\n$ docker compose pull agent\n" + separator + "Recreated\n"},
+		{name: "helper failed", helper: helper("h1", "updater", "agent", nil), exit: 1, wantStatus: protocol.ResultFailed,
+			wantErr: "docker compose up: exit status 1 (output: Recreated)"},
+		{name: "no ref label", helper: helper("h1", "updater", "agent", map[string]string{protocol.LabelSelfUpdateRef: ""}),
+			wantStatus: protocol.ResultFailed, wantErr: "update helper has no ref label"},
+		{name: "no started label", helper: helper("h1", "updater", "agent", map[string]string{protocol.LabelSelfUpdateStarted: ""}),
+			wantStatus: protocol.ResultFailed, wantErr: `update helper has no usable started label ""`},
+		{name: "no output label", helper: helper("h1", "updater", "agent", map[string]string{protocol.LabelSelfUpdateOutput: ""}),
+			wantStatus: protocol.ResultSucceeded, wantOutput: separator + "Recreated\n"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			h := &fakeHelper{exit: c.exit, logs: "Recreated\n"}
+			unrelated := []container.Summary{
+				helper("other-project", "other", "agent", nil),
+				helper("other-service", "updater", "web", nil),
+			}
+			cli := fakeHelperCompose{fakeCompose{self, append(unrelated, c.helper)}, h}
+			r := NewRunner(execFunc(func(context.Context, Job, io.Writer) (string, error) {
+				t.Error("a resumed job must not reach the executor")
+				return "", nil
+			}), time.Minute)
+
+			require.NoError(t, ResumeSelfUpdates(context.Background(), cli, "me", r, slog.New(slog.NewTextHandler(io.Discard, nil))))
+			assert.Equal(t, []string{"j-h1"}, r.Known(), "known before the runner starts, so the first hello lists it")
+
+			ctx, cancel := context.WithCancel(context.Background())
+			var wg sync.WaitGroup
+			wg.Add(1)
+			go func() { defer wg.Done(); r.Run(ctx) }()
+			t.Cleanup(func() { cancel(); wg.Wait() })
+
+			got := waitPending(t, r, 1)
+			require.Len(t, got, 1)
+			assert.Equal(t, "j-h1", got[0].ID)
+			res := got[0].Result
+			assert.Equal(t, c.wantStatus, res.Status)
+			assert.Equal(t, container.WaitConditionNotRunning, h.waitCond)
+			assert.Equal(t, []string{"h1"}, h.removed, "only the matching helper is touched")
+			if c.wantStatus == protocol.ResultSucceeded {
+				assert.Equal(t, pinned, res.Ref)
+				assert.Equal(t, c.wantOutput, res.Output)
+			} else {
+				assert.Equal(t, c.wantErr, res.Error)
+			}
+			if c.name != "no started label" {
+				assert.True(t, started.Equal(res.StartedAt), "the old agent's start time is kept")
+			}
+		})
+	}
+
+	t.Run("an agent outside Compose resumes nothing", func(t *testing.T) {
+		cli := fakeHelperCompose{fakeCompose{fakeSelf{id: "selfid"}, []container.Summary{helper("h1", "updater", "agent", nil)}}, &fakeHelper{}}
+		r := NewRunner(nil, time.Minute)
+		require.NoError(t, ResumeSelfUpdates(context.Background(), cli, "me", r, slog.New(slog.NewTextHandler(io.Discard, nil))))
+		assert.Empty(t, r.Known())
+	})
+}

internal/agent/updater/swarm.go

@@ -9,7 +9,6 @@ import (
 	"github.com/docker/docker/api/types/system"
 
 	"github.com/shcizo/package-updater/internal/imageref"
-	"github.com/shcizo/package-updater/internal/protocol"
 )
 
 // SwarmClient is the subset of the Docker SDK SwarmExecutor depends on.
@@ -40,7 +39,7 @@ func NewSwarmExecutor(cli SwarmClient, resolve *Resolver, auth *RegistryAuth) *S
 // the service that was inspected, so it changes exactly that object. Success
 // means the daemon accepted the new spec, not that the tasks have converged.
 // It returns the pinned ref.
-func (e *SwarmExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+func (e *SwarmExecutor) Execute(ctx context.Context, job Job, out io.Writer) (string, error) {
 	info, err := e.cli.Info(ctx)
 	if err != nil {
 		return "", fmt.Errorf("docker info: %w", err)

internal/agent/updater/swarm_test.go

@@ -79,7 +79,7 @@ func swarmSvc(name string) swarm.Service {
 
 func TestSwarmUsesPinnedRef(t *testing.T) {
 	const ref = "gitea/gitea:1.26@sha256:abababababababababababababababababababababababababababababababab"
-	job := protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: ref}
+	job := Job{JobStart: protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: ref}}
 	fake := &fakeSwarm{info: managerOf("c1"), svc: swarmSvc("gitea_gitea")}
 	dist := &fakeDist{digest: "sha256:other"}
 	var out bytes.Buffer
@@ -94,7 +94,7 @@ func TestSwarmUsesPinnedRef(t *testing.T) {
 }
 
 func TestSwarmExecutor(t *testing.T) {
-	job := protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"}
+	job := Job{JobStart: protocol.JobStart{Kind: protocol.KindSwarm, SwarmClusterID: "c1", SwarmService: "gitea_gitea", Ref: "gitea/gitea:1.27"}}
 	workerInfo := managerOf("c1")
 	workerInfo.Swarm.ControlAvailable = false
 	noContainer := swarmSvc("gitea_gitea")

internal/e2e/jobs_test.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"log/slog"
 	"net/http"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -46,7 +47,7 @@ func newGateExecutor() *gateExecutor {
 	return &gateExecutor{started: make(chan string, 8), release: make(chan struct{})}
 }
 
-func (g *gateExecutor) Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) {
+func (g *gateExecutor) Execute(ctx context.Context, job updater.Job, out io.Writer) (string, error) {
 	_, _ = io.WriteString(out, "pulling "+job.Ref+"\n")
 	g.started <- job.Ref
 	select {
@@ -69,11 +70,17 @@ type jobAgent struct {
 	stopRun  func()
 }
 
-func startJobAgent(t *testing.T, m testMaster, token string, exec updater.Executor) *jobAgent {
+// startJobAgent starts an agent. before, if given, runs on the new runner
+// before the worker and the connection start, like ResumeSelfUpdates in the
+// real agent.
+func startJobAgent(t *testing.T, m testMaster, token string, exec updater.Executor, before ...func(*updater.Runner)) *jobAgent {
 	t.Helper()
 	src := &staticSource{}
 	src.set(web)
 	runner := updater.NewRunner(exec, time.Minute)
+	for _, f := range before {
+		f(runner)
+	}
 	runCtx, stopRun := context.WithCancel(context.Background())
 	runDone := make(chan struct{})
 	go func() { runner.Run(runCtx); close(runDone) }()
@@ -112,7 +119,7 @@ func newJobEnv(t *testing.T) *jobEnv {
 	m := startMaster(t, store)
 	exec := newGateExecutor()
 	env := &jobEnv{store: store, m: m, exec: exec,
-		jobs:  jobs.New(store, m.hub, m.broker, "gitea.shcizo.se/shcizo/package-updater:latest", slog.New(slog.NewTextHandler(io.Discard, nil))),
+		jobs:  jobs.New(store, m.hub, m.broker, slog.New(slog.NewTextHandler(io.Discard, nil))),
 		agent: startJobAgent(t, m, "pua_ok", exec)}
 	env.waitRoutable(t)
 	return env
@@ -221,13 +228,61 @@ func TestAgentRestartMarksJobLost(t *testing.T) {
 	assert.Equal(t, lostMessage, lost.Error)
 }
 
+// TestAgentSelfUpdateResumedByNewAgent follows a self-update: the old agent
+// dies while its job runs, and the new agent finishes the job under the same
+// ID before it says hello, so the master neither loses nor re-runs it.
+func TestAgentSelfUpdateResumedByNewAgent(t *testing.T) {
+	pinned := slusk + ":1.4.2@" + e2eDigest
+	tests := []struct {
+		name      string
+		resume    updater.ResumeFunc
+		wantError string
+	}{
+		{"helper succeeded", func(_ context.Context, out io.Writer) (string, error) {
+			_, _ = io.WriteString(out, "helper output\n")
+			return pinned, nil
+		}, ""},
+		{"helper failed", func(context.Context, io.Writer) (string, error) {
+			return "", errors.New("docker compose up: exit status 1 (output: boom)")
+		}, "docker compose up: exit status 1 (output: boom)"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			e := newJobEnv(t)
+			j := e.trigger(t)
+			e.waitStarted(t)
+			started := time.Now().Add(-time.Second)
+
+			e.agent.stopConn()
+			e.agent.stopRun()
+			startJobAgent(t, e.m, "pua_ok", newGateExecutor(), func(r *updater.Runner) {
+				r.Resume(strconv.FormatInt(j.ID, 10), started, tt.resume)
+			})
+
+			if tt.wantError != "" {
+				done := e.waitStatus(t, j.ID, model.JobFailed)
+				assert.Equal(t, tt.wantError, done.Error)
+				assert.NotEqual(t, lostMessage, done.Error)
+				return
+			}
+			done := e.waitStatus(t, j.ID, model.JobSucceeded)
+			assert.Empty(t, done.Error)
+			assert.Equal(t, pinned, done.ToRef)
+			assert.Contains(t, done.Output, "helper output")
+			svc, err := e.store.GetService(context.Background(), 1)
+			require.NoError(t, err)
+			assert.Equal(t, pinned, svc.DesiredRef)
+		})
+	}
+}
+
 func TestPhaseOneAgentGetsNoJobs(t *testing.T) {
 	store := &memStore{agents: map[string]model.Agent{agenttoken.Hash("pua_old"): {ID: agentID, Name: "node-2"}}}
 	svcRow := model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
 		AgentID: ptr(agentID), ActualRef: slusk + ":1.4.2@sha256:old"}
 	store.seedService(svcRow)
 	m := startMaster(t, store)
-	js := jobs.New(store, m.hub, m.broker, "gitea.shcizo.se/shcizo/package-updater:latest", slog.New(slog.NewTextHandler(io.Discard, nil)))
+	js := jobs.New(store, m.hub, m.broker, slog.New(slog.NewTextHandler(io.Discard, nil)))
 
 	// A raw client speaking the phase-1 protocol: its hello has no
 	// protocol_version.

internal/master/hub/route.go

@@ -16,8 +16,9 @@ type candidate struct {
 
 // Route picks the connection a job for svc goes to, among agents that speak
 // at least protocol version minProtocol. It returns model.ErrAgentOffline or
-// model.ErrAgentTooOld (model.ErrAgentTooOldForPinned when minProtocol asks
-// for a pinned ref) when there is none.
+// model.ErrAgentTooOld (model.ErrAgentTooOldForSelfUpdate when minProtocol
+// asks for self-update, model.ErrAgentTooOldForPinned when it asks for a
+// pinned ref) when there is none.
 func (h *Hub) Route(svc model.Service, minProtocol int) (model.Route, error) {
 	h.mu.Lock()
 	conns := make([]candidate, 0, len(h.conns))
@@ -65,6 +66,9 @@ func route(svc model.Service, conns []candidate, minProtocol int) (model.Route,
 		}
 	}
 	if tooOld {
+		if minProtocol >= protocol.MinSelfUpdateVersion {
+			return model.Route{}, model.ErrAgentTooOldForSelfUpdate
+		}
 		if minProtocol >= protocol.MinPinnedRefVersion {
 			return model.Route{}, model.ErrAgentTooOldForPinned
 		}

internal/master/hub/route_test.go

@@ -14,9 +14,10 @@ func TestRoute(t *testing.T) {
 	swarmSvc := model.Service{Kind: protocol.KindSwarm, AgentID: ptr(int64(1)), SwarmClusterID: "c1"}
 	v2 := protocol.Hello{ProtocolVersion: 2}
 	v3 := protocol.Hello{ProtocolVersion: 3}
+	v4 := protocol.Hello{ProtocolVersion: 4}
 	mgrV2 := protocol.Hello{ProtocolVersion: 2, SwarmManager: true, SwarmClusterID: "c1"}
 	mgrV3 := protocol.Hello{ProtocolVersion: 3, SwarmManager: true, SwarmClusterID: "c1"}
-	jobs, pinned := protocol.MinJobVersion, protocol.MinPinnedRefVersion
+	jobs, pinned, selfUpdate := protocol.MinJobVersion, protocol.MinPinnedRefVersion, protocol.MinSelfUpdateVersion
 
 	tests := []struct {
 		name  string
@@ -47,6 +48,8 @@ func TestRoute(t *testing.T) {
 			[]candidate{{1, 10, protocol.Hello{ProtocolVersion: 2, SwarmManager: true}}}, jobs, model.Route{}, model.ErrAgentOffline},
 		{"swarm, pinned ref falls back to a v3 manager when the reporting one is v2", swarmSvc,
 			[]candidate{{1, 10, mgrV2}, {3, 30, mgrV3}}, pinned, model.Route{AgentID: 3, ConnID: 30}, nil},
+		{"compose, own agent is v3 and the job updates the agent itself", composeSvc, []candidate{{2, 20, v3}}, selfUpdate, model.Route{}, model.ErrAgentTooOldForSelfUpdate},
+		{"compose, v4 agent and a self-update", composeSvc, []candidate{{2, 20, v4}}, selfUpdate, model.Route{AgentID: 2, ConnID: 20}, nil},
 		{"swarm, pinned ref and only v2 managers", swarmSvc, []candidate{{1, 10, mgrV2}}, pinned, model.Route{}, model.ErrAgentTooOldForPinned},
 	}
 	for _, tt := range tests {
@@ -57,3 +60,11 @@ func TestRoute(t *testing.T) {
 		})
 	}
 }
+
+func TestRoute_TooOldForSelfUpdateIsAlsoTooOld(t *testing.T) {
+	svc := model.Service{Kind: protocol.KindCompose, AgentID: ptr(int64(2))}
+	_, err := route(svc, []candidate{{2, 20, protocol.Hello{ProtocolVersion: 3}}}, protocol.MinSelfUpdateVersion)
+	assert.ErrorIs(t, err, model.ErrAgentTooOldForSelfUpdate)
+	assert.ErrorIs(t, err, model.ErrAgentTooOld)
+	assert.EqualError(t, err, "agent too old to update itself")
+}

internal/master/jobs/drift_test.go

@@ -51,12 +51,13 @@ func TestRestorable(t *testing.T) {
 		{"never updated", func(s *model.Service) { s.DesiredRef = "" }, model.ReasonNotDrifted},
 		{"awaiting inventory", func(s *model.Service) { s.AwaitingInventory = true }, model.ReasonNotDrifted},
 		{"removed", func(s *model.Service) { s.RemovedAt = ptr(time.Now()) }, model.ReasonRemoved},
-		{"flagged self", func(s *model.Service) { s.Self = true }, model.ReasonSelf},
+		{"agent's own Compose service", func(s *model.Service) { s.Self = true }, ""},
+		{"agent's own Swarm service", func(s *model.Service) { s.Self = true; s.Kind = protocol.KindSwarm }, model.ReasonSelfSwarm},
 		{"built", func(s *model.Service) { s.Built = true }, model.ReasonBuilt},
 		{"container", func(s *model.Service) { s.Kind = protocol.KindContainer }, model.ReasonContainer},
 		{"desired ref is the agent image", func(s *model.Service) {
-			s.DesiredRef = selfImage + "@" + driftD1
-		}, model.ReasonSelf},
+			s.DesiredRef = "gitea.shcizo.se/shcizo/package-updater:latest@" + driftD1
+		}, ""},
 	}
 	s, _ := newService(&fakeStore{}, &fakeHub{})
 	for _, tt := range tests {
@@ -100,7 +101,7 @@ func TestRestoreDesired_Refusals(t *testing.T) {
 		reason string
 	}{
 		{"not drifted", func(s *model.Service) { s.ActualDigest = driftD1 }, model.ReasonNotDrifted},
-		{"self", func(s *model.Service) { s.Self = true }, model.ReasonSelf},
+		{"own Swarm service", func(s *model.Service) { s.Self = true; s.Kind = protocol.KindSwarm }, model.ReasonSelfSwarm},
 		{"built", func(s *model.Service) { s.Built = true }, model.ReasonBuilt},
 		{"container", func(s *model.Service) { s.Kind = protocol.KindContainer }, model.ReasonContainer},
 	}

internal/master/jobs/jobs.go

@@ -40,21 +40,16 @@ type Publisher interface{ Publish(t events.Topic) }
 
 // Service starts update jobs.
 type Service struct {
-	store    Store
-	hub      Hub
-	pub      Publisher
-	selfRepo string
-	log      *slog.Logger
-	now      func() time.Time
+	store Store
+	hub   Hub
+	pub   Publisher
+	log   *slog.Logger
+	now   func() time.Time
 }
 
-// New returns a Service. agentImage is the master's AGENT_IMAGE; services
-// whose image repo equals its repo are the updater itself and are refused.
-func New(store Store, hub Hub, pub Publisher, agentImage string, log *slog.Logger) *Service {
-	return &Service{
-		store: store, hub: hub, pub: pub, log: log, now: time.Now,
-		selfRepo: imageref.Parse(agentImage).Repo,
-	}
+// New returns a Service.
+func New(store Store, hub Hub, pub Publisher, log *slog.Logger) *Service {
+	return &Service{store: store, hub: hub, pub: pub, log: log, now: time.Now}
 }
 
 // target splits a service's actual ref into the repo and tag to deploy. A bare
@@ -102,8 +97,8 @@ func (s *Service) updatable(svc model.Service, explicitRef bool) error {
 	switch {
 	case svc.RemovedAt != nil:
 		return &model.NotUpdatableError{Reason: model.ReasonRemoved}
-	case svc.Self || imageref.Parse(svc.ActualRef).Repo == s.selfRepo:
-		return &model.NotUpdatableError{Reason: model.ReasonSelf}
+	case svc.Self && svc.Kind == protocol.KindSwarm:
+		return &model.NotUpdatableError{Reason: model.ReasonSelfSwarm}
 	case svc.Built:
 		return &model.NotUpdatableError{Reason: model.ReasonBuilt}
 	case svc.Kind == protocol.KindContainer:
@@ -126,13 +121,7 @@ func (s *Service) Restorable(svc model.Service) error {
 	if !svc.Drifted() {
 		return &model.NotUpdatableError{Reason: model.ReasonNotDrifted}
 	}
-	if err := s.updatable(svc, true); err != nil {
-		return err
-	}
-	if imageref.Parse(svc.DesiredRef).Repo == s.selfRepo {
-		return &model.NotUpdatableError{Reason: model.ReasonSelf}
-	}
-	return nil
+	return s.updatable(svc, true)
 }
 
 // RestoreDesired starts a manual job that deploys the drifted service's
@@ -232,9 +221,10 @@ func (s *Service) TriggerVersion(ctx context.Context, serviceID, versionID int64
 }
 
 // Start starts an update of svc. It returns the job, which may already be
-// failed ("agent offline", "agent too old for updates", or "agent too old to
+// failed ("agent offline", "agent too old for updates", "agent too old to
 // deploy a pinned version" when the ref has a digest or no tag and the agent
-// speaks protocol version 2); a
+// speaks protocol version 2, or "agent too old to update itself" for an
+// agent's own service before protocol version 4); a
 // *model.NotUpdatableError when the service cannot be updated (no job
 // created); the active job and model.ErrJobActive when one is queued or
 // running. Poll and webhook jobs are refused for a drifted service; only a
@@ -255,6 +245,9 @@ func (s *Service) Start(ctx context.Context, svc model.Service, req StartRequest
 	if p := imageref.Parse(ref); p.Digest != "" || p.Tag == "" {
 		minProtocol = protocol.MinPinnedRefVersion
 	}
+	if svc.Self {
+		minProtocol = protocol.MinSelfUpdateVersion
+	}
 	r, routeErr := s.hub.Route(svc, minProtocol)
 	nj := model.NewJob{
 		ServiceID: svc.ID, FromRef: svc.ActualRef, ToRef: ref,
@@ -276,7 +269,7 @@ func (s *Service) Start(ctx context.Context, svc model.Service, req StartRequest
 
 	failure := routeErr
 	if failure == nil {
-		start := protocol.JobStart{Kind: svc.Kind, Ref: ref}
+		start := protocol.JobStart{Kind: svc.Kind, Ref: ref, Self: svc.Self}
 		if svc.Kind == protocol.KindSwarm {
 			start.SwarmClusterID, start.SwarmService = svc.SwarmClusterID, svc.SwarmService
 		} else {

internal/master/jobs/jobs_test.go

@@ -92,6 +92,9 @@ type fakeHub struct {
 func (h *fakeHub) Route(_ model.Service, minProtocol int) (model.Route, error) {
 	h.mins = append(h.mins, minProtocol)
 	if h.agentProtocol != 0 && minProtocol > h.agentProtocol {
+		if minProtocol >= protocol.MinSelfUpdateVersion {
+			return model.Route{}, model.ErrAgentTooOldForSelfUpdate
+		}
 		if minProtocol >= protocol.MinPinnedRefVersion {
 			return model.Route{}, model.ErrAgentTooOldForPinned
 		}
@@ -109,8 +112,6 @@ type fakePub struct{ topics []events.Topic }
 
 func (p *fakePub) Publish(t events.Topic) { p.topics = append(p.topics, t) }
 
-const selfImage = "gitea.shcizo.se/shcizo/package-updater:latest"
-
 var (
 	composeSvc = model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
 		AgentID: ptr(int64(7)), ActualRef: "gitea.shcizo.se/shcizo/slusk:1.4.2@sha256:d1"}
@@ -124,7 +125,7 @@ func ptr[T any](v T) *T { return &v }
 
 func newService(st *fakeStore, h *fakeHub) (*Service, *fakePub) {
 	pub := &fakePub{}
-	return New(st, h, pub, selfImage, slog.New(slog.NewTextHandler(io.Discard, nil))), pub
+	return New(st, h, pub, slog.New(slog.NewTextHandler(io.Discard, nil))), pub
 }
 
 func TestUpdatable(t *testing.T) {
@@ -134,9 +135,10 @@ func TestUpdatable(t *testing.T) {
 		reason string
 	}{
 		{"removed", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", RemovedAt: ptr(time.Now())}, model.ReasonRemoved},
-		{"flagged self", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, model.ReasonSelf},
+		{"agent's own Compose service", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, ""},
+		{"agent's own Swarm service", model.Service{Kind: protocol.KindSwarm, ActualRef: "a:1", Self: true}, model.ReasonSelfSwarm},
 		{"same repo as the agent image", model.Service{Kind: protocol.KindCompose,
-			ActualRef: "gitea.shcizo.se/shcizo/package-updater:1.2.0@sha256:abc"}, model.ReasonSelf},
+			ActualRef: "gitea.shcizo.se/shcizo/package-updater:1.2.0@sha256:abc"}, ""},
 		{"built from source", model.Service{Kind: protocol.KindCompose, ActualRef: "slusk:1", Built: true}, model.ReasonBuilt},
 		{"container", model.Service{Kind: protocol.KindContainer, ActualRef: "redis:7"}, model.ReasonContainer},
 		{"digest without tag", model.Service{Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}, model.ReasonNoTag},
@@ -298,7 +300,8 @@ func TestStart_ExplicitRefBypassesOnlyTheNoTagRefusal(t *testing.T) {
 	}{
 		{"digest only", model.Service{Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}, ""},
 		{"removed", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", RemovedAt: ptr(time.Now())}, model.ReasonRemoved},
-		{"self", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, model.ReasonSelf},
+		{"own Swarm service", model.Service{Kind: protocol.KindSwarm, ActualRef: "a:1", Self: true}, model.ReasonSelfSwarm},
+		{"own Compose service", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, ""},
 		{"built", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Built: true}, model.ReasonBuilt},
 		{"container", model.Service{Kind: protocol.KindContainer, ActualRef: "redis:7"}, model.ReasonContainer},
 	}
@@ -342,8 +345,9 @@ func TestAutoAllowed(t *testing.T) {
 		reason string
 	}{
 		{"removed", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", RemovedAt: ptr(time.Now())}, model.ReasonRemoved},
-		{"self", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, model.ReasonSelf},
-		{"agent image repo", model.Service{Kind: protocol.KindCompose, ActualRef: selfImage}, model.ReasonSelf},
+		{"own Swarm service", model.Service{Kind: protocol.KindSwarm, ActualRef: "a:1", Self: true}, model.ReasonSelfSwarm},
+		{"own Compose service", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, ""},
+		{"agent image repo", model.Service{Kind: protocol.KindCompose, ActualRef: "gitea.shcizo.se/shcizo/package-updater:latest"}, ""},
 		{"built", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Built: true}, model.ReasonBuilt},
 		{"container", model.Service{Kind: protocol.KindContainer, ActualRef: "redis:7"}, model.ReasonContainer},
 		{"digest without tag is allowed", model.Service{Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}, ""},
@@ -396,6 +400,47 @@ func TestStart_RoutesWithTheMinimumProtocolOfTheRef(t *testing.T) {
 	}
 }
 
+func TestStart_SelfServiceNeedsASelfUpdatingAgent(t *testing.T) {
+	self := composeSvc
+	self.Self = true
+	ok := model.Route{AgentID: 7, ConnID: 70}
+
+	t.Run("routes with the self-update protocol and marks the job", func(t *testing.T) {
+		st := &fakeStore{services: map[int64]model.Service{1: self}, jobs: map[int64]model.Job{}}
+		h := &fakeHub{route: ok}
+		s, _ := newService(st, h)
+
+		_, err := s.Start(context.Background(), self, StartRequest{Trigger: model.TriggerManual, Ref: "x/y:2@sha256:a"})
+
+		require.NoError(t, err)
+		assert.Equal(t, []int{protocol.MinSelfUpdateVersion}, h.mins, "also above the pinned-ref minimum")
+		require.Len(t, h.starts, 1)
+		assert.True(t, h.starts[0].start.Self)
+	})
+	t.Run("a service that is not the agent's own is not marked", func(t *testing.T) {
+		st := &fakeStore{services: map[int64]model.Service{1: composeSvc}, jobs: map[int64]model.Job{}}
+		h := &fakeHub{route: ok}
+		s, _ := newService(st, h)
+
+		_, err := s.Start(context.Background(), composeSvc, StartRequest{Trigger: model.TriggerManual})
+
+		require.NoError(t, err)
+		assert.False(t, h.starts[0].start.Self)
+	})
+	t.Run("an older agent fails the job with a clear error", func(t *testing.T) {
+		st := &fakeStore{services: map[int64]model.Service{1: self}, jobs: map[int64]model.Job{}}
+		h := &fakeHub{route: ok, agentProtocol: 3}
+		s, _ := newService(st, h)
+
+		job, err := s.Start(context.Background(), self, StartRequest{Trigger: model.TriggerManual})
+
+		require.NoError(t, err)
+		assert.Equal(t, model.JobFailed, job.Status)
+		assert.Equal(t, "agent too old to update itself", job.Error)
+		assert.Empty(t, h.starts)
+	})
+}
+
 func TestTriggerVersion_Ref(t *testing.T) {
 	const repo = "gitea.shcizo.se/shcizo/slusk"
 	noTag := model.Service{ID: 4, Kind: protocol.KindCompose, ComposeProject: "n", ComposeService: "n",

internal/master/model/job.go

@@ -19,6 +19,10 @@ var (
 	// speaks protocol version 2, which cannot deploy a digest-pinned or
 	// tagless ref. errors.Is(err, ErrAgentTooOld) is true for it as well.
 	ErrAgentTooOldForPinned error = agentTooOldForPinned{}
+	// ErrAgentTooOldForSelfUpdate means the agent that runs the service speaks
+	// a protocol version before 4 and cannot update itself.
+	// errors.Is(err, ErrAgentTooOld) is true for it as well.
+	ErrAgentTooOldForSelfUpdate error = agentTooOldForSelfUpdate{}
 )
 
 type agentTooOldForPinned struct{}
@@ -29,6 +33,12 @@ func (agentTooOldForPinned) Error() string {
 
 func (agentTooOldForPinned) Is(target error) bool { return target == ErrAgentTooOld }
 
+type agentTooOldForSelfUpdate struct{}
+
+func (agentTooOldForSelfUpdate) Error() string { return "agent too old to update itself" }
+
+func (agentTooOldForSelfUpdate) Is(target error) bool { return target == ErrAgentTooOld }
+
 // NotUpdatableError says why a service cannot be updated. Reason is shown
 // in the UI as is.
 type NotUpdatableError struct {
@@ -44,6 +54,9 @@ const (
 	ReasonNoTag     = "follows no tag"
 	ReasonRemoved   = "service no longer exists"
 	ReasonBuilt     = "built from source (build:)"
+	// ReasonSelfSwarm is for an agent's own Swarm service, which cannot be
+	// updated by the agent that runs in it.
+	ReasonSelfSwarm = "an agent's own Swarm service cannot update itself"
 
 	ReasonDrifted    = "drifted: restore or accept the running version first"
 	ReasonNotDrifted = "runs its desired version"

internal/master/poller/poller_test.go

@@ -332,7 +332,7 @@ func TestStartErrorsDoNotAbortCycle(t *testing.T) {
 	reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
 	j := &fakeJobs{errFor: map[int64]error{
 		1: model.ErrJobActive,
-		2: &model.NotUpdatableError{Reason: model.ReasonSelf},
+		2: &model.NotUpdatableError{Reason: model.ReasonContainer},
 		3: errors.New("database down"),
 	}}
 	p, _ := newPoller(st, j, reg, Config{})

internal/master/web/server.go

@@ -883,6 +883,7 @@ func agentSnippet(cfg Config, token string) string {
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       # - /path/to/stacks:/path/to/stacks  # mount each stacks directory at the same path, read-write
+      #   (also this agent's own stack directory, or it cannot update itself)
       # - /root/.docker/config.json:/docker-config/config.json:ro
 `, cfg.AgentImage, cfg.AgentURL, cfg.MasterKey, token)
 }

internal/master/web/server_test.go

@@ -303,7 +303,8 @@ func TestOverview_GroupsServicesPerMachineAndSwarm(t *testing.T) {
 	assert.Contains(t, body, `<span class="stk">slusk</span>`)
 	assert.Contains(t, body, "jellyfin_jellyfin", "Swarm service names are shown as they are")
 	assert.Contains(t, body, "sha256:0f42497a69fa…", "digests are shortened")
-	assert.Contains(t, body, "Managed outside")
+	assert.Contains(t, body, `<span class="tag f-out">This agent</span>`)
+	assert.NotContains(t, body, "Managed outside")
 	assert.Less(t, strings.Index(body, ">Swarm</h2>"), strings.Index(body, ">node-2</h2>"), "a cluster is followed by its managers' machines")
 	assert.Contains(t, body, `class="sub" data-machine="agent-2"`, "the machine filter lists the manager under the cluster")
 }
@@ -513,6 +514,7 @@ func TestAgentSnippet(t *testing.T) {
 	assert.Contains(t, snippet, "UPDATE_TIMEOUT: 5m")
 	assert.Contains(t, snippet, "      # DOCKER_CONFIG: /docker-config\n")
 	assert.Contains(t, snippet, "      # - /path/to/stacks:/path/to/stacks  # mount each stacks directory at the same path, read-write\n")
+	assert.Contains(t, snippet, "also this agent's own stack directory")
 	assert.Contains(t, snippet, "AGENT_TOKEN: tok")
 }
 
@@ -762,7 +764,8 @@ func TestOverview_ShowsModeToggleAndAvailableBadge(t *testing.T) {
 		assert.Contains(t, body, `<input type="hidden" name="mode" value="auto">`, "manual services toggle to auto: "+path)
 		assert.Contains(t, body, `aria-checked="false" disabled title="`+model.ReasonBuilt+`">`, "blocked manual services get a disabled switch: "+path)
 		assert.Contains(t, body, `<span class="na">`+model.ReasonBuilt+`</span>`, "and say why: "+path)
-		assert.Contains(t, body, `<span class="na">n/a</span>`, "services managed outside have no switch: "+path)
+		assert.Contains(t, body, `action="/services/3/mode"`, "the agent's own service has a switch like any other: "+path)
+		assert.NotContains(t, body, `<span class="na">n/a</span>`, path)
 	}
 }
 

internal/master/web/templates/overview.html

@@ -79,11 +79,10 @@
       {{if .AvailableRef}}<span class="tag f-up" title="{{.AvailableRef}}">Update available<em>{{.AvailableTag}}</em></span>{{end}}
       {{if followsNoTag .Service}}<span class="tag f-notag">Follows no tag</span>{{end}}
       {{if .LastPollError}}<span class="tag f-err" title="{{.LastPollError}}">Poll failed</span>{{end}}
-      {{if .Self}}<span class="tag f-out">Managed outside</span>{{end}}
+      {{if .Self}}<span class="tag f-out">This agent</span>{{end}}
     </div>
     <div class="mode">
-      {{if .Self}}<span class="na">n/a</span>
-      {{else}}<form method="post" action="/services/{{.ID}}/mode">
+      <form method="post" action="/services/{{.ID}}/mode">
         <input type="hidden" name="csrf_token" value="{{$.CSRF}}">
         <input type="hidden" name="return" value="overview">
         {{if eq .UpdateMode "auto"}}<input type="hidden" name="mode" value="manual">
@@ -91,7 +90,7 @@
         {{else}}<input type="hidden" name="mode" value="auto">
         <button type="submit" class="sw" role="switch" aria-checked="false"{{if .AutoBlocked}} disabled title="{{.AutoBlocked}}"{{end}}><i></i>Manual</button>
         {{if .AutoBlocked}}<span class="na">{{.AutoBlocked}}</span>{{end}}{{end}}
-      </form>{{end}}
+      </form>
     </div>
   </div>
   {{end}}

internal/master/webhook/webhook_test.go

@@ -470,14 +470,14 @@ func TestBusyIsNotAFailure(t *testing.T) {
 
 func TestRefusedIsNotAFailure(t *testing.T) {
 	e := newEnv(t, compose(1, model.ModeAuto, slusk+":1.4.2"))
-	e.jobs.errFor[1] = &model.NotUpdatableError{Reason: model.ReasonSelf}
+	e.jobs.errFor[1] = &model.NotUpdatableError{Reason: model.ReasonContainer}
 
 	rec := e.post(`{"image":"` + slusk + `","tag":"2"}`)
 
 	assert.Equal(t, 200, rec.Code)
 	r := decode(t, rec).Results[0]
 	assert.Equal(t, statusRefused, r.Status)
-	assert.Equal(t, model.ReasonSelf, r.Error)
+	assert.Equal(t, model.ReasonContainer, r.Error)
 	assert.Zero(t, r.JobID)
 	assert.NotEmpty(t, e.store.avail)
 }

internal/protocol/protocol.go

@@ -16,7 +16,7 @@ import (
 
 // Version is the protocol version this build speaks. Agents send it in
 // Hello. Phase-1 agents do not send the field, which reads as 0.
-const Version = 3
+const Version = 4
 
 // MinJobVersion is the lowest agent protocol version the master sends jobs to.
 const MinJobVersion = 2
@@ -26,6 +26,26 @@ const MinJobVersion = 2
 // with a digest.
 const MinPinnedRefVersion = 3
 
+// MinSelfUpdateVersion is the lowest agent protocol version the master sends a
+// job for the agent's own Compose service to. Older agents neither run the
+// update in a helper container nor resume it after restarting.
+const MinSelfUpdateVersion = 4
+
+// SelfUpdateLabelPrefix starts the names of the labels on the helper
+// container that finishes an agent's self-update. The new agent finds the
+// helper by them, and the inventory skips containers that carry them.
+const SelfUpdateLabelPrefix = "se.shcizo.package-updater.self-update."
+
+// Labels on a self-update helper container.
+const (
+	LabelSelfUpdateJob     = SelfUpdateLabelPrefix + "job"
+	LabelSelfUpdateRef     = SelfUpdateLabelPrefix + "ref"
+	LabelSelfUpdateStarted = SelfUpdateLabelPrefix + "started"
+	LabelSelfUpdateProject = SelfUpdateLabelPrefix + "project"
+	LabelSelfUpdateService = SelfUpdateLabelPrefix + "service"
+	LabelSelfUpdateOutput  = SelfUpdateLabelPrefix + "output"
+)
+
 // MaxJobOutput caps the job output an agent keeps and the master stores, in
 // bytes. Only the tail is kept. With JSON escaping (worst case 6 bytes per
 // byte) a JobResult carrying this much output still fits the 1 MiB read limit.
@@ -141,6 +161,10 @@ type JobStart struct {
 	SwarmClusterID string `json:"swarm_cluster_id,omitempty"`
 	SwarmService   string `json:"swarm_service,omitempty"`
 	Ref            string `json:"ref"`
+	// Self says the service is the agent's own. The agent runs such a job
+	// after every other queued job; whether it needs a helper container is
+	// decided by the agent itself.
+	Self bool `json:"self,omitempty"`
 }
 
 // JobProgress carries output produced since the previous JobProgress for the