shcizo/package-updater
1
0
Fork 0
Code Issues 8 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: self-update of master and agents (#15)

Browse Source
This commit is contained in:
shcizo
2026-10-13 11:55:46 +02:00
parent ec28d1e027
commit fe4aca2883
34 changed files with 1167 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/master/hub/route.go
+6 -2
View File
@@ -16,8 +16,9 @@ type candidate struct {
// Route picks the connection a job for svc goes to, among agents that speak
// at least protocol version minProtocol. It returns model.ErrAgentOffline or
// model.ErrAgentTooOld (model.ErrAgentTooOldForPinned when minProtocol asks
// for a pinned ref) when there is none.
// model.ErrAgentTooOld (model.ErrAgentTooOldForSelfUpdate when minProtocol
// asks for self-update, model.ErrAgentTooOldForPinned when it asks for a
// pinned ref) when there is none.
func (h *Hub) Route(svc model.Service, minProtocol int) (model.Route, error) {
h.mu.Lock()
conns := make([]candidate, 0, len(h.conns))
@@ -65,6 +66,9 @@ func route(svc model.Service, conns []candidate, minProtocol int) (model.Route,
}
}
if tooOld {
if minProtocol >= protocol.MinSelfUpdateVersion {
return model.Route{}, model.ErrAgentTooOldForSelfUpdate
}
if minProtocol >= protocol.MinPinnedRefVersion {
return model.Route{}, model.ErrAgentTooOldForPinned
}
internal/master/hub/route_test.go
+12 -1
View File
@@ -14,9 +14,10 @@ func TestRoute(t *testing.T) {
swarmSvc := model.Service{Kind: protocol.KindSwarm, AgentID: ptr(int64(1)), SwarmClusterID: "c1"}
v2 := protocol.Hello{ProtocolVersion: 2}
v3 := protocol.Hello{ProtocolVersion: 3}
v4 := protocol.Hello{ProtocolVersion: 4}
mgrV2 := protocol.Hello{ProtocolVersion: 2, SwarmManager: true, SwarmClusterID: "c1"}
mgrV3 := protocol.Hello{ProtocolVersion: 3, SwarmManager: true, SwarmClusterID: "c1"}
jobs, pinned := protocol.MinJobVersion, protocol.MinPinnedRefVersion
jobs, pinned, selfUpdate := protocol.MinJobVersion, protocol.MinPinnedRefVersion, protocol.MinSelfUpdateVersion
tests := []struct {
name string
@@ -47,6 +48,8 @@ func TestRoute(t *testing.T) {
[]candidate{{1, 10, protocol.Hello{ProtocolVersion: 2, SwarmManager: true}}}, jobs, model.Route{}, model.ErrAgentOffline},
{"swarm, pinned ref falls back to a v3 manager when the reporting one is v2", swarmSvc,
[]candidate{{1, 10, mgrV2}, {3, 30, mgrV3}}, pinned, model.Route{AgentID: 3, ConnID: 30}, nil},
{"compose, own agent is v3 and the job updates the agent itself", composeSvc, []candidate{{2, 20, v3}}, selfUpdate, model.Route{}, model.ErrAgentTooOldForSelfUpdate},
{"compose, v4 agent and a self-update", composeSvc, []candidate{{2, 20, v4}}, selfUpdate, model.Route{AgentID: 2, ConnID: 20}, nil},
{"swarm, pinned ref and only v2 managers", swarmSvc, []candidate{{1, 10, mgrV2}}, pinned, model.Route{}, model.ErrAgentTooOldForPinned},
}
for _, tt := range tests {
@@ -57,3 +60,11 @@ func TestRoute(t *testing.T) {
})
}
}
func TestRoute_TooOldForSelfUpdateIsAlsoTooOld(t *testing.T) {
svc := model.Service{Kind: protocol.KindCompose, AgentID: ptr(int64(2))}
_, err := route(svc, []candidate{{2, 20, protocol.Hello{ProtocolVersion: 3}}}, protocol.MinSelfUpdateVersion)
assert.ErrorIs(t, err, model.ErrAgentTooOldForSelfUpdate)
assert.ErrorIs(t, err, model.ErrAgentTooOld)
assert.EqualError(t, err, "agent too old to update itself")
}
internal/master/jobs/drift_test.go
+5 -4
View File
@@ -51,12 +51,13 @@ func TestRestorable(t *testing.T) {
{"never updated", func(s *model.Service) { s.DesiredRef = "" }, model.ReasonNotDrifted},
{"awaiting inventory", func(s *model.Service) { s.AwaitingInventory = true }, model.ReasonNotDrifted},
{"removed", func(s *model.Service) { s.RemovedAt = ptr(time.Now()) }, model.ReasonRemoved},
{"flagged self", func(s *model.Service) { s.Self = true }, model.ReasonSelf},
{"agent's own Compose service", func(s *model.Service) { s.Self = true }, ""},
{"agent's own Swarm service", func(s *model.Service) { s.Self = true; s.Kind = protocol.KindSwarm }, model.ReasonSelfSwarm},
{"built", func(s *model.Service) { s.Built = true }, model.ReasonBuilt},
{"container", func(s *model.Service) { s.Kind = protocol.KindContainer }, model.ReasonContainer},
{"desired ref is the agent image", func(s *model.Service) {
s.DesiredRef = selfImage + "@" + driftD1
}, model.ReasonSelf},
s.DesiredRef = "gitea.shcizo.se/shcizo/package-updater:latest@" + driftD1
}, ""},
}
s, _ := newService(&fakeStore{}, &fakeHub{})
for _, tt := range tests {
@@ -100,7 +101,7 @@ func TestRestoreDesired_Refusals(t *testing.T) {
reason string
}{
{"not drifted", func(s *model.Service) { s.ActualDigest = driftD1 }, model.ReasonNotDrifted},
{"self", func(s *model.Service) { s.Self = true }, model.ReasonSelf},
{"own Swarm service", func(s *model.Service) { s.Self = true; s.Kind = protocol.KindSwarm }, model.ReasonSelfSwarm},
{"built", func(s *model.Service) { s.Built = true }, model.ReasonBuilt},
{"container", func(s *model.Service) { s.Kind = protocol.KindContainer }, model.ReasonContainer},
}
internal/master/jobs/jobs.go
+18 -25
View File
@@ -40,21 +40,16 @@ type Publisher interface{ Publish(t events.Topic) }
// Service starts update jobs.
type Service struct {
store Store
hub Hub
pub Publisher
selfRepo string
log *slog.Logger
now func() time.Time
store Store
hub Hub
pub Publisher
log *slog.Logger
now func() time.Time
}
// New returns a Service. agentImage is the master's AGENT_IMAGE; services
// whose image repo equals its repo are the updater itself and are refused.
func New(store Store, hub Hub, pub Publisher, agentImage string, log *slog.Logger) *Service {
return &Service{
store: store, hub: hub, pub: pub, log: log, now: time.Now,
selfRepo: imageref.Parse(agentImage).Repo,
}
// New returns a Service.
func New(store Store, hub Hub, pub Publisher, log *slog.Logger) *Service {
return &Service{store: store, hub: hub, pub: pub, log: log, now: time.Now}
}
// target splits a service's actual ref into the repo and tag to deploy. A bare
@@ -102,8 +97,8 @@ func (s *Service) updatable(svc model.Service, explicitRef bool) error {
switch {
case svc.RemovedAt != nil:
return &model.NotUpdatableError{Reason: model.ReasonRemoved}
case svc.Self || imageref.Parse(svc.ActualRef).Repo == s.selfRepo:
return &model.NotUpdatableError{Reason: model.ReasonSelf}
case svc.Self && svc.Kind == protocol.KindSwarm:
return &model.NotUpdatableError{Reason: model.ReasonSelfSwarm}
case svc.Built:
return &model.NotUpdatableError{Reason: model.ReasonBuilt}
case svc.Kind == protocol.KindContainer:
@@ -126,13 +121,7 @@ func (s *Service) Restorable(svc model.Service) error {
if !svc.Drifted() {
return &model.NotUpdatableError{Reason: model.ReasonNotDrifted}
}
if err := s.updatable(svc, true); err != nil {
return err
}
if imageref.Parse(svc.DesiredRef).Repo == s.selfRepo {
return &model.NotUpdatableError{Reason: model.ReasonSelf}
}
return nil
return s.updatable(svc, true)
}
// RestoreDesired starts a manual job that deploys the drifted service's
@@ -232,9 +221,10 @@ func (s *Service) TriggerVersion(ctx context.Context, serviceID, versionID int64
}
// Start starts an update of svc. It returns the job, which may already be
// failed ("agent offline", "agent too old for updates", or "agent too old to
// failed ("agent offline", "agent too old for updates", "agent too old to
// deploy a pinned version" when the ref has a digest or no tag and the agent
// speaks protocol version 2); a
// speaks protocol version 2, or "agent too old to update itself" for an
// agent's own service before protocol version 4); a
// *model.NotUpdatableError when the service cannot be updated (no job
// created); the active job and model.ErrJobActive when one is queued or
// running. Poll and webhook jobs are refused for a drifted service; only a
@@ -255,6 +245,9 @@ func (s *Service) Start(ctx context.Context, svc model.Service, req StartRequest
if p := imageref.Parse(ref); p.Digest != "" || p.Tag == "" {
minProtocol = protocol.MinPinnedRefVersion
}
if svc.Self {
minProtocol = protocol.MinSelfUpdateVersion
}
r, routeErr := s.hub.Route(svc, minProtocol)
nj := model.NewJob{
ServiceID: svc.ID, FromRef: svc.ActualRef, ToRef: ref,
@@ -276,7 +269,7 @@ func (s *Service) Start(ctx context.Context, svc model.Service, req StartRequest
failure := routeErr
if failure == nil {
start := protocol.JobStart{Kind: svc.Kind, Ref: ref}
start := protocol.JobStart{Kind: svc.Kind, Ref: ref, Self: svc.Self}
if svc.Kind == protocol.KindSwarm {
start.SwarmClusterID, start.SwarmService = svc.SwarmClusterID, svc.SwarmService
} else {
internal/master/jobs/jobs_test.go
+53 -8
View File
@@ -92,6 +92,9 @@ type fakeHub struct {
func (h *fakeHub) Route(_ model.Service, minProtocol int) (model.Route, error) {
h.mins = append(h.mins, minProtocol)
if h.agentProtocol != 0 && minProtocol > h.agentProtocol {
if minProtocol >= protocol.MinSelfUpdateVersion {
return model.Route{}, model.ErrAgentTooOldForSelfUpdate
}
if minProtocol >= protocol.MinPinnedRefVersion {
return model.Route{}, model.ErrAgentTooOldForPinned
}
@@ -109,8 +112,6 @@ type fakePub struct{ topics []events.Topic }
func (p *fakePub) Publish(t events.Topic) { p.topics = append(p.topics, t) }
const selfImage = "gitea.shcizo.se/shcizo/package-updater:latest"
var (
composeSvc = model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
AgentID: ptr(int64(7)), ActualRef: "gitea.shcizo.se/shcizo/slusk:1.4.2@sha256:d1"}
@@ -124,7 +125,7 @@ func ptr[T any](v T) *T { return &v }
func newService(st *fakeStore, h *fakeHub) (*Service, *fakePub) {
pub := &fakePub{}
return New(st, h, pub, selfImage, slog.New(slog.NewTextHandler(io.Discard, nil))), pub
return New(st, h, pub, slog.New(slog.NewTextHandler(io.Discard, nil))), pub
}
func TestUpdatable(t *testing.T) {
@@ -134,9 +135,10 @@ func TestUpdatable(t *testing.T) {
reason string
}{
{"removed", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", RemovedAt: ptr(time.Now())}, model.ReasonRemoved},
{"flagged self", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, model.ReasonSelf},
{"agent's own Compose service", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, ""},
{"agent's own Swarm service", model.Service{Kind: protocol.KindSwarm, ActualRef: "a:1", Self: true}, model.ReasonSelfSwarm},
{"same repo as the agent image", model.Service{Kind: protocol.KindCompose,
ActualRef: "gitea.shcizo.se/shcizo/package-updater:1.2.0@sha256:abc"}, model.ReasonSelf},
ActualRef: "gitea.shcizo.se/shcizo/package-updater:1.2.0@sha256:abc"}, ""},
{"built from source", model.Service{Kind: protocol.KindCompose, ActualRef: "slusk:1", Built: true}, model.ReasonBuilt},
{"container", model.Service{Kind: protocol.KindContainer, ActualRef: "redis:7"}, model.ReasonContainer},
{"digest without tag", model.Service{Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}, model.ReasonNoTag},
@@ -298,7 +300,8 @@ func TestStart_ExplicitRefBypassesOnlyTheNoTagRefusal(t *testing.T) {
}{
{"digest only", model.Service{Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}, ""},
{"removed", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", RemovedAt: ptr(time.Now())}, model.ReasonRemoved},
{"self", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, model.ReasonSelf},
{"own Swarm service", model.Service{Kind: protocol.KindSwarm, ActualRef: "a:1", Self: true}, model.ReasonSelfSwarm},
{"own Compose service", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, ""},
{"built", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Built: true}, model.ReasonBuilt},
{"container", model.Service{Kind: protocol.KindContainer, ActualRef: "redis:7"}, model.ReasonContainer},
}
@@ -342,8 +345,9 @@ func TestAutoAllowed(t *testing.T) {
reason string
}{
{"removed", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", RemovedAt: ptr(time.Now())}, model.ReasonRemoved},
{"self", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, model.ReasonSelf},
{"agent image repo", model.Service{Kind: protocol.KindCompose, ActualRef: selfImage}, model.ReasonSelf},
{"own Swarm service", model.Service{Kind: protocol.KindSwarm, ActualRef: "a:1", Self: true}, model.ReasonSelfSwarm},
{"own Compose service", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Self: true}, ""},
{"agent image repo", model.Service{Kind: protocol.KindCompose, ActualRef: "gitea.shcizo.se/shcizo/package-updater:latest"}, ""},
{"built", model.Service{Kind: protocol.KindCompose, ActualRef: "a:1", Built: true}, model.ReasonBuilt},
{"container", model.Service{Kind: protocol.KindContainer, ActualRef: "redis:7"}, model.ReasonContainer},
{"digest without tag is allowed", model.Service{Kind: protocol.KindCompose, ActualRef: "redis@sha256:abc"}, ""},
@@ -396,6 +400,47 @@ func TestStart_RoutesWithTheMinimumProtocolOfTheRef(t *testing.T) {
}
}
func TestStart_SelfServiceNeedsASelfUpdatingAgent(t *testing.T) {
self := composeSvc
self.Self = true
ok := model.Route{AgentID: 7, ConnID: 70}
t.Run("routes with the self-update protocol and marks the job", func(t *testing.T) {
st := &fakeStore{services: map[int64]model.Service{1: self}, jobs: map[int64]model.Job{}}
h := &fakeHub{route: ok}
s, _ := newService(st, h)
_, err := s.Start(context.Background(), self, StartRequest{Trigger: model.TriggerManual, Ref: "x/y:2@sha256:a"})
require.NoError(t, err)
assert.Equal(t, []int{protocol.MinSelfUpdateVersion}, h.mins, "also above the pinned-ref minimum")
require.Len(t, h.starts, 1)
assert.True(t, h.starts[0].start.Self)
})
t.Run("a service that is not the agent's own is not marked", func(t *testing.T) {
st := &fakeStore{services: map[int64]model.Service{1: composeSvc}, jobs: map[int64]model.Job{}}
h := &fakeHub{route: ok}
s, _ := newService(st, h)
_, err := s.Start(context.Background(), composeSvc, StartRequest{Trigger: model.TriggerManual})
require.NoError(t, err)
assert.False(t, h.starts[0].start.Self)
})
t.Run("an older agent fails the job with a clear error", func(t *testing.T) {
st := &fakeStore{services: map[int64]model.Service{1: self}, jobs: map[int64]model.Job{}}
h := &fakeHub{route: ok, agentProtocol: 3}
s, _ := newService(st, h)
job, err := s.Start(context.Background(), self, StartRequest{Trigger: model.TriggerManual})
require.NoError(t, err)
assert.Equal(t, model.JobFailed, job.Status)
assert.Equal(t, "agent too old to update itself", job.Error)
assert.Empty(t, h.starts)
})
}
func TestTriggerVersion_Ref(t *testing.T) {
const repo = "gitea.shcizo.se/shcizo/slusk"
noTag := model.Service{ID: 4, Kind: protocol.KindCompose, ComposeProject: "n", ComposeService: "n",
internal/master/model/job.go
+13
View File
@@ -19,6 +19,10 @@ var (
// speaks protocol version 2, which cannot deploy a digest-pinned or
// tagless ref. errors.Is(err, ErrAgentTooOld) is true for it as well.
ErrAgentTooOldForPinned error = agentTooOldForPinned{}
// ErrAgentTooOldForSelfUpdate means the agent that runs the service speaks
// a protocol version before 4 and cannot update itself.
// errors.Is(err, ErrAgentTooOld) is true for it as well.
ErrAgentTooOldForSelfUpdate error = agentTooOldForSelfUpdate{}
)
type agentTooOldForPinned struct{}
@@ -29,6 +33,12 @@ func (agentTooOldForPinned) Error() string {
func (agentTooOldForPinned) Is(target error) bool { return target == ErrAgentTooOld }
type agentTooOldForSelfUpdate struct{}
func (agentTooOldForSelfUpdate) Error() string { return "agent too old to update itself" }
func (agentTooOldForSelfUpdate) Is(target error) bool { return target == ErrAgentTooOld }
// NotUpdatableError says why a service cannot be updated. Reason is shown
// in the UI as is.
type NotUpdatableError struct {
@@ -44,6 +54,9 @@ const (
ReasonNoTag = "follows no tag"
ReasonRemoved = "service no longer exists"
ReasonBuilt = "built from source (build:)"
// ReasonSelfSwarm is for an agent's own Swarm service, which cannot be
// updated by the agent that runs in it.
ReasonSelfSwarm = "an agent's own Swarm service cannot update itself"
ReasonDrifted = "drifted: restore or accept the running version first"
ReasonNotDrifted = "runs its desired version"
internal/master/poller/poller_test.go
+1 -1
View File
@@ -332,7 +332,7 @@ func TestStartErrorsDoNotAbortCycle(t *testing.T) {
reg := &fakeRegistry{digests: map[string]string{ref: "sha256:new"}}
j := &fakeJobs{errFor: map[int64]error{
1: model.ErrJobActive,
2: &model.NotUpdatableError{Reason: model.ReasonSelf},
2: &model.NotUpdatableError{Reason: model.ReasonContainer},
3: errors.New("database down"),
}}
p, _ := newPoller(st, j, reg, Config{})
internal/master/web/server.go
+1
View File
@@ -883,6 +883,7 @@ func agentSnippet(cfg Config, token string) string {
volumes:
- /var/run/docker.sock:/var/run/docker.sock
# - /path/to/stacks:/path/to/stacks # mount each stacks directory at the same path, read-write
# (also this agent's own stack directory, or it cannot update itself)
# - /root/.docker/config.json:/docker-config/config.json:ro
`, cfg.AgentImage, cfg.AgentURL, cfg.MasterKey, token)
}
internal/master/web/server_test.go
+5 -2
View File
@@ -303,7 +303,8 @@ func TestOverview_GroupsServicesPerMachineAndSwarm(t *testing.T) {
assert.Contains(t, body, `<span class="stk">slusk</span>`)
assert.Contains(t, body, "jellyfin_jellyfin", "Swarm service names are shown as they are")
assert.Contains(t, body, "sha256:0f42497a69fa…", "digests are shortened")
assert.Contains(t, body, "Managed outside")
assert.Contains(t, body, `<span class="tag f-out">This agent</span>`)
assert.NotContains(t, body, "Managed outside")
assert.Less(t, strings.Index(body, ">Swarm</h2>"), strings.Index(body, ">node-2</h2>"), "a cluster is followed by its managers' machines")
assert.Contains(t, body, `class="sub" data-machine="agent-2"`, "the machine filter lists the manager under the cluster")
}
@@ -513,6 +514,7 @@ func TestAgentSnippet(t *testing.T) {
assert.Contains(t, snippet, "UPDATE_TIMEOUT: 5m")
assert.Contains(t, snippet, " # DOCKER_CONFIG: /docker-config\n")
assert.Contains(t, snippet, " # - /path/to/stacks:/path/to/stacks # mount each stacks directory at the same path, read-write\n")
assert.Contains(t, snippet, "also this agent's own stack directory")
assert.Contains(t, snippet, "AGENT_TOKEN: tok")
}
@@ -762,7 +764,8 @@ func TestOverview_ShowsModeToggleAndAvailableBadge(t *testing.T) {
assert.Contains(t, body, `<input type="hidden" name="mode" value="auto">`, "manual services toggle to auto: "+path)
assert.Contains(t, body, `aria-checked="false" disabled title="`+model.ReasonBuilt+`">`, "blocked manual services get a disabled switch: "+path)
assert.Contains(t, body, `<span class="na">`+model.ReasonBuilt+`</span>`, "and say why: "+path)
assert.Contains(t, body, `<span class="na">n/a</span>`, "services managed outside have no switch: "+path)
assert.Contains(t, body, `action="/services/3/mode"`, "the agent's own service has a switch like any other: "+path)
assert.NotContains(t, body, `<span class="na">n/a</span>`, path)
}
}
internal/master/web/templates/overview.html
+3 -4
View File
@@ -79,11 +79,10 @@
{{if .AvailableRef}}<span class="tag f-up" title="{{.AvailableRef}}">Update available<em>{{.AvailableTag}}</em></span>{{end}}
{{if followsNoTag .Service}}<span class="tag f-notag">Follows no tag</span>{{end}}
{{if .LastPollError}}<span class="tag f-err" title="{{.LastPollError}}">Poll failed</span>{{end}}
{{if .Self}}<span class="tag f-out">Managed outside</span>{{end}}
{{if .Self}}<span class="tag f-out">This agent</span>{{end}}
</div>
<div class="mode">
{{if .Self}}<span class="na">n/a</span>
{{else}}<form method="post" action="/services/{{.ID}}/mode">
<form method="post" action="/services/{{.ID}}/mode">
<input type="hidden" name="csrf_token" value="{{$.CSRF}}">
<input type="hidden" name="return" value="overview">
{{if eq .UpdateMode "auto"}}<input type="hidden" name="mode" value="manual">
@@ -91,7 +90,7 @@
{{else}}<input type="hidden" name="mode" value="auto">
<button type="submit" class="sw" role="switch" aria-checked="false"{{if .AutoBlocked}} disabled title="{{.AutoBlocked}}"{{end}}><i></i>Manual</button>
{{if .AutoBlocked}}<span class="na">{{.AutoBlocked}}</span>{{end}}{{end}}
</form>{{end}}
</form>
</div>
</div>
{{end}}
internal/master/webhook/webhook_test.go
+2 -2
View File
@@ -470,14 +470,14 @@ func TestBusyIsNotAFailure(t *testing.T) {
func TestRefusedIsNotAFailure(t *testing.T) {
e := newEnv(t, compose(1, model.ModeAuto, slusk+":1.4.2"))
e.jobs.errFor[1] = &model.NotUpdatableError{Reason: model.ReasonSelf}
e.jobs.errFor[1] = &model.NotUpdatableError{Reason: model.ReasonContainer}
rec := e.post(`{"image":"` + slusk + `","tag":"2"}`)
assert.Equal(t, 200, rec.Code)
r := decode(t, rec).Results[0]
assert.Equal(t, statusRefused, r.Status)
assert.Equal(t, model.ReasonSelf, r.Error)
assert.Equal(t, model.ReasonContainer, r.Error)
assert.Zero(t, r.JobID)
assert.NotEmpty(t, e.store.avail)
}