shcizo/package-updater
1
0
Fork 0
Code Issues 8 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(master,agent): deploy digest-pinned refs and route them to protocol 3 agents (#9)

Browse Source
This commit is contained in:
shcizo
2026-10-13 08:07:45 +02:00
parent af61f17dee
commit d0726ca992
15 changed files with 303 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/master/hub/jobs_test.go
+2 -2
View File
@@ -43,7 +43,7 @@ func waitRoute(t *testing.T, h *Hub) model.Route {
var r model.Route
require.Eventually(t, func() bool {
var err error
r, err = h.Route(svc)
r, err = h.Route(svc, protocol.MinJobVersion)
return err == nil
}, 2*time.Second, 5*time.Millisecond)
return r
@@ -61,7 +61,7 @@ func TestHub_ReconcilesBeforeRoutable(t *testing.T) {
send(t, c, protocol.TypeHello, protocol.Hello{Hostname: "node-2", ProtocolVersion: 2, KnownJobs: []string{"7", "9", "x"}})
svc := model.Service{Kind: protocol.KindCompose, AgentID: ptr(node2.ID)}
time.Sleep(50 * time.Millisecond) // hello is now blocked inside FailLostJobs
_, err = h.Route(svc)
_, err = h.Route(svc, protocol.MinJobVersion)
assert.ErrorIs(t, err, model.ErrAgentOffline)
close(store.lostGate)
internal/master/hub/route.go
+7 -6
View File
@@ -14,9 +14,10 @@ type candidate struct {
hello protocol.Hello
}
// Route picks the connection a job for svc goes to. It returns
// model.ErrAgentOffline or model.ErrAgentTooOld when there is none.
func (h *Hub) Route(svc model.Service) (model.Route, error) {
// Route picks the connection a job for svc goes to, among agents that speak
// at least protocol version minProtocol. It returns model.ErrAgentOffline or
// model.ErrAgentTooOld when there is none.
func (h *Hub) Route(svc model.Service, minProtocol int) (model.Route, error) {
h.mu.Lock()
conns := make([]candidate, 0, len(h.conns))
for id, c := range h.conns {
@@ -25,7 +26,7 @@ func (h *Hub) Route(svc model.Service) (model.Route, error) {
}
}
h.mu.Unlock()
return route(svc, conns)
return route(svc, conns, minProtocol)
}
// route picks the connection a job for svc goes to. Compose services and
@@ -33,7 +34,7 @@ func (h *Hub) Route(svc model.Service) (model.Route, error) {
// run on the reporting agent if it qualifies, else on any other connected
// manager of the same cluster (lowest agent ID first). Too-old wins over
// offline only when a connection would otherwise have qualified.
func route(svc model.Service, conns []candidate) (model.Route, error) {
func route(svc model.Service, conns []candidate, minProtocol int) (model.Route, error) {
sort.Slice(conns, func(a, b int) bool { return conns[a].agentID < conns[b].agentID })
tooOld := false
try := func(c candidate) bool {
@@ -42,7 +43,7 @@ func route(svc model.Service, conns []candidate) (model.Route, error) {
return false
}
}
if c.hello.ProtocolVersion < protocol.MinJobVersion {
if c.hello.ProtocolVersion < minProtocol {
tooOld = true
return false
}
internal/master/hub/route_test.go
+24 -15
View File
@@ -12,37 +12,46 @@ import (
func TestRoute(t *testing.T) {
composeSvc := model.Service{Kind: protocol.KindCompose, AgentID: ptr(int64(2))}
swarmSvc := model.Service{Kind: protocol.KindSwarm, AgentID: ptr(int64(1)), SwarmClusterID: "c1"}
v2 := protocol.Hello{ProtocolVersion: protocol.Version}
mgrV2 := protocol.Hello{ProtocolVersion: protocol.Version, SwarmManager: true, SwarmClusterID: "c1"}
v2 := protocol.Hello{ProtocolVersion: 2}
v3 := protocol.Hello{ProtocolVersion: 3}
mgrV2 := protocol.Hello{ProtocolVersion: 2, SwarmManager: true, SwarmClusterID: "c1"}
mgrV3 := protocol.Hello{ProtocolVersion: 3, SwarmManager: true, SwarmClusterID: "c1"}
jobs, pinned := protocol.MinJobVersion, protocol.MinPinnedRefVersion
tests := []struct {
name string
svc model.Service
conns []candidate
min int
want model.Route
err error
}{
{"compose on own agent", composeSvc, []candidate{{2, 20, v2}}, model.Route{AgentID: 2, ConnID: 20}, nil},
{"compose, own agent too old", composeSvc, []candidate{{2, 20, protocol.Hello{}}}, model.Route{}, model.ErrAgentTooOld},
{"compose, own agent missing", composeSvc, nil, model.Route{}, model.ErrAgentOffline},
{"compose, only another agent online", composeSvc, []candidate{{5, 50, v2}}, model.Route{}, model.ErrAgentOffline},
{"swarm on reporting agent", swarmSvc, []candidate{{1, 10, mgrV2}, {3, 30, mgrV2}}, model.Route{AgentID: 1, ConnID: 10}, nil},
{"swarm, reporting agent offline, other manager", swarmSvc, []candidate{{3, 30, mgrV2}}, model.Route{AgentID: 3, ConnID: 30}, nil},
{"swarm, lowest agent id first", swarmSvc, []candidate{{4, 40, mgrV2}, {3, 30, mgrV2}}, model.Route{AgentID: 3, ConnID: 30}, nil},
{"compose on own agent", composeSvc, []candidate{{2, 20, v2}}, jobs, model.Route{AgentID: 2, ConnID: 20}, nil},
{"compose, own agent too old", composeSvc, []candidate{{2, 20, protocol.Hello{}}}, jobs, model.Route{}, model.ErrAgentTooOld},
{"compose, own agent missing", composeSvc, nil, jobs, model.Route{}, model.ErrAgentOffline},
{"compose, only another agent online", composeSvc, []candidate{{5, 50, v2}}, jobs, model.Route{}, model.ErrAgentOffline},
{"compose, v2 agent and a pinned ref", composeSvc, []candidate{{2, 20, v2}}, pinned, model.Route{}, model.ErrAgentTooOld},
{"compose, v3 agent and a pinned ref", composeSvc, []candidate{{2, 20, v3}}, pinned, model.Route{AgentID: 2, ConnID: 20}, nil},
{"swarm on reporting agent", swarmSvc, []candidate{{1, 10, mgrV2}, {3, 30, mgrV2}}, jobs, model.Route{AgentID: 1, ConnID: 10}, nil},
{"swarm, reporting agent offline, other manager", swarmSvc, []candidate{{3, 30, mgrV2}}, jobs, model.Route{AgentID: 3, ConnID: 30}, nil},
{"swarm, lowest agent id first", swarmSvc, []candidate{{4, 40, mgrV2}, {3, 30, mgrV2}}, jobs, model.Route{AgentID: 3, ConnID: 30}, nil},
{"swarm, other manager in another cluster", swarmSvc,
[]candidate{{3, 30, protocol.Hello{ProtocolVersion: 2, SwarmManager: true, SwarmClusterID: "c2"}}}, model.Route{}, model.ErrAgentOffline},
[]candidate{{3, 30, protocol.Hello{ProtocolVersion: 2, SwarmManager: true, SwarmClusterID: "c2"}}}, jobs, model.Route{}, model.ErrAgentOffline},
{"swarm, only a v0 manager of the cluster", swarmSvc,
[]candidate{{1, 10, protocol.Hello{SwarmManager: true, SwarmClusterID: "c1"}}}, model.Route{}, model.ErrAgentTooOld},
[]candidate{{1, 10, protocol.Hello{SwarmManager: true, SwarmClusterID: "c1"}}}, jobs, model.Route{}, model.ErrAgentTooOld},
{"swarm, v0 manager but a v2 manager exists", swarmSvc,
[]candidate{{1, 10, protocol.Hello{SwarmManager: true, SwarmClusterID: "c1"}}, {3, 30, mgrV2}}, model.Route{AgentID: 3, ConnID: 30}, nil},
[]candidate{{1, 10, protocol.Hello{SwarmManager: true, SwarmClusterID: "c1"}}, {3, 30, mgrV2}}, jobs, model.Route{AgentID: 3, ConnID: 30}, nil},
{"swarm, worker of the cluster", swarmSvc,
[]candidate{{1, 10, protocol.Hello{ProtocolVersion: 2, SwarmClusterID: "c1"}}}, model.Route{}, model.ErrAgentOffline},
[]candidate{{1, 10, protocol.Hello{ProtocolVersion: 2, SwarmClusterID: "c1"}}}, jobs, model.Route{}, model.ErrAgentOffline},
{"swarm, manager with empty cluster id", model.Service{Kind: protocol.KindSwarm},
[]candidate{{1, 10, protocol.Hello{ProtocolVersion: 2, SwarmManager: true}}}, model.Route{}, model.ErrAgentOffline},
[]candidate{{1, 10, protocol.Hello{ProtocolVersion: 2, SwarmManager: true}}}, jobs, model.Route{}, model.ErrAgentOffline},
{"swarm, pinned ref falls back to a v3 manager when the reporting one is v2", swarmSvc,
[]candidate{{1, 10, mgrV2}, {3, 30, mgrV3}}, pinned, model.Route{AgentID: 3, ConnID: 30}, nil},
{"swarm, pinned ref and only v2 managers", swarmSvc, []candidate{{1, 10, mgrV2}}, pinned, model.Route{}, model.ErrAgentTooOld},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := route(tt.svc, tt.conns)
got, err := route(tt.svc, tt.conns, tt.min)
assert.ErrorIs(t, err, tt.err)
assert.Equal(t, tt.want, got)
})
internal/master/jobs/jobs.go
+35 -6
View File
@@ -20,13 +20,15 @@ type Store interface {
CreateJob(ctx context.Context, j model.NewJob) (model.Job, error)
FailJob(ctx context.Context, id int64, msg string, at time.Time) (bool, error)
GetJob(ctx context.Context, id int64) (model.Job, error)
GetImageVersion(ctx context.Context, id int64) (model.ImageVersion, error)
}
// Hub routes jobs to agent connections. *hub.Hub satisfies it.
type Hub interface {
// Route picks the connection for svc: model.ErrAgentOffline or
// Route picks the connection for svc among agents speaking at least
// protocol version minProtocol: model.ErrAgentOffline or
// model.ErrAgentTooOld when there is none.
Route(svc model.Service) (model.Route, error)
Route(svc model.Service, minProtocol int) (model.Route, error)
// StartJob sends job.start over exactly the routed connection.
// model.ErrAgentOffline if that connection is gone or was replaced.
StartJob(r model.Route, jobID int64, start protocol.JobStart) error
@@ -116,8 +118,9 @@ func (s *Service) updatable(svc model.Service, explicitRef bool) error {
type StartRequest struct {
Trigger model.JobTrigger
RequestedBy string // OIDC subject for manual jobs; "" otherwise
// Ref is the ref to deploy ("repo:tag"). Empty means the service's
// followed tag, derived from its actual ref.
// Ref is the ref to deploy ("repo:tag", "repo:tag@digest" or
// "repo@digest"). Empty means the service's followed tag, derived from
// its actual ref.
Ref string
}
@@ -131,8 +134,30 @@ func (s *Service) Trigger(ctx context.Context, serviceID int64, requestedBy stri
return s.Start(ctx, svc, StartRequest{Trigger: model.TriggerManual, RequestedBy: requestedBy})
}
// TriggerVersion starts a manual update of the service to exactly the image
// version versionID, which must be a version of the service's own repo; the
// ref is built from the stored version, never taken from the caller. It
// returns model.ErrNotFound for an unknown service, an unknown version or a
// version of another repo, and otherwise what Start returns. A version
// without a tag leaves the service following no tag.
func (s *Service) TriggerVersion(ctx context.Context, serviceID, versionID int64, requestedBy string) (model.Job, error) {
svc, err := s.store.GetService(ctx, serviceID)
if err != nil {
return model.Job{}, err
}
v, err := s.store.GetImageVersion(ctx, versionID)
if err != nil {
return model.Job{}, err
}
if v.Repo != svc.ImageRepo {
return model.Job{}, model.ErrNotFound
}
return s.Start(ctx, svc, StartRequest{Trigger: model.TriggerManual, RequestedBy: requestedBy, Ref: v.Ref()})
}
// Start starts an update of svc. It returns the job, which may already be
// failed ("agent offline", "agent too old for updates"); a
// failed ("agent offline", "agent too old for updates", the latter also when
// the ref has a digest or no tag and the agent speaks protocol version 2); a
// *model.NotUpdatableError when the service cannot be updated (no job
// created); the active job and model.ErrJobActive when one is queued or
// running.
@@ -145,7 +170,11 @@ func (s *Service) Start(ctx context.Context, svc model.Service, req StartRequest
ref = FollowedRef(svc.ActualRef) // updatable checked the tag
}
r, routeErr := s.hub.Route(svc)
minProtocol := protocol.MinJobVersion
if p := imageref.Parse(ref); p.Digest != "" || p.Tag == "" {
minProtocol = protocol.MinPinnedRefVersion
}
r, routeErr := s.hub.Route(svc, minProtocol)
nj := model.NewJob{
ServiceID: svc.ID, FromRef: svc.ActualRef, ToRef: ref,
Trigger: req.Trigger, RequestedBy: req.RequestedBy, CreatedAt: s.now(),
internal/master/jobs/jobs_test.go
+152 -5
View File
@@ -21,6 +21,7 @@ type fakeStore struct {
jobs map[int64]model.Job
created []model.NewJob
active *model.Job // returned with ErrJobActive when set
versions map[int64]model.ImageVersion
revokedAgents map[int64]bool // CreateJob returns ErrAgentOffline for these
}
@@ -54,6 +55,14 @@ func (f *fakeStore) FailJob(_ context.Context, id int64, msg string, at time.Tim
return true, nil
}
func (f *fakeStore) GetImageVersion(_ context.Context, id int64) (model.ImageVersion, error) {
v, ok := f.versions[id]
if !ok {
return model.ImageVersion{}, model.ErrNotFound
}
return v, nil
}
func (f *fakeStore) GetJob(_ context.Context, id int64) (model.Job, error) { return f.jobs[id], nil }
type startCall struct {
@@ -63,13 +72,21 @@ type startCall struct {
}
type fakeHub struct {
route model.Route
routeErr error
startErr error
starts []startCall
route model.Route
routeErr error
startErr error
starts []startCall
agentProtocol int // when set, Route fails with ErrAgentTooOld for a higher minProtocol
mins []int // minProtocol of every Route call
}
func (h *fakeHub) Route(model.Service) (model.Route, error) { return h.route, h.routeErr }
func (h *fakeHub) Route(_ model.Service, minProtocol int) (model.Route, error) {
h.mins = append(h.mins, minProtocol)
if h.agentProtocol != 0 && minProtocol > h.agentProtocol {
return model.Route{}, model.ErrAgentTooOld
}
return h.route, h.routeErr
}
func (h *fakeHub) StartJob(r model.Route, id int64, s protocol.JobStart) error {
h.starts = append(h.starts, startCall{r, id, s})
@@ -341,3 +358,133 @@ func TestFollowedRef(t *testing.T) {
assert.Equal(t, "localhost:5000/foo:latest", FollowedRef("localhost:5000/foo"))
assert.Empty(t, FollowedRef("redis@sha256:abc"))
}
func TestStart_RoutesWithTheMinimumProtocolOfTheRef(t *testing.T) {
tests := []struct {
name string
ref string // "" means the followed tag
want int
}{
{"followed tag", "", protocol.MinJobVersion},
{"repo:tag", "x/y:2", protocol.MinJobVersion},
{"repo:tag@digest", "x/y:2@sha256:a", protocol.MinPinnedRefVersion},
{"repo@digest", "x/y@sha256:a", protocol.MinPinnedRefVersion},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
st := &fakeStore{services: map[int64]model.Service{1: composeSvc}, jobs: map[int64]model.Job{}}
h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
s, _ := newService(st, h)
_, err := s.Start(context.Background(), composeSvc, StartRequest{Trigger: model.TriggerManual, Ref: tt.ref})
require.NoError(t, err)
assert.Equal(t, []int{tt.want}, h.mins)
})
}
}
func TestTriggerVersion_Ref(t *testing.T) {
const repo = "gitea.shcizo.se/shcizo/slusk"
noTag := model.Service{ID: 4, Kind: protocol.KindCompose, ComposeProject: "n", ComposeService: "n",
AgentID: ptr(int64(7)), ImageRepo: repo, ActualRef: repo + "@sha256:d1"}
svc := composeSvc
svc.ImageRepo = repo
tests := []struct {
name string
svc model.Service
version model.ImageVersion
wantRef string
wantMin int
}{
{"tag and digest", svc, model.ImageVersion{ID: 10, Repo: repo, Tag: "1.3.0", Digest: "sha256:old"}, repo + ":1.3.0@sha256:old", protocol.MinPinnedRefVersion},
{"tag only", svc, model.ImageVersion{ID: 10, Repo: repo, Tag: "main"}, repo + ":main", protocol.MinJobVersion},
{"digest only", svc, model.ImageVersion{ID: 10, Repo: repo, Digest: "sha256:old"}, repo + "@sha256:old", protocol.MinPinnedRefVersion},
{"service that follows no tag", noTag, model.ImageVersion{ID: 10, Repo: repo, Tag: "1.3.0", Digest: "sha256:old"}, repo + ":1.3.0@sha256:old", protocol.MinPinnedRefVersion},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
st := &fakeStore{
services: map[int64]model.Service{tt.svc.ID: tt.svc},
jobs: map[int64]model.Job{},
versions: map[int64]model.ImageVersion{10: tt.version},
}
h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
s, _ := newService(st, h)
job, err := s.TriggerVersion(context.Background(), tt.svc.ID, 10, "sub-1")
require.NoError(t, err)
assert.Equal(t, model.JobQueued, job.Status)
require.Len(t, st.created, 1)
assert.Equal(t, model.TriggerManual, st.created[0].Trigger)
assert.Equal(t, "sub-1", st.created[0].RequestedBy)
assert.Equal(t, tt.wantRef, st.created[0].ToRef)
require.Len(t, h.starts, 1)
assert.Equal(t, tt.wantRef, h.starts[0].start.Ref)
assert.Equal(t, []int{tt.wantMin}, h.mins)
})
}
}
func TestTriggerVersion_NotFound(t *testing.T) {
const repo = "gitea.shcizo.se/shcizo/slusk"
svc := composeSvc
svc.ImageRepo = repo
tests := []struct {
name string
serviceID int64
versionID int64
}{
{"unknown service", 9, 10},
{"unknown version", 1, 99},
{"version of another repo", 1, 11},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
st := &fakeStore{
services: map[int64]model.Service{1: svc},
jobs: map[int64]model.Job{},
versions: map[int64]model.ImageVersion{
10: {ID: 10, Repo: repo, Tag: "1"},
11: {ID: 11, Repo: "evil/other", Tag: "1"},
},
}
h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
s, pub := newService(st, h)
_, err := s.TriggerVersion(context.Background(), tt.serviceID, tt.versionID, "sub-1")
assert.ErrorIs(t, err, model.ErrNotFound)
assert.Empty(t, st.created, "no job")
assert.Empty(t, h.starts)
assert.Empty(t, pub.topics)
})
}
}
func TestTriggerVersion_RequiresPinnedProtocol(t *testing.T) {
const repo = "gitea.shcizo.se/shcizo/slusk"
svc := composeSvc
svc.ImageRepo = repo
st := &fakeStore{
services: map[int64]model.Service{1: svc},
jobs: map[int64]model.Job{},
versions: map[int64]model.ImageVersion{
10: {ID: 10, Repo: repo, Tag: "1.3.0", Digest: "sha256:old"},
11: {ID: 11, Repo: repo, Tag: "main"},
},
}
h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}, agentProtocol: 2}
s, _ := newService(st, h)
pinned, err := s.TriggerVersion(context.Background(), 1, 10, "sub-1")
require.NoError(t, err)
assert.Equal(t, model.JobFailed, pinned.Status)
assert.Equal(t, "agent too old for updates", pinned.Error)
assert.Empty(t, h.starts, "nothing sent to the old agent")
plain, err := s.TriggerVersion(context.Background(), 1, 11, "sub-1")
require.NoError(t, err)
assert.Equal(t, model.JobQueued, plain.Status)
}