feat: master/agent phase 5 — drift and merge, backend (#10) (#30)
Docker Build / docker (push) Successful in 42s
Docker Build / docker (push) Successful in 42s
## Summary
- Drift detection: `model.Service.Drifted()` compares desired and running version by digest (tag when the digest is unknown). `services.awaiting_inventory` hides drift between a job result and the next inventory write.
- Poll and webhook jobs are refused for a drifted service; `POST /services/{id}/restore` (manual job to `desired_ref`) and `POST /services/{id}/accept` (running version becomes desired, guarded by `actual` = `Service.RunningRef`).
- Move suggestions (`store.MoveCandidates`, Compose only) and `POST /services/{id}/merge`: the old row keeps its id, mode, `desired_ref` and history and takes the new location; the duplicate is deleted.
- Backend only. The buttons and badges come with #28, on top of the Ledger UI (#18).
Closes #10
## Test plan
- [x] `gofmt -l .`, `go vet ./...`, `go test ./...`
- [x] `go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/`
- [x] `go test -tags integration ./internal/master/store/` against postgres:17-alpine
- [ ] On the server: migration 0005 applies; no drift shown after a normal update
- [ ] On the server: manual `docker compose up` with another tag gives drift; webhook for an auto service is refused
- [ ] On the server: restore and accept via POST; merge after moving a stack
Reviewed-on: #30
This commit was merged in pull request #30.
This commit is contained in:
@@ -0,0 +1,237 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/shcizo/package-updater/internal/master/model"
|
||||
"github.com/shcizo/package-updater/internal/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
driftRepo = "gitea.shcizo.se/shcizo/slusk"
|
||||
driftD1 = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
driftD2 = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
driftD3 = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
||||
)
|
||||
|
||||
// driftedSvc was updated to 1.4.2 at D1 but now runs 1.4.2 at D2.
|
||||
func driftedSvc() model.Service {
|
||||
svc := composeSvc
|
||||
svc.ImageRepo = driftRepo
|
||||
svc.DesiredRef = driftRepo + ":1.4.2@" + driftD1
|
||||
svc.ActualRef = driftRepo + ":1.4.2"
|
||||
svc.ActualDigest = driftD2
|
||||
return svc
|
||||
}
|
||||
|
||||
func notUpdatableReason(t *testing.T, err error) string {
|
||||
t.Helper()
|
||||
var nu *model.NotUpdatableError
|
||||
require.True(t, errors.As(err, &nu), "got %v", err)
|
||||
return nu.Reason
|
||||
}
|
||||
|
||||
func TestRestorable(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*model.Service)
|
||||
reason string // "" means restorable
|
||||
}{
|
||||
{"drifted", func(*model.Service) {}, ""},
|
||||
{"drifted service that follows no tag", func(s *model.Service) {
|
||||
s.ActualRef = driftRepo + "@" + driftD2
|
||||
}, ""},
|
||||
{"not drifted", func(s *model.Service) { s.ActualDigest = driftD1 }, model.ReasonNotDrifted},
|
||||
{"never updated", func(s *model.Service) { s.DesiredRef = "" }, model.ReasonNotDrifted},
|
||||
{"awaiting inventory", func(s *model.Service) { s.AwaitingInventory = true }, model.ReasonNotDrifted},
|
||||
{"removed", func(s *model.Service) { s.RemovedAt = ptr(time.Now()) }, model.ReasonRemoved},
|
||||
{"flagged self", func(s *model.Service) { s.Self = true }, model.ReasonSelf},
|
||||
{"built", func(s *model.Service) { s.Built = true }, model.ReasonBuilt},
|
||||
{"container", func(s *model.Service) { s.Kind = protocol.KindContainer }, model.ReasonContainer},
|
||||
{"desired ref is the agent image", func(s *model.Service) {
|
||||
s.DesiredRef = selfImage + "@" + driftD1
|
||||
}, model.ReasonSelf},
|
||||
}
|
||||
s, _ := newService(&fakeStore{}, &fakeHub{})
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
svc := driftedSvc()
|
||||
tt.mutate(&svc)
|
||||
err := s.Restorable(svc)
|
||||
if tt.reason == "" {
|
||||
assert.NoError(t, err)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, tt.reason, notUpdatableReason(t, err))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDesired_StartsAManualJobToTheDesiredRef(t *testing.T) {
|
||||
svc := driftedSvc()
|
||||
st := &fakeStore{services: map[int64]model.Service{1: svc}, jobs: map[int64]model.Job{}}
|
||||
h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
|
||||
s, _ := newService(st, h)
|
||||
|
||||
job, err := s.RestoreDesired(context.Background(), 1, "sub-1")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, model.JobQueued, job.Status)
|
||||
require.Len(t, st.created, 1)
|
||||
assert.Equal(t, svc.DesiredRef, st.created[0].ToRef)
|
||||
assert.Equal(t, svc.ActualRef, st.created[0].FromRef)
|
||||
assert.Equal(t, "sub-1", st.created[0].RequestedBy)
|
||||
assert.Equal(t, model.TriggerManual, st.created[0].Trigger)
|
||||
assert.Equal(t, []int{protocol.MinPinnedRefVersion}, h.mins)
|
||||
require.Len(t, h.starts, 1)
|
||||
assert.Equal(t, svc.DesiredRef, h.starts[0].start.Ref)
|
||||
}
|
||||
|
||||
func TestRestoreDesired_Refusals(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*model.Service)
|
||||
reason string
|
||||
}{
|
||||
{"not drifted", func(s *model.Service) { s.ActualDigest = driftD1 }, model.ReasonNotDrifted},
|
||||
{"self", func(s *model.Service) { s.Self = true }, model.ReasonSelf},
|
||||
{"built", func(s *model.Service) { s.Built = true }, model.ReasonBuilt},
|
||||
{"container", func(s *model.Service) { s.Kind = protocol.KindContainer }, model.ReasonContainer},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
svc := driftedSvc()
|
||||
tt.mutate(&svc)
|
||||
st := &fakeStore{services: map[int64]model.Service{1: svc}, jobs: map[int64]model.Job{}}
|
||||
h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
|
||||
s, pub := newService(st, h)
|
||||
|
||||
_, err := s.RestoreDesired(context.Background(), 1, "sub-1")
|
||||
|
||||
assert.Equal(t, tt.reason, notUpdatableReason(t, err))
|
||||
assert.Empty(t, st.created)
|
||||
assert.Empty(t, h.starts)
|
||||
assert.Empty(t, pub.topics)
|
||||
assert.Empty(t, st.lastTag, "last_tag_digest untouched")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDesired_UnknownService(t *testing.T) {
|
||||
s, _ := newService(&fakeStore{services: map[int64]model.Service{}}, &fakeHub{})
|
||||
|
||||
_, err := s.RestoreDesired(context.Background(), 9, "sub-1")
|
||||
|
||||
assert.ErrorIs(t, err, model.ErrNotFound)
|
||||
}
|
||||
|
||||
func TestRestoreDesired_ActiveJobReturnsItAndMarksNothing(t *testing.T) {
|
||||
activeJob := model.Job{ID: 99, Status: model.JobRunning}
|
||||
st := &fakeStore{services: map[int64]model.Service{1: driftedSvc()}, jobs: map[int64]model.Job{99: activeJob}, active: &activeJob}
|
||||
s, _ := newService(st, &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}})
|
||||
|
||||
job, err := s.RestoreDesired(context.Background(), 1, "sub-1")
|
||||
|
||||
assert.ErrorIs(t, err, model.ErrJobActive)
|
||||
assert.Equal(t, int64(99), job.ID)
|
||||
assert.Empty(t, st.lastTag)
|
||||
}
|
||||
|
||||
func TestRestoreDesired_OldAgentGivesAFailedJob(t *testing.T) {
|
||||
st := &fakeStore{services: map[int64]model.Service{1: driftedSvc()}, jobs: map[int64]model.Job{}}
|
||||
h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}, agentProtocol: 2}
|
||||
s, _ := newService(st, h)
|
||||
|
||||
job, err := s.RestoreDesired(context.Background(), 1, "sub-1")
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, model.JobFailed, job.Status)
|
||||
assert.Equal(t, model.ErrAgentTooOldForPinned.Error(), job.Error)
|
||||
assert.Empty(t, h.starts)
|
||||
}
|
||||
|
||||
func TestRestoreDesired_LastTagDigest(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*model.Service)
|
||||
want string // "" means SetLastTagDigest is not called
|
||||
}{
|
||||
{"drifted to another digest of the same tag", func(*model.Service) {}, driftD2},
|
||||
{"digest in the actual ref", func(s *model.Service) {
|
||||
s.ActualRef, s.ActualDigest = driftRepo+":1.4.2@"+driftD3, ""
|
||||
}, driftD3},
|
||||
{"last tag digest holds back a newer one", func(s *model.Service) {
|
||||
s.DesiredRef = driftRepo + ":1.4.2@" + driftD1
|
||||
s.LastTagDigest = driftD3
|
||||
}, ""},
|
||||
{"last tag digest equals the desired digest", func(s *model.Service) { s.LastTagDigest = driftD1 }, driftD2},
|
||||
{"already remembered", func(s *model.Service) { s.LastTagDigest = driftD2 }, ""},
|
||||
{"different tag", func(s *model.Service) { s.ActualRef = driftRepo + ":1.5.0" }, ""},
|
||||
{"actual digest malformed", func(s *model.Service) { s.ActualDigest = "sha256:abc" }, ""},
|
||||
{"desired follows no tag", func(s *model.Service) { s.DesiredRef = driftRepo + "@" + driftD1 }, ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
svc := driftedSvc()
|
||||
tt.mutate(&svc)
|
||||
st := &fakeStore{services: map[int64]model.Service{1: svc}, jobs: map[int64]model.Job{}}
|
||||
s, _ := newService(st, &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}})
|
||||
|
||||
_, err := s.RestoreDesired(context.Background(), 1, "sub-1")
|
||||
|
||||
require.NoError(t, err)
|
||||
if tt.want == "" {
|
||||
assert.Empty(t, st.lastTag)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, map[int64]string{1: tt.want}, st.lastTag)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStart_RefusesPollAndWebhookWhileDrifted(t *testing.T) {
|
||||
tests := []struct {
|
||||
trigger model.JobTrigger
|
||||
refused bool
|
||||
}{
|
||||
{model.TriggerPoll, true},
|
||||
{model.TriggerWebhook, true},
|
||||
{model.TriggerManual, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(string(tt.trigger), func(t *testing.T) {
|
||||
svc := driftedSvc()
|
||||
st := &fakeStore{services: map[int64]model.Service{1: svc}, jobs: map[int64]model.Job{}}
|
||||
h := &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}}
|
||||
s, _ := newService(st, h)
|
||||
|
||||
_, err := s.Start(context.Background(), svc, StartRequest{Trigger: tt.trigger, Ref: driftRepo + ":2.0.0"})
|
||||
|
||||
if tt.refused {
|
||||
assert.Equal(t, model.ReasonDrifted, notUpdatableReason(t, err))
|
||||
assert.Empty(t, st.created)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, st.created, 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStart_PollAllowedWhenNotDrifted(t *testing.T) {
|
||||
svc := driftedSvc()
|
||||
svc.AwaitingInventory = true
|
||||
st := &fakeStore{services: map[int64]model.Service{1: svc}, jobs: map[int64]model.Job{}}
|
||||
s, _ := newService(st, &fakeHub{route: model.Route{AgentID: 7, ConnID: 70}})
|
||||
|
||||
_, err := s.Start(context.Background(), svc, StartRequest{Trigger: model.TriggerPoll})
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, st.created, 1)
|
||||
}
|
||||
@@ -21,6 +21,7 @@ type Store interface {
|
||||
FailJob(ctx context.Context, id int64, msg string, at time.Time) (bool, error)
|
||||
GetJob(ctx context.Context, id int64) (model.Job, error)
|
||||
GetImageVersion(ctx context.Context, id int64) (model.ImageVersion, error)
|
||||
SetLastTagDigest(ctx context.Context, id int64, digest string) error
|
||||
}
|
||||
|
||||
// Hub routes jobs to agent connections. *hub.Hub satisfies it.
|
||||
@@ -114,6 +115,81 @@ func (s *Service) updatable(svc model.Service, explicitRef bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restorable returns nil or a *model.NotUpdatableError saying why the service
|
||||
// cannot be restored to its desired version: it must have drifted, and the
|
||||
// desired version is deployed like an explicit ref, so a service that follows
|
||||
// no tag is fine. The service page calls it for every render.
|
||||
func (s *Service) Restorable(svc model.Service) error {
|
||||
if svc.RemovedAt != nil {
|
||||
return &model.NotUpdatableError{Reason: model.ReasonRemoved}
|
||||
}
|
||||
if !svc.Drifted() {
|
||||
return &model.NotUpdatableError{Reason: model.ReasonNotDrifted}
|
||||
}
|
||||
if err := s.updatable(svc, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if imageref.Parse(svc.DesiredRef).Repo == s.selfRepo {
|
||||
return &model.NotUpdatableError{Reason: model.ReasonSelf}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RestoreDesired starts a manual job that deploys the drifted service's
|
||||
// desired ref again. The ref is read from the row, never from the caller. It
|
||||
// returns model.ErrNotFound for an unknown service and otherwise what Start
|
||||
// returns, with a *model.NotUpdatableError from Restorable if it cannot be
|
||||
// restored.
|
||||
//
|
||||
// When the service had drifted to another digest of the same tag, that digest
|
||||
// becomes last_tag_digest, so the poller does not deploy it again before the
|
||||
// registry shows a newer one (the same rule as for any rollback). It is not
|
||||
// written when last_tag_digest holds back a newer digest, that is when it is
|
||||
// set and differs from the digest the desired ref names.
|
||||
func (s *Service) RestoreDesired(ctx context.Context, serviceID int64, requestedBy string) (model.Job, error) {
|
||||
svc, err := s.store.GetService(ctx, serviceID)
|
||||
if err != nil {
|
||||
return model.Job{}, err
|
||||
}
|
||||
if err := s.Restorable(svc); err != nil {
|
||||
return model.Job{}, err
|
||||
}
|
||||
job, err := s.Start(ctx, svc, StartRequest{Trigger: model.TriggerManual, RequestedBy: requestedBy, Ref: svc.DesiredRef})
|
||||
if err != nil {
|
||||
return job, err
|
||||
}
|
||||
if d := driftedTagDigest(svc); d != "" && d != svc.LastTagDigest && holdsNothingNewer(svc) {
|
||||
if err := s.store.SetLastTagDigest(ctx, svc.ID, d); err != nil {
|
||||
s.log.Warn("last_tag_digest_not_set", "service_id", svc.ID, "error", err)
|
||||
}
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
// holdsNothingNewer reports whether last_tag_digest may be overwritten by a
|
||||
// restore: it is empty or equals the digest the desired ref names. Anything
|
||||
// else is a digest the poller must keep holding back.
|
||||
func holdsNothingNewer(svc model.Service) bool {
|
||||
return svc.LastTagDigest == "" || svc.LastTagDigest == imageref.Parse(svc.DesiredRef).Digest
|
||||
}
|
||||
|
||||
// driftedTagDigest returns the digest a drifted service runs when it runs the
|
||||
// same tag as desired at another digest, else "".
|
||||
func driftedTagDigest(svc model.Service) string {
|
||||
tag := FollowedRef(svc.DesiredRef)
|
||||
if tag == "" || tag != FollowedRef(svc.ActualRef) {
|
||||
return ""
|
||||
}
|
||||
d := imageref.Parse(svc.ActualRef).Digest
|
||||
if d == "" {
|
||||
d = svc.ActualDigest
|
||||
}
|
||||
if !imageref.IsDigest(d) || d == imageref.Parse(svc.DesiredRef).Digest {
|
||||
return ""
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// StartRequest says who asks for an update and what to deploy.
|
||||
type StartRequest struct {
|
||||
Trigger model.JobTrigger
|
||||
@@ -161,8 +237,12 @@ func (s *Service) TriggerVersion(ctx context.Context, serviceID, versionID int64
|
||||
// speaks protocol version 2); a
|
||||
// *model.NotUpdatableError when the service cannot be updated (no job
|
||||
// created); the active job and model.ErrJobActive when one is queued or
|
||||
// running.
|
||||
// running. Poll and webhook jobs are refused for a drifted service; only a
|
||||
// person starts one then.
|
||||
func (s *Service) Start(ctx context.Context, svc model.Service, req StartRequest) (model.Job, error) {
|
||||
if req.Trigger != model.TriggerManual && svc.Drifted() {
|
||||
return model.Job{}, &model.NotUpdatableError{Reason: model.ReasonDrifted}
|
||||
}
|
||||
if err := s.updatable(svc, req.Ref != ""); err != nil {
|
||||
return model.Job{}, err
|
||||
}
|
||||
|
||||
@@ -23,7 +23,8 @@ type fakeStore struct {
|
||||
active *model.Job // returned with ErrJobActive when set
|
||||
versions map[int64]model.ImageVersion
|
||||
|
||||
revokedAgents map[int64]bool // CreateJob returns ErrAgentOffline for these
|
||||
revokedAgents map[int64]bool // CreateJob returns ErrAgentOffline for these
|
||||
lastTag map[int64]string // SetLastTagDigest calls: service ID to digest
|
||||
}
|
||||
|
||||
func (f *fakeStore) GetService(_ context.Context, id int64) (model.Service, error) {
|
||||
@@ -63,6 +64,14 @@ func (f *fakeStore) GetImageVersion(_ context.Context, id int64) (model.ImageVer
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) SetLastTagDigest(_ context.Context, id int64, digest string) error {
|
||||
if f.lastTag == nil {
|
||||
f.lastTag = map[int64]string{}
|
||||
}
|
||||
f.lastTag[id] = digest
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) GetJob(_ context.Context, id int64) (model.Job, error) { return f.jobs[id], nil }
|
||||
|
||||
type startCall struct {
|
||||
|
||||
Reference in New Issue
Block a user