feat(jobs): refuse poll and webhook jobs for drifted services, add restore (#10)

internal/e2e/drift_test.go

@@ -0,0 +1,113 @@
+package e2e
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/shcizo/package-updater/internal/master/model"
+	"github.com/shcizo/package-updater/internal/master/poller"
+	"github.com/shcizo/package-updater/internal/protocol"
+)
+
+const (
+	driftDesired = "sha256:" + "d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1"
+	driftOther   = "sha256:" + "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"
+)
+
+// seedDrifted replaces the environment's service with an auto service that was
+// updated to slusk:1.4.2 at driftDesired but runs driftOther now.
+func (e *jobEnv) seedDrifted() {
+	e.store.seedService(model.Service{ID: 1, Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web",
+		AgentID: ptr(agentID), ImageRepo: slusk, UpdateMode: model.ModeAuto,
+		DesiredRef: slusk + ":1.4.2@" + driftDesired,
+		ActualRef:  slusk + ":1.4.2", ActualDigest: driftOther})
+}
+
+// inventoryWrite is what the store does for an agent's upsert of the service.
+func (e *jobEnv) inventoryWrite(id int64, actualRef, actualDigest string) {
+	e.store.mu.Lock()
+	defer e.store.mu.Unlock()
+	svc := e.store.svcRows[id]
+	svc.ActualRef, svc.ActualDigest, svc.AwaitingInventory = actualRef, actualDigest, false
+	e.store.svcRows[id] = svc
+}
+
+func TestDriftedAutoServiceIsNotUpdatedByPollOrWebhook(t *testing.T) {
+	e := newJobEnv(t)
+	e.seedDrifted()
+	reg := &fixedRegistry{digests: map[string]string{slusk + ":1.4.2": "sha256:new"}}
+	p := poller.New(e.store, e.jobs, reg, e.m.broker, poller.Config{Workers: 1}, slog.New(slog.NewTextHandler(io.Discard, nil)))
+
+	p.Cycle(context.Background())
+
+	assert.Equal(t, 1, reg.calls)
+	assert.Zero(t, e.store.jobCount(), "polling starts no job for a drifted service")
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, slusk+":1.4.2", svc.AvailableRef, "the suggestion is still recorded")
+
+	code, body := callWebhook(t, e.webhookServer(t), `{"image":"`+slusk+`:ignored","tag":"2.0.0"}`)
+
+	assert.Equal(t, 200, code)
+	require.Len(t, body.Results, 1)
+	assert.Equal(t, "refused", body.Results[0].Status)
+	assert.Contains(t, body.Results[0].Error, "drifted")
+	assert.Zero(t, e.store.jobCount(), "the webhook starts no job either")
+}
+
+func TestRestoreDeploysThePinnedRefOnTheAgent(t *testing.T) {
+	e := newJobEnv(t)
+	e.seedDrifted()
+	close(e.exec.release)
+
+	job, err := e.jobs.RestoreDesired(context.Background(), 1, "alice")
+
+	require.NoError(t, err)
+	assert.Equal(t, model.TriggerManual, job.Trigger)
+	select {
+	case ref := <-e.exec.started:
+		assert.Equal(t, slusk+":1.4.2@"+driftDesired, ref, "the agent receives exactly the desired ref")
+	case <-time.After(waitFor):
+		t.Fatal("the agent never received the job")
+	}
+	done := e.waitStatus(t, job.ID, model.JobSucceeded)
+	assert.Equal(t, slusk+":1.4.2@"+driftDesired, done.ToRef)
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, driftOther, svc.LastTagDigest, "the drifted digest is remembered so polling does not redeploy it")
+	assert.False(t, svc.Drifted(), "the inventory has not caught up yet")
+
+	e.inventoryWrite(1, slusk+":1.4.2", driftDesired)
+	svc, err = e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.False(t, svc.Drifted())
+}
+
+func TestNoDriftAfterASuccessfulJob(t *testing.T) {
+	e := newJobEnv(t)
+	close(e.exec.release)
+
+	job, err := e.jobs.Trigger(context.Background(), 1, "alice")
+
+	require.NoError(t, err)
+	e.waitStatus(t, job.ID, model.JobSucceeded)
+	svc, err := e.store.GetService(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, slusk+":1.4.2@"+e2eDigest, svc.DesiredRef)
+	assert.Equal(t, slusk+":1.4.2@sha256:old", svc.ActualRef, "the agent's inventory write has not arrived")
+	assert.False(t, svc.Drifted(), "between job.result and the inventory write the row is not drifted")
+
+	e.inventoryWrite(1, slusk+":1.4.2", e2eDigest)
+	svc, _ = e.store.GetService(context.Background(), 1)
+	assert.False(t, svc.Drifted())
+
+	e.inventoryWrite(1, slusk+":1.4.2", driftOther)
+	svc, _ = e.store.GetService(context.Background(), 1)
+	assert.True(t, svc.Drifted(), "a later inventory write of another digest is drift")
+}