fix(master): keep rollbacks of auto services until a new digest (#9)

Adds services.last_auto_digest so the poller starts one automatic update per
registry digest, and re-reads the service before starting it. Also records a
bare image's running version as tag latest, only records well-formed
digests, and shows a digest-less version unless a digest row of its tag was
first seen at or after it.

internal/master/store/migrations/0004_polling.sql

@@ -3,6 +3,7 @@ ALTER TABLE services ADD CONSTRAINT services_available_source_check
     CHECK (available_source IN ('', 'webhook', 'poll'));
 ALTER TABLE services ADD COLUMN available_digest text NOT NULL DEFAULT '';
 ALTER TABLE services ADD COLUMN last_poll_error  text NOT NULL DEFAULT '';
+ALTER TABLE services ADD COLUMN last_auto_digest text NOT NULL DEFAULT '';
 
 -- Every version the master has seen. '' means "no tag" / "digest unknown";
 -- never NULL, so the unique constraint deduplicates.

internal/master/store/services.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"regexp"
 	"time"
 
 	"github.com/jackc/pgx/v5"
@@ -56,7 +57,11 @@ ON CONFLICT (location_key) DO UPDATE SET
 	updated_at    = EXCLUDED.updated_at,
 	removed_at    = NULL`
 
+// digestPattern is the form of a digest worth recording as a version.
+var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
+
 func upsertService(ctx context.Context, db execer, agentID int64, clusterID string, svc protocol.Service, at time.Time) error {
+	image := imageref.Parse(svc.Image)
 	clusterCol := ""
 	if svc.Kind == protocol.KindSwarm {
 		clusterCol = clusterID
@@ -65,14 +70,18 @@ func upsertService(ctx context.Context, db execer, agentID int64, clusterID stri
 		string(svc.Kind), model.LocationKey(agentID, clusterID, svc.Key()), agentID,
 		svc.ComposeProject, svc.ComposeService, svc.WorkingDir, svc.ConfigFiles, svc.Built,
 		clusterCol, svc.SwarmService, svc.ContainerName,
-		imageref.Parse(svc.Image).Repo, svc.Image, svc.Digest,
+		image.Repo, svc.Image, svc.Digest,
 		svc.Status, svc.Self, at)
 	if err != nil {
 		return fmt.Errorf("upsert service %s: %w", svc.Key(), err)
 	}
-	if svc.Digest != "" {
-		ref := imageref.Parse(svc.Image)
-		if err := insertVersion(ctx, db, ref.Repo, ref.Tag, svc.Digest, model.SourceRunning, at); err != nil {
+	if digestPattern.MatchString(svc.Digest) {
+		// A bare image runs "latest", as jobs.FollowedRef sees it, so it
+		// shares its row with the polled one. repo@digest has no tag.
+		if image.Tag == "" && image.Digest == "" {
+			image.Tag = "latest"
+		}
+		if err := insertVersion(ctx, db, image.Repo, image.Tag, svc.Digest, model.SourceRunning, at); err != nil {
 			return err
 		}
 	}
@@ -87,7 +96,7 @@ func insertVersion(ctx context.Context, db execer, repo, tag, digest, source str
 		INSERT INTO image_versions (repo, tag, digest, source, first_seen_at)
 		VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`,
 		repo, tag, digest, source, at); err != nil {
-		return fmt.Errorf("record version %s:%s: %w", repo, tag, err)
+		return fmt.Errorf("record version %s: %w", imageref.Ref{Repo: repo, Tag: tag, Digest: digest}, err)
 	}
 	return nil
 }
@@ -169,7 +178,7 @@ const serviceColumns = `s.id, s.kind, s.location_key, s.agent_id, COALESCE(a.nam
 	s.compose_project, s.compose_service, s.working_dir, s.config_files, s.built, s.swarm_cluster_id,
 	s.swarm_service, s.container_name, s.image_repo, s.desired_ref, s.actual_ref, s.actual_digest,
 	s.status, s.self, s.update_mode, s.available_ref, s.available_digest, s.available_source, s.available_at,
-	s.last_poll_error, s.first_seen_at, s.updated_at, s.removed_at`
+	s.last_poll_error, s.last_auto_digest, s.first_seen_at, s.updated_at, s.removed_at`
 
 func scanService(row pgx.Row) (model.Service, error) {
 	var m model.Service
@@ -178,7 +187,7 @@ func scanService(row pgx.Row) (model.Service, error) {
 		&m.ComposeProject, &m.ComposeService, &m.WorkingDir, &m.ConfigFiles, &m.Built, &m.SwarmClusterID,
 		&m.SwarmService, &m.ContainerName, &m.ImageRepo, &m.DesiredRef, &m.ActualRef, &m.ActualDigest,
 		&m.Status, &m.Self, &m.UpdateMode, &m.AvailableRef, &m.AvailableDigest, &m.AvailableSource, &m.AvailableAt,
-		&m.LastPollError, &m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt)
+		&m.LastPollError, &m.LastAutoDigest, &m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt)
 	m.Kind = protocol.Kind(kind)
 	return m, err
 }
@@ -252,16 +261,27 @@ func (s *Store) SetPollError(ctx context.Context, ids []int64, msg string) error
 	return nil
 }
 
+// SetLastAutoDigest records the digest an automatic update was last started
+// for; see model.Service.LastAutoDigest. Inventory writes never change it.
+func (s *Store) SetLastAutoDigest(ctx context.Context, id int64, digest string) error {
+	if _, err := s.pool.Exec(ctx,
+		`UPDATE services SET last_auto_digest = $2 WHERE id = $1`, id, digest); err != nil {
+		return fmt.Errorf("set last auto digest: %w", err)
+	}
+	return nil
+}
+
 // RecentVersions returns up to limit versions of repo, most recently first
 // seen first. A version without a digest is left out when a version with the
-// same tag and a digest exists.
+// same tag and a digest was first seen at or after it.
 func (s *Store) RecentVersions(ctx context.Context, repo string, limit int) ([]model.ImageVersion, error) {
 	rows, err := s.pool.Query(ctx, `
 		SELECT id, repo, tag, digest, source, first_seen_at FROM image_versions v
 		WHERE v.repo = $1
 		  AND NOT (v.digest = '' AND EXISTS (
 		        SELECT 1 FROM image_versions d
-		        WHERE d.repo = v.repo AND d.tag = v.tag AND d.digest <> ''))
+		        WHERE d.repo = v.repo AND d.tag = v.tag AND d.digest <> ''
+		          AND d.first_seen_at >= v.first_seen_at))
 		ORDER BY v.first_seen_at DESC, v.id DESC
 		LIMIT $2`, repo, limit)
 	if err != nil {

internal/master/store/services_integration_test.go

@@ -17,9 +17,9 @@ import (
 
 var (
 	t0      = time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
-	slusk   = protocol.Service{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", WorkingDir: "/srv/slusk", Image: "gitea.shcizo.se/shcizo/slusk:1.4.2", Digest: "sha256:d1", Status: "running"}
+	slusk   = protocol.Service{Kind: protocol.KindCompose, ComposeProject: "slusk", ComposeService: "web", WorkingDir: "/srv/slusk", Image: "gitea.shcizo.se/shcizo/slusk:1.4.2", Digest: digD1, Status: "running"}
 	adhoc   = protocol.Service{Kind: protocol.KindContainer, ContainerName: "adhoc", Image: "redis:alpine", Status: "stopped"}
-	gitea   = protocol.Service{Kind: protocol.KindSwarm, SwarmService: "gitea_gitea", Image: "gitea/gitea:1.27.3@sha256:87a6", Digest: "sha256:87a6", Status: "running"}
+	gitea   = protocol.Service{Kind: protocol.KindSwarm, SwarmService: "gitea_gitea", Image: "gitea/gitea:1.27.3@sha256:87a6", Digest: digGitea, Status: "running"}
 	worker  = protocol.Hello{Hostname: "node-2"}
 	manager = protocol.Hello{Hostname: "swarm-mgr", SwarmManager: true, SwarmClusterID: "c1"}
 )
@@ -53,7 +53,7 @@ func TestApplySnapshot_InsertsAsManualWithAgentName(t *testing.T) {
 	assert.Equal(t, "node-2", got[0].AgentName)
 	assert.Equal(t, "gitea.shcizo.se/shcizo/slusk", got[0].ImageRepo)
 	assert.Equal(t, "gitea.shcizo.se/shcizo/slusk:1.4.2", got[0].ActualRef)
-	assert.Equal(t, "sha256:d1", got[0].ActualDigest)
+	assert.Equal(t, digD1, got[0].ActualDigest)
 	assert.Equal(t, "compose|"+itoa(id)+"|slusk|web", got[0].LocationKey)
 }
 

internal/master/store/versions_integration_test.go

@@ -4,6 +4,7 @@ package store
 
 import (
 	"context"
+	"strings"
 	"testing"
 	"time"
 
@@ -16,6 +17,13 @@ import (
 
 const sluskRepo = "gitea.shcizo.se/shcizo/slusk"
 
+// Digests of the fixture services; only well-formed ones are recorded as versions.
+var (
+	digD1    = "sha256:" + strings.Repeat("d", 63) + "1"
+	digGitea = "sha256:" + strings.Repeat("a", 63) + "1"
+	digPg    = "sha256:" + strings.Repeat("b", 63) + "1"
+)
+
 func versionsOf(t *testing.T, s *Store, repo string) []model.ImageVersion {
 	t.Helper()
 	got, err := s.RecentVersions(context.Background(), repo, 100)
@@ -59,8 +67,8 @@ func TestSnapshotRecordsRunningVersions(t *testing.T) {
 	s := openTestStore(t)
 	ctx := context.Background()
 	id := newAgent(t, s, "node-2")
-	tagless := protocol.Service{Kind: protocol.KindSwarm, SwarmService: "g_g", Image: "gitea/gitea@sha256:87a6", Digest: "sha256:87a6", Status: "running"}
-	bare := protocol.Service{Kind: protocol.KindContainer, ContainerName: "pg", Image: "postgres", Digest: "sha256:pg1", Status: "running"}
+	tagless := protocol.Service{Kind: protocol.KindSwarm, SwarmService: "g_g", Image: "gitea/gitea@" + digGitea, Digest: digGitea, Status: "running"}
+	bare := protocol.Service{Kind: protocol.KindContainer, ContainerName: "pg", Image: "postgres", Digest: digPg, Status: "running"}
 	nodigest := protocol.Service{Kind: protocol.KindContainer, ContainerName: "nd", Image: "nodigest:1", Status: "running"}
 	hello := protocol.Hello{Hostname: "node-2", SwarmManager: true, SwarmClusterID: "c1"}
 	svcs := []protocol.Service{slusk, tagless, bare, nodigest}
@@ -71,19 +79,31 @@ func TestSnapshotRecordsRunningVersions(t *testing.T) {
 	got := versionsOf(t, s, sluskRepo)
 	require.Len(t, got, 1)
 	assert.Equal(t, "1.4.2", got[0].Tag)
-	assert.Equal(t, "sha256:d1", got[0].Digest)
+	assert.Equal(t, digD1, got[0].Digest)
 	assert.Equal(t, model.SourceRunning, got[0].Source)
 	assert.True(t, t0.Equal(got[0].FirstSeenAt), "re-snapshot does not bump first_seen_at")
 	tl := versionsOf(t, s, "gitea/gitea")
 	require.Len(t, tl, 1)
 	assert.Equal(t, "", tl[0].Tag)
-	assert.Equal(t, "sha256:87a6", tl[0].Digest)
+	assert.Equal(t, digGitea, tl[0].Digest)
 	b := versionsOf(t, s, "postgres")
 	require.Len(t, b, 1)
-	assert.Equal(t, "", b[0].Tag, "a bare image is recorded without a tag, not as latest")
+	assert.Equal(t, "latest", b[0].Tag, "a bare image runs latest, like the followed ref")
 	assert.Empty(t, versionsOf(t, s, "nodigest"))
 }
 
+func TestSnapshotRecordsOnlyWellFormedDigests(t *testing.T) {
+	s := openTestStore(t)
+	id := newAgent(t, s, "node-2")
+	bad := protocol.Service{Kind: protocol.KindContainer, ContainerName: "bad", Image: "bad:1", Digest: "sha256:short", Status: "running"}
+	upper := protocol.Service{Kind: protocol.KindContainer, ContainerName: "up", Image: "up:1", Digest: "sha256:" + strings.Repeat("A", 64), Status: "running"}
+
+	require.NoError(t, s.ApplySnapshot(context.Background(), id, protocol.Hello{Hostname: "node-2"}, []protocol.Service{bad, upper}, t0))
+
+	assert.Empty(t, versionsOf(t, s, "bad"))
+	assert.Empty(t, versionsOf(t, s, "up"))
+}
+
 func TestRecentVersions(t *testing.T) {
 	ctx := context.Background()
 	t.Run("order, limit and repo filter", func(t *testing.T) {
@@ -123,7 +143,23 @@ func TestRecentVersions(t *testing.T) {
 		for _, v := range versionsOf(t, s, sluskRepo) {
 			refs = append(refs, v.Ref())
 		}
-		assert.Equal(t, []string{sluskRepo + ":main@sha256:m1", sluskRepo + ":solo", sluskRepo + ":1.4.2@sha256:d1"}, refs)
+		assert.Equal(t, []string{sluskRepo + ":main@sha256:m1", sluskRepo + ":solo", sluskRepo + ":1.4.2@" + digD1}, refs)
+	})
+
+	t.Run("digest-less row newer than the digest row of its tag is shown", func(t *testing.T) {
+		s := openTestStore(t)
+		_, svc := jobFixture(t, s, "a")
+		add := func(tag, digest string, at time.Time) {
+			require.NoError(t, s.SetAvailable(ctx, svc, model.Available{Ref: sluskRepo + ":" + tag, Digest: digest, Source: model.SourceWebhook, At: at}))
+		}
+		add("main", "sha256:m1", t0.Add(time.Minute))
+		add("main", "", t0.Add(2*time.Minute))
+
+		var refs []string
+		for _, v := range versionsOf(t, s, sluskRepo) {
+			refs = append(refs, v.Ref())
+		}
+		assert.Equal(t, []string{sluskRepo + ":main", sluskRepo + ":main@sha256:m1", sluskRepo + ":1.4.2@" + digD1}, refs)
 	})
 }
 
@@ -158,3 +194,23 @@ func TestSetPollError(t *testing.T) {
 	require.NoError(t, err)
 	assert.Empty(t, got.LastPollError)
 }
+
+func TestSetLastAutoDigest(t *testing.T) {
+	s := openTestStore(t)
+	ctx := context.Background()
+	agent, svc := jobFixture(t, s, "a")
+	got, err := s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Empty(t, got.LastAutoDigest)
+
+	require.NoError(t, s.SetLastAutoDigest(ctx, svc, "sha256:n1"))
+	require.NoError(t, s.UpsertService(ctx, agent, "", slusk, t0.Add(time.Hour)))
+
+	got, err = s.GetService(ctx, svc)
+	require.NoError(t, err)
+	assert.Equal(t, "sha256:n1", got.LastAutoDigest, "inventory writes do not touch it")
+	list, err := s.ListServices(ctx)
+	require.NoError(t, err)
+	require.Len(t, list, 1)
+	assert.Equal(t, "sha256:n1", list[0].LastAutoDigest)
+}