feat: master/agent phase 4 — registry polling and version history (#9) (#23)

internal/master/store/services.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"regexp"
 	"time"
 
 	"github.com/jackc/pgx/v5"
@@ -56,7 +57,11 @@ ON CONFLICT (location_key) DO UPDATE SET
 	updated_at    = EXCLUDED.updated_at,
 	removed_at    = NULL`
 
+// digestPattern is the form of a digest worth recording as a version.
+var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
+
 func upsertService(ctx context.Context, db execer, agentID int64, clusterID string, svc protocol.Service, at time.Time) error {
+	image := imageref.Parse(svc.Image)
 	clusterCol := ""
 	if svc.Kind == protocol.KindSwarm {
 		clusterCol = clusterID
@@ -65,11 +70,34 @@ func upsertService(ctx context.Context, db execer, agentID int64, clusterID stri
 		string(svc.Kind), model.LocationKey(agentID, clusterID, svc.Key()), agentID,
 		svc.ComposeProject, svc.ComposeService, svc.WorkingDir, svc.ConfigFiles, svc.Built,
 		clusterCol, svc.SwarmService, svc.ContainerName,
-		imageref.Parse(svc.Image).Repo, svc.Image, svc.Digest,
+		image.Repo, svc.Image, svc.Digest,
 		svc.Status, svc.Self, at)
 	if err != nil {
 		return fmt.Errorf("upsert service %s: %w", svc.Key(), err)
 	}
+	if digestPattern.MatchString(svc.Digest) {
+		// A bare image runs "latest", as jobs.FollowedRef sees it, so it
+		// shares its row with the polled one. repo@digest has no tag.
+		if image.Tag == "" && image.Digest == "" {
+			image.Tag = "latest"
+		}
+		if err := insertVersion(ctx, db, image.Repo, image.Tag, svc.Digest, model.SourceRunning, at); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// insertVersion records a version in image_versions unless the same
+// (repo, tag, digest) is already there; the first sighting keeps its source
+// and first_seen_at.
+func insertVersion(ctx context.Context, db execer, repo, tag, digest, source string, at time.Time) error {
+	if _, err := db.Exec(ctx, `
+		INSERT INTO image_versions (repo, tag, digest, source, first_seen_at)
+		VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`,
+		repo, tag, digest, source, at); err != nil {
+		return fmt.Errorf("record version %s: %w", imageref.Ref{Repo: repo, Tag: tag, Digest: digest}, err)
+	}
 	return nil
 }
 
@@ -149,8 +177,8 @@ func (s *Store) RemoveService(ctx context.Context, agentID int64, clusterID, age
 const serviceColumns = `s.id, s.kind, s.location_key, s.agent_id, COALESCE(a.name, ''),
 	s.compose_project, s.compose_service, s.working_dir, s.config_files, s.built, s.swarm_cluster_id,
 	s.swarm_service, s.container_name, s.image_repo, s.desired_ref, s.actual_ref, s.actual_digest,
-	s.status, s.self, s.update_mode, s.available_ref, s.available_source, s.available_at,
-	s.first_seen_at, s.updated_at, s.removed_at`
+	s.status, s.self, s.update_mode, s.available_ref, s.available_digest, s.available_source, s.available_at,
+	s.last_poll_error, s.last_tag_digest, s.first_seen_at, s.updated_at, s.removed_at`
 
 func scanService(row pgx.Row) (model.Service, error) {
 	var m model.Service
@@ -158,8 +186,8 @@ func scanService(row pgx.Row) (model.Service, error) {
 	err := row.Scan(&m.ID, &kind, &m.LocationKey, &m.AgentID, &m.AgentName,
 		&m.ComposeProject, &m.ComposeService, &m.WorkingDir, &m.ConfigFiles, &m.Built, &m.SwarmClusterID,
 		&m.SwarmService, &m.ContainerName, &m.ImageRepo, &m.DesiredRef, &m.ActualRef, &m.ActualDigest,
-		&m.Status, &m.Self, &m.UpdateMode, &m.AvailableRef, &m.AvailableSource, &m.AvailableAt,
-		&m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt)
+		&m.Status, &m.Self, &m.UpdateMode, &m.AvailableRef, &m.AvailableDigest, &m.AvailableSource, &m.AvailableAt,
+		&m.LastPollError, &m.LastTagDigest, &m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt)
 	m.Kind = protocol.Kind(kind)
 	return m, err
 }
@@ -198,16 +226,96 @@ func (s *Store) queryServices(ctx context.Context, what, extraWhere string, args
 }
 
 // SetAvailable records a newer ref for the service, replacing any earlier
-// suggestion.
+// suggestion, and records the version in image_versions in the same
+// transaction (the tag is parsed from a.Ref).
 func (s *Store) SetAvailable(ctx context.Context, id int64, a model.Available) error {
-	if _, err := s.pool.Exec(ctx, `
-		UPDATE services SET available_ref = $2, available_source = $3, available_at = $4
-		WHERE id = $1`, id, a.Ref, a.Source, a.At); err != nil {
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return fmt.Errorf("set available: begin: %w", err)
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+	if _, err := tx.Exec(ctx, `
+		UPDATE services SET available_ref = $2, available_digest = $3, available_source = $4, available_at = $5
+		WHERE id = $1`, id, a.Ref, a.Digest, a.Source, a.At); err != nil {
 		return fmt.Errorf("set available: %w", err)
 	}
+	ref := imageref.Parse(a.Ref)
+	if err := insertVersion(ctx, tx, ref.Repo, ref.Tag, a.Digest, a.Source, a.At); err != nil {
+		return err
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return fmt.Errorf("set available: commit: %w", err)
+	}
 	return nil
 }
 
+// SetPollError sets last_poll_error on the given services; msg "" clears it.
+func (s *Store) SetPollError(ctx context.Context, ids []int64, msg string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	if _, err := s.pool.Exec(ctx,
+		`UPDATE services SET last_poll_error = $2 WHERE id = ANY($1)`, ids, msg); err != nil {
+		return fmt.Errorf("set poll error: %w", err)
+	}
+	return nil
+}
+
+// SetLastTagDigest sets last_tag_digest; see model.Service.LastTagDigest. The
+// poller calls it, and FinishJob sets the same column after a tag job.
+// Inventory writes never change it.
+func (s *Store) SetLastTagDigest(ctx context.Context, id int64, digest string) error {
+	if _, err := s.pool.Exec(ctx,
+		`UPDATE services SET last_tag_digest = $2 WHERE id = $1`, id, digest); err != nil {
+		return fmt.Errorf("set last tag digest: %w", err)
+	}
+	return nil
+}
+
+// RecentVersions returns up to limit versions of repo, most recently first
+// seen first. A version without a digest is left out when a version with the
+// same tag and a digest was first seen at or after it.
+func (s *Store) RecentVersions(ctx context.Context, repo string, limit int) ([]model.ImageVersion, error) {
+	rows, err := s.pool.Query(ctx, `
+		SELECT id, repo, tag, digest, source, first_seen_at FROM image_versions v
+		WHERE v.repo = $1
+		  AND NOT (v.digest = '' AND EXISTS (
+		        SELECT 1 FROM image_versions d
+		        WHERE d.repo = v.repo AND d.tag = v.tag AND d.digest <> ''
+		          AND d.first_seen_at >= v.first_seen_at))
+		ORDER BY v.first_seen_at DESC, v.id DESC
+		LIMIT $2`, repo, limit)
+	if err != nil {
+		return nil, fmt.Errorf("recent versions: %w", err)
+	}
+	defer rows.Close()
+	var out []model.ImageVersion
+	for rows.Next() {
+		var v model.ImageVersion
+		if err := rows.Scan(&v.ID, &v.Repo, &v.Tag, &v.Digest, &v.Source, &v.FirstSeenAt); err != nil {
+			return nil, fmt.Errorf("scan version: %w", err)
+		}
+		out = append(out, v)
+	}
+	return out, rows.Err()
+}
+
+// GetImageVersion returns one version. Returns model.ErrNotFound if the ID
+// does not exist.
+func (s *Store) GetImageVersion(ctx context.Context, id int64) (model.ImageVersion, error) {
+	var v model.ImageVersion
+	err := s.pool.QueryRow(ctx, `
+		SELECT id, repo, tag, digest, source, first_seen_at FROM image_versions WHERE id = $1`, id,
+	).Scan(&v.ID, &v.Repo, &v.Tag, &v.Digest, &v.Source, &v.FirstSeenAt)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return model.ImageVersion{}, model.ErrNotFound
+	}
+	if err != nil {
+		return model.ImageVersion{}, fmt.Errorf("get image version: %w", err)
+	}
+	return v, nil
+}
+
 // SetUpdateMode sets the service's update mode (model.ModeManual or
 // model.ModeAuto). Returns model.ErrNotFound if the service does not exist
 // or is marked removed.