diff --git a/CLAUDE.md b/CLAUDE.md index 42470ad..56c85c1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,8 +36,9 @@ DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/ - `internal/master/config` — env-var loading - `internal/master/store` — Postgres access and migrations - `internal/master/hub` — agent connections: auth, per-connection writer, routing, job reconcile on hello -- `internal/master/registry` — HEAD-only digest lookup against a registry (go-containerregistry, `DOCKER_CONFIG` credentials) -- `internal/master/poller` — every `POLL_INTERVAL`: one lookup per followed `repo:tag`, sets the available version, starts auto jobs +- `internal/master/registry` — HEAD-only digest lookup and tag listing against a registry (go-containerregistry, `DOCKER_CONFIG` credentials) +- `internal/master/tagver` — which tag is a newer version of a followed tag (pure functions) +- `internal/master/poller` — every `POLL_INTERVAL`: one lookup per followed `repo:tag`, sets the available version, starts auto jobs; one tag list per repo, sets the newer tag suggestion - `internal/master/jobs` — decides whether a service may be updated and starts a job on the right agent - `internal/master/webhook` — `POST /update`: records the available version, starts jobs for auto services, waits for them - `internal/master/web/icons` — embedded Simple Icons path data; slug derived from the image ref; regenerate with `go generate ./internal/master/web/icons` @@ -76,8 +77,16 @@ DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/ ## Gotchas -- **Registry access is HEAD only** (`internal/master/registry`). Docker Hub counts a manifest - GET as a pull; the client never falls back to GET, and a test asserts this. +- **Registry manifest access is HEAD only** (`internal/master/registry`). Docker Hub counts a + manifest GET as a pull; the client never GETs `/manifests/`, and a test asserts this. The + only GETs are the `/v2/` ping and `tags/list` (newer-tag detection, once per repo per cycle). +- **`services.newer_ref` is a suggestion only**: no auto jobs, not even for auto services. The + poller writes it only on change; a failed tag list keeps the old value and never sets + `last_poll_error`. The UI re-checks it at render time with `tagver.Newer`, so a stale value + is hidden until the next cycle; it is also hidden when its repo is not the one the service + follows. The suggestion is computed from the service list read at the start of the cycle, so + if the service moved on meanwhile, the render-time check hides it. Tags with one numeric component or no digits (`7`, `pg16`, + `latest`) never get suggestions. - **`image_versions` uses `''` instead of NULL** for a missing tag or digest, so `UNIQUE (repo, tag, digest)` deduplicates. The webhook records `(repo, tag, '')`; that row is hidden from `RecentVersions` when a row with the same tag and a digest was first diff --git a/README.md b/README.md index a0486fc..40a0944 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,15 @@ multi-arch image the digest is the index digest, the same one `docker inspect` s it) and starts no job for that digest again, whether the job failed or the user rolled back afterwards. If the service already had a job when the digest was found, the next cycle tries again. +- **Newer tags.** For a service that follows a version-like tag (`1.27.3`, `v1.2.3`, + `11.0-latest`; at least two numeric components), the master also lists the repo's tags + once per cycle (`GET /v2//tags/list`, which Docker Hub does not count as a pull) and + suggests the highest tag with the same prefix, suffix and number of components and the + same major number, for example `gitea:1.27.3` gives `1.28.0` but never `2.0.0`. The + overview shows a "Newer tag" badge and the service page lists the tag under Versions, from + where you deploy it. It is a suggestion only: nothing is deployed automatically, not even + for **auto** services. Tags such as `latest`, `7` or `pg16` get no suggestion. A failed tag + list keeps the old suggestion and is only logged (`poll_tags_failed`). - **What is not polled.** Services that follow no tag (`repo@digest`), `build:` services, and services without a registry digest (locally built images). The service page says which. The updater's own services and plain containers are polled, so their badge works, but diff --git a/docs/superpowers/specs/2026-10-12-master-agent-design.md b/docs/superpowers/specs/2026-10-12-master-agent-design.md index 063b511..348724d 100644 --- a/docs/superpowers/specs/2026-10-12-master-agent-design.md +++ b/docs/superpowers/specs/2026-10-12-master-agent-design.md @@ -255,6 +255,11 @@ them. An agent recreating itself would kill the job it is running. HEAD-only and never downloads a manifest. - Credentials: a Docker `config.json` (as written by `docker login`) mounted into the master. No credential UI, no encryption in Postgres. +- Newer version tags (e.g. `gitea:1.27.3` → `1.27.4`): for a followed tag with at least + two numeric components the poller lists the repo's tags once per cycle + (`GET /v2//tags/list`) and stores the highest tag with the same prefix, suffix, + component count and major number in `services.newer_ref`. A suggestion only, never + deployed automatically; deploying it is the existing version pick. ## Webhook @@ -364,10 +369,6 @@ Each phase ends with something that works on its own. ## Out of scope (later specs) -- Detecting newer version *tags* for images we do not build (e.g. `gitea:1.27.3` → - `1.27.4`, `postgres:17.10`, `loki:3.4.2`). Requires listing tags and ordering them; - major upgrades such as Postgres 17 → 18 must never be automatic. Revisit after - checking the other two servers' stack files. - Notifications. - Export/import of a fleet configuration to restore a setup. The `services` table already holds what an export needs. diff --git a/internal/e2e/agent_master_test.go b/internal/e2e/agent_master_test.go index 1568aa4..5b40a78 100644 --- a/internal/e2e/agent_master_test.go +++ b/internal/e2e/agent_master_test.go @@ -104,6 +104,19 @@ func (m *memStore) SetLastTagDigest(_ context.Context, id int64, digest string) return nil } +func (m *memStore) SetNewer(_ context.Context, id int64, ref string, at time.Time) error { + m.mu.Lock() + defer m.mu.Unlock() + svc, ok := m.svcRows[id] + if !ok { + return model.ErrNotFound + } + svc.NewerRef = ref + svc.NewerAt = &at + m.svcRows[id] = svc + return nil +} + // seedService adds a service row that jobs can be triggered for. func (m *memStore) seedService(svc model.Service) { m.mu.Lock() diff --git a/internal/e2e/versions_test.go b/internal/e2e/versions_test.go index ad94645..3530d29 100644 --- a/internal/e2e/versions_test.go +++ b/internal/e2e/versions_test.go @@ -34,6 +34,9 @@ func (f *fixedRegistry) Digest(_ context.Context, ref string) (string, error) { return f.digests[ref], nil } +// Tags lists no tags, so no newer tag is ever suggested. +func (f *fixedRegistry) Tags(context.Context, string) ([]string, error) { return nil, nil } + func TestPickPinnedVersion(t *testing.T) { env := newJobEnv(t) env.store.seedVersion(model.ImageVersion{ID: 5, Repo: slusk, Tag: "1.3.0", Digest: "sha256:old130", Source: model.SourceRunning}) diff --git a/internal/master/jobs/jobs_test.go b/internal/master/jobs/jobs_test.go index 241a59e..eb93357 100644 --- a/internal/master/jobs/jobs_test.go +++ b/internal/master/jobs/jobs_test.go @@ -411,6 +411,7 @@ func TestTriggerVersion_Ref(t *testing.T) { }{ {"tag and digest", svc, model.ImageVersion{ID: 10, Repo: repo, Tag: "1.3.0", Digest: "sha256:old"}, repo + ":1.3.0@sha256:old", protocol.MinPinnedRefVersion}, {"tag only", svc, model.ImageVersion{ID: 10, Repo: repo, Tag: "main"}, repo + ":main", protocol.MinJobVersion}, + {"digest-less version from the tag list", svc, model.ImageVersion{ID: 10, Repo: repo, Tag: "1.28.0", Source: model.SourceTags}, repo + ":1.28.0", protocol.MinJobVersion}, {"digest only", svc, model.ImageVersion{ID: 10, Repo: repo, Digest: "sha256:old"}, repo + "@sha256:old", protocol.MinPinnedRefVersion}, {"service that follows no tag", noTag, model.ImageVersion{ID: 10, Repo: repo, Tag: "1.3.0", Digest: "sha256:old"}, repo + ":1.3.0@sha256:old", protocol.MinPinnedRefVersion}, } diff --git a/internal/master/model/model.go b/internal/master/model/model.go index c105314..9cc5f30 100644 --- a/internal/master/model/model.go +++ b/internal/master/model/model.go @@ -66,6 +66,11 @@ type Service struct { AvailableSource string // SourceWebhook or SourcePoll; "" when AvailableRef is empty AvailableAt *time.Time LastPollError string // last registry error for the followed ref; "" when fine + // NewerRef is the newest repo:tag from the registry's tag list that + // tagver.Newer finds for the followed tag; "" if none. A suggestion only: + // it is never deployed automatically, not even for auto services. + NewerRef string + NewerAt *time.Time // when NewerRef was first suggested; nil when NewerRef is empty // LastTagDigest is the newest digest of the followed tag that was deployed // or started: set when the poller starts an auto job, when it sees the service // already running that digest, and when any plain tag job succeeds (the @@ -95,6 +100,9 @@ const ( // SourceRunning is used by image_versions only: an agent reported the // version as running. SourceRunning = "running" + // SourceTags is used by image_versions only: the tag appeared in the + // registry's tag list as a newer tag. + SourceTags = "tags" ) // Available is a newer ref reported for a service. @@ -113,7 +121,7 @@ type ImageVersion struct { Repo string Tag string Digest string - Source string // SourceWebhook, SourcePoll or SourceRunning: the first source that saw it + Source string // SourceWebhook, SourcePoll, SourceRunning or SourceTags: the first source that saw it FirstSeenAt time.Time } diff --git a/internal/master/poller/poller.go b/internal/master/poller/poller.go index eb1b17b..1462c9d 100644 --- a/internal/master/poller/poller.go +++ b/internal/master/poller/poller.go @@ -1,6 +1,8 @@ // Package poller checks the registry for new digests of the tags the // services follow. A new digest becomes the service's available version, and -// a service in auto mode gets an update job. +// a service in auto mode gets an update job. It also lists the tags of the +// repos whose followed tag looks like a version and suggests a newer one; +// that suggestion never starts a job. package poller import ( @@ -11,14 +13,19 @@ import ( "sync" "time" + "github.com/shcizo/package-updater/internal/imageref" "github.com/shcizo/package-updater/internal/master/events" "github.com/shcizo/package-updater/internal/master/jobs" "github.com/shcizo/package-updater/internal/master/model" + "github.com/shcizo/package-updater/internal/master/tagver" ) // lookupTimeout bounds one registry lookup. const lookupTimeout = 30 * time.Second +// listTimeout bounds one tag listing, which may take several requests. +const listTimeout = 60 * time.Second + // maxErrorBytes caps the registry error stored in services.last_poll_error. const maxErrorBytes = 500 @@ -28,6 +35,7 @@ type Store interface { GetService(ctx context.Context, id int64) (model.Service, error) SetAvailable(ctx context.Context, id int64, a model.Available) error SetLastTagDigest(ctx context.Context, id int64, digest string) error + SetNewer(ctx context.Context, id int64, ref string, at time.Time) error SetPollError(ctx context.Context, ids []int64, msg string) error } @@ -36,10 +44,11 @@ type Jobs interface { Start(ctx context.Context, svc model.Service, req jobs.StartRequest) (model.Job, error) } -// Registry looks up the digest a "repo:tag" ref points at. *registry.Client -// satisfies it. +// Registry looks up the digest a "repo:tag" ref points at and lists the tags +// of a repo. *registry.Client satisfies it. type Registry interface { Digest(ctx context.Context, ref string) (string, error) + Tags(ctx context.Context, repo string) ([]string, error) } // Publisher receives change signals. *events.Broker satisfies it. @@ -49,7 +58,7 @@ type Publisher interface{ Publish(t events.Topic) } type Config struct { Interval time.Duration // time between the end of a cycle and the start of the next (POLL_INTERVAL) FirstDelay time.Duration // wait before the first cycle; 1 minute in production - Workers int // registry lookups in flight at once; 4 in production + Workers int // registry requests in flight at once; 4 in production } // Poller runs polling cycles. @@ -112,13 +121,16 @@ type lookup struct { // cycleStats is what one cycle did, for the poll_cycle log line. type cycleStats struct { refs, ok, failed, suggested, jobs, skipped int + reposListed, tagsFailed, newer int wrote bool } // Cycle runs one poll over all services. It looks up every distinct followed // ref once, with at most Config.Workers lookups at a time, then records the -// results. Writing results and starting jobs is not cut short by ctx ending -// during the lookups, so a job that was marked is always started. +// results. Then it lists the tags of each repo that has a service following a +// version-like tag, once per repo, and records newer tags. Writing results and +// starting jobs is not cut short by ctx ending during the lookups, so a job +// that was marked is always started. func (p *Poller) Cycle(ctx context.Context) { began := p.now() svcs, err := p.store.ListServices(ctx) @@ -152,19 +164,30 @@ func (p *Poller) Cycle(ctx context.Context) { } p.record(write, l, byRef[l.ref], &stats) } - if stats.wrote { - p.pub.Publish(events.Services) - } + // Publish after the digest phase too: listing tags can take a while and + // an "Update available" should not wait for it. + p.publishIfWrote(&stats) + p.syncNewer(ctx, write, svcs, &stats) + p.publishIfWrote(&stats) p.log.Info("poll_cycle", "refs", stats.refs, "ok", stats.ok, "failed", stats.failed, "suggested", stats.suggested, "jobs", stats.jobs, "skipped", stats.skipped, + "repos_listed", stats.reposListed, "tags_failed", stats.tagsFailed, "newer", stats.newer, "duration_ms", p.now().Sub(began).Milliseconds()) } -// lookupAll asks the registry for every ref, Config.Workers at a time. It -// returns the answers in ref order and stops starting new lookups once ctx -// is done; refs never looked up are missing from the result. -func (p *Poller) lookupAll(ctx context.Context, refs []string) []lookup { - results := make([]*lookup, len(refs)) +// publishIfWrote tells the browsers to refresh if a phase of the cycle wrote +// anything, and starts the next phase with a clean slate. +func (p *Poller) publishIfWrote(stats *cycleStats) { + if stats.wrote { + p.pub.Publish(events.Services) + stats.wrote = false + } +} + +// forEach calls fn(i) for i in [0, n), Config.Workers at a time, and stops +// starting new calls once ctx is done. It returns when all started calls +// have returned. +func (p *Poller) forEach(ctx context.Context, n int, fn func(i int)) { next := make(chan int) var wg sync.WaitGroup for w := 0; w < p.cfg.Workers; w++ { @@ -172,15 +195,12 @@ func (p *Poller) lookupAll(ctx context.Context, refs []string) []lookup { go func() { defer wg.Done() for i := range next { - lctx, cancel := context.WithTimeout(ctx, lookupTimeout) - d, err := p.reg.Digest(lctx, refs[i]) - cancel() - results[i] = &lookup{ref: refs[i], digest: d, err: err} + fn(i) } }() } feed: - for i := range refs { + for i := 0; i < n; i++ { select { case next <- i: case <-ctx.Done(): @@ -189,6 +209,19 @@ feed: } close(next) wg.Wait() +} + +// lookupAll asks the registry for every ref, Config.Workers at a time. It +// returns the answers in ref order and stops starting new lookups once ctx +// is done; refs never looked up are missing from the result. +func (p *Poller) lookupAll(ctx context.Context, refs []string) []lookup { + results := make([]*lookup, len(refs)) + p.forEach(ctx, len(refs), func(i int) { + lctx, cancel := context.WithTimeout(ctx, lookupTimeout) + d, err := p.reg.Digest(lctx, refs[i]) + cancel() + results[i] = &lookup{ref: refs[i], digest: d, err: err} + }) var out []lookup for _, r := range results { if r != nil { @@ -198,6 +231,118 @@ feed: return out } +// tagList is the registry's answer for the tags of one repo. +type tagList struct { + repo string + tags []string + err error +} + +// listAll lists the tags of every repo like lookupAll looks up refs. +func (p *Poller) listAll(ctx context.Context, repos []string) []tagList { + results := make([]*tagList, len(repos)) + p.forEach(ctx, len(repos), func(i int) { + lctx, cancel := context.WithTimeout(ctx, listTimeout) + tags, err := p.reg.Tags(lctx, repos[i]) + cancel() + results[i] = &tagList{repo: repos[i], tags: tags, err: err} + }) + var out []tagList + for _, r := range results { + if r != nil { + out = append(out, *r) + } + } + return out +} + +// Followed returns the repo and tag svc follows: a bare image follows +// "latest", and a digest-only reference follows no tag. +func Followed(svc model.Service) (repo, tag string) { + ref := imageref.Parse(jobs.FollowedRef(svc.ActualRef)) + return ref.Repo, ref.Tag +} + +// followedComparable returns the repo and tag svc follows. ok is false unless +// svc is polled and the tag can have a newer tag. +func followedComparable(svc model.Service) (repo, tag string, ok bool) { + if SkipReason(svc) != "" { + return "", "", false + } + repo, tag = Followed(svc) + return repo, tag, tagver.Comparable(tag) +} + +// syncNewer lists the tags of every repo that has a service following a +// version-like tag, once per repo, and stores the newer tag of each service. +// A service whose tag cannot have a newer tag any more gets its suggestion +// cleared; one whose repo could not be listed keeps it, unless it names +// another repo than the service follows now. Nothing is written +// when the value is unchanged. It never starts a job, not even for an auto +// service: a newer tag may be a major step the user has not chosen. +func (p *Poller) syncNewer(ctx, write context.Context, svcs []model.Service, stats *cycleStats) { + seen := map[string]bool{} + var repos []string + for _, svc := range svcs { + if repo, _, ok := followedComparable(svc); ok && svc.RemovedAt == nil && !seen[repo] { + seen[repo] = true + repos = append(repos, repo) + } + } + lists := map[string][]string{} + for _, l := range p.listAll(ctx, repos) { + if l.err != nil { + p.log.Warn("poll_tags_failed", "repo", l.repo, "error", truncate(l.err.Error())) + if ctx.Err() == nil { // cancelled mid-listing: not the registry's fault + stats.tagsFailed++ + } + continue + } + stats.reposListed++ + lists[l.repo] = l.tags + } + + type repoTag struct{ repo, tag string } + newest := map[repoTag]string{} + for _, svc := range svcs { + if svc.RemovedAt != nil { + continue + } + want := "" + if repo, tag, ok := followedComparable(svc); ok { + tags, listed := lists[repo] + if !listed && imageref.Parse(svc.NewerRef).Repo == repo { + if svc.NewerRef != "" { + stats.newer++ + } + continue // not listed this cycle: keep what is stored + } + if listed { + key := repoTag{repo, tag} + n, done := newest[key] + if !done { + n, _ = tagver.Newer(tag, tags) + newest[key] = n + } + if n != "" { + want = repo + ":" + n + } + } + } + if want != "" { + stats.newer++ + } + if want == svc.NewerRef { + continue + } + if err := p.store.SetNewer(write, svc.ID, want, p.now()); err != nil { + p.log.Error("poll_set_newer_failed", "service_id", svc.ID, "ref", want, "error", err) + continue + } + stats.wrote = true + } +} + // record stores one lookup result for the services that follow l.ref. func (p *Poller) record(ctx context.Context, l lookup, svcs []model.Service, stats *cycleStats) { if l.err != nil { diff --git a/internal/master/poller/poller_test.go b/internal/master/poller/poller_test.go index 2c5fd1b..3907a61 100644 --- a/internal/master/poller/poller_test.go +++ b/internal/master/poller/poller_test.go @@ -30,6 +30,12 @@ type pollErrorCall struct { msg string } +type newerCall struct { + id int64 + ref string + at time.Time +} + type lastTagCall struct { id int64 digest string @@ -40,6 +46,7 @@ type fakeStore struct { services []model.Service fresh map[int64]model.Service // what GetService returns instead of the listed row lastTag []lastTagCall + newer []newerCall listErr error available []availableCall pollErrors []pollErrorCall @@ -86,6 +93,21 @@ func (f *fakeStore) SetAvailable(ctx context.Context, id int64, a model.Availabl return nil } +// SetNewer records the call and, like the database, changes the row the +// next cycle lists. +func (f *fakeStore) SetNewer(ctx context.Context, id int64, ref string, at time.Time) error { + f.mu.Lock() + defer f.mu.Unlock() + f.ctxErrs = append(f.ctxErrs, ctx.Err()) + f.newer = append(f.newer, newerCall{id, ref, at}) + for i := range f.services { + if f.services[i].ID == id { + f.services[i].NewerRef = ref + } + } + return nil +} + func (f *fakeStore) SetPollError(ctx context.Context, ids []int64, msg string) error { f.mu.Lock() defer f.mu.Unlock() @@ -125,6 +147,36 @@ type fakeRegistry struct { started chan struct{} // receives a value whenever a lookup starts, if set release chan struct{} // lookups wait for it to be closed, if set onResult func() // called after a lookup computed its answer + + tags map[string][]string // repo -> tags + tagErrs map[string]error // repo -> error + tagCalls []string + tagStarted chan struct{} // receives a value whenever a tag listing starts, if set + tagRelease chan struct{} // tag listings wait for it to be closed, if set + onTags func() // called after a tag listing computed its answer +} + +func (f *fakeRegistry) Tags(ctx context.Context, repo string) ([]string, error) { + f.mu.Lock() + f.tagCalls = append(f.tagCalls, repo) + f.mu.Unlock() + if f.tagStarted != nil { + f.tagStarted <- struct{}{} + } + if f.tagRelease != nil { + select { + case <-f.tagRelease: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if f.onTags != nil { + f.onTags() + } + if err := f.tagErrs[repo]; err != nil { + return nil, err + } + return f.tags[repo], nil } func (f *fakeRegistry) Digest(ctx context.Context, ref string) (string, error) { @@ -649,3 +701,222 @@ func TestSeedsTheTagDigestOfARunningService(t *testing.T) { assert.Empty(t, j.starts, "the rollback stays") assert.Len(t, st.lastTag, 1, "nothing more to seed") } + +// newTagRegistry returns a fake registry that lists tags by repo. +func newTagRegistry(tags map[string][]string) *fakeRegistry { + return &fakeRegistry{digests: map[string]string{}, tags: tags} +} + +// newerSvc is a polled service running tag of repo that already runs the +// registry's digest, so the digest phase writes nothing. +func newerSvc(id int64, repo, tag string) model.Service { + s := svc(id, repo+":"+tag) + s.ActualDigest = "sha256:new" + return s +} + +// newerRegistry is newTagRegistry that also answers sha256:new for refs. +func newerRegistry(tags map[string][]string, refs ...string) *fakeRegistry { + reg := newTagRegistry(tags) + for _, r := range refs { + reg.digests[r] = "sha256:new" + } + return reg +} + +func TestNewerTagSuggested(t *testing.T) { + ref := repoB + ":1.27.3" + st := &fakeStore{services: []model.Service{newerSvc(1, repoB, "1.27.3")}} + reg := newerRegistry(map[string][]string{repoB: {"1.27.4", "1.28.0", "2.0.0", "latest", "1.28.1-rootless"}}, ref) + p, pub := newPoller(st, &fakeJobs{}, reg, Config{}) + + p.Cycle(context.Background()) + + assert.Equal(t, []newerCall{{1, repoB + ":1.28.0", time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)}}, st.newer) + assert.Equal(t, []events.Topic{events.Services}, pub.topics) +} + +func TestOneTagListPerRepo(t *testing.T) { + st := &fakeStore{services: []model.Service{ + newerSvc(1, repoB, "1.27.3"), + newerSvc(2, repoB, "2.1.0"), + newerSvc(3, repoB, "1.27.3"), + }} + reg := newerRegistry(map[string][]string{repoB: {"2.1.1", "1.27.4"}}, + repoB+":1.27.3", repoB+":2.1.0") + p, _ := newPoller(st, &fakeJobs{}, reg, Config{}) + + p.Cycle(context.Background()) + + assert.Equal(t, []string{repoB}, reg.tagCalls) + got := map[int64]string{} + for _, c := range st.newer { + got[c.id] = c.ref + } + assert.Equal(t, map[int64]string{1: repoB + ":1.27.4", 2: repoB + ":2.1.1", 3: repoB + ":1.27.4"}, got) +} + +func TestNoTagListWithoutComparableTag(t *testing.T) { + built := newerSvc(5, repoB, "1.2.3") + built.Built = true + noDigest := newerSvc(6, repoB, "1.2.3") + noDigest.ActualDigest = "" + st := &fakeStore{services: []model.Service{ + newerSvc(1, repoB, "latest"), + newerSvc(2, repoB, "7"), + newerSvc(3, repoB, "pg16"), + newerSvc(4, repoB+"@sha256:abc", ""), + built, + noDigest, + }} + st.services[3].ActualRef = repoB + "@sha256:abc" + reg := newTagRegistry(map[string][]string{repoB: {"1.2.4"}}) + p, _ := newPoller(st, &fakeJobs{}, reg, Config{}) + + p.Cycle(context.Background()) + + assert.Empty(t, reg.tagCalls) + assert.Empty(t, st.newer) +} + +func TestNewerClearedWhenNoCandidate(t *testing.T) { + s := newerSvc(1, repoB, "1.27.3") + s.NewerRef = repoB + ":1.27.4" + st := &fakeStore{services: []model.Service{s}} + reg := newerRegistry(map[string][]string{repoB: {"1.27.3", "2.0.0"}}, repoB+":1.27.3") + p, _ := newPoller(st, &fakeJobs{}, reg, Config{}) + + p.Cycle(context.Background()) + + assert.Equal(t, []newerCall{{1, "", time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)}}, st.newer) +} + +func TestNewerClearedWhenTagNoLongerComparable(t *testing.T) { + moved := newerSvc(1, repoB, "latest") + moved.NewerRef = repoB + ":1.27.4" + built := newerSvc(2, repoB, "1.27.3") + built.Built = true + built.NewerRef = repoB + ":1.27.4" + st := &fakeStore{services: []model.Service{moved, built}} + reg := newerRegistry(map[string][]string{repoB: {"1.27.4"}}, repoB+":latest") + p, pub := newPoller(st, &fakeJobs{}, reg, Config{}) + + p.Cycle(context.Background()) + + require.Len(t, st.newer, 2) + assert.Equal(t, "", st.newer[0].ref) + assert.Equal(t, "", st.newer[1].ref) + assert.Empty(t, reg.tagCalls) + assert.Equal(t, []events.Topic{events.Services}, pub.topics) +} + +func TestTagListErrorKeepsNewerAndPollError(t *testing.T) { + s := newerSvc(1, repoB, "1.27.3") + s.NewerRef = repoB + ":1.27.4" + st := &fakeStore{services: []model.Service{s}} + reg := newerRegistry(nil, repoB+":1.27.3") + reg.tagErrs = map[string]error{repoB: errors.New("list x: registry answered 500 Internal Server Error")} + p, pub := newPoller(st, &fakeJobs{}, reg, Config{}) + + p.Cycle(context.Background()) + + assert.Empty(t, st.newer, "the stored suggestion stays") + assert.Empty(t, st.pollErrors, "a failed tag list is not a poll error") + assert.Empty(t, pub.topics) +} + +func TestNewerIdempotent(t *testing.T) { + s := newerSvc(1, repoB, "1.27.3") + s.NewerRef = repoB + ":1.28.0" + st := &fakeStore{services: []model.Service{s}} + reg := newerRegistry(map[string][]string{repoB: {"1.28.0"}}, repoB+":1.27.3") + p, pub := newPoller(st, &fakeJobs{}, reg, Config{}) + + p.Cycle(context.Background()) + + assert.Empty(t, st.newer) + assert.Empty(t, pub.topics) +} + +func TestNewerNeverStartsJobs(t *testing.T) { + s := newerSvc(1, repoB, "1.27.3") + s.UpdateMode = model.ModeAuto + st := &fakeStore{services: []model.Service{s}} + reg := newerRegistry(map[string][]string{repoB: {"1.27.4"}}, repoB+":1.27.3") + j := &fakeJobs{} + p, _ := newPoller(st, j, reg, Config{}) + + p.Cycle(context.Background()) + + require.Len(t, st.newer, 1) + assert.Empty(t, j.starts) +} + +func TestNewerWritesSurviveCancel(t *testing.T) { + st := &fakeStore{services: []model.Service{newerSvc(1, repoB, "1.27.3")}} + ctx, cancel := context.WithCancel(context.Background()) + reg := newerRegistry(map[string][]string{repoB: {"1.27.4"}}, repoB+":1.27.3") + reg.onTags = cancel + p, _ := newPoller(st, &fakeJobs{}, reg, Config{}) + + p.Cycle(ctx) + + require.Len(t, st.newer, 1) + assert.NoError(t, st.ctxErrs[0], "the write does not see the cancelled context") +} + +func TestCancelledListKeepsNewer(t *testing.T) { + s := newerSvc(1, repoB, "1.27.3") + s.NewerRef = repoB + ":1.27.4" + st := &fakeStore{services: []model.Service{s}} + reg := newerRegistry(map[string][]string{repoB: {"1.28.0"}}, repoB+":1.27.3") + reg.tagRelease = make(chan struct{}) + reg.tagStarted = make(chan struct{}, 1) + p, pub := newPoller(st, &fakeJobs{}, reg, Config{}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { p.Cycle(ctx); close(done) }() + <-reg.tagStarted + + cancel() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("Cycle did not return after cancel") + } + assert.Empty(t, st.newer) + assert.Empty(t, pub.topics) +} + +func TestFailedListClearsNewerFromAnotherRepo(t *testing.T) { + s := newerSvc(1, repoB, "1.27.3") + s.NewerRef = repoA + ":1.27.4" + st := &fakeStore{services: []model.Service{s}} + reg := newerRegistry(nil, repoB+":1.27.3") + reg.tagErrs = map[string]error{repoB: errors.New("list x: registry answered 500 Internal Server Error")} + p, _ := newPoller(st, &fakeJobs{}, reg, Config{}) + + p.Cycle(context.Background()) + + assert.Equal(t, []newerCall{{1, "", time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)}}, st.newer) +} + +func TestDigestWritesArePublishedBeforeTagsAreListed(t *testing.T) { + st := &fakeStore{services: []model.Service{svc(1, repoB+":1.27.3")}} + reg := newerRegistry(map[string][]string{repoB: {"1.27.3"}}, repoB+":1.27.3") + reg.tagRelease = make(chan struct{}) + reg.tagStarted = make(chan struct{}, 1) + p, pub := newPoller(st, &fakeJobs{}, reg, Config{}) + done := make(chan struct{}) + go func() { p.Cycle(context.Background()); close(done) }() + <-reg.tagStarted + + pub.mu.Lock() + published := append([]events.Topic(nil), pub.topics...) + pub.mu.Unlock() + assert.Equal(t, []events.Topic{events.Services}, published, "published while the listing is still blocked") + + close(reg.tagRelease) + <-done +} diff --git a/internal/master/registry/registry.go b/internal/master/registry/registry.go index 3674c3e..e8d2d55 100644 --- a/internal/master/registry/registry.go +++ b/internal/master/registry/registry.go @@ -1,5 +1,6 @@ // Package registry looks up the digest a tag points at in a container -// registry, using HEAD requests only. +// registry and lists the tags of a repository. Manifests are only ever +// requested with HEAD. package registry import ( @@ -21,9 +22,10 @@ import ( "github.com/shcizo/package-updater/internal/imageref" ) -// Client looks up tag digests with HEAD requests only. Docker Hub counts a -// GET of a manifest as a pull against the rate limit and a HEAD as nothing, -// so Client never issues a manifest GET. +// Client looks up tag digests and lists tags. Docker Hub counts a GET of a +// manifest as a pull against the rate limit and a HEAD as nothing, so Client +// never issues a manifest GET. The only GETs it sends are the /v2/ ping and +// tags/list, which Docker Hub does not count as a pull. type Client struct { keychain authn.Keychain userAgent string @@ -65,16 +67,46 @@ func (c *Client) Digest(ctx context.Context, ref string) (string, error) { remote.WithAuth(auth), remote.WithUserAgent(c.userAgent)) if err != nil { - var terr *transport.Error - if errors.As(err, &terr) { - return "", fmt.Errorf("lookup %s: %s", r, describe(terr)) - } - c.log.Warn("registry_lookup_failed", "ref", ref, "error", err) - return "", fmt.Errorf("lookup %s: %s", r, classify(err)) + return "", c.lookupError("lookup", r.String(), ref, err) } return desc.Digest.String(), nil } +// Tags returns every tag of repo ("host/path", or a Docker Hub name), in the +// order the registry lists them. It sends GET /v2//tags/list, following +// the registry's pagination links. Errors are sanitised like Digest's. +func (c *Client) Tags(ctx context.Context, repo string) ([]string, error) { + r, err := name.NewRepository(repo) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", repo, err) + } + auth, err := c.keychain.Resolve(r) + if err != nil { + return nil, fmt.Errorf("list %s: %w", r, err) + } + tags, err := remote.List(r, + remote.WithContext(ctx), + remote.WithAuth(auth), + remote.WithUserAgent(c.userAgent)) + if err != nil { + return nil, c.lookupError("list", r.String(), repo, err) + } + return tags, nil +} + +// lookupError turns a failed registry request into an error that is safe to +// show: a registry answer is described by its status and codes, anything +// else by a fixed text. The full error goes to the log. verb and target +// start the message; ref is what the log line names. +func (c *Client) lookupError(verb, target, ref string, err error) error { + var terr *transport.Error + if errors.As(err, &terr) { + return fmt.Errorf("%s %s: %s", verb, target, describe(terr)) + } + c.log.Warn("registry_request_failed", "ref", ref, "error", err) + return fmt.Errorf("%s %s: %s", verb, target, classify(err)) +} + // classify returns a fixed text for an error that is not a registry answer. // ggcr copies registry-controlled text (token responses, authentication // challenges, redirect URLs, certificate names) into such errors, so none of diff --git a/internal/master/registry/registry_test.go b/internal/master/registry/registry_test.go index e9565c0..e2f4ca1 100644 --- a/internal/master/registry/registry_test.go +++ b/internal/master/registry/registry_test.go @@ -134,6 +134,8 @@ func TestDigestIgnoresDigestSuffix(t *testing.T) { assert.Equal(t, want, got) } +// Digest asks the manifest endpoint with HEAD only; the one other request is +// the GET /v2/ ping that every client sends first. func TestDigestUsesHeadOnly(t *testing.T) { dockerConfig(t, "") reg := newTestRegistry(t, "", "") @@ -236,7 +238,7 @@ func TestCredsStoreWithoutHelperFailsTheLookup(t *testing.T) { } // A registry that leaves out Docker-Content-Digest must not make the client -// retry with a manifest download, which Docker Hub counts as a pull. +// retry with a manifest GET, which Docker Hub counts as a pull. func TestMissingDigestHeaderDoesNotFallBackToDownload(t *testing.T) { dockerConfig(t, "") var mu sync.Mutex @@ -334,3 +336,101 @@ func TestClassify(t *testing.T) { }) } } + +// pushTags stores a random index under each tag of repo. +func (tr *testRegistry) pushTags(t *testing.T, repo string, tags ...string) { + t.Helper() + for _, tag := range tags { + tr.pushIndex(t, repo, tag) + } +} + +// Digest and Tags together never GET a manifest: every manifest request is a +// HEAD, and the only other requests are the /v2/ ping and tags/list. +func TestNoManifestGet(t *testing.T) { + dockerConfig(t, "") + reg := newTestRegistry(t, "", "") + reg.pushTags(t, "team/app", "1.2", "1.3") + reg.reset() + + _, err := newClient().Digest(context.Background(), reg.host+"/team/app:1.2") + require.NoError(t, err) + _, err = newClient().Tags(context.Background(), reg.host+"/team/app") + require.NoError(t, err) + + reg.mu.Lock() + defer reg.mu.Unlock() + for _, r := range reg.requests { + switch { + case strings.Contains(r.path, "/manifests/"): + assert.Equal(t, http.MethodHead, r.method, r.path) + case r.path == "/v2/", r.path == "/v2/team/app/tags/list": + assert.Equal(t, http.MethodGet, r.method, r.path) + default: + assert.Failf(t, "unexpected request", "%s %s", r.method, r.path) + } + } +} + +func TestTagsListsAllTags(t *testing.T) { + dockerConfig(t, "") + reg := newTestRegistry(t, "", "") + reg.pushTags(t, "team/app", "1.2", "1.3", "latest") + + got, err := newClient().Tags(context.Background(), reg.host+"/team/app") + + require.NoError(t, err) + assert.ElementsMatch(t, []string{"1.2", "1.3", "latest"}, got) +} + +// ggcr's in-memory registry sends no Link header, so pagination is tested +// against a handler that does. +func TestTagsFollowsLinkPagination(t *testing.T) { + dockerConfig(t, "") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/v2/": + case r.URL.Path == "/v2/a/b/tags/list" && r.URL.Query().Get("last") == "": + w.Header().Set("Link", `; rel="next"`) + _, _ = w.Write([]byte(`{"name":"a/b","tags":["1.2","1.3"]}`)) + case r.URL.Path == "/v2/a/b/tags/list": + _, _ = w.Write([]byte(`{"name":"a/b","tags":["1.4"]}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + got, err := newClient().Tags(context.Background(), strings.TrimPrefix(srv.URL, "http://")+"/a/b") + + require.NoError(t, err) + assert.Equal(t, []string{"1.2", "1.3", "1.4"}, got) +} + +func TestTagsUsesDockerConfigCredentials(t *testing.T) { + reg := newTestRegistry(t, "bot", "s3cret") + reg.pushIndex(t, "team/app", "1.2", remote.WithAuth(&authn.Basic{Username: "bot", Password: "s3cret"})) + auth := base64.StdEncoding.EncodeToString([]byte("bot:s3cret")) + dockerConfig(t, `{"auths":{"`+reg.host+`":{"auth":"`+auth+`"}}}`) + + got, err := newClient().Tags(context.Background(), reg.host+"/team/app") + + require.NoError(t, err) + assert.Equal(t, []string{"1.2"}, got) +} + +func TestTagsErrorBodyDoesNotLeak(t *testing.T) { + dockerConfig(t, "") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"errors":[{"code":"NAME_UNKNOWN","message":"internal-secret-body"}]}`)) + })) + t.Cleanup(srv.Close) + host := strings.TrimPrefix(srv.URL, "http://") + + _, err := newClient().Tags(context.Background(), host+"/a/b") + + require.Error(t, err) + assert.NotContains(t, err.Error(), "internal-secret-body") + assert.ErrorContains(t, err, "list "+host+"/a/b: registry answered 400 Bad Request") +} diff --git a/internal/master/store/drift_merge_integration_test.go b/internal/master/store/drift_merge_integration_test.go index 1433920..e315f3d 100644 --- a/internal/master/store/drift_merge_integration_test.go +++ b/internal/master/store/drift_merge_integration_test.go @@ -275,6 +275,7 @@ func TestMergeService_MovesHistoryModeAndDesiredToTheNewLocation(t *testing.T) { require.NoError(t, s.SetLastTagDigest(ctx, f.to, "sha256:fromto")) require.NoError(t, s.SetAvailable(ctx, f.to, model.Available{Ref: sluskDrift + ":3", Digest: digNew, Source: model.SourcePoll, At: t0})) require.NoError(t, s.SetPollError(ctx, []int64{f.to}, "registry down")) + require.NoError(t, s.SetNewer(ctx, f.to, sluskDrift+":3.1", t0)) // A job on the old row is finished, and awaiting_inventory is set on both. before, err := s.GetService(ctx, f.from) require.NoError(t, err) @@ -297,6 +298,9 @@ func TestMergeService_MovesHistoryModeAndDesiredToTheNewLocation(t *testing.T) { assert.Equal(t, sluskDrift+":3", got.AvailableRef) assert.Equal(t, digNew, got.AvailableDigest) assert.Equal(t, "registry down", got.LastPollError) + assert.Equal(t, sluskDrift+":3.1", got.NewerRef, "newer tag from the duplicate") + require.NotNil(t, got.NewerAt) + assert.True(t, t0.Equal(*got.NewerAt)) assert.Equal(t, before.LastTagDigest, got.LastTagDigest, "an existing last_tag_digest wins") _, err = s.GetService(ctx, f.to) diff --git a/internal/master/store/merge.go b/internal/master/store/merge.go index 0a4f0a7..6a23570 100644 --- a/internal/master/store/merge.go +++ b/internal/master/store/merge.go @@ -106,9 +106,9 @@ type mergeRow struct { // MergeService moves the live service toID onto the removed service fromID: // the old row keeps its id, first_seen_at, update mode and desired ref, takes -// the location and inventory of the live row (and its available suggestion), -// becomes live, and the live row is deleted. The jobs of the live row move to -// the old one. This is the only place a services row is deleted. +// the location and inventory of the live row (and its available and newer-tag +// suggestions), becomes live, and the live row is deleted. The jobs of the live +// row move to the old one. This is the only place a services row is deleted. // // It returns *model.MergeRefusedError when the pair cannot be merged, // model.ErrJobActive when either row has a queued or running job, @@ -195,12 +195,12 @@ func (s *Store) MergeService(ctx context.Context, fromID, toID int64) error { SELECT agent_id, location_key, compose_project, compose_service, working_dir, config_files, built, swarm_cluster_id, swarm_service, container_name, image_repo, actual_ref, actual_digest, status, self, updated_at, available_ref, available_digest, available_source, available_at, last_poll_error, - last_tag_digest + last_tag_digest, newer_ref, newer_at FROM services WHERE id = $1`, toID).Scan( &t.AgentID, &t.LocationKey, &t.ComposeProject, &t.ComposeService, &t.WorkingDir, &t.ConfigFiles, &t.Built, &t.SwarmClusterID, &t.SwarmService, &t.ContainerName, &t.ImageRepo, &t.ActualRef, &t.ActualDigest, &t.Status, &t.Self, &t.UpdatedAt, &t.AvailableRef, &t.AvailableDigest, &t.AvailableSource, &t.AvailableAt, &t.LastPollError, - &lastTagDigest) + &lastTagDigest, &t.NewerRef, &t.NewerAt) if err != nil { return fmt.Errorf("merge: read new service: %w", err) } @@ -215,13 +215,14 @@ func (s *Store) MergeService(ctx context.Context, fromID, toID int64) error { available_ref = $18, available_digest = $19, available_source = $20, available_at = $21, last_poll_error = $22, last_tag_digest = CASE WHEN last_tag_digest <> '' THEN last_tag_digest ELSE $23 END, + newer_ref = $24, newer_at = $25, removed_at = NULL, awaiting_inventory = false WHERE id = $1`, fromID, t.AgentID, t.LocationKey, t.ComposeProject, t.ComposeService, t.WorkingDir, t.ConfigFiles, t.Built, t.SwarmClusterID, t.SwarmService, t.ContainerName, t.ImageRepo, t.ActualRef, t.ActualDigest, t.Status, t.Self, t.UpdatedAt, t.AvailableRef, t.AvailableDigest, t.AvailableSource, t.AvailableAt, - t.LastPollError, lastTagDigest); err != nil { + t.LastPollError, lastTagDigest, t.NewerRef, t.NewerAt); err != nil { return fmt.Errorf("merge: update old service: %w", err) } if err := tx.Commit(ctx); err != nil { diff --git a/internal/master/store/migrations/0007_newer_tag.sql b/internal/master/store/migrations/0007_newer_tag.sql new file mode 100644 index 0000000..374e154 --- /dev/null +++ b/internal/master/store/migrations/0007_newer_tag.sql @@ -0,0 +1,9 @@ +-- Newer version tag of the followed tag, found in the registry's tag list. +-- A suggestion only: nothing deploys it automatically. +ALTER TABLE services ADD COLUMN newer_ref text NOT NULL DEFAULT ''; +ALTER TABLE services ADD COLUMN newer_at timestamptz; + +-- A suggested tag is recorded in image_versions with source 'tags'. +ALTER TABLE image_versions DROP CONSTRAINT image_versions_source_check; +ALTER TABLE image_versions ADD CONSTRAINT image_versions_source_check + CHECK (source IN ('webhook', 'poll', 'running', 'tags')); diff --git a/internal/master/store/services.go b/internal/master/store/services.go index 58358d4..7d2a378 100644 --- a/internal/master/store/services.go +++ b/internal/master/store/services.go @@ -175,7 +175,7 @@ const serviceColumns = `s.id, s.kind, s.location_key, s.agent_id, COALESCE(a.nam s.compose_project, s.compose_service, s.working_dir, s.config_files, s.built, s.swarm_cluster_id, s.swarm_service, s.container_name, s.image_repo, s.desired_ref, s.actual_ref, s.actual_digest, s.status, s.self, s.update_mode, s.available_ref, s.available_digest, s.available_source, s.available_at, - s.last_poll_error, s.last_tag_digest, s.awaiting_inventory, s.first_seen_at, s.updated_at, s.removed_at` + s.last_poll_error, s.newer_ref, s.newer_at, s.last_tag_digest, s.awaiting_inventory, s.first_seen_at, s.updated_at, s.removed_at` func scanService(row pgx.Row) (model.Service, error) { var m model.Service @@ -184,7 +184,7 @@ func scanService(row pgx.Row) (model.Service, error) { &m.ComposeProject, &m.ComposeService, &m.WorkingDir, &m.ConfigFiles, &m.Built, &m.SwarmClusterID, &m.SwarmService, &m.ContainerName, &m.ImageRepo, &m.DesiredRef, &m.ActualRef, &m.ActualDigest, &m.Status, &m.Self, &m.UpdateMode, &m.AvailableRef, &m.AvailableDigest, &m.AvailableSource, &m.AvailableAt, - &m.LastPollError, &m.LastTagDigest, &m.AwaitingInventory, &m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt) + &m.LastPollError, &m.NewerRef, &m.NewerAt, &m.LastTagDigest, &m.AwaitingInventory, &m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt) m.Kind = protocol.Kind(kind) return m, err } @@ -246,6 +246,33 @@ func (s *Store) SetAvailable(ctx context.Context, id int64, a model.Available) e return nil } +// SetNewer sets the service's newer-tag suggestion (model.Service.NewerRef); +// ref "" clears it. A non-empty ref is also recorded in image_versions with +// source "tags", in the same transaction, unless that (repo, tag) is already +// there. The digest is unknown, so the row has none. +func (s *Store) SetNewer(ctx context.Context, id int64, ref string, at time.Time) error { + tx, err := s.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("set newer: begin: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + if _, err := tx.Exec(ctx, ` + UPDATE services SET newer_ref = $2, newer_at = CASE WHEN $2 = '' THEN NULL ELSE $3::timestamptz END + WHERE id = $1`, id, ref, at); err != nil { + return fmt.Errorf("set newer: %w", err) + } + if ref != "" { + p := imageref.Parse(ref) + if err := insertVersion(ctx, tx, p.Repo, p.Tag, "", model.SourceTags, at); err != nil { + return err + } + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("set newer: commit: %w", err) + } + return nil +} + // SetPollError sets last_poll_error on the given services; msg "" clears it. func (s *Store) SetPollError(ctx context.Context, ids []int64, msg string) error { if len(ids) == 0 { @@ -313,6 +340,23 @@ func (s *Store) GetImageVersion(ctx context.Context, id int64) (model.ImageVersi return v, nil } +// GetTagVersion returns the version row of repo and tag with an empty digest, which +// SetNewer and the webhook create. Returns model.ErrNotFound if there is none. +func (s *Store) GetTagVersion(ctx context.Context, repo, tag string) (model.ImageVersion, error) { + var v model.ImageVersion + err := s.pool.QueryRow(ctx, ` + SELECT id, repo, tag, digest, source, first_seen_at FROM image_versions + WHERE repo = $1 AND tag = $2 AND digest = ''`, repo, tag, + ).Scan(&v.ID, &v.Repo, &v.Tag, &v.Digest, &v.Source, &v.FirstSeenAt) + if errors.Is(err, pgx.ErrNoRows) { + return model.ImageVersion{}, model.ErrNotFound + } + if err != nil { + return model.ImageVersion{}, fmt.Errorf("get tag version: %w", err) + } + return v, nil +} + // SetUpdateMode sets the service's update mode (model.ModeManual or // model.ModeAuto). Returns model.ErrNotFound if the service does not exist // or is marked removed. diff --git a/internal/master/store/versions_integration_test.go b/internal/master/store/versions_integration_test.go index 1e108f7..5fc5c64 100644 --- a/internal/master/store/versions_integration_test.go +++ b/internal/master/store/versions_integration_test.go @@ -214,3 +214,79 @@ func TestSetLastTagDigest(t *testing.T) { require.Len(t, list, 1) assert.Equal(t, "sha256:n1", list[0].LastTagDigest) } + +func TestSetNewer_SetsAndClears(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + _, svc := jobFixture(t, s, "a") + + require.NoError(t, s.SetNewer(ctx, svc, sluskRepo+":1.5.0", t0)) + + got, err := s.GetService(ctx, svc) + require.NoError(t, err) + assert.Equal(t, sluskRepo+":1.5.0", got.NewerRef) + require.NotNil(t, got.NewerAt) + assert.True(t, t0.Equal(*got.NewerAt)) + v, err := s.GetTagVersion(ctx, sluskRepo, "1.5.0") + require.NoError(t, err) + assert.Equal(t, model.SourceTags, v.Source) + assert.Empty(t, v.Digest) + assert.True(t, t0.Equal(v.FirstSeenAt)) + + require.NoError(t, s.SetNewer(ctx, svc, "", t0.Add(time.Hour))) + + got, err = s.GetService(ctx, svc) + require.NoError(t, err) + assert.Empty(t, got.NewerRef) + assert.Nil(t, got.NewerAt) + _, err = s.GetTagVersion(ctx, sluskRepo, "1.5.0") + assert.NoError(t, err, "clearing the suggestion keeps the version") +} + +func TestSetNewer_KeepsEarlierVersionSource(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + _, svc := jobFixture(t, s, "a") + require.NoError(t, s.SetAvailable(ctx, svc, model.Available{Ref: sluskRepo + ":1.5.0", Source: model.SourceWebhook, At: t0})) + + require.NoError(t, s.SetNewer(ctx, svc, sluskRepo+":1.5.0", t0.Add(time.Hour))) + + v, err := s.GetTagVersion(ctx, sluskRepo, "1.5.0") + require.NoError(t, err) + assert.Equal(t, model.SourceWebhook, v.Source) + assert.True(t, t0.Equal(v.FirstSeenAt)) +} + +func TestRecentVersions_TagsRowHiddenOnceDigestKnown(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + _, svc := jobFixture(t, s, "a") + require.NoError(t, s.SetNewer(ctx, svc, sluskRepo+":1.5.0", t0.Add(time.Minute))) + require.Len(t, versionsOf(t, s, sluskRepo), 2, "running 1.4.2 and the tags row") + + require.NoError(t, s.SetAvailable(ctx, svc, model.Available{Ref: sluskRepo + ":1.5.0", Digest: "sha256:n1", Source: model.SourcePoll, At: t0.Add(2 * time.Minute)})) + + var refs []string + for _, v := range versionsOf(t, s, sluskRepo) { + refs = append(refs, v.Ref()) + } + assert.Equal(t, []string{sluskRepo + ":1.5.0@sha256:n1", sluskRepo + ":1.4.2@" + digD1}, refs) +} + +func TestGetTagVersion(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + _, svc := jobFixture(t, s, "a") + require.NoError(t, s.SetNewer(ctx, svc, sluskRepo+":1.5.0", t0)) + + got, err := s.GetTagVersion(ctx, sluskRepo, "1.5.0") + require.NoError(t, err) + assert.Equal(t, sluskRepo+":1.5.0", got.Ref()) + + _, err = s.GetTagVersion(ctx, sluskRepo, "9.9.9") + assert.ErrorIs(t, err, model.ErrNotFound) + _, err = s.GetTagVersion(ctx, "other/repo", "1.5.0") + assert.ErrorIs(t, err, model.ErrNotFound) + _, err = s.GetTagVersion(ctx, sluskRepo, "1.4.2") + assert.ErrorIs(t, err, model.ErrNotFound, "a row with a digest is not a tag version") +} diff --git a/internal/master/tagver/tagver.go b/internal/master/tagver/tagver.go new file mode 100644 index 0000000..c5aa15d --- /dev/null +++ b/internal/master/tagver/tagver.go @@ -0,0 +1,101 @@ +// Package tagver decides which tag of a repository is a newer version of the +// tag a service follows, using only the shape of the tag names. +package tagver + +import ( + "regexp" + "strconv" + "strings" +) + +// tagPattern splits a tag into a prefix without digits, a dotted numeric +// part and whatever follows it. "v1.2.3-alpine" gives "v", "1.2.3", "-alpine". +var tagPattern = regexp.MustCompile(`^([^0-9]*)([0-9]+(?:\.[0-9]+)*)(.*)$`) + +// cosignPattern matches the tags cosign pushes next to an image to store its +// signature, attestation or SBOM. They look like versions to a naive check. +// The pattern is taken from Diun (crazy-max/diun, MIT, pkg/registry/tags.go). +var cosignPattern = regexp.MustCompile(`^sha256-[0-9a-f]{64}(\.att|\.sbom|\.sig)?$`) + +// parsed is a tag split into its parts. +type parsed struct { + prefix, suffix string + nums []int +} + +// parse splits tag. It returns false for a cosign tag, a tag without digits +// and a numeric component that does not fit an int. +func parse(tag string) (parsed, bool) { + if cosignPattern.MatchString(tag) { + return parsed{}, false + } + m := tagPattern.FindStringSubmatch(tag) + if m == nil { + return parsed{}, false + } + var nums []int + for _, part := range strings.Split(m[2], ".") { + n, err := strconv.Atoi(part) + if err != nil { + return parsed{}, false + } + nums = append(nums, n) + } + return parsed{prefix: m[1], suffix: m[3], nums: nums}, true +} + +// Comparable reports whether tag can ever get a newer tag: it splits into +// prefix, numeric part and suffix, has at least two numeric components, and +// is not a cosign artifact tag. A tag with one component ("7", "pg16", +// "8-bookworm") is a moving tag, not a version. +func Comparable(tag string) bool { + p, ok := parse(tag) + return ok && len(p.nums) >= 2 +} + +// Newer returns the highest candidate that is newer than current, and false +// when there is none. A candidate qualifies when it has the same prefix and +// suffix and the same number of components as current, the same first +// component, and a greater numeric part. Of two candidates with the same +// numeric value the lexically smaller string wins, so the result does not +// depend on the order of candidates. A tag such as "name2-v1.2.3", where a +// digit comes before the version, is not comparable. +func Newer(current string, candidates []string) (string, bool) { + cur, ok := parse(current) + if !ok || len(cur.nums) < 2 { + return "", false + } + best, found := "", false + var bestNums []int + for _, c := range candidates { + p, ok := parse(c) + if !ok || p.prefix != cur.prefix || p.suffix != cur.suffix || + len(p.nums) != len(cur.nums) || p.nums[0] != cur.nums[0] { + continue + } + if compare(p.nums, cur.nums) <= 0 { + continue + } + if !found { + best, bestNums, found = c, p.nums, true + continue + } + if cmp := compare(p.nums, bestNums); cmp > 0 || (cmp == 0 && c < best) { + best, bestNums = c, p.nums + } + } + return best, found +} + +// compare compares two numeric parts of equal length. +func compare(a, b []int) int { + for i := range a { + switch { + case a[i] < b[i]: + return -1 + case a[i] > b[i]: + return 1 + } + } + return 0 +} diff --git a/internal/master/tagver/tagver_test.go b/internal/master/tagver/tagver_test.go new file mode 100644 index 0000000..e207064 --- /dev/null +++ b/internal/master/tagver/tagver_test.go @@ -0,0 +1,71 @@ +package tagver + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestComparable(t *testing.T) { + sig := "sha256-" + strings.Repeat("a", 64) + ".sig" + tests := []struct { + tag string + want bool + }{ + {"latest", false}, + {"release", false}, + {"stable", false}, + {"edge", false}, + {"alpine", false}, + {"7", false}, + {"7-alpine", false}, + {"8-bookworm", false}, + {"pg16", false}, + {"14-vectorchord0.4.3-pgvectors0.2.0", false}, + {"name2-v1.2.3", false}, + {sig, false}, + {"99999999999999999999.1", false}, + {"1.2", true}, + {"1.2.3", true}, + {"v1.2.3", true}, + {"11.0-latest", true}, + } + for _, tt := range tests { + t.Run(tt.tag, func(t *testing.T) { + assert.Equal(t, tt.want, Comparable(tt.tag)) + }) + } +} + +func TestNewer(t *testing.T) { + tests := []struct { + name string + current string + candidates []string + want string + ok bool + }{ + {"highest within major", "1.2.3", []string{"1.2.4", "1.3.0", "2.0.0", "1.2.4-rc1", "1.3", "v1.2.4"}, "1.3.0", true}, + {"v prefix kept", "v1.2.3", []string{"1.2.4", "v1.2.4"}, "v1.2.4", true}, + {"name prefix kept", "name-v1.2.3", []string{"v1.2.4", "name-v1.2.4"}, "name-v1.2.4", true}, + {"suffix kept", "11.0-latest", []string{"11.1", "11.1-latest"}, "11.1-latest", true}, + {"two components", "7.0", []string{"8.0", "7.0.1", "7.2", "7.1"}, "7.2", true}, + {"equal is not newer", "1.2.3", []string{"1.2.3"}, "", false}, + {"older is not newer", "1.2.3", []string{"1.2.2", "1.1.9"}, "", false}, + {"numeric not lexical", "1.9.0", []string{"1.10.0"}, "1.10.0", true}, + {"empty candidates", "1.2.3", nil, "", false}, + {"moving tag", "latest", []string{"1.2.3"}, "", false}, + {"single component", "7", []string{"8"}, "", false}, + {"digit before version", "name2-v1.2.3", []string{"name2-v1.2.4"}, "", false}, + {"cosign tag ignored", "1.2.3", []string{"sha256-" + strings.Repeat("0", 64) + ".sig"}, "", false}, + {"equal value picks smaller string", "1.2.3", []string{"1.02.4", "1.2.4"}, "1.02.4", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := Newer(tt.current, tt.candidates) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/internal/master/web/server.go b/internal/master/web/server.go index b41cfc6..fa17540 100644 --- a/internal/master/web/server.go +++ b/internal/master/web/server.go @@ -38,6 +38,9 @@ var agentNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) type Store interface { ListServices(ctx context.Context) ([]model.Service, error) GetService(ctx context.Context, id int64) (model.Service, error) + // GetTagVersion returns the version of repo with this tag and no digest; + // model.ErrNotFound if there is none. + GetTagVersion(ctx context.Context, repo, tag string) (model.ImageVersion, error) // RecentVersions returns up to limit versions of repo the master has // seen, most recently first seen first. RecentVersions(ctx context.Context, repo string, limit int) ([]model.ImageVersion, error) @@ -202,6 +205,7 @@ type pageData struct { Reason string // why Service cannot be updated; "" when it can AutoBlocked string // why Service cannot be set to auto; "" when it can Versions []versionRow + NewerTag string // Service's newer tag while it still holds; "" otherwise Polling string // what the service page says about registry polling PollFailed bool // Polling is the last registry error // RestoreBlocked is why a drifted Service cannot be restored; "" when it @@ -388,6 +392,17 @@ func (s *Server) serviceDetail(r *http.Request, id int64) (pageData, error) { if err != nil { return data, err } + data.NewerTag = newerTag(svc) + if data.NewerTag != "" && !slices.ContainsFunc(versions, func(v model.ImageVersion) bool { return v.Tag == data.NewerTag && v.Digest == "" }) { + // The suggested tag can be older than the listed versions; it must + // still be pickable. + v, err := s.store.GetTagVersion(r.Context(), svc.ImageRepo, data.NewerTag) + if err == nil { + versions = append(versions, v) + } else if !errors.Is(err, model.ErrNotFound) { + return data, err + } + } data.Versions = versionRows(versions, svc.ActualDigest) data.Polling, data.PollFailed = pollingStatus(svc, s.cfg.PollInterval) data.Jobs, err = s.store.ListJobs(r.Context(), model.JobFilter{ServiceID: id, Limit: serviceJobsLimit}) diff --git a/internal/master/web/server_test.go b/internal/master/web/server_test.go index 11b05f1..24d87b7 100644 --- a/internal/master/web/server_test.go +++ b/internal/master/web/server_test.go @@ -787,3 +787,34 @@ func TestServicePage_AutoBlockedDisablesToggle(t *testing.T) { _, body := get(t, srv, "/services/1") assert.Contains(t, body, "Auto unavailable: "+model.ReasonContainer+".") } + +// GetTagVersion returns the digest-less version of repo with this tag. +func (f *fakeStore) GetTagVersion(_ context.Context, repo, tag string) (model.ImageVersion, error) { + for _, v := range f.versions { + if v.Repo == repo && v.Tag == tag && v.Digest == "" { + return v, nil + } + } + return model.ImageVersion{}, model.ErrNotFound +} + +func TestOverview_ShowsNewerTagChipBesideUpdateChip(t *testing.T) { + srv, store, _, _, _ := newTestEnv(t) + store.services[0].AvailableRef = "gitea.shcizo.se/shcizo/slusk:1.4.2" + store.services[0].NewerRef = "gitea.shcizo.se/shcizo/slusk:1.5.0" + + for _, path := range []string{"/", "/fragments/services"} { + _, body := get(t, srv, path) + assert.Contains(t, body, `class="tag f-newer" title="gitea.shcizo.se/shcizo/slusk:1.5.0">Newer tag1.5.0`, path) + assert.Contains(t, body, `class="tag f-up"`, path) + } +} + +func TestOverview_HidesStaleNewerTag(t *testing.T) { + srv, store, _, _, _ := newTestEnv(t) + store.services[0].NewerRef = "gitea.shcizo.se/shcizo/slusk:1.4.1" // not newer than 1.4.2 + + _, body := get(t, srv, "/") + + assert.NotContains(t, body, "f-newer") +} diff --git a/internal/master/web/static/app.css b/internal/master/web/static/app.css index b877aae..fcccf00 100644 --- a/internal/master/web/static/app.css +++ b/internal/master/web/static/app.css @@ -106,8 +106,9 @@ summary.btn::-webkit-details-marker{display:none} .tag em{font:500 11.5px var(--mono);font-style:normal;margin-left:6px} .tag em:empty{display:none} .f-up{color:var(--orange);background:var(--orangebg)} +.f-newer em{color:var(--orange)} .f-err{color:var(--red);background:var(--redbg)} -.f-notag{color:var(--mut);box-shadow:inset 0 0 0 1px var(--line2)} +.f-newer,.f-notag{color:var(--mut);box-shadow:inset 0 0 0 1px var(--line2)} .f-out{color:var(--dim);box-shadow:inset 0 0 0 1px var(--line)} .mode{display:flex;flex-direction:column;align-items:flex-start;gap:2px} .mode form{margin:0;display:flex;flex-direction:column;align-items:flex-start;gap:2px} diff --git a/internal/master/web/templates/overview.html b/internal/master/web/templates/overview.html index fdefe5c..049c749 100644 --- a/internal/master/web/templates/overview.html +++ b/internal/master/web/templates/overview.html @@ -77,6 +77,7 @@
{{if .AvailableRef}}Update available{{.AvailableTag}}{{end}} + {{if .NewerTag}}Newer tag{{.NewerTag}}{{end}} {{if followsNoTag .Service}}Follows no tag{{end}} {{if .LastPollError}}Poll failed{{end}} {{if .Self}}Managed outside{{end}} diff --git a/internal/master/web/templates/service.html b/internal/master/web/templates/service.html index 0e46159..43681c1 100644 --- a/internal/master/web/templates/service.html +++ b/internal/master/web/templates/service.html @@ -9,6 +9,7 @@ Status{{.Service.Status}} {{if .Service.DesiredRef}}Desired image{{shortRef .Service.DesiredRef}}{{end}} {{if .Service.AvailableRef}}Available{{.Service.AvailableRef}} via {{.Service.AvailableSource}}{{with .Service.AvailableAt}}, {{.Format "2006-01-02 15:04"}}{{end}}{{end}} + {{if .NewerTag}}Newer tag{{.Service.NewerRef}} from the registry's tag list{{with .Service.NewerAt}}, {{.Format "2006-01-02 15:04"}}{{end}}; deploy it from Versions{{end}} Mode{{.Service.UpdateMode}} Polling{{.Polling}} {{if .ConfigFiles}}Config files{{range .ConfigFiles}}{{.}}
{{end}}{{end}} diff --git a/internal/master/web/versions_test.go b/internal/master/web/versions_test.go index 0b29796..525f24e 100644 --- a/internal/master/web/versions_test.go +++ b/internal/master/web/versions_test.go @@ -219,3 +219,53 @@ func TestOverview_ShowsFollowsNoTagAndPollFailedBadges(t *testing.T) { assert.Equal(t, 1, strings.Count(body, "Poll failed"), path) } } + +func TestServicePage_VersionPickerAppendsNewerTagBeyondLimit(t *testing.T) { + srv, store, _, _, _ := newTestEnv(t) + store.services[0].ImageRepo = sluskRepo + store.services[0].NewerRef = sluskRepo + ":1.5.0" + for i := int64(1); i <= 12; i++ { + store.versions = append(store.versions, model.ImageVersion{ID: i, Repo: sluskRepo, Tag: "t", Digest: "sha256:d", Source: "poll"}) + } + store.versions = append(store.versions, model.ImageVersion{ID: 99, Repo: sluskRepo, Tag: "1.5.0", Source: "tags"}) + + _, body := get(t, srv, "/services/1") + + assert.Equal(t, 11, strings.Count(body, `name="version"`), "the ten recent versions and the newer tag") + assert.Contains(t, body, ``) + assert.Contains(t, body, "Newer tag") +} + +func TestServicePage_NewerTagAlreadyListedIsNotAppendedTwice(t *testing.T) { + srv, store, _, _, _ := newTestEnv(t) + store.services[0].ImageRepo = sluskRepo + store.services[0].NewerRef = sluskRepo + ":1.5.0" + store.versions = []model.ImageVersion{{ID: 31, Repo: sluskRepo, Tag: "1.5.0", Source: "tags"}} + + _, body := get(t, srv, "/services/1") + + assert.Equal(t, 1, strings.Count(body, `name="version"`)) +} + +func TestServicePage_PinnedRowOfTheNewerTagDoesNotHideThePlainRow(t *testing.T) { + srv, store, _, _, _ := newTestEnv(t) + store.services[0].ImageRepo = sluskRepo + store.services[0].NewerRef = sluskRepo + ":1.5.0" + store.versions = []model.ImageVersion{ + {ID: 31, Repo: sluskRepo, Tag: "1.5.0", Digest: "sha256:d", Source: "poll"}, + {ID: 32, Repo: sluskRepo, Tag: "1.5.0", Source: "tags"}, + } + + _, body := get(t, srv, "/services/1") + + assert.Equal(t, 2, strings.Count(body, `name="version"`)) + assert.Contains(t, body, ``) +} + +func TestServicePage_NoNewerTagRowWithoutSuggestion(t *testing.T) { + srv, _, _, _, _ := newTestEnv(t) + + _, body := get(t, srv, "/services/1") + + assert.NotContains(t, body, "Newer tag") +} diff --git a/internal/master/web/views.go b/internal/master/web/views.go index 95347c4..2639f3d 100644 --- a/internal/master/web/views.go +++ b/internal/master/web/views.go @@ -12,6 +12,7 @@ import ( "github.com/shcizo/package-updater/internal/master/jobs" "github.com/shcizo/package-updater/internal/master/model" "github.com/shcizo/package-updater/internal/master/poller" + "github.com/shcizo/package-updater/internal/master/tagver" "github.com/shcizo/package-updater/internal/master/web/icons" "github.com/shcizo/package-updater/internal/protocol" ) @@ -45,6 +46,7 @@ type serviceRow struct { Tag string // ":tag", or "" Digest string // "@sha256:<12 hex>…" when the image has no tag, else "" AvailableTag string // the tag of AvailableRef, or "" + NewerTag string // the tag of NewerRef while it is still newer than the followed tag, or "" Icon string // Simple Icons slug; "" when there is no icon Letter string // shown instead of an icon LetterClass string // lc-0 .. lc-3 @@ -66,6 +68,7 @@ func newServiceRow(s model.Service, autoBlocked string) serviceRow { } r.Repo, r.Tag, r.Digest = imageParts(s.ActualRef) r.AvailableTag = imageref.Parse(s.AvailableRef).Tag + r.NewerTag = newerTag(s) r.Icon = icons.ForImage(s.ActualRef) r.Letter = "?" if first, _ := utf8.DecodeRuneInString(r.Name); first != utf8.RuneError { @@ -243,6 +246,26 @@ func followsNoTag(svc model.Service) bool { return jobs.FollowedRef(svc.ActualRef) == "" } +// newerTag returns the tag of the service's newer-tag suggestion, or "" when +// there is none or it no longer holds. The poller writes the suggestion once +// per cycle, so after the service moved to another tag the stored value can +// be stale until the next cycle; checking again here hides it meanwhile. A +// suggestion from another repo than the service follows now is hidden too. +func newerTag(s model.Service) string { + if s.RemovedAt != nil || s.NewerRef == "" { + return "" + } + newer := imageref.Parse(s.NewerRef) + repo, followed := poller.Followed(s) + if newer.Repo != repo { + return "" + } + if _, ok := tagver.Newer(followed, []string{newer.Tag}); !ok { + return "" + } + return newer.Tag +} + // versionRow is an image version as shown on the service page. type versionRow struct { model.ImageVersion diff --git a/internal/master/web/views_test.go b/internal/master/web/views_test.go index f68a6b5..ce515b2 100644 --- a/internal/master/web/views_test.go +++ b/internal/master/web/views_test.go @@ -211,3 +211,25 @@ func TestGroupServices(t *testing.T) { assert.Equal(t, "because", groups[0].Services[0].AutoBlocked) }) } + +func TestNewerTag(t *testing.T) { + removed := time.Now() + tests := []struct { + name string + svc model.Service + want string + }{ + {"newer", model.Service{ActualRef: "gitea/gitea:1.27.3", NewerRef: "gitea/gitea:1.28.0"}, "1.28.0"}, + {"bare image follows latest", model.Service{ActualRef: "gitea/gitea", NewerRef: "gitea/gitea:1.28.0"}, ""}, + {"stale after the tag changed", model.Service{ActualRef: "gitea/gitea:1.28.0", NewerRef: "gitea/gitea:1.28.0"}, ""}, + {"stale after the tag moved past it", model.Service{ActualRef: "gitea/gitea:1.29.0", NewerRef: "gitea/gitea:1.28.0"}, ""}, + {"other repo", model.Service{ActualRef: "redis:7.2.4", NewerRef: "bitnami/redis:7.2.5"}, ""}, + {"removed", model.Service{ActualRef: "gitea/gitea:1.27.3", NewerRef: "gitea/gitea:1.28.0", RemovedAt: &removed}, ""}, + {"empty", model.Service{ActualRef: "gitea/gitea:1.27.3"}, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, newerTag(tt.svc)) + }) + } +}