feat(master): suggest newer version tags for third-party images (#11)

This commit is contained in:
2026-10-13 11:50:22 +02:00
parent ec28d1e027
commit c9cb1103ad
26 changed files with 1013 additions and 44 deletions
+11 -4
View File
@@ -36,8 +36,9 @@ DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/
- `internal/master/config` — env-var loading
- `internal/master/store` — Postgres access and migrations
- `internal/master/hub` — agent connections: auth, per-connection writer, routing, job reconcile on hello
- `internal/master/registry` — HEAD-only digest lookup against a registry (go-containerregistry, `DOCKER_CONFIG` credentials)
- `internal/master/poller` — every `POLL_INTERVAL`: one lookup per followed `repo:tag`, sets the available version, starts auto jobs
- `internal/master/registry` — HEAD-only digest lookup and tag listing against a registry (go-containerregistry, `DOCKER_CONFIG` credentials)
- `internal/master/tagver` — which tag is a newer version of a followed tag (pure functions)
- `internal/master/poller` — every `POLL_INTERVAL`: one lookup per followed `repo:tag`, sets the available version, starts auto jobs; one tag list per repo, sets the newer tag suggestion
- `internal/master/jobs` — decides whether a service may be updated and starts a job on the right agent
- `internal/master/webhook` — `POST /update`: records the available version, starts jobs for auto services, waits for them
- `internal/master/web/icons` — embedded Simple Icons path data; slug derived from the image ref; regenerate with `go generate ./internal/master/web/icons`
@@ -76,8 +77,14 @@ DATABASE_URL=postgres://... go test -tags integration ./internal/master/store/
## Gotchas
- **Registry access is HEAD only** (`internal/master/registry`). Docker Hub counts a manifest
GET as a pull; the client never falls back to GET, and a test asserts this.
- **Registry manifest access is HEAD only** (`internal/master/registry`). Docker Hub counts a
manifest GET as a pull; the client never GETs `/manifests/`, and a test asserts this. The
only GETs are the `/v2/` ping and `tags/list` (newer-tag detection, once per repo per cycle).
- **`services.newer_ref` is a suggestion only**: no auto jobs, not even for auto services. The
poller writes it only on change; a failed tag list keeps the old value and never sets
`last_poll_error`. The UI re-checks it at render time with `tagver.Newer`, so a stale value
is hidden until the next cycle. Tags with one numeric component or no digits (`7`, `pg16`,
`latest`) never get suggestions.
- **`image_versions` uses `''` instead of NULL** for a missing tag or digest, so
`UNIQUE (repo, tag, digest)` deduplicates. The webhook records `(repo, tag, '')`; that
row is hidden from `RecentVersions` when a row with the same tag and a digest was first
+9
View File
@@ -143,6 +143,15 @@ multi-arch image the digest is the index digest, the same one `docker inspect` s
it) and starts no job for that digest again, whether the job
failed or the user rolled back afterwards. If the service already had a job when the
digest was found, the next cycle tries again.
- **Newer tags.** For a service that follows a version-like tag (`1.27.3`, `v1.2.3`,
`11.0-latest`; at least two numeric components), the master also lists the repo's tags
once per cycle (`GET /v2/<repo>/tags/list`, which Docker Hub does not count as a pull) and
suggests the highest tag with the same prefix, suffix and number of components and the
same major number, for example `gitea:1.27.3` gives `1.28.0` but never `2.0.0`. The
overview shows a "Newer tag" badge and the service page lists the tag under Versions, from
where you deploy it. It is a suggestion only: nothing is deployed automatically, not even
for **auto** services. Tags such as `latest`, `7` or `pg16` get no suggestion. A failed tag
list keeps the old suggestion and is only logged (`poll_tags_failed`).
- **What is not polled.** Services that follow no tag (`repo@digest`), `build:` services, and
services without a registry digest (locally built images). The service page says which.
The updater's own services and plain containers are polled, so their badge works, but
@@ -255,6 +255,11 @@ them. An agent recreating itself would kill the job it is running.
HEAD-only and never downloads a manifest.
- Credentials: a Docker `config.json` (as written by `docker login`) mounted into the
master. No credential UI, no encryption in Postgres.
- Newer version tags (e.g. `gitea:1.27.3` → `1.27.4`): for a followed tag with at least
two numeric components the poller lists the repo's tags once per cycle
(`GET /v2/<repo>/tags/list`) and stores the highest tag with the same prefix, suffix,
component count and major number in `services.newer_ref`. A suggestion only, never
deployed automatically; deploying it is the existing version pick.
## Webhook
@@ -364,10 +369,6 @@ Each phase ends with something that works on its own.
## Out of scope (later specs)
- Detecting newer version *tags* for images we do not build (e.g. `gitea:1.27.3` →
`1.27.4`, `postgres:17.10`, `loki:3.4.2`). Requires listing tags and ordering them;
major upgrades such as Postgres 17 → 18 must never be automatic. Revisit after
checking the other two servers' stack files.
- Notifications.
- Export/import of a fleet configuration to restore a setup. The `services` table
already holds what an export needs.
+13
View File
@@ -104,6 +104,19 @@ func (m *memStore) SetLastTagDigest(_ context.Context, id int64, digest string)
return nil
}
func (m *memStore) SetNewer(_ context.Context, id int64, ref string, at time.Time) error {
m.mu.Lock()
defer m.mu.Unlock()
svc, ok := m.svcRows[id]
if !ok {
return model.ErrNotFound
}
svc.NewerRef = ref
svc.NewerAt = &at
m.svcRows[id] = svc
return nil
}
// seedService adds a service row that jobs can be triggered for.
func (m *memStore) seedService(svc model.Service) {
m.mu.Lock()
+3
View File
@@ -34,6 +34,9 @@ func (f *fixedRegistry) Digest(_ context.Context, ref string) (string, error) {
return f.digests[ref], nil
}
// Tags lists no tags, so no newer tag is ever suggested.
func (f *fixedRegistry) Tags(context.Context, string) ([]string, error) { return nil, nil }
func TestPickPinnedVersion(t *testing.T) {
env := newJobEnv(t)
env.store.seedVersion(model.ImageVersion{ID: 5, Repo: slusk, Tag: "1.3.0", Digest: "sha256:old130", Source: model.SourceRunning})
+1
View File
@@ -411,6 +411,7 @@ func TestTriggerVersion_Ref(t *testing.T) {
}{
{"tag and digest", svc, model.ImageVersion{ID: 10, Repo: repo, Tag: "1.3.0", Digest: "sha256:old"}, repo + ":1.3.0@sha256:old", protocol.MinPinnedRefVersion},
{"tag only", svc, model.ImageVersion{ID: 10, Repo: repo, Tag: "main"}, repo + ":main", protocol.MinJobVersion},
{"digest-less version from the tag list", svc, model.ImageVersion{ID: 10, Repo: repo, Tag: "1.28.0", Source: model.SourceTags}, repo + ":1.28.0", protocol.MinJobVersion},
{"digest only", svc, model.ImageVersion{ID: 10, Repo: repo, Digest: "sha256:old"}, repo + "@sha256:old", protocol.MinPinnedRefVersion},
{"service that follows no tag", noTag, model.ImageVersion{ID: 10, Repo: repo, Tag: "1.3.0", Digest: "sha256:old"}, repo + ":1.3.0@sha256:old", protocol.MinPinnedRefVersion},
}
+9 -1
View File
@@ -66,6 +66,11 @@ type Service struct {
AvailableSource string // SourceWebhook or SourcePoll; "" when AvailableRef is empty
AvailableAt *time.Time
LastPollError string // last registry error for the followed ref; "" when fine
// NewerRef is the newest repo:tag from the registry's tag list that
// tagver.Newer finds for the followed tag; "" if none. A suggestion only:
// it is never deployed automatically, not even for auto services.
NewerRef string
NewerAt *time.Time // when NewerRef was first suggested; nil when NewerRef is empty
// LastTagDigest is the newest digest of the followed tag that was deployed
// or started: set when the poller starts an auto job, when it sees the service
// already running that digest, and when any plain tag job succeeds (the
@@ -95,6 +100,9 @@ const (
// SourceRunning is used by image_versions only: an agent reported the
// version as running.
SourceRunning = "running"
// SourceTags is used by image_versions only: the tag appeared in the
// registry's tag list as a newer tag.
SourceTags = "tags"
)
// Available is a newer ref reported for a service.
@@ -113,7 +121,7 @@ type ImageVersion struct {
Repo string
Tag string
Digest string
Source string // SourceWebhook, SourcePoll or SourceRunning: the first source that saw it
Source string // SourceWebhook, SourcePoll, SourceRunning or SourceTags: the first source that saw it
FirstSeenAt time.Time
}
+142 -16
View File
@@ -1,6 +1,8 @@
// Package poller checks the registry for new digests of the tags the
// services follow. A new digest becomes the service's available version, and
// a service in auto mode gets an update job.
// a service in auto mode gets an update job. It also lists the tags of the
// repos whose followed tag looks like a version and suggests a newer one;
// that suggestion never starts a job.
package poller
import (
@@ -11,14 +13,19 @@ import (
"sync"
"time"
"github.com/shcizo/package-updater/internal/imageref"
"github.com/shcizo/package-updater/internal/master/events"
"github.com/shcizo/package-updater/internal/master/jobs"
"github.com/shcizo/package-updater/internal/master/model"
"github.com/shcizo/package-updater/internal/master/tagver"
)
// lookupTimeout bounds one registry lookup.
const lookupTimeout = 30 * time.Second
// listTimeout bounds one tag listing, which may take several requests.
const listTimeout = 60 * time.Second
// maxErrorBytes caps the registry error stored in services.last_poll_error.
const maxErrorBytes = 500
@@ -28,6 +35,7 @@ type Store interface {
GetService(ctx context.Context, id int64) (model.Service, error)
SetAvailable(ctx context.Context, id int64, a model.Available) error
SetLastTagDigest(ctx context.Context, id int64, digest string) error
SetNewer(ctx context.Context, id int64, ref string, at time.Time) error
SetPollError(ctx context.Context, ids []int64, msg string) error
}
@@ -36,10 +44,11 @@ type Jobs interface {
Start(ctx context.Context, svc model.Service, req jobs.StartRequest) (model.Job, error)
}
// Registry looks up the digest a "repo:tag" ref points at. *registry.Client
// satisfies it.
// Registry looks up the digest a "repo:tag" ref points at and lists the tags
// of a repo. *registry.Client satisfies it.
type Registry interface {
Digest(ctx context.Context, ref string) (string, error)
Tags(ctx context.Context, repo string) ([]string, error)
}
// Publisher receives change signals. *events.Broker satisfies it.
@@ -49,7 +58,7 @@ type Publisher interface{ Publish(t events.Topic) }
type Config struct {
Interval time.Duration // time between the end of a cycle and the start of the next (POLL_INTERVAL)
FirstDelay time.Duration // wait before the first cycle; 1 minute in production
Workers int // registry lookups in flight at once; 4 in production
Workers int // registry requests in flight at once; 4 in production
}
// Poller runs polling cycles.
@@ -112,13 +121,16 @@ type lookup struct {
// cycleStats is what one cycle did, for the poll_cycle log line.
type cycleStats struct {
refs, ok, failed, suggested, jobs, skipped int
reposListed, tagsFailed, newer int
wrote bool
}
// Cycle runs one poll over all services. It looks up every distinct followed
// ref once, with at most Config.Workers lookups at a time, then records the
// results. Writing results and starting jobs is not cut short by ctx ending
// during the lookups, so a job that was marked is always started.
// results. Then it lists the tags of each repo that has a service following a
// version-like tag, once per repo, and records newer tags. Writing results and
// starting jobs is not cut short by ctx ending during the lookups, so a job
// that was marked is always started.
func (p *Poller) Cycle(ctx context.Context) {
began := p.now()
svcs, err := p.store.ListServices(ctx)
@@ -152,19 +164,20 @@ func (p *Poller) Cycle(ctx context.Context) {
}
p.record(write, l, byRef[l.ref], &stats)
}
p.syncNewer(ctx, write, svcs, &stats)
if stats.wrote {
p.pub.Publish(events.Services)
}
p.log.Info("poll_cycle", "refs", stats.refs, "ok", stats.ok, "failed", stats.failed,
"suggested", stats.suggested, "jobs", stats.jobs, "skipped", stats.skipped,
"repos_listed", stats.reposListed, "tags_failed", stats.tagsFailed, "newer", stats.newer,
"duration_ms", p.now().Sub(began).Milliseconds())
}
// lookupAll asks the registry for every ref, Config.Workers at a time. It
// returns the answers in ref order and stops starting new lookups once ctx
// is done; refs never looked up are missing from the result.
func (p *Poller) lookupAll(ctx context.Context, refs []string) []lookup {
results := make([]*lookup, len(refs))
// forEach calls fn(i) for i in [0, n), Config.Workers at a time, and stops
// starting new calls once ctx is done. It returns when all started calls
// have returned.
func (p *Poller) forEach(ctx context.Context, n int, fn func(i int)) {
next := make(chan int)
var wg sync.WaitGroup
for w := 0; w < p.cfg.Workers; w++ {
@@ -172,15 +185,12 @@ func (p *Poller) lookupAll(ctx context.Context, refs []string) []lookup {
go func() {
defer wg.Done()
for i := range next {
lctx, cancel := context.WithTimeout(ctx, lookupTimeout)
d, err := p.reg.Digest(lctx, refs[i])
cancel()
results[i] = &lookup{ref: refs[i], digest: d, err: err}
fn(i)
}
}()
}
feed:
for i := range refs {
for i := 0; i < n; i++ {
select {
case next <- i:
case <-ctx.Done():
@@ -189,6 +199,19 @@ feed:
}
close(next)
wg.Wait()
}
// lookupAll asks the registry for every ref, Config.Workers at a time. It
// returns the answers in ref order and stops starting new lookups once ctx
// is done; refs never looked up are missing from the result.
func (p *Poller) lookupAll(ctx context.Context, refs []string) []lookup {
results := make([]*lookup, len(refs))
p.forEach(ctx, len(refs), func(i int) {
lctx, cancel := context.WithTimeout(ctx, lookupTimeout)
d, err := p.reg.Digest(lctx, refs[i])
cancel()
results[i] = &lookup{ref: refs[i], digest: d, err: err}
})
var out []lookup
for _, r := range results {
if r != nil {
@@ -198,6 +221,109 @@ feed:
return out
}
// tagList is the registry's answer for the tags of one repo.
type tagList struct {
repo string
tags []string
err error
}
// listAll lists the tags of every repo like lookupAll looks up refs.
func (p *Poller) listAll(ctx context.Context, repos []string) []tagList {
results := make([]*tagList, len(repos))
p.forEach(ctx, len(repos), func(i int) {
lctx, cancel := context.WithTimeout(ctx, listTimeout)
tags, err := p.reg.Tags(lctx, repos[i])
cancel()
results[i] = &tagList{repo: repos[i], tags: tags, err: err}
})
var out []tagList
for _, r := range results {
if r != nil {
out = append(out, *r)
}
}
return out
}
// followedTag returns the repo and tag svc follows. ok is false unless svc is
// polled and the tag can have a newer tag.
func followedTag(svc model.Service) (repo, tag string, ok bool) {
if SkipReason(svc) != "" {
return "", "", false
}
ref := imageref.Parse(jobs.FollowedRef(svc.ActualRef))
return ref.Repo, ref.Tag, tagver.Comparable(ref.Tag)
}
// syncNewer lists the tags of every repo that has a service following a
// version-like tag, once per repo, and stores the newer tag of each service.
// A service whose tag cannot have a newer tag any more gets its suggestion
// cleared; one whose repo could not be listed keeps it. Nothing is written
// when the value is unchanged. It never starts a job, not even for an auto
// service: a newer tag may be a major step the user has not chosen.
func (p *Poller) syncNewer(ctx, write context.Context, svcs []model.Service, stats *cycleStats) {
seen := map[string]bool{}
var repos []string
for _, svc := range svcs {
if repo, _, ok := followedTag(svc); ok && svc.RemovedAt == nil && !seen[repo] {
seen[repo] = true
repos = append(repos, repo)
}
}
lists := map[string][]string{}
for _, l := range p.listAll(ctx, repos) {
if l.err != nil {
if ctx.Err() != nil {
continue // cancelled mid-listing: not the registry's fault
}
stats.tagsFailed++
p.log.Warn("poll_tags_failed", "repo", l.repo, "error", truncate(l.err.Error()))
continue
}
stats.reposListed++
lists[l.repo] = l.tags
}
type repoTag struct{ repo, tag string }
newest := map[repoTag]string{}
for _, svc := range svcs {
if svc.RemovedAt != nil {
continue
}
want := ""
if repo, tag, ok := followedTag(svc); ok {
tags, listed := lists[repo]
if !listed {
if svc.NewerRef != "" {
stats.newer++
}
continue // not listed this cycle: keep what is stored
}
key := repoTag{repo, tag}
n, done := newest[key]
if !done {
n, _ = tagver.Newer(tag, tags)
newest[key] = n
}
if n != "" {
want = repo + ":" + n
}
}
if want != "" {
stats.newer++
}
if want == svc.NewerRef {
continue
}
if err := p.store.SetNewer(write, svc.ID, want, p.now()); err != nil {
p.log.Error("poll_set_newer_failed", "service_id", svc.ID, "ref", want, "error", err)
continue
}
stats.wrote = true
}
}
// record stores one lookup result for the services that follow l.ref.
func (p *Poller) record(ctx context.Context, l lookup, svcs []model.Service, stats *cycleStats) {
if l.err != nil {
+239
View File
@@ -30,6 +30,12 @@ type pollErrorCall struct {
msg string
}
type newerCall struct {
id int64
ref string
at time.Time
}
type lastTagCall struct {
id int64
digest string
@@ -40,6 +46,7 @@ type fakeStore struct {
services []model.Service
fresh map[int64]model.Service // what GetService returns instead of the listed row
lastTag []lastTagCall
newer []newerCall
listErr error
available []availableCall
pollErrors []pollErrorCall
@@ -86,6 +93,21 @@ func (f *fakeStore) SetAvailable(ctx context.Context, id int64, a model.Availabl
return nil
}
// SetNewer records the call and, like the database, changes the row the
// next cycle lists.
func (f *fakeStore) SetNewer(ctx context.Context, id int64, ref string, at time.Time) error {
f.mu.Lock()
defer f.mu.Unlock()
f.ctxErrs = append(f.ctxErrs, ctx.Err())
f.newer = append(f.newer, newerCall{id, ref, at})
for i := range f.services {
if f.services[i].ID == id {
f.services[i].NewerRef = ref
}
}
return nil
}
func (f *fakeStore) SetPollError(ctx context.Context, ids []int64, msg string) error {
f.mu.Lock()
defer f.mu.Unlock()
@@ -125,6 +147,36 @@ type fakeRegistry struct {
started chan struct{} // receives a value whenever a lookup starts, if set
release chan struct{} // lookups wait for it to be closed, if set
onResult func() // called after a lookup computed its answer
tags map[string][]string // repo -> tags
tagErrs map[string]error // repo -> error
tagCalls []string
tagStarted chan struct{} // receives a value whenever a tag listing starts, if set
tagRelease chan struct{} // tag listings wait for it to be closed, if set
onTags func() // called after a tag listing computed its answer
}
func (f *fakeRegistry) Tags(ctx context.Context, repo string) ([]string, error) {
f.mu.Lock()
f.tagCalls = append(f.tagCalls, repo)
f.mu.Unlock()
if f.tagStarted != nil {
f.tagStarted <- struct{}{}
}
if f.tagRelease != nil {
select {
case <-f.tagRelease:
case <-ctx.Done():
return nil, ctx.Err()
}
}
if f.onTags != nil {
f.onTags()
}
if err := f.tagErrs[repo]; err != nil {
return nil, err
}
return f.tags[repo], nil
}
func (f *fakeRegistry) Digest(ctx context.Context, ref string) (string, error) {
@@ -649,3 +701,190 @@ func TestSeedsTheTagDigestOfARunningService(t *testing.T) {
assert.Empty(t, j.starts, "the rollback stays")
assert.Len(t, st.lastTag, 1, "nothing more to seed")
}
// newTagRegistry returns a fake registry that lists tags by repo.
func newTagRegistry(tags map[string][]string) *fakeRegistry {
return &fakeRegistry{digests: map[string]string{}, tags: tags}
}
// newerSvc is a polled service running tag of repo that already runs the
// registry's digest, so the digest phase writes nothing.
func newerSvc(id int64, repo, tag string) model.Service {
s := svc(id, repo+":"+tag)
s.ActualDigest = "sha256:new"
return s
}
// newerRegistry is newTagRegistry that also answers sha256:new for refs.
func newerRegistry(tags map[string][]string, refs ...string) *fakeRegistry {
reg := newTagRegistry(tags)
for _, r := range refs {
reg.digests[r] = "sha256:new"
}
return reg
}
func TestNewerTagSuggested(t *testing.T) {
ref := repoB + ":1.27.3"
st := &fakeStore{services: []model.Service{newerSvc(1, repoB, "1.27.3")}}
reg := newerRegistry(map[string][]string{repoB: {"1.27.4", "1.28.0", "2.0.0", "latest", "1.28.1-rootless"}}, ref)
p, pub := newPoller(st, &fakeJobs{}, reg, Config{})
p.Cycle(context.Background())
assert.Equal(t, []newerCall{{1, repoB + ":1.28.0", time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)}}, st.newer)
assert.Equal(t, []events.Topic{events.Services}, pub.topics)
}
func TestOneTagListPerRepo(t *testing.T) {
st := &fakeStore{services: []model.Service{
newerSvc(1, repoB, "1.27.3"),
newerSvc(2, repoB, "2.1.0"),
newerSvc(3, repoB, "1.27.3"),
}}
reg := newerRegistry(map[string][]string{repoB: {"2.1.1", "1.27.4"}},
repoB+":1.27.3", repoB+":2.1.0")
p, _ := newPoller(st, &fakeJobs{}, reg, Config{})
p.Cycle(context.Background())
assert.Equal(t, []string{repoB}, reg.tagCalls)
got := map[int64]string{}
for _, c := range st.newer {
got[c.id] = c.ref
}
assert.Equal(t, map[int64]string{1: repoB + ":1.27.4", 2: repoB + ":2.1.1", 3: repoB + ":1.27.4"}, got)
}
func TestNoTagListWithoutComparableTag(t *testing.T) {
built := newerSvc(5, repoB, "1.2.3")
built.Built = true
noDigest := newerSvc(6, repoB, "1.2.3")
noDigest.ActualDigest = ""
st := &fakeStore{services: []model.Service{
newerSvc(1, repoB, "latest"),
newerSvc(2, repoB, "7"),
newerSvc(3, repoB, "pg16"),
newerSvc(4, repoB+"@sha256:abc", ""),
built,
noDigest,
}}
st.services[3].ActualRef = repoB + "@sha256:abc"
reg := newTagRegistry(map[string][]string{repoB: {"1.2.4"}})
p, _ := newPoller(st, &fakeJobs{}, reg, Config{})
p.Cycle(context.Background())
assert.Empty(t, reg.tagCalls)
assert.Empty(t, st.newer)
}
func TestNewerClearedWhenNoCandidate(t *testing.T) {
s := newerSvc(1, repoB, "1.27.3")
s.NewerRef = repoB + ":1.27.4"
st := &fakeStore{services: []model.Service{s}}
reg := newerRegistry(map[string][]string{repoB: {"1.27.3", "2.0.0"}}, repoB+":1.27.3")
p, _ := newPoller(st, &fakeJobs{}, reg, Config{})
p.Cycle(context.Background())
assert.Equal(t, []newerCall{{1, "", time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)}}, st.newer)
}
func TestNewerClearedWhenTagNoLongerComparable(t *testing.T) {
moved := newerSvc(1, repoB, "latest")
moved.NewerRef = repoB + ":1.27.4"
built := newerSvc(2, repoB, "1.27.3")
built.Built = true
built.NewerRef = repoB + ":1.27.4"
st := &fakeStore{services: []model.Service{moved, built}}
reg := newerRegistry(map[string][]string{repoB: {"1.27.4"}}, repoB+":latest")
p, pub := newPoller(st, &fakeJobs{}, reg, Config{})
p.Cycle(context.Background())
require.Len(t, st.newer, 2)
assert.Equal(t, "", st.newer[0].ref)
assert.Equal(t, "", st.newer[1].ref)
assert.Empty(t, reg.tagCalls)
assert.Equal(t, []events.Topic{events.Services}, pub.topics)
}
func TestTagListErrorKeepsNewerAndPollError(t *testing.T) {
s := newerSvc(1, repoB, "1.27.3")
s.NewerRef = repoB + ":1.27.4"
st := &fakeStore{services: []model.Service{s}}
reg := newerRegistry(nil, repoB+":1.27.3")
reg.tagErrs = map[string]error{repoB: errors.New("list x: registry answered 500 Internal Server Error")}
p, pub := newPoller(st, &fakeJobs{}, reg, Config{})
p.Cycle(context.Background())
assert.Empty(t, st.newer, "the stored suggestion stays")
assert.Empty(t, st.pollErrors, "a failed tag list is not a poll error")
assert.Empty(t, pub.topics)
}
func TestNewerIdempotent(t *testing.T) {
s := newerSvc(1, repoB, "1.27.3")
s.NewerRef = repoB + ":1.28.0"
st := &fakeStore{services: []model.Service{s}}
reg := newerRegistry(map[string][]string{repoB: {"1.28.0"}}, repoB+":1.27.3")
p, pub := newPoller(st, &fakeJobs{}, reg, Config{})
p.Cycle(context.Background())
assert.Empty(t, st.newer)
assert.Empty(t, pub.topics)
}
func TestNewerNeverStartsJobs(t *testing.T) {
s := newerSvc(1, repoB, "1.27.3")
s.UpdateMode = model.ModeAuto
st := &fakeStore{services: []model.Service{s}}
reg := newerRegistry(map[string][]string{repoB: {"1.27.4"}}, repoB+":1.27.3")
j := &fakeJobs{}
p, _ := newPoller(st, j, reg, Config{})
p.Cycle(context.Background())
require.Len(t, st.newer, 1)
assert.Empty(t, j.starts)
}
func TestNewerWritesSurviveCancel(t *testing.T) {
st := &fakeStore{services: []model.Service{newerSvc(1, repoB, "1.27.3")}}
ctx, cancel := context.WithCancel(context.Background())
reg := newerRegistry(map[string][]string{repoB: {"1.27.4"}}, repoB+":1.27.3")
reg.onTags = cancel
p, _ := newPoller(st, &fakeJobs{}, reg, Config{})
p.Cycle(ctx)
require.Len(t, st.newer, 1)
assert.NoError(t, st.ctxErrs[0], "the write does not see the cancelled context")
}
func TestCancelledListKeepsNewer(t *testing.T) {
s := newerSvc(1, repoB, "1.27.3")
s.NewerRef = repoB + ":1.27.4"
st := &fakeStore{services: []model.Service{s}}
reg := newerRegistry(map[string][]string{repoB: {"1.28.0"}}, repoB+":1.27.3")
reg.tagRelease = make(chan struct{})
reg.tagStarted = make(chan struct{}, 1)
p, pub := newPoller(st, &fakeJobs{}, reg, Config{})
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { p.Cycle(ctx); close(done) }()
<-reg.tagStarted
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("Cycle did not return after cancel")
}
assert.Empty(t, st.newer)
assert.Empty(t, pub.topics)
}
+42 -10
View File
@@ -1,5 +1,6 @@
// Package registry looks up the digest a tag points at in a container
// registry, using HEAD requests only.
// registry and lists the tags of a repository. Manifests are only ever
// requested with HEAD.
package registry
import (
@@ -21,9 +22,10 @@ import (
"github.com/shcizo/package-updater/internal/imageref"
)
// Client looks up tag digests with HEAD requests only. Docker Hub counts a
// GET of a manifest as a pull against the rate limit and a HEAD as nothing,
// so Client never issues a manifest GET.
// Client looks up tag digests and lists tags. Docker Hub counts a GET of a
// manifest as a pull against the rate limit and a HEAD as nothing, so Client
// never issues a manifest GET. The only GETs it sends are the /v2/ ping and
// tags/list, which Docker Hub does not count as a pull.
type Client struct {
keychain authn.Keychain
userAgent string
@@ -65,16 +67,46 @@ func (c *Client) Digest(ctx context.Context, ref string) (string, error) {
remote.WithAuth(auth),
remote.WithUserAgent(c.userAgent))
if err != nil {
var terr *transport.Error
if errors.As(err, &terr) {
return "", fmt.Errorf("lookup %s: %s", r, describe(terr))
}
c.log.Warn("registry_lookup_failed", "ref", ref, "error", err)
return "", fmt.Errorf("lookup %s: %s", r, classify(err))
return "", c.lookupError("lookup", r.String(), ref, err)
}
return desc.Digest.String(), nil
}
// Tags returns every tag of repo ("host/path", or a Docker Hub name), in the
// order the registry lists them. It sends GET /v2/<repo>/tags/list, following
// the registry's pagination links. Errors are sanitised like Digest's.
func (c *Client) Tags(ctx context.Context, repo string) ([]string, error) {
r, err := name.NewRepository(repo)
if err != nil {
return nil, fmt.Errorf("parse %s: %w", repo, err)
}
auth, err := c.keychain.Resolve(r)
if err != nil {
return nil, fmt.Errorf("list %s: %w", r, err)
}
tags, err := remote.List(r,
remote.WithContext(ctx),
remote.WithAuth(auth),
remote.WithUserAgent(c.userAgent))
if err != nil {
return nil, c.lookupError("list", r.String(), repo, err)
}
return tags, nil
}
// lookupError turns a failed registry request into an error that is safe to
// show: a registry answer is described by its status and codes, anything
// else by a fixed text. The full error goes to the log. verb and target
// start the message; ref is what the log line names.
func (c *Client) lookupError(verb, target, ref string, err error) error {
var terr *transport.Error
if errors.As(err, &terr) {
return fmt.Errorf("%s %s: %s", verb, target, describe(terr))
}
c.log.Warn("registry_lookup_failed", "ref", ref, "error", err)
return fmt.Errorf("%s %s: %s", verb, target, classify(err))
}
// classify returns a fixed text for an error that is not a registry answer.
// ggcr copies registry-controlled text (token responses, authentication
// challenges, redirect URLs, certificate names) into such errors, so none of
+101 -1
View File
@@ -134,6 +134,8 @@ func TestDigestIgnoresDigestSuffix(t *testing.T) {
assert.Equal(t, want, got)
}
// Digest asks the manifest endpoint with HEAD only; the one other request is
// the GET /v2/ ping that every client sends first.
func TestDigestUsesHeadOnly(t *testing.T) {
dockerConfig(t, "")
reg := newTestRegistry(t, "", "")
@@ -236,7 +238,7 @@ func TestCredsStoreWithoutHelperFailsTheLookup(t *testing.T) {
}
// A registry that leaves out Docker-Content-Digest must not make the client
// retry with a manifest download, which Docker Hub counts as a pull.
// retry with a manifest GET, which Docker Hub counts as a pull.
func TestMissingDigestHeaderDoesNotFallBackToDownload(t *testing.T) {
dockerConfig(t, "")
var mu sync.Mutex
@@ -334,3 +336,101 @@ func TestClassify(t *testing.T) {
})
}
}
// pushTags stores a random index under each tag of repo.
func (tr *testRegistry) pushTags(t *testing.T, repo string, tags ...string) {
t.Helper()
for _, tag := range tags {
tr.pushIndex(t, repo, tag)
}
}
// Digest and Tags together never GET a manifest: every manifest request is a
// HEAD, and the only other requests are the /v2/ ping and tags/list.
func TestNoManifestGet(t *testing.T) {
dockerConfig(t, "")
reg := newTestRegistry(t, "", "")
reg.pushTags(t, "team/app", "1.2", "1.3")
reg.reset()
_, err := newClient().Digest(context.Background(), reg.host+"/team/app:1.2")
require.NoError(t, err)
_, err = newClient().Tags(context.Background(), reg.host+"/team/app")
require.NoError(t, err)
reg.mu.Lock()
defer reg.mu.Unlock()
for _, r := range reg.requests {
switch {
case strings.Contains(r.path, "/manifests/"):
assert.Equal(t, http.MethodHead, r.method, r.path)
case r.path == "/v2/", r.path == "/v2/team/app/tags/list":
assert.Equal(t, http.MethodGet, r.method, r.path)
default:
assert.Failf(t, "unexpected request", "%s %s", r.method, r.path)
}
}
}
func TestTagsListsAllTags(t *testing.T) {
dockerConfig(t, "")
reg := newTestRegistry(t, "", "")
reg.pushTags(t, "team/app", "1.2", "1.3", "latest")
got, err := newClient().Tags(context.Background(), reg.host+"/team/app")
require.NoError(t, err)
assert.ElementsMatch(t, []string{"1.2", "1.3", "latest"}, got)
}
// ggcr's in-memory registry sends no Link header, so pagination is tested
// against a handler that does.
func TestTagsFollowsLinkPagination(t *testing.T) {
dockerConfig(t, "")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.URL.Path == "/v2/":
case r.URL.Path == "/v2/a/b/tags/list" && r.URL.Query().Get("last") == "":
w.Header().Set("Link", `</v2/a/b/tags/list?last=1.3&n=2>; rel="next"`)
_, _ = w.Write([]byte(`{"name":"a/b","tags":["1.2","1.3"]}`))
case r.URL.Path == "/v2/a/b/tags/list":
_, _ = w.Write([]byte(`{"name":"a/b","tags":["1.4"]}`))
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(srv.Close)
got, err := newClient().Tags(context.Background(), strings.TrimPrefix(srv.URL, "http://")+"/a/b")
require.NoError(t, err)
assert.Equal(t, []string{"1.2", "1.3", "1.4"}, got)
}
func TestTagsUsesDockerConfigCredentials(t *testing.T) {
reg := newTestRegistry(t, "bot", "s3cret")
reg.pushIndex(t, "team/app", "1.2", remote.WithAuth(&authn.Basic{Username: "bot", Password: "s3cret"}))
auth := base64.StdEncoding.EncodeToString([]byte("bot:s3cret"))
dockerConfig(t, `{"auths":{"`+reg.host+`":{"auth":"`+auth+`"}}}`)
got, err := newClient().Tags(context.Background(), reg.host+"/team/app")
require.NoError(t, err)
assert.Equal(t, []string{"1.2"}, got)
}
func TestTagsErrorBodyDoesNotLeak(t *testing.T) {
dockerConfig(t, "")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(`{"errors":[{"code":"NAME_UNKNOWN","message":"internal-secret-body"}]}`))
}))
t.Cleanup(srv.Close)
host := strings.TrimPrefix(srv.URL, "http://")
_, err := newClient().Tags(context.Background(), host+"/a/b")
require.Error(t, err)
assert.NotContains(t, err.Error(), "internal-secret-body")
assert.ErrorContains(t, err, "list "+host+"/a/b: registry answered 400 Bad Request")
}
@@ -275,6 +275,7 @@ func TestMergeService_MovesHistoryModeAndDesiredToTheNewLocation(t *testing.T) {
require.NoError(t, s.SetLastTagDigest(ctx, f.to, "sha256:fromto"))
require.NoError(t, s.SetAvailable(ctx, f.to, model.Available{Ref: sluskDrift + ":3", Digest: digNew, Source: model.SourcePoll, At: t0}))
require.NoError(t, s.SetPollError(ctx, []int64{f.to}, "registry down"))
require.NoError(t, s.SetNewer(ctx, f.to, sluskDrift+":3.1", t0))
// A job on the old row is finished, and awaiting_inventory is set on both.
before, err := s.GetService(ctx, f.from)
require.NoError(t, err)
@@ -297,6 +298,9 @@ func TestMergeService_MovesHistoryModeAndDesiredToTheNewLocation(t *testing.T) {
assert.Equal(t, sluskDrift+":3", got.AvailableRef)
assert.Equal(t, digNew, got.AvailableDigest)
assert.Equal(t, "registry down", got.LastPollError)
assert.Equal(t, sluskDrift+":3.1", got.NewerRef, "newer tag from the duplicate")
require.NotNil(t, got.NewerAt)
assert.True(t, t0.Equal(*got.NewerAt))
assert.Equal(t, before.LastTagDigest, got.LastTagDigest, "an existing last_tag_digest wins")
_, err = s.GetService(ctx, f.to)
+6 -6
View File
@@ -106,8 +106,8 @@ type mergeRow struct {
// MergeService moves the live service toID onto the removed service fromID:
// the old row keeps its id, first_seen_at, update mode and desired ref, takes
// the location and inventory of the live row (and its available suggestion),
// becomes live, and the live row is deleted. The jobs of the live row move to
// the location and inventory of the live row (and its available and newer-tag
// suggestions), becomes live, and the live row is deleted. The jobs of the live row move to
// the old one. This is the only place a services row is deleted.
//
// It returns *model.MergeRefusedError when the pair cannot be merged,
@@ -195,12 +195,12 @@ func (s *Store) MergeService(ctx context.Context, fromID, toID int64) error {
SELECT agent_id, location_key, compose_project, compose_service, working_dir, config_files, built,
swarm_cluster_id, swarm_service, container_name, image_repo, actual_ref, actual_digest, status, self,
updated_at, available_ref, available_digest, available_source, available_at, last_poll_error,
last_tag_digest
last_tag_digest, newer_ref, newer_at
FROM services WHERE id = $1`, toID).Scan(
&t.AgentID, &t.LocationKey, &t.ComposeProject, &t.ComposeService, &t.WorkingDir, &t.ConfigFiles, &t.Built,
&t.SwarmClusterID, &t.SwarmService, &t.ContainerName, &t.ImageRepo, &t.ActualRef, &t.ActualDigest, &t.Status, &t.Self,
&t.UpdatedAt, &t.AvailableRef, &t.AvailableDigest, &t.AvailableSource, &t.AvailableAt, &t.LastPollError,
&lastTagDigest)
&lastTagDigest, &t.NewerRef, &t.NewerAt)
if err != nil {
return fmt.Errorf("merge: read new service: %w", err)
}
@@ -213,7 +213,7 @@ func (s *Store) MergeService(ctx context.Context, fromID, toID int64) error {
config_files = $7, built = $8, swarm_cluster_id = $9, swarm_service = $10, container_name = $11,
image_repo = $12, actual_ref = $13, actual_digest = $14, status = $15, self = $16, updated_at = $17,
available_ref = $18, available_digest = $19, available_source = $20, available_at = $21,
last_poll_error = $22,
last_poll_error = $22, newer_ref = $24, newer_at = $25,
last_tag_digest = CASE WHEN last_tag_digest <> '' THEN last_tag_digest ELSE $23 END,
removed_at = NULL, awaiting_inventory = false
WHERE id = $1`,
@@ -221,7 +221,7 @@ func (s *Store) MergeService(ctx context.Context, fromID, toID int64) error {
t.ConfigFiles, t.Built, t.SwarmClusterID, t.SwarmService, t.ContainerName,
t.ImageRepo, t.ActualRef, t.ActualDigest, t.Status, t.Self, t.UpdatedAt,
t.AvailableRef, t.AvailableDigest, t.AvailableSource, t.AvailableAt,
t.LastPollError, lastTagDigest); err != nil {
t.LastPollError, lastTagDigest, t.NewerRef, t.NewerAt); err != nil {
return fmt.Errorf("merge: update old service: %w", err)
}
if err := tx.Commit(ctx); err != nil {
@@ -0,0 +1,9 @@
-- Newer version tag of the followed tag, found in the registry's tag list.
-- A suggestion only: nothing deploys it automatically.
ALTER TABLE services ADD COLUMN newer_ref text NOT NULL DEFAULT '';
ALTER TABLE services ADD COLUMN newer_at timestamptz;
-- A suggested tag is recorded in image_versions with source 'tags'.
ALTER TABLE image_versions DROP CONSTRAINT image_versions_source_check;
ALTER TABLE image_versions ADD CONSTRAINT image_versions_source_check
CHECK (source IN ('webhook', 'poll', 'running', 'tags'));
+46 -2
View File
@@ -175,7 +175,7 @@ const serviceColumns = `s.id, s.kind, s.location_key, s.agent_id, COALESCE(a.nam
s.compose_project, s.compose_service, s.working_dir, s.config_files, s.built, s.swarm_cluster_id,
s.swarm_service, s.container_name, s.image_repo, s.desired_ref, s.actual_ref, s.actual_digest,
s.status, s.self, s.update_mode, s.available_ref, s.available_digest, s.available_source, s.available_at,
s.last_poll_error, s.last_tag_digest, s.awaiting_inventory, s.first_seen_at, s.updated_at, s.removed_at`
s.last_poll_error, s.newer_ref, s.newer_at, s.last_tag_digest, s.awaiting_inventory, s.first_seen_at, s.updated_at, s.removed_at`
func scanService(row pgx.Row) (model.Service, error) {
var m model.Service
@@ -184,7 +184,7 @@ func scanService(row pgx.Row) (model.Service, error) {
&m.ComposeProject, &m.ComposeService, &m.WorkingDir, &m.ConfigFiles, &m.Built, &m.SwarmClusterID,
&m.SwarmService, &m.ContainerName, &m.ImageRepo, &m.DesiredRef, &m.ActualRef, &m.ActualDigest,
&m.Status, &m.Self, &m.UpdateMode, &m.AvailableRef, &m.AvailableDigest, &m.AvailableSource, &m.AvailableAt,
&m.LastPollError, &m.LastTagDigest, &m.AwaitingInventory, &m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt)
&m.LastPollError, &m.NewerRef, &m.NewerAt, &m.LastTagDigest, &m.AwaitingInventory, &m.FirstSeenAt, &m.UpdatedAt, &m.RemovedAt)
m.Kind = protocol.Kind(kind)
return m, err
}
@@ -246,6 +246,33 @@ func (s *Store) SetAvailable(ctx context.Context, id int64, a model.Available) e
return nil
}
// SetNewer sets the service's newer-tag suggestion (model.Service.NewerRef);
// ref "" clears it. A non-empty ref is also recorded in image_versions with
// source "tags", in the same transaction, unless that (repo, tag) is already
// there. The digest is unknown, so the row has none.
func (s *Store) SetNewer(ctx context.Context, id int64, ref string, at time.Time) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return fmt.Errorf("set newer: begin: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := tx.Exec(ctx, `
UPDATE services SET newer_ref = $2, newer_at = CASE WHEN $2 = '' THEN NULL ELSE $3::timestamptz END
WHERE id = $1`, id, ref, at); err != nil {
return fmt.Errorf("set newer: %w", err)
}
if ref != "" {
p := imageref.Parse(ref)
if err := insertVersion(ctx, tx, p.Repo, p.Tag, "", model.SourceTags, at); err != nil {
return err
}
}
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("set newer: commit: %w", err)
}
return nil
}
// SetPollError sets last_poll_error on the given services; msg "" clears it.
func (s *Store) SetPollError(ctx context.Context, ids []int64, msg string) error {
if len(ids) == 0 {
@@ -313,6 +340,23 @@ func (s *Store) GetImageVersion(ctx context.Context, id int64) (model.ImageVersi
return v, nil
}
// GetTagVersion returns the digest-less version row with an empty digest, which
// SetNewer and the webhook create. Returns model.ErrNotFound if there is none.
func (s *Store) GetTagVersion(ctx context.Context, repo, tag string) (model.ImageVersion, error) {
var v model.ImageVersion
err := s.pool.QueryRow(ctx, `
SELECT id, repo, tag, digest, source, first_seen_at FROM image_versions
WHERE repo = $1 AND tag = $2 AND digest = ''`, repo, tag,
).Scan(&v.ID, &v.Repo, &v.Tag, &v.Digest, &v.Source, &v.FirstSeenAt)
if errors.Is(err, pgx.ErrNoRows) {
return model.ImageVersion{}, model.ErrNotFound
}
if err != nil {
return model.ImageVersion{}, fmt.Errorf("get tag version: %w", err)
}
return v, nil
}
// SetUpdateMode sets the service's update mode (model.ModeManual or
// model.ModeAuto). Returns model.ErrNotFound if the service does not exist
// or is marked removed.
@@ -214,3 +214,79 @@ func TestSetLastTagDigest(t *testing.T) {
require.Len(t, list, 1)
assert.Equal(t, "sha256:n1", list[0].LastTagDigest)
}
func TestSetNewer_SetsAndClears(t *testing.T) {
s := openTestStore(t)
ctx := context.Background()
_, svc := jobFixture(t, s, "a")
require.NoError(t, s.SetNewer(ctx, svc, sluskRepo+":1.5.0", t0))
got, err := s.GetService(ctx, svc)
require.NoError(t, err)
assert.Equal(t, sluskRepo+":1.5.0", got.NewerRef)
require.NotNil(t, got.NewerAt)
assert.True(t, t0.Equal(*got.NewerAt))
v, err := s.GetTagVersion(ctx, sluskRepo, "1.5.0")
require.NoError(t, err)
assert.Equal(t, model.SourceTags, v.Source)
assert.Empty(t, v.Digest)
assert.True(t, t0.Equal(v.FirstSeenAt))
require.NoError(t, s.SetNewer(ctx, svc, "", t0.Add(time.Hour)))
got, err = s.GetService(ctx, svc)
require.NoError(t, err)
assert.Empty(t, got.NewerRef)
assert.Nil(t, got.NewerAt)
_, err = s.GetTagVersion(ctx, sluskRepo, "1.5.0")
assert.NoError(t, err, "clearing the suggestion keeps the version")
}
func TestSetNewer_KeepsEarlierVersionSource(t *testing.T) {
s := openTestStore(t)
ctx := context.Background()
_, svc := jobFixture(t, s, "a")
require.NoError(t, s.SetAvailable(ctx, svc, model.Available{Ref: sluskRepo + ":1.5.0", Source: model.SourceWebhook, At: t0}))
require.NoError(t, s.SetNewer(ctx, svc, sluskRepo+":1.5.0", t0.Add(time.Hour)))
v, err := s.GetTagVersion(ctx, sluskRepo, "1.5.0")
require.NoError(t, err)
assert.Equal(t, model.SourceWebhook, v.Source)
assert.True(t, t0.Equal(v.FirstSeenAt))
}
func TestRecentVersions_TagsRowHiddenOnceDigestKnown(t *testing.T) {
s := openTestStore(t)
ctx := context.Background()
_, svc := jobFixture(t, s, "a")
require.NoError(t, s.SetNewer(ctx, svc, sluskRepo+":1.5.0", t0.Add(time.Minute)))
require.Len(t, versionsOf(t, s, sluskRepo), 2, "running 1.4.2 and the tags row")
require.NoError(t, s.SetAvailable(ctx, svc, model.Available{Ref: sluskRepo + ":1.5.0", Digest: "sha256:n1", Source: model.SourcePoll, At: t0.Add(2 * time.Minute)}))
var refs []string
for _, v := range versionsOf(t, s, sluskRepo) {
refs = append(refs, v.Ref())
}
assert.Equal(t, []string{sluskRepo + ":1.5.0@sha256:n1", sluskRepo + ":1.4.2@" + digD1}, refs)
}
func TestGetTagVersion(t *testing.T) {
s := openTestStore(t)
ctx := context.Background()
_, svc := jobFixture(t, s, "a")
require.NoError(t, s.SetNewer(ctx, svc, sluskRepo+":1.5.0", t0))
got, err := s.GetTagVersion(ctx, sluskRepo, "1.5.0")
require.NoError(t, err)
assert.Equal(t, sluskRepo+":1.5.0", got.Ref())
_, err = s.GetTagVersion(ctx, sluskRepo, "9.9.9")
assert.ErrorIs(t, err, model.ErrNotFound)
_, err = s.GetTagVersion(ctx, "other/repo", "1.5.0")
assert.ErrorIs(t, err, model.ErrNotFound)
_, err = s.GetTagVersion(ctx, sluskRepo, "1.4.2")
assert.ErrorIs(t, err, model.ErrNotFound, "a row with a digest is not a tag version")
}
+101
View File
@@ -0,0 +1,101 @@
// Package tagver decides which tag of a repository is a newer version of the
// tag a service follows, using only the shape of the tag names.
package tagver
import (
"regexp"
"strconv"
"strings"
)
// tagPattern splits a tag into a prefix without digits, a dotted numeric
// part and whatever follows it. "v1.2.3-alpine" gives "v", "1.2.3", "-alpine".
var tagPattern = regexp.MustCompile(`^([^0-9]*)([0-9]+(?:\.[0-9]+)*)(.*)$`)
// cosignPattern matches the tags cosign pushes next to an image to store its
// signature, attestation or SBOM. They look like versions to a naive check.
// The pattern is taken from Diun (crazy-max/diun, MIT, pkg/registry/tags.go).
var cosignPattern = regexp.MustCompile(`^sha256-[0-9a-f]{64}(\.att|\.sbom|\.sig)?$`)
// parsed is a tag split into its parts.
type parsed struct {
prefix, suffix string
nums []int
}
// parse splits tag. It returns false for a cosign tag, a tag without digits
// and a numeric component that does not fit an int.
func parse(tag string) (parsed, bool) {
if cosignPattern.MatchString(tag) {
return parsed{}, false
}
m := tagPattern.FindStringSubmatch(tag)
if m == nil {
return parsed{}, false
}
var nums []int
for _, part := range strings.Split(m[2], ".") {
n, err := strconv.Atoi(part)
if err != nil {
return parsed{}, false
}
nums = append(nums, n)
}
return parsed{prefix: m[1], suffix: m[3], nums: nums}, true
}
// Comparable reports whether tag can ever get a newer tag: it splits into
// prefix, numeric part and suffix, has at least two numeric components, and
// is not a cosign artifact tag. A tag with one component ("7", "pg16",
// "8-bookworm") is a moving tag, not a version.
func Comparable(tag string) bool {
p, ok := parse(tag)
return ok && len(p.nums) >= 2
}
// Newer returns the highest candidate that is newer than current, and false
// when there is none. A candidate qualifies when it has the same prefix and
// suffix and the same number of components as current, the same first
// component, and a greater numeric part. Of two candidates with the same
// numeric value the lexically smaller string wins, so the result does not
// depend on the order of candidates. A tag such as "name2-v1.2.3", where a
// digit comes before the version, is not comparable.
func Newer(current string, candidates []string) (string, bool) {
cur, ok := parse(current)
if !ok || len(cur.nums) < 2 {
return "", false
}
best, found := "", false
var bestNums []int
for _, c := range candidates {
p, ok := parse(c)
if !ok || p.prefix != cur.prefix || p.suffix != cur.suffix ||
len(p.nums) != len(cur.nums) || p.nums[0] != cur.nums[0] {
continue
}
if compare(p.nums, cur.nums) <= 0 {
continue
}
if !found {
best, bestNums, found = c, p.nums, true
continue
}
if cmp := compare(p.nums, bestNums); cmp > 0 || (cmp == 0 && c < best) {
best, bestNums = c, p.nums
}
}
return best, found
}
// compare compares two numeric parts of equal length.
func compare(a, b []int) int {
for i := range a {
switch {
case a[i] < b[i]:
return -1
case a[i] > b[i]:
return 1
}
}
return 0
}
+71
View File
@@ -0,0 +1,71 @@
package tagver
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestComparable(t *testing.T) {
sig := "sha256-" + strings.Repeat("a", 64) + ".sig"
tests := []struct {
tag string
want bool
}{
{"latest", false},
{"release", false},
{"stable", false},
{"edge", false},
{"alpine", false},
{"7", false},
{"7-alpine", false},
{"8-bookworm", false},
{"pg16", false},
{"14-vectorchord0.4.3-pgvectors0.2.0", false},
{"name2-v1.2.3", false},
{sig, false},
{"99999999999999999999.1", false},
{"1.2", true},
{"1.2.3", true},
{"v1.2.3", true},
{"11.0-latest", true},
}
for _, tt := range tests {
t.Run(tt.tag, func(t *testing.T) {
assert.Equal(t, tt.want, Comparable(tt.tag))
})
}
}
func TestNewer(t *testing.T) {
tests := []struct {
name string
current string
candidates []string
want string
ok bool
}{
{"highest within major", "1.2.3", []string{"1.2.4", "1.3.0", "2.0.0", "1.2.4-rc1", "1.3", "v1.2.4"}, "1.3.0", true},
{"v prefix kept", "v1.2.3", []string{"1.2.4", "v1.2.4"}, "v1.2.4", true},
{"name prefix kept", "name-v1.2.3", []string{"v1.2.4", "name-v1.2.4"}, "name-v1.2.4", true},
{"suffix kept", "11.0-latest", []string{"11.1", "11.1-latest"}, "11.1-latest", true},
{"two components", "7.0", []string{"8.0", "7.0.1", "7.2", "7.1"}, "7.2", true},
{"equal is not newer", "1.2.3", []string{"1.2.3"}, "", false},
{"older is not newer", "1.2.3", []string{"1.2.2", "1.1.9"}, "", false},
{"numeric not lexical", "1.9.0", []string{"1.10.0"}, "1.10.0", true},
{"empty candidates", "1.2.3", nil, "", false},
{"moving tag", "latest", []string{"1.2.3"}, "", false},
{"single component", "7", []string{"8"}, "", false},
{"digit before version", "name2-v1.2.3", []string{"name2-v1.2.4"}, "", false},
{"cosign tag ignored", "1.2.3", []string{"sha256-" + strings.Repeat("0", 64) + ".sig"}, "", false},
{"equal value picks smaller string", "1.2.3", []string{"1.02.4", "1.2.4"}, "1.02.4", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, ok := Newer(tt.current, tt.candidates)
assert.Equal(t, tt.ok, ok)
assert.Equal(t, tt.want, got)
})
}
}
+15
View File
@@ -38,6 +38,9 @@ var agentNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
type Store interface {
ListServices(ctx context.Context) ([]model.Service, error)
GetService(ctx context.Context, id int64) (model.Service, error)
// GetTagVersion returns the version of repo with this tag and no digest;
// model.ErrNotFound if there is none.
GetTagVersion(ctx context.Context, repo, tag string) (model.ImageVersion, error)
// RecentVersions returns up to limit versions of repo the master has
// seen, most recently first seen first.
RecentVersions(ctx context.Context, repo string, limit int) ([]model.ImageVersion, error)
@@ -202,6 +205,7 @@ type pageData struct {
Reason string // why Service cannot be updated; "" when it can
AutoBlocked string // why Service cannot be set to auto; "" when it can
Versions []versionRow
NewerTag string // Service's newer tag while it still holds; "" otherwise
Polling string // what the service page says about registry polling
PollFailed bool // Polling is the last registry error
// RestoreBlocked is why a drifted Service cannot be restored; "" when it
@@ -388,6 +392,17 @@ func (s *Server) serviceDetail(r *http.Request, id int64) (pageData, error) {
if err != nil {
return data, err
}
data.NewerTag = newerTag(svc)
if data.NewerTag != "" && !slices.ContainsFunc(versions, func(v model.ImageVersion) bool { return v.Tag == data.NewerTag }) {
// The suggested tag can be older than the listed versions; it must
// still be pickable.
v, err := s.store.GetTagVersion(r.Context(), svc.ImageRepo, data.NewerTag)
if err == nil {
versions = append(versions, v)
} else if !errors.Is(err, model.ErrNotFound) {
return data, err
}
}
data.Versions = versionRows(versions, svc.ActualDigest)
data.Polling, data.PollFailed = pollingStatus(svc, s.cfg.PollInterval)
data.Jobs, err = s.store.ListJobs(r.Context(), model.JobFilter{ServiceID: id, Limit: serviceJobsLimit})
+31
View File
@@ -787,3 +787,34 @@ func TestServicePage_AutoBlockedDisablesToggle(t *testing.T) {
_, body := get(t, srv, "/services/1")
assert.Contains(t, body, "Auto unavailable: "+model.ReasonContainer+".")
}
// GetTagVersion returns the digest-less version of repo with this tag.
func (f *fakeStore) GetTagVersion(_ context.Context, repo, tag string) (model.ImageVersion, error) {
for _, v := range f.versions {
if v.Repo == repo && v.Tag == tag && v.Digest == "" {
return v, nil
}
}
return model.ImageVersion{}, model.ErrNotFound
}
func TestOverview_ShowsNewerTagChipBesideUpdateChip(t *testing.T) {
srv, store, _, _, _ := newTestEnv(t)
store.services[0].AvailableRef = "gitea.shcizo.se/shcizo/slusk:1.4.2"
store.services[0].NewerRef = "gitea.shcizo.se/shcizo/slusk:1.5.0"
for _, path := range []string{"/", "/fragments/services"} {
_, body := get(t, srv, path)
assert.Contains(t, body, `class="tag f-newer" title="gitea.shcizo.se/shcizo/slusk:1.5.0">Newer tag<em>1.5.0</em>`, path)
assert.Contains(t, body, `class="tag f-up"`, path)
}
}
func TestOverview_HidesStaleNewerTag(t *testing.T) {
srv, store, _, _, _ := newTestEnv(t)
store.services[0].NewerRef = "gitea.shcizo.se/shcizo/slusk:1.4.1" // not newer than 1.4.2
_, body := get(t, srv, "/")
assert.NotContains(t, body, "f-newer")
}
+2
View File
@@ -106,6 +106,8 @@ summary.btn::-webkit-details-marker{display:none}
.tag em{font:500 11.5px var(--mono);font-style:normal;margin-left:6px}
.tag em:empty{display:none}
.f-up{color:var(--orange);background:var(--orangebg)}
.f-newer{color:var(--mut);box-shadow:inset 0 0 0 1px var(--line2)}
.f-newer em{color:var(--orange)}
.f-err{color:var(--red);background:var(--redbg)}
.f-notag{color:var(--mut);box-shadow:inset 0 0 0 1px var(--line2)}
.f-out{color:var(--dim);box-shadow:inset 0 0 0 1px var(--line)}
@@ -77,6 +77,7 @@
</div>
<div class="flags">
{{if .AvailableRef}}<span class="tag f-up" title="{{.AvailableRef}}">Update available<em>{{.AvailableTag}}</em></span>{{end}}
{{if .NewerTag}}<span class="tag f-newer" title="{{.NewerRef}}">Newer tag<em>{{.NewerTag}}</em></span>{{end}}
{{if followsNoTag .Service}}<span class="tag f-notag">Follows no tag</span>{{end}}
{{if .LastPollError}}<span class="tag f-err" title="{{.LastPollError}}">Poll failed</span>{{end}}
{{if .Self}}<span class="tag f-out">Managed outside</span>{{end}}
@@ -9,6 +9,7 @@
<tr><th>Status</th><td class="status status-{{.Service.Status}}">{{.Service.Status}}</td></tr>
{{if .Service.DesiredRef}}<tr><th>Desired image</th><td><code>{{shortRef .Service.DesiredRef}}</code></td></tr>{{end}}
{{if .Service.AvailableRef}}<tr><th>Available</th><td><code>{{.Service.AvailableRef}}</code> <span class="muted">via {{.Service.AvailableSource}}{{with .Service.AvailableAt}}, {{.Format "2006-01-02 15:04"}}{{end}}</span></td></tr>{{end}}
{{if .NewerTag}}<tr><th>Newer tag</th><td><code>{{.Service.NewerRef}}</code> <span class="muted">from the registry's tag list{{with .Service.NewerAt}}, {{.Format "2006-01-02 15:04"}}{{end}}; deploy it from Versions</span></td></tr>{{end}}
<tr><th>Mode</th><td>{{.Service.UpdateMode}}</td></tr>
<tr><th>Polling</th><td{{if .PollFailed}} class="error"{{end}}>{{.Polling}}</td></tr>
{{if .ConfigFiles}}<tr><th>Config files</th><td>{{range .ConfigFiles}}<code>{{.}}</code><br>{{end}}</td></tr>{{end}}
+35
View File
@@ -219,3 +219,38 @@ func TestOverview_ShowsFollowsNoTagAndPollFailedBadges(t *testing.T) {
assert.Equal(t, 1, strings.Count(body, "Poll failed"), path)
}
}
func TestServicePage_VersionPickerAppendsNewerTagBeyondLimit(t *testing.T) {
srv, store, _, _, _ := newTestEnv(t)
store.services[0].ImageRepo = sluskRepo
store.services[0].NewerRef = sluskRepo + ":1.5.0"
for i := int64(1); i <= 12; i++ {
store.versions = append(store.versions, model.ImageVersion{ID: i, Repo: sluskRepo, Tag: "t", Digest: "sha256:d", Source: "poll"})
}
store.versions = append(store.versions, model.ImageVersion{ID: 99, Repo: sluskRepo, Tag: "1.5.0", Source: "tags"})
_, body := get(t, srv, "/services/1")
assert.Equal(t, 11, strings.Count(body, `name="version"`), "the ten recent versions and the newer tag")
assert.Contains(t, body, `<input type="hidden" name="version" value="99">`)
assert.Contains(t, body, "<th>Newer tag</th>")
}
func TestServicePage_NewerTagAlreadyListedIsNotAppendedTwice(t *testing.T) {
srv, store, _, _, _ := newTestEnv(t)
store.services[0].ImageRepo = sluskRepo
store.services[0].NewerRef = sluskRepo + ":1.5.0"
store.versions = []model.ImageVersion{{ID: 31, Repo: sluskRepo, Tag: "1.5.0", Source: "tags"}}
_, body := get(t, srv, "/services/1")
assert.Equal(t, 1, strings.Count(body, `name="version"`))
}
func TestServicePage_NoNewerTagRowWithoutSuggestion(t *testing.T) {
srv, _, _, _, _ := newTestEnv(t)
_, body := get(t, srv, "/services/1")
assert.NotContains(t, body, "Newer tag")
}
+19
View File
@@ -12,6 +12,7 @@ import (
"github.com/shcizo/package-updater/internal/master/jobs"
"github.com/shcizo/package-updater/internal/master/model"
"github.com/shcizo/package-updater/internal/master/poller"
"github.com/shcizo/package-updater/internal/master/tagver"
"github.com/shcizo/package-updater/internal/master/web/icons"
"github.com/shcizo/package-updater/internal/protocol"
)
@@ -45,6 +46,7 @@ type serviceRow struct {
Tag string // ":tag", or ""
Digest string // "@sha256:<12 hex>…" when the image has no tag, else ""
AvailableTag string // the tag of AvailableRef, or ""
NewerTag string // the tag of NewerRef while it is still newer than the followed tag, or ""
Icon string // Simple Icons slug; "" when there is no icon
Letter string // shown instead of an icon
LetterClass string // lc-0 .. lc-3
@@ -66,6 +68,7 @@ func newServiceRow(s model.Service, autoBlocked string) serviceRow {
}
r.Repo, r.Tag, r.Digest = imageParts(s.ActualRef)
r.AvailableTag = imageref.Parse(s.AvailableRef).Tag
r.NewerTag = newerTag(s)
r.Icon = icons.ForImage(s.ActualRef)
r.Letter = "?"
if first, _ := utf8.DecodeRuneInString(r.Name); first != utf8.RuneError {
@@ -243,6 +246,22 @@ func followsNoTag(svc model.Service) bool {
return jobs.FollowedRef(svc.ActualRef) == ""
}
// newerTag returns the tag of the service's newer-tag suggestion, or "" when
// there is none or it no longer holds. The poller writes the suggestion once
// per cycle, so after the service moved to another tag the stored value can
// be stale until the next cycle; checking again here hides it meanwhile.
func newerTag(s model.Service) string {
if s.RemovedAt != nil || s.NewerRef == "" {
return ""
}
tag := imageref.Parse(s.NewerRef).Tag
followed := imageref.Parse(jobs.FollowedRef(s.ActualRef)).Tag
if _, ok := tagver.Newer(followed, []string{tag}); !ok {
return ""
}
return tag
}
// versionRow is an image version as shown on the service page.
type versionRow struct {
model.ImageVersion
+21
View File
@@ -211,3 +211,24 @@ func TestGroupServices(t *testing.T) {
assert.Equal(t, "because", groups[0].Services[0].AutoBlocked)
})
}
func TestNewerTag(t *testing.T) {
removed := time.Now()
tests := []struct {
name string
svc model.Service
want string
}{
{"newer", model.Service{ActualRef: "gitea/gitea:1.27.3", NewerRef: "gitea/gitea:1.28.0"}, "1.28.0"},
{"bare image follows latest", model.Service{ActualRef: "gitea/gitea", NewerRef: "gitea/gitea:1.28.0"}, ""},
{"stale after the tag changed", model.Service{ActualRef: "gitea/gitea:1.28.0", NewerRef: "gitea/gitea:1.28.0"}, ""},
{"stale after the tag moved past it", model.Service{ActualRef: "gitea/gitea:1.29.0", NewerRef: "gitea/gitea:1.28.0"}, ""},
{"removed", model.Service{ActualRef: "gitea/gitea:1.27.3", NewerRef: "gitea/gitea:1.28.0", RemovedAt: &removed}, ""},
{"empty", model.Service{ActualRef: "gitea/gitea:1.27.3"}, ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, newerTag(tt.svc))
})
}
}