Docker Build / docker (push) Successful in 38s
## Summary
- `POST /update` on the master (outside OIDC, bearer `UPDATER_API_KEY`; unset → 503). Matches `image` against `image_repo` on all agents, marks `available_*` on every match, starts webhook-triggered jobs for `auto` services targeting `<image_repo>:<tag>` (followed tag when no tag is given), and waits for them up to `UPDATE_TIMEOUT` (default 5m). 404 on no match, 500 if any auto job failed or timed out, otherwise 200.
- Manual/auto toggle per service on the overview and service page (`POST /services/{id}/mode`, CSRF); auto is blocked for self, `build:` and container services with the reason shown. "Update available" badge and an "Available" row.
- `FinishJob` clears `available_*` only when the successful job was created for that exact ref, after it was reported. Migration `0003_webhook.sql`, phase-3 plan doc, README / gitea-action README / CLAUDE.md / deploy example updated. `action.yml` unchanged.
Closes #8
## Behaviour changes for CI
- Mixed results now return 500 (the old 207 is gone); no match returns 404 instead of 200.
- With a `tag`, Compose services get pinned to `<image_repo>:<tag>` via `compose.override.yaml`; the tag must be a pushed tag.
- `busy` (job already active) and `refused` (not updatable) are reported but return 200.
- The reverse proxy read timeout must exceed `UPDATE_TIMEOUT`.
## Test plan
- [x] `go test ./...` and `go test -race ./internal/agent/... ./internal/master/... ./internal/e2e/`
- [x] `go test -tags integration ./internal/master/store/` against throwaway Postgres
- [x] UI verified in browser (mode toggle, disabled reason, badge via SSE, Available row)
- [ ] Manual: webhook with manual service → `marked` + badge
- [ ] Manual: webhook with auto service → waits, `succeeded`, badge cleared, override pinned
- [ ] Manual: wrong token 401, unknown image 404, offline agent 500
- [ ] Gitea action pointed at the master's `/update` in a real CI run
Reviewed-on: #22
351 lines
12 KiB
Go
351 lines
12 KiB
Go
// Package webhook implements POST /update, the endpoint CI calls after
|
|
// pushing an image. It records the pushed tag as "available" on every service
|
|
// that uses the image, starts an update for the services in auto mode and
|
|
// waits for those updates to finish.
|
|
package webhook
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"crypto/subtle"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"log/slog"
|
|
"net/http"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/shcizo/package-updater/internal/imageref"
|
|
"github.com/shcizo/package-updater/internal/logging"
|
|
"github.com/shcizo/package-updater/internal/master/events"
|
|
"github.com/shcizo/package-updater/internal/master/jobs"
|
|
"github.com/shcizo/package-updater/internal/master/model"
|
|
)
|
|
|
|
// maxBodyBytes caps the request body; a valid request is a few hundred bytes.
|
|
const maxBodyBytes = 64 << 10
|
|
|
|
// defaultPollEvery is how often pending jobs are re-read from the store when
|
|
// no broker signal arrives (signals to a slow subscriber can be dropped).
|
|
const defaultPollEvery = 5 * time.Second
|
|
|
|
// finalReadTimeout bounds the last read of pending jobs after the wait ends.
|
|
const finalReadTimeout = 2 * time.Second
|
|
|
|
// tagPattern is Docker's tag grammar. It keeps spaces, colons and "@" out of
|
|
// the ref the agent is asked to deploy.
|
|
var tagPattern = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`)
|
|
|
|
// requestIDPattern limits a caller-supplied X-Request-ID to what is safe to
|
|
// echo in a header and write to the log.
|
|
var requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9._-]{1,128}$`)
|
|
|
|
// Result statuses.
|
|
const (
|
|
statusSucceeded = "succeeded"
|
|
statusFailed = "failed"
|
|
statusTimeout = "timeout"
|
|
statusMarked = "marked"
|
|
statusBusy = "busy"
|
|
statusRefused = "refused"
|
|
)
|
|
|
|
// Store is the storage the webhook needs. *store.Store satisfies it.
|
|
type Store interface {
|
|
ServicesByRepo(ctx context.Context, repo string) ([]model.Service, error)
|
|
SetAvailable(ctx context.Context, id int64, a model.Available) error
|
|
GetJob(ctx context.Context, id int64) (model.Job, error)
|
|
}
|
|
|
|
// Jobs starts updates. *jobs.Service satisfies it.
|
|
type Jobs interface {
|
|
Start(ctx context.Context, svc model.Service, req jobs.StartRequest) (model.Job, error)
|
|
}
|
|
|
|
// Broker delivers change signals. *events.Broker satisfies it.
|
|
type Broker interface {
|
|
Subscribe() (<-chan events.Topic, func())
|
|
Publish(t events.Topic)
|
|
}
|
|
|
|
// Config holds the webhook settings.
|
|
type Config struct {
|
|
Token string // bearer token; empty disables the webhook
|
|
Timeout time.Duration // bounds the whole wait of one request
|
|
PublicURL string // base of job_url, without trailing slash
|
|
}
|
|
|
|
// Handler serves POST /update.
|
|
type Handler struct {
|
|
store Store
|
|
jobs Jobs
|
|
broker Broker
|
|
cfg Config
|
|
log *slog.Logger
|
|
|
|
now func() time.Time
|
|
pollEvery time.Duration
|
|
}
|
|
|
|
// New returns the handler for POST /update. It does its own bearer-token
|
|
// check, so mount it outside the OIDC login.
|
|
func New(store Store, jobs Jobs, broker Broker, cfg Config, log *slog.Logger) *Handler {
|
|
return &Handler{store: store, jobs: jobs, broker: broker, cfg: cfg, log: log,
|
|
now: time.Now, pollEvery: defaultPollEvery}
|
|
}
|
|
|
|
type request struct {
|
|
Image string `json:"image"`
|
|
Tag string `json:"tag"`
|
|
}
|
|
|
|
type result struct {
|
|
ServiceID int64 `json:"service_id"`
|
|
Service string `json:"service"`
|
|
Kind string `json:"kind"`
|
|
Mode string `json:"mode"`
|
|
Status string `json:"status"`
|
|
JobID int64 `json:"job_id,omitempty"`
|
|
JobURL string `json:"job_url,omitempty"`
|
|
ToRef string `json:"to_ref,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
}
|
|
|
|
type response struct {
|
|
RequestID string `json:"request_id"`
|
|
Image string `json:"image"`
|
|
Tag string `json:"tag,omitempty"`
|
|
Matched int `json:"matched"`
|
|
Results []result `json:"results"`
|
|
}
|
|
|
|
// ServeHTTP handles one webhook call. See the README for the contract.
|
|
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
started := h.now()
|
|
requestID := r.Header.Get("X-Request-ID")
|
|
if !requestIDPattern.MatchString(requestID) {
|
|
requestID = newRequestID()
|
|
}
|
|
w.Header().Set("X-Request-ID", requestID)
|
|
|
|
if h.cfg.Token == "" {
|
|
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "webhook disabled: UPDATER_API_KEY not set"})
|
|
return
|
|
}
|
|
if !h.authorized(r) {
|
|
writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
|
|
return
|
|
}
|
|
|
|
var req request
|
|
r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON body"})
|
|
return
|
|
}
|
|
if req.Image == "" {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "image is required"})
|
|
return
|
|
}
|
|
if req.Tag != "" && !tagPattern.MatchString(req.Tag) {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid tag"})
|
|
return
|
|
}
|
|
|
|
log := logging.FromContext(logging.WithRequestID(r.Context(), requestID), h.log)
|
|
repo := imageref.Parse(req.Image).Repo
|
|
// A client that disconnects must not abort marking or starting: that
|
|
// could leave a queued job no agent knows about.
|
|
work := context.WithoutCancel(r.Context())
|
|
|
|
services, err := h.store.ServicesByRepo(work, repo)
|
|
if err != nil {
|
|
log.Error("webhook_lookup_failed", "image", req.Image, "error", err)
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal error"})
|
|
return
|
|
}
|
|
if len(services) == 0 {
|
|
writeJSON(w, http.StatusNotFound, map[string]string{
|
|
"error": "no service uses image " + repo, "request_id": requestID})
|
|
return
|
|
}
|
|
|
|
// Subscribe before any job exists so no completion signal is missed.
|
|
sub, cancel := h.broker.Subscribe()
|
|
defer cancel()
|
|
|
|
resp := response{RequestID: requestID, Image: req.Image, Tag: req.Tag, Matched: len(services)}
|
|
pending := map[int]int64{} // index into resp.Results -> job ID
|
|
for _, svc := range services {
|
|
res, job := h.handle(work, svc, req.Tag, started, log)
|
|
resp.Results = append(resp.Results, res)
|
|
if job != nil {
|
|
pending[len(resp.Results)-1] = job.ID
|
|
}
|
|
}
|
|
h.broker.Publish(events.Services)
|
|
|
|
h.wait(r.Context(), sub, &resp, pending, started, log)
|
|
|
|
status := http.StatusOK
|
|
counts := map[string]int{}
|
|
for _, res := range resp.Results {
|
|
counts[res.Status]++
|
|
if res.Status == statusFailed || res.Status == statusTimeout {
|
|
status = http.StatusInternalServerError
|
|
}
|
|
}
|
|
log.Info("webhook_request", "image", req.Image, "tag", req.Tag, "matched", resp.Matched,
|
|
"succeeded", counts[statusSucceeded], "failed", counts[statusFailed], "timeout", counts[statusTimeout],
|
|
"marked", counts[statusMarked], "busy", counts[statusBusy], "refused", counts[statusRefused])
|
|
writeJSON(w, status, resp)
|
|
}
|
|
|
|
// handle records the available ref on svc and, in auto mode, starts an update.
|
|
// It returns the result and, if the job is still active, that job.
|
|
func (h *Handler) handle(ctx context.Context, svc model.Service, tag string, started time.Time, log *slog.Logger) (result, *model.Job) {
|
|
res := result{ServiceID: svc.ID, Service: svc.DisplayName(), Kind: string(svc.Kind), Mode: svc.UpdateMode}
|
|
|
|
// The ref is built from the service's own repo, never from the request
|
|
// string, so a request cannot point a service at another registry.
|
|
var ref string
|
|
if tag != "" {
|
|
ref = svc.ImageRepo + ":" + tag
|
|
} else {
|
|
ref = jobs.FollowedRef(svc.ActualRef)
|
|
}
|
|
if ref == "" {
|
|
return h.finish(res, statusRefused, "no tag given and the service follows no tag", started), nil
|
|
}
|
|
res.ToRef = ref
|
|
err := h.store.SetAvailable(ctx, svc.ID, model.Available{Ref: ref, Source: model.SourceWebhook, At: h.now()})
|
|
if err != nil {
|
|
log.Error("webhook_set_available_failed", "service_id", svc.ID, "error", err)
|
|
return h.finish(res, statusFailed, "could not record available version", started), nil
|
|
}
|
|
if svc.UpdateMode != model.ModeAuto {
|
|
return h.finish(res, statusMarked, "", started), nil
|
|
}
|
|
|
|
job, err := h.jobs.Start(ctx, svc, jobs.StartRequest{Trigger: model.TriggerWebhook, Ref: ref})
|
|
var notUpdatable *model.NotUpdatableError
|
|
switch {
|
|
case errors.As(err, ¬Updatable):
|
|
return h.finish(res, statusRefused, notUpdatable.Reason, started), nil
|
|
case errors.Is(err, model.ErrJobActive):
|
|
h.setJob(&res, job)
|
|
return h.finish(res, statusBusy, err.Error(), started), nil
|
|
case err != nil:
|
|
log.Error("webhook_start_failed", "service_id", svc.ID, "error", err)
|
|
return h.finish(res, statusFailed, "could not start job", started), nil
|
|
}
|
|
h.setJob(&res, job)
|
|
if job.Active() {
|
|
return res, &job
|
|
}
|
|
h.resolveJob(&res, job, started)
|
|
return res, nil
|
|
}
|
|
|
|
func (h *Handler) setJob(res *result, job model.Job) {
|
|
res.JobID = job.ID
|
|
res.JobURL = h.cfg.PublicURL + "/jobs/" + strconv.FormatInt(job.ID, 10)
|
|
}
|
|
|
|
func (h *Handler) finish(res result, status, msg string, started time.Time) result {
|
|
res.Status, res.Error = status, msg
|
|
res.DurationMS = h.now().Sub(started).Milliseconds()
|
|
return res
|
|
}
|
|
|
|
// resolveJob sets the result from a job that is no longer active.
|
|
func (h *Handler) resolveJob(res *result, job model.Job, started time.Time) {
|
|
if job.Status == model.JobSucceeded {
|
|
*res = h.finish(*res, statusSucceeded, "", started)
|
|
return
|
|
}
|
|
*res = h.finish(*res, statusFailed, job.Error, started)
|
|
}
|
|
|
|
// wait blocks until every pending job has finished, the timeout passes or
|
|
// the client goes away. Jobs still pending then get status "timeout"; the
|
|
// master never cancels or fails them.
|
|
func (h *Handler) wait(parent context.Context, sub <-chan events.Topic, resp *response,
|
|
pending map[int]int64, started time.Time, log *slog.Logger) {
|
|
if len(pending) == 0 {
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(parent, h.cfg.Timeout)
|
|
defer cancel()
|
|
ticker := time.NewTicker(h.pollEvery)
|
|
defer ticker.Stop()
|
|
|
|
for len(pending) > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
// A job may have finished while the timer fired: look once more,
|
|
// on a context that outlives the request.
|
|
final, stop := context.WithTimeout(context.WithoutCancel(parent), finalReadTimeout)
|
|
h.readPending(final, resp, pending, started, log)
|
|
stop()
|
|
for i := range pending {
|
|
resp.Results[i] = h.finish(resp.Results[i], statusTimeout, "still running when the request timed out", started)
|
|
}
|
|
return
|
|
case t := <-sub:
|
|
if t != events.Jobs {
|
|
continue
|
|
}
|
|
case <-ticker.C:
|
|
}
|
|
h.readPending(ctx, resp, pending, started, log)
|
|
}
|
|
}
|
|
|
|
// readPending re-reads every pending job and resolves the ones that are done.
|
|
func (h *Handler) readPending(ctx context.Context, resp *response, pending map[int]int64,
|
|
started time.Time, log *slog.Logger) {
|
|
for i, id := range pending {
|
|
job, err := h.store.GetJob(ctx, id)
|
|
if err != nil {
|
|
log.Warn("webhook_get_job_failed", "job_id", id, "error", err)
|
|
continue
|
|
}
|
|
if !job.Active() {
|
|
h.resolveJob(&resp.Results[i], job, started)
|
|
delete(pending, i)
|
|
}
|
|
}
|
|
}
|
|
|
|
// authorized reports whether the request carries the configured bearer token.
|
|
// Both sides are hashed so the comparison takes the same time whatever the
|
|
// length of the supplied token.
|
|
func (h *Handler) authorized(r *http.Request) bool {
|
|
const prefix = "Bearer "
|
|
got, ok := strings.CutPrefix(r.Header.Get("Authorization"), prefix)
|
|
if !ok || got == "" {
|
|
return false
|
|
}
|
|
a, b := sha256.Sum256([]byte(got)), sha256.Sum256([]byte(h.cfg.Token))
|
|
return subtle.ConstantTimeCompare(a[:], b[:]) == 1
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, status int, body any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(body)
|
|
}
|
|
|
|
// newRequestID returns a random 128-bit hex ID.
|
|
func newRequestID() string {
|
|
var b [16]byte
|
|
_, _ = rand.Read(b[:])
|
|
return hex.EncodeToString(b[:])
|
|
}
|