Files
package-updater/internal/master/webhook/webhook.go
T
shcizo b1d6e5e705 fix: review fixes for webhook and auto mode (#8)
Clear available_* only when the deployed ref equals it, drop the unused
poll source, stop leaking internal errors to CI, use the request-scoped
logger, validate X-Request-ID, re-read jobs once after the timeout,
report refused when no target ref exists, and tidy UI, deploy and docs.
2026-10-12 20:48:43 +02:00

351 lines
12 KiB
Go

// Package webhook implements POST /update, the endpoint CI calls after
// pushing an image. It records the pushed tag as "available" on every service
// that uses the image, starts an update for the services in auto mode and
// waits for those updates to finish.
package webhook
import (
"context"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"encoding/json"
"errors"
"log/slog"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/shcizo/package-updater/internal/imageref"
"github.com/shcizo/package-updater/internal/logging"
"github.com/shcizo/package-updater/internal/master/events"
"github.com/shcizo/package-updater/internal/master/jobs"
"github.com/shcizo/package-updater/internal/master/model"
)
// maxBodyBytes caps the request body; a valid request is a few hundred bytes.
const maxBodyBytes = 64 << 10
// defaultPollEvery is how often pending jobs are re-read from the store when
// no broker signal arrives (signals to a slow subscriber can be dropped).
const defaultPollEvery = 5 * time.Second
// finalReadTimeout bounds the last read of pending jobs after the wait ends.
const finalReadTimeout = 2 * time.Second
// tagPattern is Docker's tag grammar. It keeps spaces, colons and "@" out of
// the ref the agent is asked to deploy.
var tagPattern = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`)
// requestIDPattern limits a caller-supplied X-Request-ID to what is safe to
// echo in a header and write to the log.
var requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9._-]{1,128}$`)
// Result statuses.
const (
statusSucceeded = "succeeded"
statusFailed = "failed"
statusTimeout = "timeout"
statusMarked = "marked"
statusBusy = "busy"
statusRefused = "refused"
)
// Store is the storage the webhook needs. *store.Store satisfies it.
type Store interface {
ServicesByRepo(ctx context.Context, repo string) ([]model.Service, error)
SetAvailable(ctx context.Context, id int64, a model.Available) error
GetJob(ctx context.Context, id int64) (model.Job, error)
}
// Jobs starts updates. *jobs.Service satisfies it.
type Jobs interface {
Start(ctx context.Context, svc model.Service, req jobs.StartRequest) (model.Job, error)
}
// Broker delivers change signals. *events.Broker satisfies it.
type Broker interface {
Subscribe() (<-chan events.Topic, func())
Publish(t events.Topic)
}
// Config holds the webhook settings.
type Config struct {
Token string // bearer token; empty disables the webhook
Timeout time.Duration // bounds the whole wait of one request
PublicURL string // base of job_url, without trailing slash
}
// Handler serves POST /update.
type Handler struct {
store Store
jobs Jobs
broker Broker
cfg Config
log *slog.Logger
now func() time.Time
pollEvery time.Duration
}
// New returns the handler for POST /update. It does its own bearer-token
// check, so mount it outside the OIDC login.
func New(store Store, jobs Jobs, broker Broker, cfg Config, log *slog.Logger) *Handler {
return &Handler{store: store, jobs: jobs, broker: broker, cfg: cfg, log: log,
now: time.Now, pollEvery: defaultPollEvery}
}
type request struct {
Image string `json:"image"`
Tag string `json:"tag"`
}
type result struct {
ServiceID int64 `json:"service_id"`
Service string `json:"service"`
Kind string `json:"kind"`
Mode string `json:"mode"`
Status string `json:"status"`
JobID int64 `json:"job_id,omitempty"`
JobURL string `json:"job_url,omitempty"`
ToRef string `json:"to_ref,omitempty"`
Error string `json:"error,omitempty"`
DurationMS int64 `json:"duration_ms"`
}
type response struct {
RequestID string `json:"request_id"`
Image string `json:"image"`
Tag string `json:"tag,omitempty"`
Matched int `json:"matched"`
Results []result `json:"results"`
}
// ServeHTTP handles one webhook call. See the README for the contract.
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
started := h.now()
requestID := r.Header.Get("X-Request-ID")
if !requestIDPattern.MatchString(requestID) {
requestID = newRequestID()
}
w.Header().Set("X-Request-ID", requestID)
if h.cfg.Token == "" {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "webhook disabled: UPDATER_API_KEY not set"})
return
}
if !h.authorized(r) {
writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
return
}
var req request
r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON body"})
return
}
if req.Image == "" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "image is required"})
return
}
if req.Tag != "" && !tagPattern.MatchString(req.Tag) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid tag"})
return
}
log := logging.FromContext(logging.WithRequestID(r.Context(), requestID), h.log)
repo := imageref.Parse(req.Image).Repo
// A client that disconnects must not abort marking or starting: that
// could leave a queued job no agent knows about.
work := context.WithoutCancel(r.Context())
services, err := h.store.ServicesByRepo(work, repo)
if err != nil {
log.Error("webhook_lookup_failed", "image", req.Image, "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal error"})
return
}
if len(services) == 0 {
writeJSON(w, http.StatusNotFound, map[string]string{
"error": "no service uses image " + repo, "request_id": requestID})
return
}
// Subscribe before any job exists so no completion signal is missed.
sub, cancel := h.broker.Subscribe()
defer cancel()
resp := response{RequestID: requestID, Image: req.Image, Tag: req.Tag, Matched: len(services)}
pending := map[int]int64{} // index into resp.Results -> job ID
for _, svc := range services {
res, job := h.handle(work, svc, req.Tag, started, log)
resp.Results = append(resp.Results, res)
if job != nil {
pending[len(resp.Results)-1] = job.ID
}
}
h.broker.Publish(events.Services)
h.wait(r.Context(), sub, &resp, pending, started, log)
status := http.StatusOK
counts := map[string]int{}
for _, res := range resp.Results {
counts[res.Status]++
if res.Status == statusFailed || res.Status == statusTimeout {
status = http.StatusInternalServerError
}
}
log.Info("webhook_request", "image", req.Image, "tag", req.Tag, "matched", resp.Matched,
"succeeded", counts[statusSucceeded], "failed", counts[statusFailed], "timeout", counts[statusTimeout],
"marked", counts[statusMarked], "busy", counts[statusBusy], "refused", counts[statusRefused])
writeJSON(w, status, resp)
}
// handle records the available ref on svc and, in auto mode, starts an update.
// It returns the result and, if the job is still active, that job.
func (h *Handler) handle(ctx context.Context, svc model.Service, tag string, started time.Time, log *slog.Logger) (result, *model.Job) {
res := result{ServiceID: svc.ID, Service: svc.DisplayName(), Kind: string(svc.Kind), Mode: svc.UpdateMode}
// The ref is built from the service's own repo, never from the request
// string, so a request cannot point a service at another registry.
var ref string
if tag != "" {
ref = svc.ImageRepo + ":" + tag
} else {
ref = jobs.FollowedRef(svc.ActualRef)
}
if ref == "" {
return h.finish(res, statusRefused, "no tag given and the service follows no tag", started), nil
}
res.ToRef = ref
err := h.store.SetAvailable(ctx, svc.ID, model.Available{Ref: ref, Source: model.SourceWebhook, At: h.now()})
if err != nil {
log.Error("webhook_set_available_failed", "service_id", svc.ID, "error", err)
return h.finish(res, statusFailed, "could not record available version", started), nil
}
if svc.UpdateMode != model.ModeAuto {
return h.finish(res, statusMarked, "", started), nil
}
job, err := h.jobs.Start(ctx, svc, jobs.StartRequest{Trigger: model.TriggerWebhook, Ref: ref})
var notUpdatable *model.NotUpdatableError
switch {
case errors.As(err, &notUpdatable):
return h.finish(res, statusRefused, notUpdatable.Reason, started), nil
case errors.Is(err, model.ErrJobActive):
h.setJob(&res, job)
return h.finish(res, statusBusy, err.Error(), started), nil
case err != nil:
log.Error("webhook_start_failed", "service_id", svc.ID, "error", err)
return h.finish(res, statusFailed, "could not start job", started), nil
}
h.setJob(&res, job)
if job.Active() {
return res, &job
}
h.resolveJob(&res, job, started)
return res, nil
}
func (h *Handler) setJob(res *result, job model.Job) {
res.JobID = job.ID
res.JobURL = h.cfg.PublicURL + "/jobs/" + strconv.FormatInt(job.ID, 10)
}
func (h *Handler) finish(res result, status, msg string, started time.Time) result {
res.Status, res.Error = status, msg
res.DurationMS = h.now().Sub(started).Milliseconds()
return res
}
// resolveJob sets the result from a job that is no longer active.
func (h *Handler) resolveJob(res *result, job model.Job, started time.Time) {
if job.Status == model.JobSucceeded {
*res = h.finish(*res, statusSucceeded, "", started)
return
}
*res = h.finish(*res, statusFailed, job.Error, started)
}
// wait blocks until every pending job has finished, the timeout passes or
// the client goes away. Jobs still pending then get status "timeout"; the
// master never cancels or fails them.
func (h *Handler) wait(parent context.Context, sub <-chan events.Topic, resp *response,
pending map[int]int64, started time.Time, log *slog.Logger) {
if len(pending) == 0 {
return
}
ctx, cancel := context.WithTimeout(parent, h.cfg.Timeout)
defer cancel()
ticker := time.NewTicker(h.pollEvery)
defer ticker.Stop()
for len(pending) > 0 {
select {
case <-ctx.Done():
// A job may have finished while the timer fired: look once more,
// on a context that outlives the request.
final, stop := context.WithTimeout(context.WithoutCancel(parent), finalReadTimeout)
h.readPending(final, resp, pending, started, log)
stop()
for i := range pending {
resp.Results[i] = h.finish(resp.Results[i], statusTimeout, "still running when the request timed out", started)
}
return
case t := <-sub:
if t != events.Jobs {
continue
}
case <-ticker.C:
}
h.readPending(ctx, resp, pending, started, log)
}
}
// readPending re-reads every pending job and resolves the ones that are done.
func (h *Handler) readPending(ctx context.Context, resp *response, pending map[int]int64,
started time.Time, log *slog.Logger) {
for i, id := range pending {
job, err := h.store.GetJob(ctx, id)
if err != nil {
log.Warn("webhook_get_job_failed", "job_id", id, "error", err)
continue
}
if !job.Active() {
h.resolveJob(&resp.Results[i], job, started)
delete(pending, i)
}
}
}
// authorized reports whether the request carries the configured bearer token.
// Both sides are hashed so the comparison takes the same time whatever the
// length of the supplied token.
func (h *Handler) authorized(r *http.Request) bool {
const prefix = "Bearer "
got, ok := strings.CutPrefix(r.Header.Get("Authorization"), prefix)
if !ok || got == "" {
return false
}
a, b := sha256.Sum256([]byte(got)), sha256.Sum256([]byte(h.cfg.Token))
return subtle.ConstantTimeCompare(a[:], b[:]) == 1
}
func writeJSON(w http.ResponseWriter, status int, body any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(body)
}
// newRequestID returns a random 128-bit hex ID.
func newRequestID() string {
var b [16]byte
_, _ = rand.Read(b[:])
return hex.EncodeToString(b[:])
}