Files
package-updater/internal/agent/conn/conn.go

341 lines
9.7 KiB
Go

// Package conn keeps an agent connected to the master: it dials over pinned
// TLS, announces itself, sends its inventory, runs the jobs the master sends
// and streams changes and job results until the connection drops, then
// reconnects with backoff.
package conn
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"sync"
"time"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"github.com/shcizo/package-updater/internal/agent/inventory"
"github.com/shcizo/package-updater/internal/agent/updater"
"github.com/shcizo/package-updater/internal/pin"
"github.com/shcizo/package-updater/internal/protocol"
)
// readLimit caps a single incoming message. Matches the master.
const readLimit = 1 << 20
// dialTimeout bounds the TLS + WebSocket handshake so a master that accepts
// TCP but never answers does not hang the agent forever.
const dialTimeout = 30 * time.Second
// writeTimeout bounds one websocket write so a stalled peer ends the session
// instead of blocking the writer forever.
const writeTimeout = 10 * time.Second
// defaultProgressInterval is how often job output is batched to the master.
const defaultProgressInterval = 500 * time.Millisecond
// Source supplies what the agent reports. *inventory.Collector satisfies it.
type Source interface {
Hello(ctx context.Context) (protocol.Hello, error)
Snapshot(ctx context.Context) ([]protocol.Service, error)
}
// Jobs is the agent's job queue as the connection sees it. *updater.Runner
// satisfies it.
type Jobs interface {
// Submit queues a job; a job ID already held is ignored.
Submit(id string, start protocol.JobStart)
// Ack forgets a finished job the master has stored.
Ack(id string)
// Known lists every job ID held: queued, running, or finished unacked.
Known() []string
// TakeProgress returns output produced since the previous call.
TakeProgress() []updater.Progress
// Pending returns results the master has not acknowledged yet.
Pending() []updater.Finished
// Changed receives when there is new progress or a new result.
Changed() <-chan struct{}
}
// Config holds connection settings. Token is sent as a bearer header and must
// never be logged.
type Config struct {
URL string // wss://host:port/agent
Token string // AGENT_TOKEN
Fingerprint string // MASTER_KEY
Version string
MinBackoff time.Duration // default 1s
MaxBackoff time.Duration // default 30s
ProgressInterval time.Duration // default 500ms
}
// Agent is the agent's side of the master connection.
type Agent struct {
cfg Config
src Source
jobs Jobs
log *slog.Logger
changes chan struct{}
}
// New returns an Agent. Call Run to connect.
func New(cfg Config, src Source, jobs Jobs, log *slog.Logger) *Agent {
if cfg.MinBackoff == 0 {
cfg.MinBackoff = time.Second
}
if cfg.MaxBackoff == 0 {
cfg.MaxBackoff = 30 * time.Second
}
if cfg.ProgressInterval == 0 {
cfg.ProgressInterval = defaultProgressInterval
}
return &Agent{cfg: cfg, src: src, jobs: jobs, log: log, changes: make(chan struct{}, 1)}
}
// Notify tells the agent its inventory may have changed. It never blocks;
// several calls before the agent gets to them collapse into one refresh.
func (a *Agent) Notify() {
select {
case a.changes <- struct{}{}:
default:
}
}
// Run keeps the agent connected until ctx ends, then returns nil.
func (a *Agent) Run(ctx context.Context) error {
backoff := a.cfg.MinBackoff
for {
connected, err := a.session(ctx)
if ctx.Err() != nil {
return nil
}
if connected {
backoff = a.cfg.MinBackoff
}
a.log.Warn("master_connection_lost", "error", err, "retry_in", backoff.String())
select {
case <-ctx.Done():
return nil
case <-time.After(backoff):
}
backoff = min(backoff*2, a.cfg.MaxBackoff)
}
}
// session runs one connection until it fails. connected reports whether the
// hello was delivered, so Run resets its backoff only after real progress.
//
// Two goroutines serve the connection: a reader, which handles what the
// master sends, and a writer, the only goroutine that writes to the socket.
func (a *Agent) session(parent context.Context) (connected bool, err error) {
httpClient := &http.Client{Transport: &http.Transport{TLSClientConfig: pin.ClientConfig(a.cfg.Fingerprint)}}
// The dial context only bounds the handshake; coder/websocket does not
// tie the established connection to it, so cancelling afterwards is safe.
dialCtx, cancelDial := context.WithTimeout(parent, dialTimeout)
c, _, err := websocket.Dial(dialCtx, a.cfg.URL, &websocket.DialOptions{
HTTPClient: httpClient,
HTTPHeader: http.Header{"Authorization": []string{"Bearer " + a.cfg.Token}},
})
cancelDial()
if err != nil {
return false, fmt.Errorf("dial master: %w", err)
}
c.SetReadLimit(readLimit)
ctx, cancel := context.WithCancelCause(parent)
out := make(chan protocol.Envelope, 64)
var wg sync.WaitGroup
defer func() {
cancel(nil)
if parent.Err() != nil {
_ = c.Close(websocket.StatusNormalClosure, "")
}
c.CloseNow()
wg.Wait()
}()
wg.Add(2)
go func() { defer wg.Done(); a.writeLoop(ctx, cancel, c, out) }()
go func() { defer wg.Done(); a.readLoop(ctx, cancel, c) }()
// send queues an envelope for the writer. It gives up when the session ends.
send := func(typ string, payload any) error {
env, err := protocol.Encode(typ, payload)
if err != nil {
return err
}
return enqueue(ctx, out, env)
}
sendJob := func(typ, id string, payload any) error {
env, err := protocol.EncodeJob(typ, id, payload)
if err != nil {
return err
}
return enqueue(ctx, out, env)
}
fail := func(connected bool, err error) (bool, error) {
if cause := context.Cause(ctx); cause != nil && !errors.Is(cause, context.Canceled) {
return connected, cause
}
return connected, err
}
hello, err := a.src.Hello(ctx)
if err != nil {
return fail(false, err)
}
hello.AgentVersion = a.cfg.Version
hello.ProtocolVersion = protocol.Version
hello.KnownJobs = a.jobs.Known()
if err := send(protocol.TypeHello, hello); err != nil {
return fail(false, err)
}
last, err := a.src.Snapshot(ctx)
if err != nil {
return fail(true, err)
}
if err := send(protocol.TypeSnapshot, protocol.Snapshot{Services: last}); err != nil {
return fail(true, err)
}
a.log.Info("master_connected", "services", len(last), "known_jobs", len(hello.KnownJobs))
sent := make(map[string]bool) // job results already sent on this connection
flush := func() error {
for _, p := range a.jobs.TakeProgress() {
if err := sendJob(protocol.TypeJobProgress, p.ID, protocol.JobProgress{Output: p.Output}); err != nil {
return err
}
}
for _, f := range a.jobs.Pending() {
if sent[f.ID] {
continue
}
if err := sendJob(protocol.TypeJobResult, f.ID, f.Result); err != nil {
return err
}
sent[f.ID] = true
}
return nil
}
// flush is throttled to one call per ProgressInterval; a signal that
// arrives early arms the timer for the remainder instead of being lost.
lastFlush := time.Time{}
var timer *time.Timer
var timerC <-chan time.Time
defer func() {
if timer != nil {
timer.Stop()
}
}()
maybeFlush := func() error {
wait := a.cfg.ProgressInterval - time.Since(lastFlush)
if wait > 0 {
if timerC == nil {
timer = time.NewTimer(wait)
timerC = timer.C
}
return nil
}
lastFlush = time.Now()
return flush()
}
if err := maybeFlush(); err != nil {
return fail(true, err)
}
jobsChanged := a.jobs.Changed()
for {
select {
case <-ctx.Done():
return fail(true, ctx.Err())
case <-jobsChanged:
if err := maybeFlush(); err != nil {
return fail(true, err)
}
case <-timerC:
timerC = nil
lastFlush = time.Now()
if err := flush(); err != nil {
return fail(true, err)
}
case <-a.changes:
next, err := a.src.Snapshot(ctx)
if err != nil {
return fail(true, err)
}
upserts, removes := inventory.Diff(last, next)
for _, s := range upserts {
if err := send(protocol.TypeUpsert, protocol.Upsert{Service: s}); err != nil {
return fail(true, err)
}
}
for _, k := range removes {
if err := send(protocol.TypeRemove, protocol.Remove{Key: k}); err != nil {
return fail(true, err)
}
}
last = next
}
}
}
// enqueue hands env to the writer, or gives up when the session ends.
func enqueue(ctx context.Context, out chan<- protocol.Envelope, env protocol.Envelope) error {
select {
case out <- env:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// writeLoop is the only goroutine that writes to c. A failed or timed-out
// write ends the session.
func (a *Agent) writeLoop(ctx context.Context, cancel context.CancelCauseFunc, c *websocket.Conn, out <-chan protocol.Envelope) {
for {
select {
case <-ctx.Done():
return
case env := <-out:
wctx, wcancel := context.WithTimeout(ctx, writeTimeout)
err := wsjson.Write(wctx, c, env)
wcancel()
if err != nil {
cancel(fmt.Errorf("send %s: %w", env.Type, err))
return
}
}
}
}
// readLoop handles what the master sends. A message it cannot parse ends the
// session; a message type it does not know is logged and ignored, so a newer
// master can add message types without breaking this agent.
func (a *Agent) readLoop(ctx context.Context, cancel context.CancelCauseFunc, c *websocket.Conn) {
for {
var env protocol.Envelope
if err := wsjson.Read(ctx, c, &env); err != nil {
cancel(fmt.Errorf("read: %w", err))
return
}
switch env.Type {
case protocol.TypeJobStart:
start, err := protocol.Decode[protocol.JobStart](env)
if err != nil || env.ID == "" {
cancel(fmt.Errorf("malformed job.start (id %q): %v", env.ID, err))
return
}
a.jobs.Submit(env.ID, start)
case protocol.TypeJobAck:
// JobAck has no fields, so the payload is not inspected.
a.jobs.Ack(env.ID)
default:
a.log.Warn("unknown_message_type", "type", env.Type)
}
}
}