341 lines
9.7 KiB
Go
341 lines
9.7 KiB
Go
// Package conn keeps an agent connected to the master: it dials over pinned
|
|
// TLS, announces itself, sends its inventory, runs the jobs the master sends
|
|
// and streams changes and job results until the connection drops, then
|
|
// reconnects with backoff.
|
|
package conn
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/coder/websocket"
|
|
"github.com/coder/websocket/wsjson"
|
|
|
|
"github.com/shcizo/package-updater/internal/agent/inventory"
|
|
"github.com/shcizo/package-updater/internal/agent/updater"
|
|
"github.com/shcizo/package-updater/internal/pin"
|
|
"github.com/shcizo/package-updater/internal/protocol"
|
|
)
|
|
|
|
// readLimit caps a single incoming message. Matches the master.
|
|
const readLimit = 1 << 20
|
|
|
|
// dialTimeout bounds the TLS + WebSocket handshake so a master that accepts
|
|
// TCP but never answers does not hang the agent forever.
|
|
const dialTimeout = 30 * time.Second
|
|
|
|
// writeTimeout bounds one websocket write so a stalled peer ends the session
|
|
// instead of blocking the writer forever.
|
|
const writeTimeout = 10 * time.Second
|
|
|
|
// defaultProgressInterval is how often job output is batched to the master.
|
|
const defaultProgressInterval = 500 * time.Millisecond
|
|
|
|
// Source supplies what the agent reports. *inventory.Collector satisfies it.
|
|
type Source interface {
|
|
Hello(ctx context.Context) (protocol.Hello, error)
|
|
Snapshot(ctx context.Context) ([]protocol.Service, error)
|
|
}
|
|
|
|
// Jobs is the agent's job queue as the connection sees it. *updater.Runner
|
|
// satisfies it.
|
|
type Jobs interface {
|
|
// Submit queues a job; a job ID already held is ignored.
|
|
Submit(id string, start protocol.JobStart)
|
|
// Ack forgets a finished job the master has stored.
|
|
Ack(id string)
|
|
// Known lists every job ID held: queued, running, or finished unacked.
|
|
Known() []string
|
|
// TakeProgress returns output produced since the previous call.
|
|
TakeProgress() []updater.Progress
|
|
// Pending returns results the master has not acknowledged yet.
|
|
Pending() []updater.Finished
|
|
// Changed receives when there is new progress or a new result.
|
|
Changed() <-chan struct{}
|
|
}
|
|
|
|
// Config holds connection settings. Token is sent as a bearer header and must
|
|
// never be logged.
|
|
type Config struct {
|
|
URL string // wss://host:port/agent
|
|
Token string // AGENT_TOKEN
|
|
Fingerprint string // MASTER_KEY
|
|
Version string
|
|
MinBackoff time.Duration // default 1s
|
|
MaxBackoff time.Duration // default 30s
|
|
ProgressInterval time.Duration // default 500ms
|
|
}
|
|
|
|
// Agent is the agent's side of the master connection.
|
|
type Agent struct {
|
|
cfg Config
|
|
src Source
|
|
jobs Jobs
|
|
log *slog.Logger
|
|
changes chan struct{}
|
|
}
|
|
|
|
// New returns an Agent. Call Run to connect.
|
|
func New(cfg Config, src Source, jobs Jobs, log *slog.Logger) *Agent {
|
|
if cfg.MinBackoff == 0 {
|
|
cfg.MinBackoff = time.Second
|
|
}
|
|
if cfg.MaxBackoff == 0 {
|
|
cfg.MaxBackoff = 30 * time.Second
|
|
}
|
|
if cfg.ProgressInterval == 0 {
|
|
cfg.ProgressInterval = defaultProgressInterval
|
|
}
|
|
return &Agent{cfg: cfg, src: src, jobs: jobs, log: log, changes: make(chan struct{}, 1)}
|
|
}
|
|
|
|
// Notify tells the agent its inventory may have changed. It never blocks;
|
|
// several calls before the agent gets to them collapse into one refresh.
|
|
func (a *Agent) Notify() {
|
|
select {
|
|
case a.changes <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// Run keeps the agent connected until ctx ends, then returns nil.
|
|
func (a *Agent) Run(ctx context.Context) error {
|
|
backoff := a.cfg.MinBackoff
|
|
for {
|
|
connected, err := a.session(ctx)
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
if connected {
|
|
backoff = a.cfg.MinBackoff
|
|
}
|
|
a.log.Warn("master_connection_lost", "error", err, "retry_in", backoff.String())
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case <-time.After(backoff):
|
|
}
|
|
backoff = min(backoff*2, a.cfg.MaxBackoff)
|
|
}
|
|
}
|
|
|
|
// session runs one connection until it fails. connected reports whether the
|
|
// hello was delivered, so Run resets its backoff only after real progress.
|
|
//
|
|
// Two goroutines serve the connection: a reader, which handles what the
|
|
// master sends, and a writer, the only goroutine that writes to the socket.
|
|
func (a *Agent) session(parent context.Context) (connected bool, err error) {
|
|
httpClient := &http.Client{Transport: &http.Transport{TLSClientConfig: pin.ClientConfig(a.cfg.Fingerprint)}}
|
|
// The dial context only bounds the handshake; coder/websocket does not
|
|
// tie the established connection to it, so cancelling afterwards is safe.
|
|
dialCtx, cancelDial := context.WithTimeout(parent, dialTimeout)
|
|
c, _, err := websocket.Dial(dialCtx, a.cfg.URL, &websocket.DialOptions{
|
|
HTTPClient: httpClient,
|
|
HTTPHeader: http.Header{"Authorization": []string{"Bearer " + a.cfg.Token}},
|
|
})
|
|
cancelDial()
|
|
if err != nil {
|
|
return false, fmt.Errorf("dial master: %w", err)
|
|
}
|
|
c.SetReadLimit(readLimit)
|
|
|
|
ctx, cancel := context.WithCancelCause(parent)
|
|
out := make(chan protocol.Envelope, 64)
|
|
var wg sync.WaitGroup
|
|
defer func() {
|
|
cancel(nil)
|
|
if parent.Err() != nil {
|
|
_ = c.Close(websocket.StatusNormalClosure, "")
|
|
}
|
|
c.CloseNow()
|
|
wg.Wait()
|
|
}()
|
|
|
|
wg.Add(2)
|
|
go func() { defer wg.Done(); a.writeLoop(ctx, cancel, c, out) }()
|
|
go func() { defer wg.Done(); a.readLoop(ctx, cancel, c) }()
|
|
|
|
// send queues an envelope for the writer. It gives up when the session ends.
|
|
send := func(typ string, payload any) error {
|
|
env, err := protocol.Encode(typ, payload)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return enqueue(ctx, out, env)
|
|
}
|
|
sendJob := func(typ, id string, payload any) error {
|
|
env, err := protocol.EncodeJob(typ, id, payload)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return enqueue(ctx, out, env)
|
|
}
|
|
fail := func(connected bool, err error) (bool, error) {
|
|
if cause := context.Cause(ctx); cause != nil && !errors.Is(cause, context.Canceled) {
|
|
return connected, cause
|
|
}
|
|
return connected, err
|
|
}
|
|
|
|
hello, err := a.src.Hello(ctx)
|
|
if err != nil {
|
|
return fail(false, err)
|
|
}
|
|
hello.AgentVersion = a.cfg.Version
|
|
hello.ProtocolVersion = protocol.Version
|
|
hello.KnownJobs = a.jobs.Known()
|
|
if err := send(protocol.TypeHello, hello); err != nil {
|
|
return fail(false, err)
|
|
}
|
|
|
|
last, err := a.src.Snapshot(ctx)
|
|
if err != nil {
|
|
return fail(true, err)
|
|
}
|
|
if err := send(protocol.TypeSnapshot, protocol.Snapshot{Services: last}); err != nil {
|
|
return fail(true, err)
|
|
}
|
|
a.log.Info("master_connected", "services", len(last), "known_jobs", len(hello.KnownJobs))
|
|
|
|
sent := make(map[string]bool) // job results already sent on this connection
|
|
flush := func() error {
|
|
for _, p := range a.jobs.TakeProgress() {
|
|
if err := sendJob(protocol.TypeJobProgress, p.ID, protocol.JobProgress{Output: p.Output}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, f := range a.jobs.Pending() {
|
|
if sent[f.ID] {
|
|
continue
|
|
}
|
|
if err := sendJob(protocol.TypeJobResult, f.ID, f.Result); err != nil {
|
|
return err
|
|
}
|
|
sent[f.ID] = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// flush is throttled to one call per ProgressInterval; a signal that
|
|
// arrives early arms the timer for the remainder instead of being lost.
|
|
lastFlush := time.Time{}
|
|
var timer *time.Timer
|
|
var timerC <-chan time.Time
|
|
defer func() {
|
|
if timer != nil {
|
|
timer.Stop()
|
|
}
|
|
}()
|
|
maybeFlush := func() error {
|
|
wait := a.cfg.ProgressInterval - time.Since(lastFlush)
|
|
if wait > 0 {
|
|
if timerC == nil {
|
|
timer = time.NewTimer(wait)
|
|
timerC = timer.C
|
|
}
|
|
return nil
|
|
}
|
|
lastFlush = time.Now()
|
|
return flush()
|
|
}
|
|
if err := maybeFlush(); err != nil {
|
|
return fail(true, err)
|
|
}
|
|
|
|
jobsChanged := a.jobs.Changed()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return fail(true, ctx.Err())
|
|
case <-jobsChanged:
|
|
if err := maybeFlush(); err != nil {
|
|
return fail(true, err)
|
|
}
|
|
case <-timerC:
|
|
timerC = nil
|
|
lastFlush = time.Now()
|
|
if err := flush(); err != nil {
|
|
return fail(true, err)
|
|
}
|
|
case <-a.changes:
|
|
next, err := a.src.Snapshot(ctx)
|
|
if err != nil {
|
|
return fail(true, err)
|
|
}
|
|
upserts, removes := inventory.Diff(last, next)
|
|
for _, s := range upserts {
|
|
if err := send(protocol.TypeUpsert, protocol.Upsert{Service: s}); err != nil {
|
|
return fail(true, err)
|
|
}
|
|
}
|
|
for _, k := range removes {
|
|
if err := send(protocol.TypeRemove, protocol.Remove{Key: k}); err != nil {
|
|
return fail(true, err)
|
|
}
|
|
}
|
|
last = next
|
|
}
|
|
}
|
|
}
|
|
|
|
// enqueue hands env to the writer, or gives up when the session ends.
|
|
func enqueue(ctx context.Context, out chan<- protocol.Envelope, env protocol.Envelope) error {
|
|
select {
|
|
case out <- env:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
// writeLoop is the only goroutine that writes to c. A failed or timed-out
|
|
// write ends the session.
|
|
func (a *Agent) writeLoop(ctx context.Context, cancel context.CancelCauseFunc, c *websocket.Conn, out <-chan protocol.Envelope) {
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case env := <-out:
|
|
wctx, wcancel := context.WithTimeout(ctx, writeTimeout)
|
|
err := wsjson.Write(wctx, c, env)
|
|
wcancel()
|
|
if err != nil {
|
|
cancel(fmt.Errorf("send %s: %w", env.Type, err))
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// readLoop handles what the master sends. A message it cannot parse ends the
|
|
// session; a message type it does not know is logged and ignored, so a newer
|
|
// master can add message types without breaking this agent.
|
|
func (a *Agent) readLoop(ctx context.Context, cancel context.CancelCauseFunc, c *websocket.Conn) {
|
|
for {
|
|
var env protocol.Envelope
|
|
if err := wsjson.Read(ctx, c, &env); err != nil {
|
|
cancel(fmt.Errorf("read: %w", err))
|
|
return
|
|
}
|
|
switch env.Type {
|
|
case protocol.TypeJobStart:
|
|
start, err := protocol.Decode[protocol.JobStart](env)
|
|
if err != nil || env.ID == "" {
|
|
cancel(fmt.Errorf("malformed job.start (id %q): %v", env.ID, err))
|
|
return
|
|
}
|
|
a.jobs.Submit(env.ID, start)
|
|
case protocol.TypeJobAck:
|
|
// JobAck has no fields, so the payload is not inspected.
|
|
a.jobs.Ack(env.ID)
|
|
default:
|
|
a.log.Warn("unknown_message_type", "type", env.Type)
|
|
}
|
|
}
|
|
}
|