test: jobs end to end over pinned TLS (#7)
This commit is contained in:
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/shcizo/package-updater/internal/agent/conn"
|
||||
"github.com/shcizo/package-updater/internal/agent/updater"
|
||||
"github.com/shcizo/package-updater/internal/agenttoken"
|
||||
"github.com/shcizo/package-updater/internal/master/events"
|
||||
"github.com/shcizo/package-updater/internal/master/hub"
|
||||
@@ -35,6 +36,129 @@ type memStore struct {
|
||||
lookups int
|
||||
snapshots int
|
||||
services map[string]protocol.Service
|
||||
|
||||
// Jobs: in-memory twins of the store methods in C5, with the same
|
||||
// ownership and "only active jobs change" rules.
|
||||
svcRows map[int64]model.Service
|
||||
jobs map[int64]*model.Job
|
||||
nextJob int64
|
||||
finishApplied int // FinishJob calls that returned true
|
||||
}
|
||||
|
||||
// seedService adds a service row that jobs can be triggered for.
|
||||
func (m *memStore) seedService(svc model.Service) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.svcRows == nil {
|
||||
m.svcRows = map[int64]model.Service{}
|
||||
}
|
||||
m.svcRows[svc.ID] = svc
|
||||
}
|
||||
|
||||
func (m *memStore) GetService(_ context.Context, id int64) (model.Service, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
svc, ok := m.svcRows[id]
|
||||
if !ok {
|
||||
return model.Service{}, model.ErrNotFound
|
||||
}
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
func (m *memStore) CreateJob(_ context.Context, nj model.NewJob) (model.Job, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.jobs == nil {
|
||||
m.jobs = map[int64]*model.Job{}
|
||||
}
|
||||
for _, j := range m.jobs {
|
||||
if j.ServiceID == nj.ServiceID && j.Active() {
|
||||
return *j, model.ErrJobActive
|
||||
}
|
||||
}
|
||||
m.nextJob++
|
||||
j := &model.Job{ID: m.nextJob, ServiceID: nj.ServiceID, AgentID: nj.AgentID, FromRef: nj.FromRef, ToRef: nj.ToRef,
|
||||
Trigger: nj.Trigger, RequestedBy: nj.RequestedBy, Status: model.JobQueued, CreatedAt: nj.CreatedAt}
|
||||
m.jobs[j.ID] = j
|
||||
return *j, nil
|
||||
}
|
||||
|
||||
func (m *memStore) FailJob(_ context.Context, id int64, msg string, at time.Time) (bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
j, ok := m.jobs[id]
|
||||
if !ok || !j.Active() {
|
||||
return false, nil
|
||||
}
|
||||
j.Status, j.Error, j.FinishedAt = model.JobFailed, msg, &at
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *memStore) GetJob(_ context.Context, id int64) (model.Job, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
j, ok := m.jobs[id]
|
||||
if !ok {
|
||||
return model.Job{}, model.ErrNotFound
|
||||
}
|
||||
return *j, nil
|
||||
}
|
||||
|
||||
func (m *memStore) FailLostJobs(_ context.Context, agentID int64, known []int64, at time.Time) ([]int64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var lost []int64
|
||||
for _, j := range m.jobs {
|
||||
if j.AgentID == nil || *j.AgentID != agentID || !j.Active() {
|
||||
continue
|
||||
}
|
||||
isKnown := false
|
||||
for _, k := range known {
|
||||
isKnown = isKnown || k == j.ID
|
||||
}
|
||||
if !isKnown {
|
||||
j.Status, j.Error, j.FinishedAt = model.JobFailed, "lost: the agent did not know this job when it reconnected", &at
|
||||
lost = append(lost, j.ID)
|
||||
}
|
||||
}
|
||||
return lost, nil
|
||||
}
|
||||
|
||||
func (m *memStore) JobProgress(_ context.Context, agentID, jobID int64, chunk string, at time.Time) (bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
j, ok := m.jobs[jobID]
|
||||
if !ok || j.AgentID == nil || *j.AgentID != agentID || !j.Active() {
|
||||
return false, nil
|
||||
}
|
||||
j.Status, j.Output = model.JobRunning, j.Output+chunk
|
||||
if j.StartedAt == nil {
|
||||
j.StartedAt = &at
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *memStore) FinishJob(_ context.Context, agentID, jobID int64, f model.JobFinish) (bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
j, ok := m.jobs[jobID]
|
||||
if !ok || j.AgentID == nil || *j.AgentID != agentID || !j.Active() {
|
||||
return false, nil
|
||||
}
|
||||
j.Status, j.Error, j.Output, j.OutputTruncated, j.FinishedAt = f.Status, f.Error, f.Output, f.OutputTruncated, &f.FinishedAt
|
||||
if f.Ref != "" {
|
||||
j.ToRef = f.Ref
|
||||
}
|
||||
if j.StartedAt == nil {
|
||||
j.StartedAt = &f.StartedAt
|
||||
}
|
||||
if f.Status == model.JobSucceeded {
|
||||
svc := m.svcRows[j.ServiceID]
|
||||
svc.DesiredRef = f.Ref
|
||||
m.svcRows[j.ServiceID] = svc
|
||||
}
|
||||
m.finishApplied++
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *memStore) AgentByTokenHash(_ context.Context, hash string) (model.Agent, error) {
|
||||
@@ -112,8 +236,21 @@ func (s *staticSource) set(v ...protocol.Service) {
|
||||
type testMaster struct {
|
||||
url string
|
||||
fingerprint string
|
||||
hub *hub.Hub
|
||||
hub *currentHub // the hub behind /agent; follows restart()
|
||||
attempts *atomic.Int64 // TCP connections accepted by the server
|
||||
broker *events.Broker
|
||||
restart func() // replaces the hub and drops every connection
|
||||
}
|
||||
|
||||
// currentHub forwards to whichever hub is live, so a master restart can swap
|
||||
// the hub without the mux or the jobs service noticing.
|
||||
type currentHub struct{ p atomic.Pointer[hub.Hub] }
|
||||
|
||||
func (c *currentHub) ServeHTTP(w http.ResponseWriter, r *http.Request) { c.p.Load().ServeHTTP(w, r) }
|
||||
func (c *currentHub) Disconnect(id int64) { c.p.Load().Disconnect(id) }
|
||||
func (c *currentHub) Route(svc model.Service) (model.Route, error) { return c.p.Load().Route(svc) }
|
||||
func (c *currentHub) StartJob(r model.Route, id int64, s protocol.JobStart) error {
|
||||
return c.p.Load().StartJob(r, id, s)
|
||||
}
|
||||
|
||||
func startMaster(t *testing.T, store *memStore) testMaster {
|
||||
@@ -125,7 +262,10 @@ func startMaster(t *testing.T, store *memStore) testMaster {
|
||||
fingerprint, err := pin.Fingerprint(key.Public())
|
||||
require.NoError(t, err)
|
||||
|
||||
h := hub.New(store, events.NewBroker(), slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
quiet := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
broker := events.NewBroker()
|
||||
h := ¤tHub{}
|
||||
h.p.Store(hub.New(store, broker, quiet))
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("GET /agent", h)
|
||||
attempts := &atomic.Int64{}
|
||||
@@ -139,14 +279,27 @@ func startMaster(t *testing.T, store *memStore) testMaster {
|
||||
srv.TLS = &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13}
|
||||
srv.StartTLS()
|
||||
t.Cleanup(srv.Close)
|
||||
return testMaster{url: "wss" + strings.TrimPrefix(srv.URL, "https") + "/agent", fingerprint: fingerprint, hub: h, attempts: attempts}
|
||||
restart := func() {
|
||||
// The hub holds no job state, so a new hub over the same store is a
|
||||
// faithful master restart.
|
||||
old := h.p.Swap(hub.New(store, broker, quiet))
|
||||
// CloseClientConnections skips hijacked (WebSocket) connections, so
|
||||
// cut each agent through the old hub.
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
for _, a := range store.agents {
|
||||
old.Disconnect(a.ID)
|
||||
}
|
||||
}
|
||||
return testMaster{url: "wss" + strings.TrimPrefix(srv.URL, "https") + "/agent", fingerprint: fingerprint,
|
||||
hub: h, attempts: attempts, broker: broker, restart: restart}
|
||||
}
|
||||
|
||||
func runAgent(t *testing.T, url, fingerprint, token string, src conn.Source) *conn.Agent {
|
||||
t.Helper()
|
||||
a := conn.New(conn.Config{URL: url, Token: token, Fingerprint: fingerprint, Version: "e2e",
|
||||
MinBackoff: 10 * time.Millisecond, MaxBackoff: 50 * time.Millisecond,
|
||||
}, src, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
}, src, updater.NewRunner(nil, time.Minute), slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { _ = a.Run(ctx); close(done) }()
|
||||
|
||||
Reference in New Issue
Block a user