shcizo/package-updater
1
0
Fork 0
Code Issues 8 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(master): refuse jobs for revoked agents, signal revoked jobs, agent start time wins (#7)

Browse Source
This commit is contained in:
shcizo
2026-10-12 19:14:32 +02:00
parent 93ba6915b9
commit c09e774d74
6 changed files with 138 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/master/store/jobs.go
+49 -24
View File
@@ -19,7 +19,8 @@ const lostJobMessage = "lost: the agent did not know this job when it reconnecte
// message $2 at time $3.
const failActiveJobsSQL = `
UPDATE update_jobs SET status = 'failed', error = $2, finished_at = $3
WHERE agent_id = $1 AND status IN ('queued', 'running')`
WHERE agent_id = $1 AND status IN ('queued', 'running')
RETURNING id`
const jobSelectSQL = `
SELECT j.id, j.service_id, s.kind, s.compose_project, s.compose_service, s.swarm_service, s.container_name,
@@ -56,27 +57,44 @@ func (s *Store) getJob(ctx context.Context, id int64) (model.Job, error) {
// CreateJob inserts a queued job. If the service already has a queued or
// running job, it returns that job and model.ErrJobActive (detected via the
// unique index, so it holds under concurrent calls).
// unique index, so it holds under concurrent calls). A job for a revoked
// agent is refused with model.ErrAgentOffline: the agent's revoke fails its
// jobs once, so a job created afterwards would never be settled and would
// block the service. The share lock on the agent row makes a concurrent
// revoke wait for this insert, so its job cleanup sees the new row.
func (s *Store) CreateJob(ctx context.Context, j model.NewJob) (model.Job, error) {
var id int64
err := s.pool.QueryRow(ctx, `
INSERT INTO update_jobs (service_id, agent_id, from_ref, to_ref, trigger, requested_by, status, created_at)
VALUES ($1, $2, $3, $4, $5, $6, 'queued', $7)
RETURNING id`,
j.ServiceID, j.AgentID, j.FromRef, j.ToRef, string(j.Trigger), j.RequestedBy, j.CreatedAt).Scan(&id)
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "23505" && pgErr.ConstraintName == "update_jobs_one_active" {
active, err := scanJob(s.pool.QueryRow(ctx, fmt.Sprintf(jobSelectSQL, "j.output")+
` WHERE j.service_id = $1 AND j.status IN ('queued', 'running')`, j.ServiceID))
if err != nil {
return model.Job{}, fmt.Errorf("find active job: %w", err)
// The active job can finish between the unique violation and the lookup
// below; then the insert is simply tried once more.
for attempt := 0; attempt < 2; attempt++ {
var id int64
err := s.pool.QueryRow(ctx, `
INSERT INTO update_jobs (service_id, agent_id, from_ref, to_ref, trigger, requested_by, status, created_at)
SELECT $1::bigint, $2::bigint, $3::text, $4::text, $5::text, $6::text, 'queued', $7::timestamptz
WHERE $2::bigint IS NULL
OR EXISTS (SELECT 1 FROM agents WHERE id = $2 AND revoked_at IS NULL FOR SHARE)
RETURNING id`,
j.ServiceID, j.AgentID, j.FromRef, j.ToRef, string(j.Trigger), j.RequestedBy, j.CreatedAt).Scan(&id)
if errors.Is(err, pgx.ErrNoRows) {
return model.Job{}, model.ErrAgentOffline
}
return active, model.ErrJobActive
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "23505" && pgErr.ConstraintName == "update_jobs_one_active" {
active, err := scanJob(s.pool.QueryRow(ctx, fmt.Sprintf(jobSelectSQL, "j.output")+
` WHERE j.service_id = $1 AND j.status IN ('queued', 'running')`, j.ServiceID))
if errors.Is(err, pgx.ErrNoRows) {
continue
}
if err != nil {
return model.Job{}, fmt.Errorf("find active job: %w", err)
}
return active, model.ErrJobActive
}
if err != nil {
return model.Job{}, fmt.Errorf("create job: %w", err)
}
return s.getJob(ctx, id)
}
if err != nil {
return model.Job{}, fmt.Errorf("create job: %w", err)
}
return s.getJob(ctx, id)
return model.Job{}, fmt.Errorf("create job: active job vanished twice")
}
// FailJob sets a queued or running job to failed with msg and finished_at =
@@ -167,9 +185,10 @@ func (s *Store) JobProgress(ctx context.Context, agentID, jobID int64, chunk str
// FinishJob records the agent's result for a queued or running job sent to
// agentID: status, error, output (replacing what progress assembled),
// started_at (kept if already set), finished_at; to_ref = f.Ref when set. On
// success it also sets the service's desired_ref = f.Ref, in the same
// transaction. Returns false if no such active job belongs to agentID.
// started_at (the agent's, or the existing one if the agent sent none),
// finished_at; to_ref = f.Ref when set. On success it also sets the
// service's desired_ref = f.Ref, in the same transaction. Returns false if no
// such active job belongs to agentID.
func (s *Store) FinishJob(ctx context.Context, agentID, jobID int64, f model.JobFinish) (bool, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
@@ -177,15 +196,21 @@ func (s *Store) FinishJob(ctx context.Context, agentID, jobID int64, f model.Job
}
defer func() { _ = tx.Rollback(ctx) }()
// The agent's clock wins when it sent a start time; otherwise the time the
// first progress arrived stays.
var startedAt *time.Time
if !f.StartedAt.IsZero() {
startedAt = &f.StartedAt
}
var serviceID int64
err = tx.QueryRow(ctx, `
UPDATE update_jobs SET
status = $3, error = $4, output = $5, output_truncated = $6,
to_ref = CASE WHEN $7::text <> '' THEN $7::text ELSE to_ref END,
started_at = COALESCE(started_at, $8), finished_at = $9
started_at = COALESCE($8, started_at), finished_at = $9
WHERE id = $1 AND agent_id = $2 AND status IN ('queued', 'running')
RETURNING service_id`,
jobID, agentID, string(f.Status), f.Error, f.Output, f.OutputTruncated, f.Ref, f.StartedAt, f.FinishedAt).Scan(&serviceID)
jobID, agentID, string(f.Status), f.Error, f.Output, f.OutputTruncated, f.Ref, startedAt, f.FinishedAt).Scan(&serviceID)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}