feat(agent): job runner, compose override executor and swarm executor (#7)
This commit is contained in:
@@ -0,0 +1,272 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/shcizo/package-updater/internal/protocol"
|
||||
)
|
||||
|
||||
// maxProgress caps the output buffered between two TakeProgress calls, so a
|
||||
// chatty job cannot grow a single job.progress message without bound.
|
||||
const maxProgress = 32 << 10
|
||||
|
||||
// Executor deploys one job and returns the "repo:tag@digest" it deployed.
|
||||
// Everything it writes to out is streamed to the master as job output.
|
||||
type Executor interface {
|
||||
Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error)
|
||||
}
|
||||
|
||||
// Progress is output a running job produced since the previous TakeProgress.
|
||||
type Progress struct{ ID, Output string }
|
||||
|
||||
// Finished is a job's outcome, kept until the master acknowledges it.
|
||||
type Finished struct {
|
||||
ID string
|
||||
Result protocol.JobResult
|
||||
}
|
||||
|
||||
// tailBuffer keeps the last max bytes written to it.
|
||||
type tailBuffer struct {
|
||||
max int
|
||||
buf []byte
|
||||
truncated bool
|
||||
}
|
||||
|
||||
func (b *tailBuffer) Write(p []byte) {
|
||||
b.buf = append(b.buf, p...)
|
||||
if over := len(b.buf) - b.max; over > 0 {
|
||||
b.buf = append(b.buf[:0], b.buf[over:]...)
|
||||
b.truncated = true
|
||||
}
|
||||
}
|
||||
|
||||
func (b *tailBuffer) String() string { return string(b.buf) }
|
||||
func (b *tailBuffer) Truncated() bool { return b.truncated }
|
||||
func (b *tailBuffer) Reset() { b.buf, b.truncated = nil, false }
|
||||
|
||||
type jobState struct {
|
||||
start protocol.JobStart
|
||||
running bool
|
||||
result *protocol.JobResult
|
||||
output tailBuffer // everything, capped at protocol.MaxJobOutput
|
||||
unsent tailBuffer // since the last TakeProgress, capped at maxProgress
|
||||
reported bool // a Progress has been taken since the job started
|
||||
finished time.Time // orders Pending
|
||||
}
|
||||
|
||||
// Runner executes jobs one at a time in the order they arrive and keeps each
|
||||
// job's result until Ack. Two docker compose calls against the same stack
|
||||
// would race, so there is deliberately exactly one worker.
|
||||
//
|
||||
// Runner satisfies conn.Jobs.
|
||||
type Runner struct {
|
||||
exec Executor
|
||||
timeout time.Duration
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex // guards jobs and queue
|
||||
jobs map[string]*jobState
|
||||
queue []string
|
||||
|
||||
wake chan struct{} // wakes the worker
|
||||
changed chan struct{} // tells the connection there is something to send
|
||||
}
|
||||
|
||||
// NewRunner returns a Runner that gives every job at most timeout. Call Run
|
||||
// to start the worker.
|
||||
func NewRunner(exec Executor, timeout time.Duration) *Runner {
|
||||
return &Runner{
|
||||
exec: exec,
|
||||
timeout: timeout,
|
||||
now: time.Now,
|
||||
jobs: make(map[string]*jobState),
|
||||
wake: make(chan struct{}, 1),
|
||||
changed: make(chan struct{}, 1),
|
||||
}
|
||||
}
|
||||
|
||||
// signal does a non-blocking send so neither side ever waits on the other.
|
||||
func signal(ch chan struct{}) {
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Submit queues a job. A job whose ID the runner already holds (queued,
|
||||
// running, or finished and not yet acknowledged) is ignored.
|
||||
func (r *Runner) Submit(id string, start protocol.JobStart) {
|
||||
r.mu.Lock()
|
||||
if _, ok := r.jobs[id]; ok {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.jobs[id] = &jobState{
|
||||
start: start,
|
||||
output: tailBuffer{max: protocol.MaxJobOutput},
|
||||
unsent: tailBuffer{max: maxProgress},
|
||||
}
|
||||
r.queue = append(r.queue, id)
|
||||
r.mu.Unlock()
|
||||
signal(r.wake)
|
||||
}
|
||||
|
||||
// Ack forgets a finished job. Acks for unknown, queued or running jobs are
|
||||
// ignored.
|
||||
func (r *Runner) Ack(id string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if j, ok := r.jobs[id]; ok && j.result != nil {
|
||||
delete(r.jobs, id)
|
||||
}
|
||||
}
|
||||
|
||||
// Known returns the sorted IDs of every job the runner holds.
|
||||
func (r *Runner) Known() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
ids := make([]string, 0, len(r.jobs))
|
||||
for id := range r.jobs {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
// TakeProgress returns the output running jobs produced since the previous
|
||||
// call. A job's first entry (possibly with empty Output) tells the master it
|
||||
// is running.
|
||||
func (r *Runner) TakeProgress() []Progress {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var out []Progress
|
||||
for id, j := range r.jobs {
|
||||
if !j.running {
|
||||
continue
|
||||
}
|
||||
s := j.unsent.String()
|
||||
if j.reported && s == "" {
|
||||
continue
|
||||
}
|
||||
j.reported = true
|
||||
j.unsent.Reset()
|
||||
out = append(out, Progress{ID: id, Output: s})
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID })
|
||||
return out
|
||||
}
|
||||
|
||||
// Pending returns the results not yet acknowledged, oldest first.
|
||||
func (r *Runner) Pending() []Finished {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var out []Finished
|
||||
var at []time.Time
|
||||
for id, j := range r.jobs {
|
||||
if j.result != nil {
|
||||
out = append(out, Finished{ID: id, Result: *j.result})
|
||||
at = append(at, j.finished)
|
||||
}
|
||||
}
|
||||
idx := make([]int, len(out))
|
||||
for i := range idx {
|
||||
idx[i] = i
|
||||
}
|
||||
sort.Slice(idx, func(a, b int) bool {
|
||||
if !at[idx[a]].Equal(at[idx[b]]) {
|
||||
return at[idx[a]].Before(at[idx[b]])
|
||||
}
|
||||
return out[idx[a]].ID < out[idx[b]].ID
|
||||
})
|
||||
sorted := make([]Finished, len(out))
|
||||
for i, k := range idx {
|
||||
sorted[i] = out[k]
|
||||
}
|
||||
return sorted
|
||||
}
|
||||
|
||||
// Changed returns a channel that receives when there is new progress or a
|
||||
// new result to send.
|
||||
func (r *Runner) Changed() <-chan struct{} { return r.changed }
|
||||
|
||||
// Run works through the queue until ctx ends. Jobs still queued then are
|
||||
// dropped; the master marks them lost on the agent's next hello.
|
||||
func (r *Runner) Run(ctx context.Context) {
|
||||
for {
|
||||
id, start, ok := r.next()
|
||||
if !ok {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-r.wake:
|
||||
continue
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
r.runOne(ctx, id, start)
|
||||
}
|
||||
}
|
||||
|
||||
// next pops the next queued job and marks it running.
|
||||
func (r *Runner) next() (string, protocol.JobStart, bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if len(r.queue) == 0 {
|
||||
return "", protocol.JobStart{}, false
|
||||
}
|
||||
id := r.queue[0]
|
||||
r.queue = r.queue[1:]
|
||||
j := r.jobs[id]
|
||||
j.running = true
|
||||
signal(r.changed)
|
||||
return id, j.start, true
|
||||
}
|
||||
|
||||
func (r *Runner) runOne(ctx context.Context, id string, start protocol.JobStart) {
|
||||
started := r.now()
|
||||
jobCtx, cancel := context.WithTimeout(ctx, r.timeout)
|
||||
ref, err := r.exec.Execute(jobCtx, start, &jobWriter{r: r, id: id})
|
||||
timedOut := errors.Is(jobCtx.Err(), context.DeadlineExceeded)
|
||||
cancel()
|
||||
|
||||
res := protocol.JobResult{Status: protocol.ResultSucceeded, Ref: ref, StartedAt: started, FinishedAt: r.now()}
|
||||
switch {
|
||||
case timedOut:
|
||||
res = protocol.JobResult{Status: protocol.ResultFailed, Error: fmt.Sprintf("timed out after %s", r.timeout), StartedAt: started, FinishedAt: res.FinishedAt}
|
||||
case err != nil:
|
||||
res = protocol.JobResult{Status: protocol.ResultFailed, Error: err.Error(), StartedAt: started, FinishedAt: res.FinishedAt}
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
j := r.jobs[id]
|
||||
res.Output, res.OutputTruncated = j.output.String(), j.output.Truncated()
|
||||
j.running, j.result, j.finished = false, &res, res.FinishedAt
|
||||
j.unsent.Reset()
|
||||
r.mu.Unlock()
|
||||
signal(r.changed)
|
||||
}
|
||||
|
||||
// jobWriter feeds an executor's output into the job's buffers.
|
||||
type jobWriter struct {
|
||||
r *Runner
|
||||
id string
|
||||
}
|
||||
|
||||
func (w *jobWriter) Write(p []byte) (int, error) {
|
||||
w.r.mu.Lock()
|
||||
if j, ok := w.r.jobs[w.id]; ok {
|
||||
j.output.Write(p)
|
||||
j.unsent.Write(p)
|
||||
}
|
||||
w.r.mu.Unlock()
|
||||
signal(w.r.changed)
|
||||
return len(p), nil
|
||||
}
|
||||
Reference in New Issue
Block a user