package updater import ( "context" "errors" "fmt" "io" "sort" "sync" "time" "github.com/shcizo/package-updater/internal/protocol" ) // maxProgress caps the output buffered between two TakeProgress calls, so a // chatty job cannot grow a single job.progress message without bound. const maxProgress = 32 << 10 // Executor deploys one job and returns the "repo:tag@digest" it deployed. // Everything it writes to out is streamed to the master as job output. type Executor interface { Execute(ctx context.Context, job protocol.JobStart, out io.Writer) (string, error) } // Progress is output a running job produced since the previous TakeProgress. type Progress struct{ ID, Output string } // Finished is a job's outcome, kept until the master acknowledges it. type Finished struct { ID string Result protocol.JobResult } // tailBuffer keeps the last max bytes written to it. type tailBuffer struct { max int buf []byte truncated bool } func (b *tailBuffer) Write(p []byte) { b.buf = append(b.buf, p...) if over := len(b.buf) - b.max; over > 0 { b.buf = append(b.buf[:0], b.buf[over:]...) b.truncated = true } } func (b *tailBuffer) String() string { return string(b.buf) } func (b *tailBuffer) Truncated() bool { return b.truncated } func (b *tailBuffer) Reset() { b.buf, b.truncated = nil, false } type jobState struct { start protocol.JobStart running bool result *protocol.JobResult output tailBuffer // everything, capped at protocol.MaxJobOutput unsent tailBuffer // since the last TakeProgress, capped at maxProgress reported bool // a Progress has been taken since the job started finished time.Time // orders Pending } // Runner executes jobs one at a time in the order they arrive and keeps each // job's result until Ack. Two docker compose calls against the same stack // would race, so there is deliberately exactly one worker. // // Runner satisfies conn.Jobs. type Runner struct { exec Executor timeout time.Duration now func() time.Time mu sync.Mutex // guards jobs and queue jobs map[string]*jobState queue []string wake chan struct{} // wakes the worker changed chan struct{} // tells the connection there is something to send } // NewRunner returns a Runner that gives every job at most timeout. Call Run // to start the worker. func NewRunner(exec Executor, timeout time.Duration) *Runner { return &Runner{ exec: exec, timeout: timeout, now: time.Now, jobs: make(map[string]*jobState), wake: make(chan struct{}, 1), changed: make(chan struct{}, 1), } } // signal does a non-blocking send so neither side ever waits on the other. func signal(ch chan struct{}) { select { case ch <- struct{}{}: default: } } // Submit queues a job. A job whose ID the runner already holds (queued, // running, or finished and not yet acknowledged) is ignored. func (r *Runner) Submit(id string, start protocol.JobStart) { r.mu.Lock() if _, ok := r.jobs[id]; ok { r.mu.Unlock() return } r.jobs[id] = &jobState{ start: start, output: tailBuffer{max: protocol.MaxJobOutput}, unsent: tailBuffer{max: maxProgress}, } r.queue = append(r.queue, id) r.mu.Unlock() signal(r.wake) } // Ack forgets a finished job. Acks for unknown, queued or running jobs are // ignored. func (r *Runner) Ack(id string) { r.mu.Lock() defer r.mu.Unlock() if j, ok := r.jobs[id]; ok && j.result != nil { delete(r.jobs, id) } } // Known returns the sorted IDs of every job the runner holds. func (r *Runner) Known() []string { r.mu.Lock() defer r.mu.Unlock() ids := make([]string, 0, len(r.jobs)) for id := range r.jobs { ids = append(ids, id) } sort.Strings(ids) return ids } // TakeProgress returns the output running jobs produced since the previous // call. A job's first entry (possibly with empty Output) tells the master it // is running. func (r *Runner) TakeProgress() []Progress { r.mu.Lock() defer r.mu.Unlock() var out []Progress for id, j := range r.jobs { if !j.running { continue } s := j.unsent.String() if j.reported && s == "" { continue } j.reported = true j.unsent.Reset() out = append(out, Progress{ID: id, Output: s}) } sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID }) return out } // Pending returns the results not yet acknowledged, oldest first. func (r *Runner) Pending() []Finished { r.mu.Lock() defer r.mu.Unlock() var out []Finished var at []time.Time for id, j := range r.jobs { if j.result != nil { out = append(out, Finished{ID: id, Result: *j.result}) at = append(at, j.finished) } } idx := make([]int, len(out)) for i := range idx { idx[i] = i } sort.Slice(idx, func(a, b int) bool { if !at[idx[a]].Equal(at[idx[b]]) { return at[idx[a]].Before(at[idx[b]]) } return out[idx[a]].ID < out[idx[b]].ID }) sorted := make([]Finished, len(out)) for i, k := range idx { sorted[i] = out[k] } return sorted } // Changed returns a channel that receives when there is new progress or a // new result to send. func (r *Runner) Changed() <-chan struct{} { return r.changed } // Run works through the queue until ctx ends. Jobs still queued then are // dropped; the master marks them lost on the agent's next hello. func (r *Runner) Run(ctx context.Context) { for { id, start, ok := r.next() if !ok { select { case <-ctx.Done(): return case <-r.wake: continue } } if ctx.Err() != nil { return } r.runOne(ctx, id, start) } } // next pops the next queued job and marks it running. func (r *Runner) next() (string, protocol.JobStart, bool) { r.mu.Lock() defer r.mu.Unlock() if len(r.queue) == 0 { return "", protocol.JobStart{}, false } id := r.queue[0] r.queue = r.queue[1:] j := r.jobs[id] j.running = true signal(r.changed) return id, j.start, true } func (r *Runner) runOne(ctx context.Context, id string, start protocol.JobStart) { started := r.now() jobCtx, cancel := context.WithTimeout(ctx, r.timeout) ref, err := r.exec.Execute(jobCtx, start, &jobWriter{r: r, id: id}) timedOut := errors.Is(jobCtx.Err(), context.DeadlineExceeded) cancel() res := protocol.JobResult{Status: protocol.ResultSucceeded, Ref: ref, StartedAt: started, FinishedAt: r.now()} switch { case timedOut: res = protocol.JobResult{Status: protocol.ResultFailed, Error: fmt.Sprintf("timed out after %s", r.timeout), StartedAt: started, FinishedAt: res.FinishedAt} case err != nil: res = protocol.JobResult{Status: protocol.ResultFailed, Error: err.Error(), StartedAt: started, FinishedAt: res.FinishedAt} } r.mu.Lock() j := r.jobs[id] res.Output, res.OutputTruncated = j.output.String(), j.output.Truncated() j.running, j.result, j.finished = false, &res, res.FinishedAt j.unsent.Reset() r.mu.Unlock() signal(r.changed) } // jobWriter feeds an executor's output into the job's buffers. type jobWriter struct { r *Runner id string } func (w *jobWriter) Write(p []byte) (int, error) { w.r.mu.Lock() if j, ok := w.r.jobs[w.id]; ok { j.output.Write(p) j.unsent.Write(p) } w.r.mu.Unlock() signal(w.r.changed) return len(p), nil }